Add actions to reweight and remove OSDs.

Adds action change-osd-weight, to allow changing the weight of an OSD.

Adds action purge-osd, to remove an OSD entirely from the crush map.

Change-Id: Ic1f468c96931a136ba897d4a02b8cd43dfdf056b
Closes-bug: #1761048

README.md

@@ -147,6 +147,7 @@ Actions allow specific operations to be performed on a per-unit basis. To
 display action descriptions run `juju actions ceph-mon`. If the charm is not
 deployed then see file `actions.yaml`.
 
+* `change-osd-weight`
 * `copy-pool`
 * `create-cache-tier`
 * `create-crush-rule`
@@ -163,6 +164,7 @@ deployed then see file `actions.yaml`.
 * `pool-get`
 * `pool-set`
 * `pool-statistics`
+* `purge-osd`
 * `remove-cache-tier`
 * `remove-pool-snapshot`
 * `rename-pool`

actions.yaml

@@ -350,3 +350,27 @@ unset-noout:
   description: "Unset ceph noout across the cluster."
 security-checklist:
   description: Validate the running configuration against the OpenStack security guides checklist
+purge-osd:
+  description: "Removes an OSD from a cluster map, removes its authentication key, removes the OSD from the OSD map. The OSD must have zero weight before running this action, to avoid excessive I/O on the cluster."
+  params:
+    osd:
+      type: integer
+      description: "ID of the OSD to remove, e.g. for osd.53, supply 53."
+    i-really-mean-it:
+      type: boolean
+      description: "This must be toggled to enable actually performing this action."
+  required:
+    - osd
+    - i-really-mean-it
+change-osd-weight:
+  description: "Set the crush weight of an OSD to the new value supplied."
+  params:
+    osd:
+      type: integer
+      description: "ID of the OSD to operate on, e.g. for osd.53, supply 53."
+    weight:
+      type: number
+      description: "The new weight of the OSD, must be a decimal number, e.g. 1.04"
+  required:
+    - osd
+    - weight

actions/change-osd-weight

@@ -0,0 +1 @@
+change_osd_weight.py

actions/change_osd_weight.py

@@ -0,0 +1,45 @@
+#! /usr/bin/env python3
+#
+# Copyright 2020 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Changes the crush weight of an OSD."""
+
+import sys
+
+sys.path.append("lib")
+sys.path.append("hooks")
+
+from charmhelpers.core.hookenv import function_fail, function_get, log
+from charms_ceph.utils import reweight_osd
+
+
+def crush_reweight(osd_num, new_weight):
+    """Run reweight_osd to change OSD weight."""
+    try:
+        result = reweight_osd(str(osd_num), str(new_weight))
+    except Exception as e:
+        log(e)
+        function_fail("Reweight failed due to exception")
+        return
+
+    if not result:
+        function_fail("Reweight failed to complete")
+        return
+
+
+if __name__ == "__main__":
+    osd_num = function_get("osd")
+    new_weight = function_get("weight")
+    crush_reweight(osd_num, new_weight)

actions/purge-osd

@@ -0,0 +1 @@
+purge_osd.py

actions/purge_osd.py

@@ -0,0 +1,90 @@
+#! /usr/bin/env python3
+#
+# Copyright 2020 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Removes an OSD from a cluster map.
+
+Runs the ceph osd purge command, or earlier equivalents, removing an OSD from
+the cluster map, removes its authentication key, removes the OSD from the OSD
+map.
+"""
+
+from subprocess import (
+    check_call,
+    CalledProcessError,
+)
+
+import sys
+sys.path.append('lib')
+sys.path.append('hooks')
+
+
+from charmhelpers.core.hookenv import (
+    function_get,
+    log,
+    function_fail
+)
+from charmhelpers.core.host import cmp_pkgrevno
+from charmhelpers.contrib.storage.linux import ceph
+from charms_ceph.utils import get_osd_weight
+
+
+def purge_osd(osd):
+    """Run the OSD purge action.
+
+    :param osd: the OSD ID to operate on
+    """
+    svc = 'admin'
+    osd_str = str(osd)
+    osd_name = "osd.{}".format(osd_str)
+    current_osds = ceph.get_osds(svc)
+    if osd not in current_osds:
+        function_fail("OSD {} is not in the current list of OSDs".format(osd))
+        return
+
+    osd_weight = get_osd_weight(osd_name)
+    if osd_weight > 0:
+        function_fail("OSD has weight {}, must have zero weight before "
+                      "this operation".format(osd_weight))
+        return
+
+    luminous_or_later = cmp_pkgrevno('ceph-common', '12.0.0') >= 0
+    if not function_get('i-really-mean-it'):
+        function_fail('i-really-mean-it is a required parameter')
+        return
+    if luminous_or_later:
+        cmds = [
+            ["ceph", "osd", "out", osd_name],
+            ['ceph', 'osd', 'purge', osd_str, '--yes-i-really-mean-it']
+        ]
+    else:
+        cmds = [
+            ["ceph", "osd", "out", osd_name],
+            ["ceph", "osd", "crush", "remove", "osd.{}".format(osd)],
+            ["ceph", "auth", "del", osd_name],
+            ['ceph', 'osd', 'rm', osd_str],
+        ]
+    for cmd in cmds:
+        try:
+            check_call(cmd)
+        except CalledProcessError as e:
+            log(e)
+            function_fail("OSD Purge for OSD {} failed".format(osd))
+            return
+
+
+if __name__ == '__main__':
+    osd = function_get("osd")
+    purge_osd(osd)

unit_tests/test_action_change_osd_weight.py

@@ -0,0 +1,38 @@
+# Copyright 2020 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for reweight_osd action."""
+
+from actions import change_osd_weight as action
+from mock import mock
+from test_utils import CharmTestCase
+
+
+class ReweightTestCase(CharmTestCase):
+    """Run tests for action."""
+
+    def setUp(self):
+        """Init mocks for test cases."""
+        super(ReweightTestCase, self).setUp(
+            action, ["function_get", "function_fail"]
+        )
+
+    @mock.patch("actions.change_osd_weight.reweight_osd")
+    def test_reweight_osd(self, _reweight_osd):
+        """Test reweight_osd action has correct calls."""
+        _reweight_osd.return_value = True
+        osd_num = 4
+        new_weight = 1.2
+        action.crush_reweight(osd_num, new_weight)
+        print(_reweight_osd.calls)
+        _reweight_osd.assert_has_calls([mock.call("4", "1.2")])

unit_tests/test_action_purge_osd.py

@@ -0,0 +1,74 @@
+# Copyright 2020 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for purge_osd action."""
+
+from actions import purge_osd as action
+from mock import mock
+from test_utils import CharmTestCase
+
+
+class PurgeTestCase(CharmTestCase):
+    """Run tests for action."""
+
+    def setUp(self):
+        """Init mocks for test cases."""
+        super(PurgeTestCase, self).setUp(
+            action, ["check_call", "function_get", "function_fail", "open"]
+        )
+
+    @mock.patch("actions.purge_osd.get_osd_weight")
+    @mock.patch("actions.purge_osd.cmp_pkgrevno")
+    @mock.patch("charmhelpers.contrib.storage.linux.ceph.get_osds")
+    def test_purge_osd(self, _get_osds, _cmp_pkgrevno, _get_osd_weight):
+        """Test purge_osd action has correct calls."""
+        _get_osds.return_value = [0, 1, 2, 3, 4, 5]
+        _cmp_pkgrevno.return_value = 1
+        _get_osd_weight.return_value = 0
+        osd = 4
+        action.purge_osd(osd)
+        cmds = [
+            mock.call(["ceph", "osd", "out", "osd.4"]),
+            mock.call(
+                ["ceph", "osd", "purge", str(osd), "--yes-i-really-mean-it"]
+            ),
+        ]
+        self.check_call.assert_has_calls(cmds)
+
+    @mock.patch("actions.purge_osd.get_osd_weight")
+    @mock.patch("actions.purge_osd.cmp_pkgrevno")
+    @mock.patch("charmhelpers.contrib.storage.linux.ceph.get_osds")
+    def test_purge_invalid_osd(
+        self, _get_osds, _cmp_pkgrevno, _get_osd_weight
+    ):
+        """Test purge_osd action captures bad OSD string."""
+        _get_osds.return_value = [0, 1, 2, 3, 4, 5]
+        _cmp_pkgrevno.return_value = 1
+        _get_osd_weight.return_value = 0
+        osd = 99
+        action.purge_osd(osd)
+        self.function_fail.assert_called()
+
+    @mock.patch("actions.purge_osd.get_osd_weight")
+    @mock.patch("actions.purge_osd.cmp_pkgrevno")
+    @mock.patch("charmhelpers.contrib.storage.linux.ceph.get_osds")
+    def test_purge_osd_weight_high(
+        self, _get_osds, _cmp_pkgrevno, _get_osd_weight
+    ):
+        """Test purge_osd action fails when OSD has weight >0."""
+        _get_osds.return_value = [0, 1, 2, 3, 4, 5]
+        _cmp_pkgrevno.return_value = 1
+        _get_osd_weight.return_value = 2.5
+        osd = "4"
+        action.purge_osd(osd)
+        self.function_fail.assert_called()