Implement the 'list-inconsistent-objs' action

Previously, if we wanted to list the inconsistent objects per PG,
we needed to call 'ceph health detail' to get the list of inconsistent
PGs and then 'rados list-inconsistent-obj $PG' for each PG. This patch
aims to implement a single action that does all that and formats it
as a pretty JSON.

Closes-Bug: #1931751
Change-Id: I05bf90ff274e4f1b7ee9e278d62894b68ba2e787

README.md

@@ -159,6 +159,7 @@ deployed then see file `actions.yaml`.
 * `get-erasure-profile`
 * `get-health`
 * `list-erasure-profiles`
+* `list-inconsistent-objs`
 * `list-pools`
 * `pause-health`
 * `pool-get`

actions.yaml

@@ -208,6 +208,18 @@ delete-erasure-profile:
 list-erasure-profiles:
   description: "List the names of all erasure code profiles"
   additionalProperties: false
+list-inconsistent-objs:
+  description: "List the names of the inconsistent objecs per PG"
+  params:
+    format:
+      type: string
+      enum:
+        - json
+        - yaml
+        - text
+      default: text
+      description: "The output format, either json, yaml or text (default)"
+  additionalProperties: false
 list-pools:
   description: "List your cluster's pools"
   additionalProperties: false

actions/list-inconsistent-objs

@@ -0,0 +1 @@
+list_inconsistent_objs.py

actions/list_inconsistent_objs.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+#
+# Copyright 2021 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import re
+import sys
+from subprocess import check_output, CalledProcessError
+import yaml
+
+sys.path.append('hooks')
+
+from charmhelpers.core.hookenv import function_fail, function_get, \
+    function_set, log
+
+
+VALID_FORMATS = ('text', 'json', 'yaml')
+
+
+def get_health_detail():
+    return check_output(['ceph', 'health', 'detail']).decode('UTF-8')
+
+
+def get_rados_inconsistent(pg):
+    return check_output(['rados', 'list-inconsistent-obj', pg]).decode('UTF-8')
+
+
+def get_inconsistent_objs():
+    # For the call to 'ceph health detail' we are interested in
+    # lines with the form:
+    # pg $PG is ...inconsistent...
+    rx = re.compile('pg (\\S+) .+inconsistent')
+    out = get_health_detail()
+    msg = {}   # Maps PG -> object name list.
+
+    for line in out.split('\n'):
+        res = rx.search(line)
+        if res is None:
+            continue
+
+        pg = res.groups()[0]
+        out = get_rados_inconsistent(pg)
+        js = json.loads(out)
+        inconsistents = js.get('inconsistents')
+
+        if not inconsistents:
+            continue
+
+        msg.setdefault(pg, []).extend(x['object']['name']
+                                      for x in inconsistents)
+
+    return msg
+
+
+def text_format(obj):
+    ret = ''
+    for pg, objs in obj.items():
+        ret += '{}: {}'.format(pg, ','.join(objs))
+    return ret
+
+
+if __name__ == '__main__':
+    try:
+        fmt = function_get('format')
+        if fmt and fmt not in VALID_FORMATS:
+            function_fail('Unknown format specified: {}'.format(fmt))
+        else:
+            msg = get_inconsistent_objs()
+            if fmt == 'yaml':
+                msg = yaml.dump(msg)
+            elif fmt == 'json':
+                msg = json.dumps(msg, indent=4, sort_keys=True)
+            else:
+                msg = text_format(msg)
+            function_set({'message': msg})
+    except CalledProcessError as e:
+        log(e)
+        function_fail("Listing inconsistent objects failed with error {}"
+                      .format(str(e)))

unit_tests/test_action_list_inconsistent.py

@@ -0,0 +1,89 @@
+# Copyright 2021 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for the list_inconsistent_objs action."""
+
+from actions import list_inconsistent_objs as action
+from mock import mock
+from test_utils import CharmTestCase
+
+
+class ListInconsistentTestCase(CharmTestCase):
+    """Run tests for the action."""
+
+    def setUp(self):
+        """Init mocks for test cases."""
+        super(ListInconsistentTestCase, self).setUp(
+            action, ["get_health_detail", "get_rados_inconsistent"]
+        )
+
+    @mock.patch("actions.list_inconsistent_objs.get_rados_inconsistent")
+    @mock.patch("actions.list_inconsistent_objs.get_health_detail")
+    def test_inconsistent_empty(
+        self, _get_health_detail, _get_rados_inconsistent
+    ):
+        """Test that the returned object is empty."""
+        _get_health_detail.return_value = "nothing to see here"
+        _get_rados_inconsistent.return_value = """
+            {"epoch": 0, "inconsistents": {1: 1}}
+        """
+        ret = action.get_inconsistent_objs()
+        _get_health_detail.assert_called_once()
+        _get_rados_inconsistent.assert_not_called()
+        self.assertEqual(len(ret), 0)
+        self.assertEqual('', action.text_format(ret))
+
+    @mock.patch("actions.list_inconsistent_objs.get_rados_inconsistent")
+    @mock.patch("actions.list_inconsistent_objs.get_health_detail")
+    def test_inconsistent_entry(
+        self, _get_health_detail, _get_rados_inconsistent
+    ):
+        """Test that expected PG is in the returned value."""
+        pg_id = '3.9'
+        _get_health_detail.return_value = """
+        pg 2.1 is active
+        pg {} is active+inconsistent+clean
+        """.format(pg_id)
+
+        _get_rados_inconsistent.return_value = """{
+            "epoch": 95,
+            "inconsistents": [ { "errors": [ "size_mismatch" ],
+                                 "object": { "locator": "", "name": "testfile",
+                                             "nspace": "", "snap": "head" },
+                                 "shards": [ { "data_digest": "0xa3ba020a",
+                                               "errors": [ "size_mismatch" ],
+                                               "omap_digest": "0xffffffff",
+                                               "osd": 0, "size": 21 },
+                                            { "data_digest": "0xa3ba020a",
+                                              "errors": [ "size_mismatch" ],
+                                              "omap_digest": "0xffffffff",
+                                              "osd": 1, "size": 22 },
+                                            { "data_digest": "0xa3ba020a",
+                                              "errors": [],
+                                              "omap_digest": "0xffffffff",
+                                              "osd": 2, "size": 23 }
+                                           ]}]
+        }"""
+
+        ret = action.get_inconsistent_objs()
+        _get_health_detail.assert_called_once()
+        _get_rados_inconsistent.assert_called()
+        self.assertNotEqual(len(ret), 0)
+        self.assertIn(pg_id, ret)
+
+        js = action.json.loads(_get_rados_inconsistent.return_value)
+        obj_name = js["inconsistents"][0]["object"]["name"]
+
+        self.assertIn(obj_name, ret[pg_id])
+        self.assertEqual(action.text_format(ret),
+                         '{}: {}'.format(pg_id, obj_name))