Fix ceph-mon upgrade path

This PR makes some small changes in the upgrade path logic by
providing a fallback method of fetching the current ceph-mon
version and adding additional checks to see if the upgrade can
be done in a sane way.

Closes-Bug: #2024253
Change-Id: I1ca4316aaf4f0b855a12aa582a8188c88e926fa6

src/ceph_hooks.py

@@ -112,6 +112,19 @@ STATUS_CRONFILE = '/etc/cron.d/cat-ceph-health'
 HOST_OSD_COUNT_REPORT = '{}/host-osd-report.json'.format(NAGIOS_FILE_FOLDER)
 
 
+def get_current_ceph_version():
+    try:
+        out = subprocess.check_output(['ceph-mon', '-v']).decode('utf-8')
+    except subprocess.CalledProcessError as exc:
+        log(("failed to get ceph version: %s. check that the ceph-mon "
+            "binary is installed and runs correctly") % str(exc),
+            level=ERROR)
+        return ''
+
+    # ceph version X.Y.Z (HASH) version-name (stable)
+    return out.split()[4]
+
+
 def check_for_upgrade():
     if not ceph.is_bootstrapped():
         log("Ceph is not bootstrapped, skipping upgrade checks.")
@@ -120,9 +133,21 @@ def check_for_upgrade():
     c = hookenv.config()
     old_version = ceph.resolve_ceph_version(c.previous('source') or
                                             'distro')
+
+    if not old_version:
+        old_version = get_current_ceph_version()
+        if not old_version:
+            log(("failed to get ceph version. check that the ceph-mon "
+                 "binary is installed and runs correctly"), level=ERROR)
+            return
+
     log('old_version: {}'.format(old_version))
-    # Strip all whitespace
+
     new_version = ceph.resolve_ceph_version(hookenv.config('source'))
+    if not new_version:
+        log(("new version not found. make sure the 'source' option has "
+             "been set and try again (using 'distro' may help"), level=WARNING)
+        return
 
     old_version_os = get_os_codename_install_source(c.previous('source') or
                                                     'distro')
@@ -137,6 +162,8 @@ def check_for_upgrade():
         ceph.roll_monitor_cluster(new_version=new_version,
                                   upgrade_key='admin')
     elif (old_version == new_version and
+          old_version_os is not None and
+          new_version_os is not None and
           old_version_os < new_version_os):
         # See LP: #1778823
         add_source(hookenv.config('source'), hookenv.config('key'))

unit_tests/test_upgrade.py

@@ -1,6 +1,7 @@
 from unittest.mock import patch
 from ceph_hooks import check_for_upgrade
 from test_utils import CharmTestCase
+from charms_ceph.utils import resolve_ceph_version as resolve_ceph_version_orig
 
 
 __author__ = 'Chris Holcombe <chris.holcombe@canonical.com>'
@@ -76,3 +77,51 @@ class UpgradeRollingTestCase(CharmTestCase):
         check_for_upgrade()
         roll_monitor_cluster.assert_not_called()
         add_source.assert_called_with('cloud:bionic-stein', 'some-key')
+
+    @patch('ceph_hooks.ceph.resolve_ceph_version')
+    @patch('ceph_hooks.subprocess.check_output')
+    @patch('ceph_hooks.add_source')
+    @patch('ceph_hooks.ceph.is_bootstrapped')
+    @patch('ceph_hooks.hookenv')
+    @patch('ceph_hooks.ceph.roll_monitor_cluster')
+    def test_check_for_upgrade_no_current_version(self, roll_monitor_cluster,
+                                                  hookenv, is_bootstrapped,
+                                                  add_source, check_output,
+                                                  resolve_ceph_version):
+        _resolve_first = True
+
+        def _resolve_version(arg):
+            nonlocal _resolve_first
+            if _resolve_first:
+                _resolve_first = False
+                return None
+            return resolve_ceph_version_orig(arg)
+
+        resolve_ceph_version.side_effect = _resolve_version
+        check_output.return_value = b"""
+ceph version 16.2.13 (123) pacific (stable)"""
+        is_bootstrapped.return_value = True
+        hookenv.config.side_effect = self.test_config
+        self.test_config.set('source', 'cloud:focal-yoga')
+        check_for_upgrade()
+        roll_monitor_cluster.assert_called()
+        add_source.assert_not_called()
+
+    @patch('ceph_hooks.ceph.resolve_ceph_version')
+    @patch('ceph_hooks.subprocess.check_output')
+    @patch('ceph_hooks.add_source')
+    @patch('ceph_hooks.ceph.is_bootstrapped')
+    @patch('ceph_hooks.hookenv')
+    @patch('ceph_hooks.ceph.roll_monitor_cluster')
+    def test_check_for_upgrade_no_versions(self, roll_monitor_cluster,
+                                           hookenv, is_bootstrapped,
+                                           add_source, check_output,
+                                           resolve_ceph_version):
+        resolve_ceph_version.return_value = None
+        check_output.return_value = b"""
+ceph version 17.2.5 (456) quincy (stable)"""
+        is_bootstrapped.return_value = True
+        hookenv.config.side_effect = self.test_config
+        check_for_upgrade()
+        roll_monitor_cluster.assert_not_called()
+        add_source.assert_not_called()