From dd4b04bdd6e083aae9bcdc8aefafe9e7cdba33f0 Mon Sep 17 00:00:00 2001 From: Seyeong Kim Date: Wed, 26 Jun 2024 00:02:24 +0000 Subject: [PATCH] Fixing broken mysql-router configuration A customer faced an issue when they face full disk. Mysql router configuration file was broken. As this charm doesn't handle whole configuration file, charm wrote just part of them based on template. So re-bootstrapping is needed when this symptom happens. To do that, I put code to check configuration file size, and if it is under 1000bytes, run bootstrap. func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1196 Closes-Bug: #2004088 Change-Id: I02863c6afa0b4d95d3dbf550339a1860fe5ea8c1 --- src/lib/charm/openstack/mysql_router.py | 35 +++++- src/reactive/mysql_router_handlers.py | 9 +- .../test_lib_charm_openstack_mysql_router.py | 102 ++++++++++++++++++ unit_tests/test_mysql_router_handlers.py | 6 +- 4 files changed, 146 insertions(+), 6 deletions(-) diff --git a/src/lib/charm/openstack/mysql_router.py b/src/lib/charm/openstack/mysql_router.py index 6853189..d3f19f6 100644 --- a/src/lib/charm/openstack/mysql_router.py +++ b/src/lib/charm/openstack/mysql_router.py @@ -479,7 +479,25 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm): return None, None - def bootstrap_mysqlrouter(self): + def validate_configuration(self): + """Validate Configuration + + Check if mysql router configuration file is less than 1024 bytes. + If so, then the configuration file is probably damaged, and then + re-run the `bootstrap_mysqlrouter()` function with True to force + it. + """ + + if os.path.exists(self.mysqlrouter_conf): + conf_size = os.path.getsize(self.mysqlrouter_conf) + if conf_size <= 1024: + self.bootstrap_mysqlrouter(True) + else: + ch_core.hookenv.log( + "mysql router configuration file is not exist yet.", + "WARNING") + + def bootstrap_mysqlrouter(self, force=False): """Bootstrap MySQL Router. Execute the mysqlrouter bootstrap command. MySQL Router bootstraps into @@ -494,7 +512,7 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm): :rtype: None """ - if reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAPPED): + if not force and reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAPPED): ch_core.hookenv.log( "Bootstrap mysqlrouter is being called after we set the " "bootstrapped flag: {}. This may require manual intervention," @@ -522,8 +540,19 @@ class MySQLRouterCharm(charms_openstack.charm.OpenStackCharm): # If we have attempted to bootstrap before but unsuccessfully, # use the force option to avoid LP Bug#1919560 - if reactive.flags.is_flag_set(MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED): + is_bootstrap_attempted = reactive.flags.is_flag_set( + MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED) + if is_bootstrap_attempted or force: cmd.append("--force") + # clear configuration before force bootstrap + # because if there are some regex string, it fails + try: + with open(self.mysqlrouter_conf, "wt") as f: + f.write("[DEFAULT]\n") + except Exception: + ch_core.hookenv.log( + "ignored, because the bootstrap will overwrite the file.") + pass # Set and attempt the bootstrap reactive.flags.set_flag(MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED) diff --git a/src/reactive/mysql_router_handlers.py b/src/reactive/mysql_router_handlers.py index af0e6ca..83820b0 100644 --- a/src/reactive/mysql_router_handlers.py +++ b/src/reactive/mysql_router_handlers.py @@ -11,7 +11,6 @@ charms_openstack.bus.discover() charm.use_defaults( 'charm.installed', 'config.changed', - 'update-status', 'upgrade-charm') @@ -99,6 +98,7 @@ def proxy_shared_db_responses(shared_db, db_router): :type db_router_interface: MySQLRouterRequires object """ with charm.provide_charm_instance() as instance: + instance.validate_configuration() instance.config_changed() instance.proxy_db_and_user_responses(db_router, shared_db) instance.assess_status() @@ -112,3 +112,10 @@ def stop_charm(): with charm.provide_charm_instance() as instance: instance.stop_mysqlrouter() instance.config_cleanup() + + +@reactive.hook('update-status') +def update_status(): + with charm.provide_charm_instance() as instance: + instance.validate_configuration() + instance.assess_status() diff --git a/unit_tests/test_lib_charm_openstack_mysql_router.py b/unit_tests/test_lib_charm_openstack_mysql_router.py index d0cde5f..001d21c 100644 --- a/unit_tests/test_lib_charm_openstack_mysql_router.py +++ b/unit_tests/test_lib_charm_openstack_mysql_router.py @@ -484,6 +484,108 @@ class TestMySQLRouterCharm(test_utils.PatchHelper): self.clear_flag.assert_called_once_with( mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED) + def test_bootstrap_mysqlrouter_force(self): + _json_addr = '"10.10.10.60"' + _json_pass = '"clusterpass"' + _pass = json.loads(_json_pass) + _addr = json.loads(_json_addr) + _user = "mysql" + _port = "3006" + self.patch_object(mysql_router.reactive.flags, "is_flag_set") + self.endpoint_from_flag.return_value = self.db_router + self.db_router.password.return_value = _json_pass + self.db_router.db_host.return_value = _json_addr + self.is_flag_set.return_value = False + + mrc = mysql_router.MySQLRouterCharm() + mrc.options.system_user = _user + mrc.options.base_port = _port + + _relations = ["relid"] + + self.patch_object(mysql_router.ch_core.hookenv, "relation_ids") + self.relation_ids.return_value = _relations + + _related_units = ["relunits"] + + self.patch_object(mysql_router.ch_core.hookenv, "related_units") + self.related_units.return_value = _related_units + + _config_data = { + "mysqlrouter_password": json.dumps(_pass), + "db_host": json.dumps(_addr), + } + + self.patch_object(mysql_router.ch_core.hookenv, "relation_get") + self.relation_get.return_value = _config_data + + self.cmp_pkgrevno.return_value = 1 + self.is_flag_set.side_effect = [False, True] + self.subprocess.check_output.side_effect = None + mrc.bootstrap_mysqlrouter(True) + self.subprocess.check_output.assert_called_once_with( + [mrc.mysqlrouter_bin, "--user", _user, "--name", mrc.name, + "--bootstrap", "{}:{}@{}" + .format(mrc.db_router_user, _pass, _addr), + "--directory", mrc.mysqlrouter_working_dir, + "--conf-use-sockets", + "--conf-bind-address", mrc.shared_db_address, + "--report-host", mrc.db_router_address, + "--conf-base-port", _port, + "--disable-rest", "--force"], + stderr=self.stdout) + self.set_flag.assert_has_calls([ + mock.call(mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED), + mock.call(mysql_router.MYSQL_ROUTER_BOOTSTRAPPED)]) + self.clear_flag.assert_called_once_with( + mysql_router.MYSQL_ROUTER_BOOTSTRAP_ATTEMPTED) + + def test_validate_configuration_file_exists_and_small_size(self): + self.patch_object(mysql_router.os.path, "exists", + return_value=True) + self.patch_object(mysql_router.os.path, "getsize", + return_value=500) + self.patch_object(mysql_router.ch_core.hookenv, "log") + self.patch_object(mysql_router.MySQLRouterCharm, + 'bootstrap_mysqlrouter') + + mrc = mysql_router.MySQLRouterCharm() + mrc.validate_configuration() + + self.bootstrap_mysqlrouter.assert_called_once_with(True) + self.log.assert_not_called() + + def test_validate_configuration_file_exists_and_large_size(self): + self.patch_object(mysql_router.os.path, "exists", + return_value=True) + self.patch_object(mysql_router.os.path, "getsize", + return_value=1500) + self.patch_object(mysql_router.ch_core.hookenv, "log") + self.patch_object(mysql_router.MySQLRouterCharm, + 'bootstrap_mysqlrouter') + + mrc = mysql_router.MySQLRouterCharm() + mrc.validate_configuration() + + self.bootstrap_mysqlrouter.assert_not_called() + self.log.assert_not_called() + + def test_validate_configuration_file_not_exists(self): + self.patch_object(mysql_router.os.path, "exists", + return_value=False) + self.patch_object(mysql_router.ch_core.hookenv, "log") + self.patch_object(mysql_router.MySQLRouterCharm, + 'bootstrap_mysqlrouter') + + mrc = mysql_router.MySQLRouterCharm() + mrc.validate_configuration() + + self.bootstrap_mysqlrouter.assert_not_called() + self.log.assert_called_once_with( + "mysql router configuration file is not exist yet.", + "WARNING" + ) + def test_start_mysqlrouter(self): self.patch_object(mysql_router.ch_core.host, "service_start") _name = "keystone-mysql-router" diff --git a/unit_tests/test_mysql_router_handlers.py b/unit_tests/test_mysql_router_handlers.py index 705451c..b65a814 100644 --- a/unit_tests/test_mysql_router_handlers.py +++ b/unit_tests/test_mysql_router_handlers.py @@ -25,7 +25,6 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks): def test_hooks(self): defaults = [ "config.changed", - "update-status", "upgrade-charm", "charm.installed", ] @@ -56,7 +55,10 @@ class TestRegisteredHooks(test_utils.TestRegisteredHooks): "hook": { "stop_charm": ( "stop", - ) + ), + "update_status": ( + "update-status", + ), } } # test that the hooks were registered via the