From 3ff94b3581abcc7a29feaf45f4df324f14c7f272 Mon Sep 17 00:00:00 2001 From: Marius Oprin Date: Mon, 23 Nov 2020 17:18:01 +0200 Subject: [PATCH] Implement new relation: 'ceph-replication-device' Add CephReplicationDeviceContext Class Add CinderCephContext Class Co-authored-by: Ionut Balutoiu Change-Id: I5f9139294c63f1adec3bbbe3bb9c99b2e7f5413b --- hooks/ceph-replication-device-relation-broken | 1 + .../ceph-replication-device-relation-changed | 1 + hooks/ceph-replication-device-relation-joined | 1 + hooks/cinder_contexts.py | 56 +++++++++ hooks/cinder_hooks.py | 108 +++++++++++++++--- hooks/cinder_utils.py | 23 +++- metadata.yaml | 2 + unit_tests/test_cinder_contexts.py | 27 +++++ unit_tests/test_cinder_hooks.py | 84 ++++++++++++++ unit_tests/test_cinder_utils.py | 1 + 10 files changed, 289 insertions(+), 15 deletions(-) create mode 120000 hooks/ceph-replication-device-relation-broken create mode 120000 hooks/ceph-replication-device-relation-changed create mode 120000 hooks/ceph-replication-device-relation-joined diff --git a/hooks/ceph-replication-device-relation-broken b/hooks/ceph-replication-device-relation-broken new file mode 120000 index 0000000..6dcd008 --- /dev/null +++ b/hooks/ceph-replication-device-relation-broken @@ -0,0 +1 @@ +cinder_hooks.py \ No newline at end of file diff --git a/hooks/ceph-replication-device-relation-changed b/hooks/ceph-replication-device-relation-changed new file mode 120000 index 0000000..6dcd008 --- /dev/null +++ b/hooks/ceph-replication-device-relation-changed @@ -0,0 +1 @@ +cinder_hooks.py \ No newline at end of file diff --git a/hooks/ceph-replication-device-relation-joined b/hooks/ceph-replication-device-relation-joined new file mode 120000 index 0000000..6dcd008 --- /dev/null +++ b/hooks/ceph-replication-device-relation-joined @@ -0,0 +1 @@ +cinder_hooks.py \ No newline at end of file diff --git a/hooks/cinder_contexts.py b/hooks/cinder_contexts.py index e0cb384..6919119 100644 --- a/hooks/cinder_contexts.py +++ b/hooks/cinder_contexts.py @@ -17,11 +17,15 @@ from charmhelpers.core.hookenv import ( service_name, is_relation_made, leader_get, + log, + relation_get, relation_ids, related_units, + DEBUG, ) from charmhelpers.contrib.openstack.context import ( + CephContext, OSContextGenerator, ) @@ -30,6 +34,10 @@ from charmhelpers.contrib.openstack.utils import ( CompareOpenStackReleases, ) +from charmhelpers.contrib.network.ip import ( + format_ipv6_addr, +) + CHARM_CEPH_CONF = '/var/lib/charm/{}/ceph.conf' @@ -99,3 +107,51 @@ class CephSubordinateContext(OSContextGenerator): config('rbd-flatten-volume-from-snapshot'))) return {'cinder': {'/etc/cinder/cinder.conf': {'sections': section}}} + + +class CephReplicationDeviceContext(CephContext): + """Generates context for /etc/ceph/ceph.conf templates.""" + + interfaces = ['ceph-replication-device'] + + def __call__(self): + if not relation_ids('ceph-replication-device'): + return {} + log('Generating template context for ceph-replication-device', + level=DEBUG) + mon_hosts = [] + ctxt = { + 'use_syslog': str(config('use-syslog')).lower() + } + for rid in relation_ids('ceph-replication-device'): + for unit in related_units(rid): + if not ctxt.get('auth'): + ctxt['auth'] = relation_get('auth', rid=rid, unit=unit) + if not ctxt.get('key'): + ctxt['key'] = relation_get('key', rid=rid, unit=unit) + ceph_addrs = relation_get('ceph-public-address', rid=rid, + unit=unit) + if ceph_addrs: + for addr in ceph_addrs.split(' '): + mon_hosts.append(format_ipv6_addr(addr) or addr) + else: + priv_addr = relation_get('private-address', rid=rid, + unit=unit) + mon_hosts.append(format_ipv6_addr(priv_addr) or priv_addr) + + ctxt['mon_hosts'] = ' '.join(sorted(mon_hosts)) + if not self.context_complete(ctxt): + return {} + + return ctxt + + +class CinderCephContext(CephContext): + + def __call__(self): + ctxt = super(CinderCephContext, self).__call__() + # NOTE: If "rbd-mirroring-mode" is set to "image" we are going + # to ignore default 'rbd_features' that are set in the context + if config('rbd-mirror-mode') == "image": + ctxt.pop('rbd_features', None) + return ctxt diff --git a/hooks/cinder_hooks.py b/hooks/cinder_hooks.py index 891ce81..029a51d 100755 --- a/hooks/cinder_hooks.py +++ b/hooks/cinder_hooks.py @@ -62,6 +62,7 @@ from charmhelpers.core.hookenv import ( log, relation_ids, relation_set, + application_name, service_name, status_set, UnregisteredHookError, @@ -76,6 +77,7 @@ from charmhelpers.payload.execd import execd_preinstall from cinder_contexts import ( ceph_config_file, CephSubordinateContext, + CephReplicationDeviceContext, ) from cinder_utils import ( CEPH_CONF, @@ -85,6 +87,7 @@ from cinder_utils import ( restart_map, scrub_old_style_ceph, VERSION_PACKAGE, + ceph_replication_device_config_file, ) @@ -109,6 +112,13 @@ def ceph_joined(): send_application_name() +@hooks.hook('ceph-replication-device-relation-joined') +def ceph_replication_device_joined(): + data = {'application-name': '{}-replication-device'.format( + application_name())} + relation_set(relation_settings=data) + + def get_ceph_request(): rq = CephBrokerRq() service = service_name() @@ -249,6 +259,26 @@ def ceph_changed(): level=DEBUG) +@hooks.hook('ceph-replication-device-relation-changed') +@restart_on_change(restart_map()) +def ceph_replication_device_changed(): + if 'ceph-replication-device' not in CONFIGS.complete_contexts(): + log('ceph-replication-device relation incomplete.') + return + + app_name = '{}-replication-device'.format(application_name()) + if not ensure_ceph_keyring(service=app_name, + relation='ceph-replication-device', + user='cinder', group='cinder'): + log('Could not create ceph keyring.') + return + + CONFIGS.write_all() + + for rid in relation_ids('storage-backend'): + storage_backend(rid) + + @hooks.hook('ceph-relation-broken') def ceph_broken(): service = service_name() @@ -257,6 +287,13 @@ def ceph_broken(): remove_alternative(os.path.basename(CEPH_CONF), ceph_config_file()) +@hooks.hook('ceph-replication-device-relation-broken') +def ceph_replication_device_broken(): + app_name = '{}-replication-device'.format(application_name()) + delete_keyring(service=app_name) + CONFIGS.write_all() + + @hooks.hook('config-changed') @restart_on_change(restart_map()) def write_and_restart(): @@ -274,13 +311,28 @@ def write_and_restart(): def storage_backend(rel_id=None): if 'ceph' not in CONFIGS.complete_contexts(): log('ceph relation incomplete. Peer not ready?') - else: - relation_set( - relation_id=rel_id, - backend_name=service_name(), - subordinate_configuration=json.dumps(CephSubordinateContext()()), - stateless=True, - ) + return + + subordinate_config = CephSubordinateContext()() + + if 'ceph-replication-device' in CONFIGS.complete_contexts(): + replication_device = { + 'backend_id': 'ceph', + 'conf': ceph_replication_device_config_file(), + 'user': '{}-replication-device'.format(application_name()) + } + replication_device_str = ','.join( + ['{}:{}'.format(k, v) for k, v in replication_device.items()]) + subordinate_config['cinder'][ + '/etc/cinder/cinder.conf']['sections'][application_name()].append( + ('replication_device', replication_device_str)) + + relation_set( + relation_id=rel_id, + backend_name=service_name(), + subordinate_configuration=json.dumps(subordinate_config), + stateless=True, + ) @hooks.hook('storage-backend-relation-changed') @@ -328,12 +380,42 @@ def ceph_access_joined(relation_id=None): # NOTE(jamespage): get key from ceph using a context ceph_keys = CephContext()() - - relation_set( - relation_id=relation_id, - relation_settings={'key': ceph_keys.get('key'), - 'secret-uuid': leader_get('secret-uuid')} - ) + if 'ceph-replication-device' not in CONFIGS.complete_contexts(): + relation_data = { + 'key': ceph_keys.get('key'), + 'secret-uuid': leader_get('secret-uuid') + } + relation_set( + relation_id=relation_id, + relation_settings=relation_data + ) + else: + replication_secret_uuid = leader_get('replication-device-secret-uuid') + if not replication_secret_uuid: + if is_leader(): + leader_set( + {'replication-device-secret-uuid': str(uuid.uuid4())}) + else: + log('Deferring keyrings provision until ' + 'leader seeds replication device uuid') + return + ceph_replication_keys = CephReplicationDeviceContext()() + keyrings = [ + { + 'name': application_name(), + 'key': ceph_keys.get('key'), + 'secret-uuid': leader_get('secret-uuid') + }, + { + 'name': '{}-replication-device'.format(application_name()), + 'key': ceph_replication_keys.get('key'), + 'secret-uuid': leader_get('replication-device-secret-uuid') + } + ] + relation_set( + relation_id=relation_id, + keyrings=json.dumps(keyrings) + ) @hooks.hook('pre-series-upgrade') diff --git a/hooks/cinder_utils.py b/hooks/cinder_utils.py index ea9f8e4..bdcf37c 100644 --- a/hooks/cinder_utils.py +++ b/hooks/cinder_utils.py @@ -18,7 +18,6 @@ from collections import OrderedDict from tempfile import NamedTemporaryFile from charmhelpers.contrib.openstack import ( - context, templating, ) from charmhelpers.contrib.openstack.alternatives import install_alternative @@ -26,6 +25,7 @@ from charmhelpers.contrib.openstack.utils import get_os_codename_package from charmhelpers.core.hookenv import ( hook_name, relation_ids, + application_name, service_name, ) from charmhelpers.core.host import mkdir @@ -58,6 +58,11 @@ def ceph_config_file(): return CHARM_CEPH_CONF.format(service_name()) +def ceph_replication_device_config_file(): + return CHARM_CEPH_CONF.format( + '{}-replication-device'.format(application_name())) + + def register_configs(): """ Register config files with their respective contexts. @@ -87,12 +92,26 @@ def register_configs(): install_alternative(os.path.basename(CEPH_CONF), CEPH_CONF, ceph_config_file()) CONFIG_FILES[ceph_config_file()] = { - 'hook_contexts': [context.CephContext(), + 'hook_contexts': [cinder_contexts.CinderCephContext(), cinder_contexts.CephAccessContext()], 'services': ['cinder-volume'], } confs.append(ceph_config_file()) + relation_present = relation_ids('ceph-replication-device') and \ + hook_name() != 'ceph-replication-device-relation-broken' + if relation_present: + mkdir(os.path.dirname(ceph_replication_device_config_file())) + + if not os.path.exists(ceph_replication_device_config_file()): + open(ceph_replication_device_config_file(), 'wt').close() + + CONFIG_FILES[ceph_replication_device_config_file()] = { + 'hook_contexts': [cinder_contexts.CephReplicationDeviceContext()], + 'services': ['cinder-volume'], + } + confs.append(ceph_replication_device_config_file()) + for conf in confs: configs.register(conf, CONFIG_FILES[conf]['hook_contexts']) diff --git a/metadata.yaml b/metadata.yaml index 23b07c8..124f642 100644 --- a/metadata.yaml +++ b/metadata.yaml @@ -30,3 +30,5 @@ requires: scope: container ceph: interface: ceph-client + ceph-replication-device: + interface: ceph-client diff --git a/unit_tests/test_cinder_contexts.py b/unit_tests/test_cinder_contexts.py index 1032874..e35bba4 100644 --- a/unit_tests/test_cinder_contexts.py +++ b/unit_tests/test_cinder_contexts.py @@ -24,6 +24,7 @@ TO_PATCH = [ 'service_name', 'get_os_codename_package', 'leader_get', + 'relation_get', 'relation_ids', 'related_units', ] @@ -241,3 +242,29 @@ class TestCinderContext(CharmTestCase): contexts.CephAccessContext()(), {'complete': True} ) + + def test_ceph_replication_device(self): + '''Test ceph context with ceph-replication-device relation''' + self.relation_get.side_effect = ['foo', 'bar', + 'ceph-mon-0', 'ceph-mon-1'] + self.relation_ids.return_value = ['ceph-replication-device:1'] + self.related_units.return_value = ['ceph-mon/0', 'ceph-mon/1'] + ctxt = contexts.CephReplicationDeviceContext() + ctxt_dict = ctxt() + self.assertEqual( + ctxt.context_complete(ctxt_dict), + True + ) + self.assertEqual( + ctxt_dict, + { + 'use_syslog': 'false', + 'auth': 'foo', + 'key': 'bar', + 'mon_hosts': 'ceph-mon-0 ceph-mon-1' + } + ) + self.assertEquals( + contexts.CephReplicationDeviceContext.interfaces, + ['ceph-replication-device'] + ) diff --git a/unit_tests/test_cinder_hooks.py b/unit_tests/test_cinder_hooks.py index ecae0bb..582a99a 100644 --- a/unit_tests/test_cinder_hooks.py +++ b/unit_tests/test_cinder_hooks.py @@ -38,7 +38,9 @@ TO_PATCH = [ 'CONFIGS', 'CEPH_CONF', 'ceph_config_file', + 'ceph_replication_device_config_file', # charmhelpers.core.hookenv + 'application_name', 'config', 'relation_ids', 'relation_set', @@ -327,6 +329,53 @@ class TestCinderHooks(CharmTestCase): stateless=True, ) + @patch('charmhelpers.core.hookenv.config') + def test_storage_backend_replication_device(self, mock_config): + self.application_name.return_value = 'test' + app_name = '{}-replication-device'.format(self.application_name()) + self.service_name.return_value = app_name + self.CONFIGS.complete_contexts.return_value = [ + 'ceph', 'ceph-replication-device'] + + def func(): + return { + 'cinder': { + '/etc/cinder/cinder.conf': { + 'sections': { + 'test': [] + } + } + } + } + + self.CephSubordinateContext.return_value = func + hooks.hooks.execute(['hooks/storage-backend-relation-joined']) + + replication_device = { + 'backend_id': 'ceph', + 'conf': self.ceph_replication_device_config_file(), + 'user': 'test-replication-device' + } + replication_device_str = ','.join( + ['{}:{}'.format(k, v) for k, v in replication_device.items()]) + expected_config = { + 'cinder': { + '/etc/cinder/cinder.conf': { + 'sections': { + 'test': [ + ('replication_device', replication_device_str) + ] + } + } + } + } + self.relation_set.assert_called_with( + relation_id=None, + backend_name='test-replication-device', + subordinate_configuration=json.dumps(expected_config), + stateless=True, + ) + @patch.object(hooks, 'ceph_access_joined') @patch.object(hooks, 'storage_backend') def test_leader_settings_changed(self, @@ -441,3 +490,38 @@ class TestCinderHooks(CharmTestCase): hooks.assess_status() self.status_set.assert_called_once_with( 'blocked', 'Invalid configuration: fake message') + + @patch('charmhelpers.core.hookenv.config') + @patch.object(hooks, 'storage_backend') + def test_ceph_replication_device_changed(self, + storage_backend, + mock_config): + self.CONFIGS.complete_contexts.return_value = [ + 'ceph-replication-device'] + self.ensure_ceph_keyring.return_value = True + self.relation_ids.return_value = ['storage-backend:1'] + app_name = '{}-replication-device'.format(self.application_name()) + hooks.hooks.execute(['hooks/ceph-replication-device-relation-changed']) + self.ensure_ceph_keyring.assert_called_with( + service=app_name, + relation='ceph-replication-device', + user='cinder', + group='cinder') + self.assertTrue(self.CONFIGS.write_all.called) + storage_backend.assert_called_with('storage-backend:1') + + @patch('charmhelpers.core.hookenv.config') + def test_ceph_replication_device_broken(self, mock_config): + app_name = '{}-replication-device'.format(self.application_name()) + self.service_name.return_value = app_name + hooks.hooks.execute(['hooks/ceph-replication-device-relation-broken']) + self.delete_keyring.assert_called_with( + service=self.service_name.return_value) + self.assertTrue(self.CONFIGS.write_all.called) + + @patch('charmhelpers.core.hookenv.config') + def test_ceph_replication_device_joined(self, mock_config): + data = {'application-name': '{}-replication-device'.format( + self.application_name())} + hooks.hooks.execute(['hooks/ceph-replication-device-relation-joined']) + self.relation_set.assert_called_with(relation_settings=data) diff --git a/unit_tests/test_cinder_utils.py b/unit_tests/test_cinder_utils.py index 3499217..fdd67d0 100644 --- a/unit_tests/test_cinder_utils.py +++ b/unit_tests/test_cinder_utils.py @@ -24,6 +24,7 @@ TO_PATCH = [ # helpers.core.hookenv 'relation_ids', 'service_name', + 'application_name', # storage_utils 'get_os_codename_package', 'templating',