diff --git a/.gitmodules b/.gitmodules index 6386c4f..41ffcfc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,12 +1,15 @@ [submodule "mod/operator"] path = mod/operator url = https://github.com/canonical/operator -[submodule "mod/interface-ceph-client"] - path = mod/interface-ceph-client - url = https://github.com/gnuoy/oper-interface-ceph-client.git +[submodule "mod/ops-interface-ceph-client"] + path = mod/ops-interface-ceph-client + url = https://github.com/openstack-charmers/ops-interface-ceph-client.git [submodule "mod/ops-openstack"] path = mod/ops-openstack url = https://github.com/openstack-charmers/ops-openstack.git [submodule "mod/charm-helpers"] path = mod/charm-helpers url = https://github.com/juju/charm-helpers.git +[submodule "mod/ops-interface-tls-certificates"] + path = mod/ops-interface-tls-certificates + url = https://github.com/openstack-charmers/ops-interface-tls-certificates.git diff --git a/charm-init.sh b/charm-init.sh index 6437235..06fa76c 100755 --- a/charm-init.sh +++ b/charm-init.sh @@ -14,11 +14,14 @@ if [[ -z "$UPDATE" ]]; then else git -C mod/operator pull origin master git -C mod/ops-openstack pull origin master + git -C mod/ops-interface-ceph-client pull origin master +# git -C mod/ops-interface-tls-certificates pull origin master git -C mod/charm-helpers pull origin master pip install -t lib -r build-requirements.txt --upgrade fi ln -f -t lib -s ../mod/operator/ops -ln -f -t lib -s ../mod/interface-ceph-client/interface_ceph_client.py +ln -f -t lib -s ../mod/ops-interface-ceph-client/interface_ceph_client.py ln -f -t lib -s ../mod/ops-openstack/ops_openstack.py ln -f -t lib -s ../mod/ops-openstack/adapters.py +ln -f -t lib -s ../mod/ops-interface-tls-certificates/ca_client.py diff --git a/mod/charm-helpers b/mod/charm-helpers index f3f36f8..b4aa4e3 160000 --- a/mod/charm-helpers +++ b/mod/charm-helpers @@ -1 +1 @@ -Subproject commit f3f36f85f54380a651ba05972e78467ad22468e3 +Subproject commit b4aa4e3398e7406dbf0f76a23f91afa6a72aed1a diff --git a/mod/interface-ceph-client b/mod/interface-ceph-client deleted file mode 160000 index 4f84bca..0000000 --- a/mod/interface-ceph-client +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4f84bcad2d4b3ea415b5eccc850c85b9f4fc172e diff --git a/mod/operator b/mod/operator index bb9b534..47af0fc 160000 --- a/mod/operator +++ b/mod/operator @@ -1 +1 @@ -Subproject commit bb9b534e68b04286eef78a36ed930815ffb1968d +Subproject commit 47af0fc9f86dc5b16e88a83a00377864a1541734 diff --git a/mod/ops-interface-ceph-client b/mod/ops-interface-ceph-client new file mode 160000 index 0000000..30213fa --- /dev/null +++ b/mod/ops-interface-ceph-client @@ -0,0 +1 @@ +Subproject commit 30213fa66f979eb93473242297677ea6554984df diff --git a/mod/ops-interface-tls-certificates b/mod/ops-interface-tls-certificates new file mode 160000 index 0000000..d03a251 --- /dev/null +++ b/mod/ops-interface-tls-certificates @@ -0,0 +1 @@ +Subproject commit d03a251e87f02528789af0eb4cce88e471847e68 diff --git a/src/charm.py b/src/charm.py index bef471b..ab76b07 100755 --- a/src/charm.py +++ b/src/charm.py @@ -7,11 +7,15 @@ import subprocess import sys import string import secrets +from pathlib import Path sys.path.append('lib') from ops.framework import ( StoredState, + EventSource, + EventBase, + ObjectEvents, ) from ops.main import main import ops.model @@ -19,12 +23,12 @@ import charmhelpers.core.host as ch_host import charmhelpers.core.templating as ch_templating import interface_ceph_client import interface_ceph_iscsi_peer -import interface_tls_certificates +import ca_client import adapters import ops_openstack import gwcli_client - +import cryptography.hazmat.primitives.serialization as serialization logger = logging.getLogger(__name__) @@ -71,7 +75,10 @@ class TLSCertificatesAdapter(adapters.OpenStackOperRelationAdapter): @property def enable_tls(self): - return bool(self.relation.application_certs) + try: + return bool(self.relation.application_certificate) + except ca_client.CAClientError: + return False class CephISCSIGatewayAdapters(adapters.OpenStackRelationAdapters): @@ -92,44 +99,81 @@ class CephISCSIGatewayCharmBase(ops_openstack.OSBaseCharm): "mon", "allow *", "mgr", "allow r"] - RESTART_MAP = { - '/etc/ceph/ceph.conf': ['rbd-target-api', 'rbd-target-gw'], - '/etc/ceph/iscsi-gateway.cfg': ['rbd-target-api'], - '/etc/ceph/ceph.client.ceph-iscsi.keyring': ['rbd-target-api']} DEFAULT_TARGET = "iqn.2003-01.com.ubuntu.iscsi-gw:iscsi-igw" REQUIRED_RELATIONS = ['ceph-client', 'cluster'] - # Two has been tested before is probably fine too but needs + + # Two has been tested but four is probably fine too but needs # validating ALLOWED_UNIT_COUNTS = [2] + + CEPH_CONFIG_PATH = Path('/etc/ceph') + CEPH_ISCSI_CONFIG_PATH = CEPH_CONFIG_PATH / 'iscsi' + GW_CONF = CEPH_CONFIG_PATH / 'iscsi-gateway.cfg' + CEPH_CONF = CEPH_ISCSI_CONFIG_PATH / 'ceph.conf' + GW_KEYRING = CEPH_ISCSI_CONFIG_PATH / 'ceph.client.ceph-iscsi.keyring' + TLS_KEY_PATH = CEPH_CONFIG_PATH / 'iscsi-gateway.key' + TLS_PUB_KEY_PATH = CEPH_CONFIG_PATH / 'iscsi-gateway-pub.key' + TLS_CERT_PATH = CEPH_CONFIG_PATH / 'iscsi-gateway.crt' + TLS_KEY_AND_CERT_PATH = CEPH_CONFIG_PATH / 'iscsi-gateway.pem' + TLS_CA_CERT_PATH = Path( + '/usr/local/share/ca-certificates/vault_ca_cert.crt') + + GW_SERVICES = ['rbd-target-api', 'rbd-target-gw'] + + RESTART_MAP = { + str(GW_CONF): GW_SERVICES, + str(CEPH_CONF): GW_SERVICES, + str(GW_KEYRING): GW_SERVICES} + release = 'default' def __init__(self, framework, key): super().__init__(framework, key) logging.info("Using {} class".format(self.release)) - self.state.set_default(target_created=False) - self.state.set_default(enable_tls=False) - self.state.set_default(additional_trusted_ips=[]) + self.state.set_default( + target_created=False, + enable_tls=False, + additional_trusted_ips=[]) self.ceph_client = interface_ceph_client.CephClientRequires( self, 'ceph-client') self.peers = interface_ceph_iscsi_peer.CephISCSIGatewayPeers( self, 'cluster') - self.tls = interface_tls_certificates.TlsRequires(self, "certificates") + self.ca_client = ca_client.CAClient( + self, + 'certificates') self.adapters = CephISCSIGatewayAdapters( - (self.ceph_client, self.peers, self.tls), + (self.ceph_client, self.peers, self.ca_client), + self) + self.framework.observe( + self.ceph_client.on.broker_available, + self.request_ceph_pool) + self.framework.observe( + self.ceph_client.on.pools_available, + self.render_config) + self.framework.observe( + self.peers.on.has_peers, + self) + self.framework.observe( + self.ca_client.on.tls_app_config_ready, + self.on_tls_app_config_ready) + self.framework.observe( + self.ca_client.on.ca_available, + self.on_ca_available) + self.framework.observe( + self.on.config_changed, + self.render_config) + self.framework.observe( + self.on.upgrade_charm, + self.render_config) + self.framework.observe( + self.on.create_target_action, + self) + self.framework.observe( + self.on.add_trusted_ip_action, self) - self.framework.observe(self.on.ceph_client_relation_joined, self) - self.framework.observe(self.ceph_client.on.pools_available, self) - self.framework.observe(self.peers.on.has_peers, self) - self.framework.observe(self.peers.on.ready_peers, self) - self.framework.observe(self.on.create_target_action, self) - self.framework.observe(self.on.add_trusted_ip_action, self) - self.framework.observe(self.on.certificates_relation_joined, self) - self.framework.observe(self.on.certificates_relation_changed, self) - self.framework.observe(self.on.config_changed, self) - self.framework.observe(self.on.upgrade_charm, self) def on_install(self, event): if ch_host.is_container(): @@ -138,10 +182,127 @@ class CephISCSIGatewayCharmBase(ops_openstack.OSBaseCharm): else: self.install_pkgs() + + def on_has_peers(self, event): + logging.info("Unit has peers") + if self.unit.is_leader() and not self.peers.admin_password: + logging.info("Setting admin password") + alphabet = string.ascii_letters + string.digits + password = ''.join(secrets.choice(alphabet) for i in range(8)) + self.peers.set_admin_password(password) + + def request_ceph_pool(self, event): + logging.info("Requesting replicated pool") + self.ceph_client.create_replicated_pool( + self.model.config['rbd-metadata-pool']) + logging.info("Requesting permissions") + self.ceph_client.request_ceph_permissions( + 'ceph-iscsi', + self.CEPH_CAPABILITIES) + self.ceph_client.request_osd_settings({ + 'osd heartbeat grace': 20, + 'osd heartbeat interval': 5}) + + def refresh_request(self, event): + self.render_config(event) + self.request_ceph_pool(event) + + def render_config(self, event): + if not self.peers.admin_password: + logging.info("Defering setup") + event.defer() + return + if not self.ceph_client.pools_available: + logging.info("Defering setup") + event.defer() + return + + self.CEPH_ISCSI_CONFIG_PATH.mkdir( + exist_ok=True, + mode=0o750) + + def daemon_reload_and_restart(service_name): + subprocess.check_call(['systemctl', 'daemon-reload']) + subprocess.check_call(['systemctl', 'restart', service_name]) + + rfuncs = { + 'rbd-target-api': daemon_reload_and_restart} + + @ch_host.restart_on_change(self.RESTART_MAP, restart_functions=rfuncs) + def _render_configs(): + for config_file in self.RESTART_MAP.keys(): + ch_templating.render( + os.path.basename(config_file), + config_file, + self.adapters) + logging.info("Rendering config") + _render_configs() + logging.info("Setting started state") + self.peers.announce_ready() + self.state.is_started = True + self.update_status() + logging.info("on_pools_available: status updated") + + def on_ca_available(self, event): + addresses = set() + for binding_name in ['public', 'cluster']: + binding = self.model.get_binding(binding_name) + addresses.add(binding.network.ingress_address) + addresses.add(binding.network.bind_address) + sans = [str(s) for s in addresses] + sans.append(socket.gethostname()) + self.ca_client.request_application_certificate(socket.getfqdn(), sans) + + def on_tls_app_config_ready(self, event): + self.TLS_KEY_PATH.write_bytes( + self.ca_client.application_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption())) + self.TLS_CERT_PATH.write_bytes( + self.ca_client.application_certificate.public_bytes( + encoding=serialization.Encoding.PEM)) + self.TLS_CA_CERT_PATH.write_bytes( + self.ca_client.ca_certificate.public_bytes( + encoding=serialization.Encoding.PEM)) + self.TLS_KEY_AND_CERT_PATH.write_bytes( + self.ca_client.application_certificate.public_bytes( + encoding=serialization.Encoding.PEM) + + b'\n' + + self.ca_client.application_key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption()) + ) + self.TLS_PUB_KEY_PATH.write_bytes( + self.ca_client.application_key.public_key().public_bytes( + format=serialization.PublicFormat.SubjectPublicKeyInfo, + encoding=serialization.Encoding.PEM)) + subprocess.check_call(['update-ca-certificates']) + self.state.enable_tls = True + self.refresh_request(event) + + def custom_status_check(self): + if ch_host.is_container(): + self.unit.status = ops.model.BlockedStatus( + 'Charm cannot be deployed into a container') + return False + if self.peers.unit_count not in self.ALLOWED_UNIT_COUNTS: + self.unit.status = ops.model.BlockedStatus( + '{} is an invalid unit count'.format(self.peers.unit_count)) + return False + return True + + # Actions + def on_add_trusted_ip_action(self, event): - self.state.additional_trusted_ips.append( - event.params['ips'].split(' ')) - logging.info(self.state.additional_trusted_ips) + if self.unit.is_leader(): + self.state.additional_trusted_ips = event.params.get('ips') + logging.info(len(self.state.additional_trusted_ips)) + self.peers.set_allowed_ips( + self.state.additional_trusted_ips) + else: + event.fail("Action must be run on leader") def on_create_target_action(self, event): gw_client = gwcli_client.GatewayClient() @@ -177,142 +338,6 @@ class CephISCSIGatewayCharmBase(ops_openstack.OSBaseCharm): event.params['image-name']) event.set_results({'iqn': target}) - def setup_default_target(self): - gw_client = gwcli_client.GatewayClient() - gw_client.create_target(self.DEFAULT_TARGET) - for gw_unit, gw_config in self.peers.ready_peer_details.items(): - gw_client.add_gateway_to_target( - self.DEFAULT_TARGET, - gw_config['ip'], - gw_config['fqdn']) - self.state.target_created = True - - def on_ready_peers(self, event): - if not self.unit.is_leader(): - logging.info("Leader should do setup") - return - if not self.state.is_started: - logging.info("Cannot perform setup yet, not started") - event.defer() - return - if self.state.target_created: - logging.info("Initial target setup already complete") - return - else: - # This appears to race and sometime runs before the - # peer is 100% ready. There is probably little value - # in this anyway so may just remove it. - # self.setup_default_target() - return - - def on_has_peers(self, event): - logging.info("Unit has peers") - if self.unit.is_leader() and not self.peers.admin_password: - logging.info("Setting admin password") - alphabet = string.ascii_letters + string.digits - password = ''.join(secrets.choice(alphabet) for i in range(8)) - self.peers.set_admin_password(password) - - def on_ceph_client_relation_joined(self, event): - logging.info("Requesting replicated pool") - self.ceph_client.create_replicated_pool( - self.model.config['rbd-metadata-pool']) - logging.info("Requesting permissions") - self.ceph_client.request_ceph_permissions( - 'ceph-iscsi', - self.CEPH_CAPABILITIES) - self.ceph_client.request_osd_settings({ - 'osd heartbeat grace': 20, - 'osd heartbeat interval': 5}) - - def on_config_changed(self, event): - if self.state.is_started: - self.on_pools_available(event) - self.on_ceph_client_relation_joined(event) - - def on_upgrade_charm(self, event): - if self.state.is_started: - self.on_pools_available(event) - self.on_ceph_client_relation_joined(event) - - def on_pools_available(self, event): - logging.info("on_pools_available") - if not self.peers.admin_password: - logging.info("Defering setup") - event.defer() - return - - def daemon_reload_and_restart(service_name): - subprocess.check_call(['systemctl', 'daemon-reload']) - subprocess.check_call(['systemctl', 'restart', service_name]) - - rfuncs = { - 'rbd-target-api': daemon_reload_and_restart} - - @ch_host.restart_on_change(self.RESTART_MAP, restart_functions=rfuncs) - def render_configs(): - for config_file in self.RESTART_MAP.keys(): - ch_templating.render( - os.path.basename(config_file), - config_file, - self.adapters) - logging.info("Rendering config") - render_configs() - logging.info("Setting started state") - self.peers.announce_ready() - self.state.is_started = True - self.update_status() - logging.info("on_pools_available: status updated") - - def on_certificates_relation_joined(self, event): - addresses = set() - for binding_name in ['public', 'cluster']: - binding = self.model.get_binding(binding_name) - addresses.add(binding.network.ingress_address) - addresses.add(binding.network.bind_address) - sans = [str(s) for s in addresses] - sans.append(socket.gethostname()) - self.tls.request_application_cert(socket.getfqdn(), sans) - - def on_certificates_relation_changed(self, event): - app_certs = self.tls.application_certs - if not all([self.tls.root_ca_cert, app_certs]): - return - if self.tls.chain: - # Append chain file so that clients that trust the root CA will - # trust certs signed by an intermediate in the chain - ca_cert_data = self.tls.root_ca_cert + os.linesep + self.tls.chain - else: - ca_cert_data = self.tls.root_ca_cert - pem_data = app_certs['cert'] + os.linesep + app_certs['key'] - tls_files = { - '/etc/ceph/iscsi-gateway.crt': app_certs['cert'], - '/etc/ceph/iscsi-gateway.key': app_certs['key'], - '/etc/ceph/iscsi-gateway.pem': pem_data, - '/usr/local/share/ca-certificates/vault_ca_cert.crt': ca_cert_data} - for tls_file, tls_data in sorted(tls_files.items()): - with open(tls_file, 'w') as f: - f.write(tls_data) - subprocess.check_call(['update-ca-certificates']) - cert_out = subprocess.check_output( - ('openssl x509 -inform pem -in /etc/ceph/iscsi-gateway.pem ' - '-pubkey -noout').split()) - with open('/etc/ceph/iscsi-gateway-pub.key', 'w') as f: - f.write(cert_out.decode('UTF-8')) - self.state.enable_tls = True - self.on_pools_available(event) - - def custom_status_check(self): - if ch_host.is_container(): - self.unit.status = ops.model.BlockedStatus( - 'Charm cannot be deployed into a container') - return False - if self.peers.unit_count not in self.ALLOWED_UNIT_COUNTS: - self.unit.status = ops.model.BlockedStatus( - '{} is an invalid unit count'.format(self.peers.unit_count)) - return False - return True - @ops_openstack.charm_class class CephISCSIGatewayCharmJewel(CephISCSIGatewayCharmBase): diff --git a/src/interface_ceph_iscsi_peer.py b/src/interface_ceph_iscsi_peer.py index 20eba6d..e8a9df2 100644 --- a/src/interface_ceph_iscsi_peer.py +++ b/src/interface_ceph_iscsi_peer.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import json import logging import socket @@ -31,6 +32,7 @@ class CephISCSIGatewayPeers(Object): PASSWORD_KEY = 'admin_password' READY_KEY = 'gateway_ready' FQDN_KEY = 'gateway_fqdn' + ALLOWED_IPS_KEY = 'allowed_ips' def __init__(self, charm, relation_name): super().__init__(charm, relation_name) @@ -50,6 +52,11 @@ class CephISCSIGatewayPeers(Object): logging.info("Setting admin password") self.peer_rel.data[self.peer_rel.app][self.PASSWORD_KEY] = password + def set_allowed_ips(self, ips): + logging.info("Setting allowed ips") + ip_str = json.dumps(ips) + self.peer_rel.data[self.peer_rel.app][self.ALLOWED_IPS_KEY] = ip_str + def announce_ready(self): logging.info("announcing ready") self.peer_rel.data[self.this_unit][self.READY_KEY] = 'True' @@ -90,9 +97,18 @@ class CephISCSIGatewayPeers(Object): @property def admin_password(self): - # https://github.com/canonical/operator/issues/148 + if not self.peer_rel: + return None return self.peer_rel.data[self.peer_rel.app].get(self.PASSWORD_KEY) + @property + def allowed_ips(self): + if not self.peer_rel: + return None + ip_str = self.peer_rel.data[self.peer_rel.app].get( + self.ALLOWED_IPS_KEY) + return json.loads(ip_str) + @property def peer_addresses(self): addresses = [self.cluster_bind_address] diff --git a/src/interface_tls_certificates.py b/src/interface_tls_certificates.py deleted file mode 100644 index 0e12adb..0000000 --- a/src/interface_tls_certificates.py +++ /dev/null @@ -1,116 +0,0 @@ -import json -from ops.framework import ( - Object, -) - - -class TlsRequires(Object): - def __init__(self, parent, key): - super().__init__(parent, key) - self.name = self.relation_name = key - - def request_application_cert(self, cn, sans): - """ - Request a client certificate and key be generated for the given - common name (`cn`) and list of alternative names (`sans`). - - This can be called multiple times to request more than one client - certificate, although the common names must be unique. If called - again with the same common name, it will be ignored. - """ - relations = self.framework.model.relations[self.name] - if not relations: - return - # assume we'll only be connected to one provider - relation = relations[0] - unit = self.framework.model.unit - requests = relation.data[unit].get('application_cert_requests', '{}') - requests = json.loads(requests) - requests[cn] = {'sans': sans} - relation.data[unit]['application_cert_requests'] = json.dumps( - requests, - sort_keys=True) - - @property - def root_ca_cert(self): - """ - Root CA certificate. - """ - # only the leader of the provider should set the CA, or all units - # had better agree - for relation in self.framework.model.relations[self.name]: - for unit in relation.units: - if relation.data[unit].get('ca'): - return relation.data[unit].get('ca') - - @property - def chain(self): - """ - Root CA certificate. - """ - # only the leader of the provider should set the CA, or all units - # had better agree - for relation in self.framework.model.relations[self.name]: - for unit in relation.units: - if relation.data[unit].get('chain'): - return relation.data[unit].get('chain') - - @property - def server_certs(self): - """ - List of [Certificate][] instances for all available server certs. - """ - unit_name = self.framework.model.unit.name.replace('/', '_') - field = '{}.processed_requests'.format(unit_name) - - for relation in self.framework.model.relations[self.name]: - for unit in relation.units: - if field not in relation.data[unit]: - continue - certs_data = relation.data[unit][field] - if not certs_data: - continue - certs_data = json.loads(certs_data) - if not certs_data: - continue - return list(certs_data.values())[0] - - @property - def client_certs(self): - """ - List of [Certificate][] instances for all available client certs. - """ - unit_name = self.framework.model.unit.name.replace('/', '_') - field = '{}.processed_client_requests'.format(unit_name) - - for relation in self.framework.model.relations[self.name]: - for unit in relation.units: - if field not in relation.data[unit]: - continue - certs_data = relation.data[unit][field] - if not certs_data: - continue - certs_data = json.loads(certs_data) - if not certs_data: - continue - return list(certs_data.values())[0] - - @property - def application_certs(self): - """ - List of [Certificate][] instances for all available application certs. - """ - unit_name = self.framework.model.unit.name.replace('/', '_') - field = '{}.processed_application_requests'.format(unit_name) - - for relation in self.framework.model.relations[self.name]: - for unit in relation.units: - if field not in relation.data[unit]: - continue - certs_data = relation.data[unit][field] - if not certs_data: - continue - certs_data = json.loads(certs_data) - if not certs_data: - continue - return certs_data['app_data'] diff --git a/templates/ceph.conf b/templates/ceph.conf index a27fd58..5d0227b 100644 --- a/templates/ceph.conf +++ b/templates/ceph.conf @@ -6,7 +6,7 @@ [global] auth supported = {{ ceph_client.auth_supported }} mon host = {{ ceph_client.mon_hosts }} -keyring = /etc/ceph/$cluster.$name.keyring +keyring = /etc/ceph/iscsi/$cluster.$name.keyring [client.ceph-iscsi] client mount uid = 0 diff --git a/templates/iscsi-gateway.cfg b/templates/iscsi-gateway.cfg index 3c06842..c8db2bb 100644 --- a/templates/iscsi-gateway.cfg +++ b/templates/iscsi-gateway.cfg @@ -1,32 +1,14 @@ [config] -# Name of the Ceph storage cluster. A suitable Ceph configuration file allowing -# # access to the Ceph storage cluster from the gateway node is required, if not -# # colocated on an OSD node. logger_level = DEBUG cluster_name = ceph cluster_client_name = client.ceph-iscsi pool = {{ options.rbd_metadata_pool }} -# -# # Place a copy of the ceph cluster's admin keyring in the gateway's /etc/ceph -# # drectory and reference the filename here -#gateway_keyring = ceph.client.admin.keyring + gateway_keyring = ceph.client.ceph-iscsi.keyring -# -# -# # API settings. -# # The API supports a number of options that allow you to tailor it to your -# # local environment. If you want to run the API under https, you will need to -# # create cert/key files that are compatible for each iSCSI gateway node, that is -# # not locked to a specific node. SSL cert and key files *must* be called -# # 'iscsi-gateway.crt' and 'iscsi-gateway.key' and placed in the '/etc/ceph/' directory -# # on *each* gateway node. With the SSL files in place, you can use 'api_secure = true' -# # to switch to https mode. -# -# # To support the API, the bear minimum settings are: +ceph_config_dir = /etc/ceph/iscsi + api_secure = {{ certificates.enable_tls }} api_user = admin api_password = {{ cluster.admin_password }} api_port = 5000 trusted_ip_list = {{ cluster.gw_hosts }} -# -# diff --git a/tests/bundles/focal.yaml b/tests/bundles/focal.yaml index cc7d12a..598a5ad 100644 --- a/tests/bundles/focal.yaml +++ b/tests/bundles/focal.yaml @@ -30,7 +30,7 @@ applications: - '0' - '1' ceph-osd: - charm: cs:~gnuoy/ceph-osd-5 + charm: cs:~openstack-charmers-next/ceph-osd num_units: 3 storage: osd-devices: 'cinder,10G' @@ -41,7 +41,7 @@ applications: - '1' - '2' ceph-mon: - charm: cs:~gnuoy/ceph-mon-6 + charm: cs:~openstack-charmers-next/ceph-mon num_units: 3 options: monitor-count: '3' diff --git a/unit_tests/test_ceph_iscsi_charm.py b/unit_tests/test_ceph_iscsi_charm.py index b3352d0..2f43250 100644 --- a/unit_tests/test_ceph_iscsi_charm.py +++ b/unit_tests/test_ceph_iscsi_charm.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import json import unittest import sys @@ -31,6 +32,78 @@ from ops import framework, model import charm +TEST_CA = '''-----BEGIN CERTIFICATE----- +MIIC8TCCAdmgAwIBAgIUIchLT42Gy3QexrQbppgWb+xF2SgwDQYJKoZIhvcNAQEL +BQAwGjEYMBYGA1UEAwwPRGl2aW5lQXV0aG9yaXR5MB4XDTIwMDUwNTA5NDIzMVoX +DTIwMDYwNDA5NDIzMlowGjEYMBYGA1UEAwwPRGl2aW5lQXV0aG9yaXR5MIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA54oZkgz+xpaM8AKfHTT19lwqvVSr +W3uZiyyiNAWBX+Ru5/5RqQONKmjPqU3Bh966IBxo8hGYsk7MJ3LobvuG6j497SUc +nn4JECm/mOKGeQvSSGnor93ropyWAQDQ3U1JVxV/K4sw2EpwwxfaJAM4L5rVi9EK +TsN23cPI81DKLuDxeXGGDPXMgQuTqfGD74jk6oTpfEHNmQB1Lcj+t+HxQqyoHyo5 +RPNRpntgPAvrF8i1ktJ/EH4GJxSBwm7098JcMgQSif9PHzL0UKehC2mlNX7ljGQ+ +eOLo6XNHYnq6DfxO6c3TbOIYt7VSc8K3IG500/4IzIT3+mtZ3rrM3mQWDwIDAQAB +oy8wLTAaBgNVHREEEzARgg9EaXZpbmVBdXRob3JpdHkwDwYDVR0TAQH/BAUwAwEB +/zANBgkqhkiG9w0BAQsFAAOCAQEAfzQSUzfaUv5Q4Eqz2YiWFx2zRYi0mUjYrGf9 +1qcprgpAq7F72+ed3uLGEmMr53+wgL4XdzLnSZwpYRFNBI7/t6hU3kxw9fJC5wMg +LHLdNlNqXAfoGVVTjcWPiQDF6tguccqyE3UWksl+2fncgkkcUpH4IP0AZVYlCsrz +mzs5P3ATpdTE1BZiw4WEiE4+N8ZC7Rcz0icfCEbKJduMkkxpJlvp5LwSsmtrpS3v +IZvomDHx8ypr+byzUTsfbAExdXVpctkG/zLMAi6/ZApO8GlD8ga8BUn2NGfBO5Q8 +28kEjS5DV835Re4hHE6pTC4HEjq0D2r1/4OG7ijt8emO5XPoMg== +-----END CERTIFICATE-----''' + +TEST_APP_CERT = '''-----BEGIN CERTIFICATE----- +MIID9jCCAt6gAwIBAgIUX5lsqmlS3aFLw7+IqSqadI7W1yswDQYJKoZIhvcNAQEL +BQAwRTFDMEEGA1UEAxM6VmF1bHQgSW50ZXJtZWRpYXRlIENlcnRpZmljYXRlIEF1 +dGhvcml0eSAoY2hhcm0tcGtpLWxvY2FsKTAeFw0yMDA1MDUwOTQyMTdaFw0yMTA1 +MDUwODQyNDdaMA4xDDAKBgNVBAMTA2FwcDCCASIwDQYJKoZIhvcNAQEBBQADggEP +ADCCAQoCggEBALfmMzGbbShmQGduZImaGsJWd6vGriVwgYlIV60Kb1MLxuLvMyzV +tBseRH1izKgPDEmMRafU9N4DC0jRb+04APBM8QBWEDrrYgRQQSNxlCDVMn4Q4iHO +72FwCqI1HuW0R5J3yik4FkW3Kb8Uq5KDsKWqTLtaBW5X40toi1bkyFTnRZ6/3vmt +9arAfqmZyXlZK3rN+uiznLx8/rYU5umkicNGfDcWI37wjdYvK/tIE79vPom5VhGb +R+rz+hri7JmiaYkzrTWWibyjPNK0aGHa5OUIiFJfAtfyjoT1d/pxwS301BWLicw1 +vSzCJcTwpkzh2EWvuquK2sUjgHNR1qAkGIECAwEAAaOCARMwggEPMA4GA1UdDwEB +/wQEAwIDqDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwHQYDVR0OBBYE +FL0B0hMaFwG0I0WR4CiOZnrqRHoLMEkGCCsGAQUFBwEBBD0wOzA5BggrBgEFBQcw +AoYtaHR0cDovLzE3Mi4yMC4wLjE5OjgyMDAvdjEvY2hhcm0tcGtpLWxvY2FsL2Nh +MDMGA1UdEQQsMCqCA2FwcIIDYXBwgghhcHB1bml0MYIIYXBwdW5pdDKHBKwAAAGH +BKwAAAIwPwYDVR0fBDgwNjA0oDKgMIYuaHR0cDovLzE3Mi4yMC4wLjE5OjgyMDAv +djEvY2hhcm0tcGtpLWxvY2FsL2NybDANBgkqhkiG9w0BAQsFAAOCAQEAbf6kIurd +pBs/84YD59bgeytlo8RatUzquwCRgRSv6N81+dYFBHtEVOoLwy/4wJAH2uMSKK+/ +C13vTBj/cx+SxWSIccPS0rglwEKhRF/u3n9hrFAL3QMLQPEXAJ5rJtapZ7a8uIWy +bChTMhoL4bApCXG+SH4mbhkD6SWQ1zPgfXD4ZiVtjEVIdyn63/fbNFUfhFKba8BE +wQUYw0yWq0/8ILq/WPyjKBvhSinIauy+ybdzaDMEg0Grq1n0K5l/WyK+t9tQd+UG +cLjamd6EKZ2OvOxZN6/cJlHDY2NKfjGF6KhQ5D2cseYK7dhOQ9AFjUCB/NgIAH9D +8vVp8VJOx6plOw== +-----END CERTIFICATE-----''' + +TEST_APP_KEY = '''-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAt+YzMZttKGZAZ25kiZoawlZ3q8auJXCBiUhXrQpvUwvG4u8z +LNW0Gx5EfWLMqA8MSYxFp9T03gMLSNFv7TgA8EzxAFYQOutiBFBBI3GUINUyfhDi +Ic7vYXAKojUe5bRHknfKKTgWRbcpvxSrkoOwpapMu1oFblfjS2iLVuTIVOdFnr/e ++a31qsB+qZnJeVkres366LOcvHz+thTm6aSJw0Z8NxYjfvCN1i8r+0gTv28+iblW +EZtH6vP6GuLsmaJpiTOtNZaJvKM80rRoYdrk5QiIUl8C1/KOhPV3+nHBLfTUFYuJ +zDW9LMIlxPCmTOHYRa+6q4raxSOAc1HWoCQYgQIDAQABAoIBAD92GUSNNmYyoxcO +aXNy0rktza5hqccRxCHz7Q2yBCjMb53wneBi/vw8vbXnWmjEiKD43zDDtJzIwCQo +4k8ifHBwnNpY2ND8WZ7TcycgEtYhvIL0oJS6LLGbUJAZdMggJnLNE96VlFoKk0V1 +hJ/TAiqpUkF1F1q0yaNEOJGL8fYaI5Mz1pU+rspxS2uURFYGcD78Ouda5Pruwcp3 +A0Sbo+5P0FZRy79zpZbIzlvcS9R7wKuDJExCXXCsoZ+G0BWwTJPsDhkmcuXdS7f3 +3k3VO4Y8rcsOIHtI0Gj38yhO6giDjPeZWmXF6h7+zSWPaZydswTqtyS2BbvUmE3N +t/HYCOECgYEA2AYQZqAeFk5i7Qnb80pG9q1THZOM4V/FQsyfb9Bzw+nANP6LMd3D +tnY7BUNj0vTJVy/wnwFSmryQn3OqsxHYbOaor9xjuCauAGzp/4cj0anTySz0pZiQ +TzVepB35bj8ghRsQ1TO+7FQtMMZQGrNf1i6e3p9+hpKUA6ZwP0OEbpMCgYEA2e5E +Uqqj1u0pnUAeXp/2VbQS4rmxUrRsbdbiyoypNJOp+Olfi2DjQNgji0XDBdTLhDNv +nFtHY7TW4HJrwVAAqBlYKkunf6zGlP3iEGhk7RF1LSyGZXjfLACe7kzqlAx34Ue9 +9ynkesNKeT8kOOCC08llHuInMjfgfN0c7jWYNRsCgYEAgzBrlWd33iQMf9eU89MP +9Y6dA0EwNU5sBX0u9kCpjTjPuV88OTRsPsreXPvoC50NCR3cCzRKbh5F1g/wgn87 +6CbMGsDE7njPAwMhuEThw9pW+72JdWeJfBD1QMXTTNiZbzxYpKGgOPWF3DETRKPa +d8AoSxqhRCiQKwdQ85qVOnECgYAu6dfTY+B5N/ypWVAwVocU0/rsy8ScZTKiQov3 +xmf2ZYNFjhd/TZAeOWkNZishajmVb+0q34tyr09Cad9AchRyG2KbWEXqeisVj8HG +fnKbhhKPcvJLjcWdF1UfP3eP/08fM+508pO4yamSiEEn7Uy8grI9/7koWlb9Cixc +KzVk2QKBgQCdA3eoJHu4nTHRNgcvU3pxbRU4HQV8e+Hiw1tcxjprkACrNVvd7wZS +wULKjMb8z0RZyTBXLdNw3YKYOk/B7e/e9D+Zve4PTEL23Fcdt532x/7hBQ+7o6/4 +7RxsGx5/PXZI0/YKMKk9hsrdMl4/UAd0izvwPCQbB3eisuZYU/i8Jw== +-----END RSA PRIVATE KEY-----''' + class CharmTestCase(unittest.TestCase): @@ -57,6 +130,7 @@ class TestCephISCSIGatewayCharmBase(CharmTestCase): 'ch_templating', 'gwcli_client', 'subprocess', + 'os', ] def setUp(self): @@ -67,15 +141,9 @@ class TestCephISCSIGatewayCharmBase(CharmTestCase): self.gwc = MagicMock() self.gwcli_client.GatewayClient.return_value = self.gwc - # BEGIN: Workaround until - # https://github.com/canonical/operator/pull/196 lands + # BEGIN: Workaround until network_get is implemented class _TestingOPSModelBackend(_TestingModelBackend): - def relation_ids(self, relation_name): - return self._relation_ids_map.get(relation_name, []) - - # Hardcoded until network_get is implemented in - # _TestingModelBackend def network_get(self, endpoint_name, relation_id=None): network_data = { 'bind-addresses': [{ @@ -88,7 +156,7 @@ class TestCephISCSIGatewayCharmBase(CharmTestCase): return network_data self.harness._backend = _TestingOPSModelBackend( - self.harness._unit_name) + self.harness._unit_name, self.harness._meta) self.harness._model = model.Model( self.harness._unit_name, self.harness._meta, @@ -253,6 +321,8 @@ class TestCephISCSIGatewayCharmBase(CharmTestCase): 'allow r']}]) def test_on_pools_available(self): + self.os.path.exists.return_value = False + self.os.path.basename = os.path.basename rel_id = self.add_cluster_relation() self.harness.update_relation_data( rel_id, @@ -262,14 +332,16 @@ class TestCephISCSIGatewayCharmBase(CharmTestCase): self.harness.begin() self.harness.charm.ceph_client.on.pools_available.emit() self.ch_templating.render.assert_has_calls([ - call('ceph.conf', '/etc/ceph/ceph.conf', ANY), + call('ceph.conf', '/etc/ceph/iscsi/ceph.conf', ANY), call('iscsi-gateway.cfg', '/etc/ceph/iscsi-gateway.cfg', ANY), call( 'ceph.client.ceph-iscsi.keyring', - '/etc/ceph/ceph.client.ceph-iscsi.keyring', ANY)]) + '/etc/ceph/iscsi/ceph.client.ceph-iscsi.keyring', ANY)], + any_order=True) self.assertTrue(self.harness.charm.state.is_started) rel_data = self.harness.get_relation_data(rel_id, 'ceph-iscsi/0') self.assertEqual(rel_data['gateway_ready'], 'True') + self.os.mkdir.assert_called_once_with('/etc/ceph/iscsi', 488) @patch('socket.gethostname') def test_on_certificates_relation_joined(self, _gethostname): @@ -290,41 +362,42 @@ class TestCephISCSIGatewayCharmBase(CharmTestCase): @patch('socket.gethostname') def test_on_certificates_relation_changed(self, _gethostname): + mock_TLS_CERT_PATH = MagicMock() + mock_TLS_CA_CERT_PATH = MagicMock() + mock_TLS_KEY_PATH = MagicMock() + mock_KEY_AND_CERT_PATH = MagicMock() + mock_TLS_PUB_KEY_PATH = MagicMock() _gethostname.return_value = 'server1' self.subprocess.check_output.return_value = b'pubkey' rel_id = self.harness.add_relation('certificates', 'vault') self.add_cluster_relation() self.harness.begin() - with patch('builtins.open', unittest.mock.mock_open()) as _open: - self.harness.add_relation_unit( - rel_id, - 'vault/0') - self.harness.update_relation_data( - rel_id, - 'vault/0', - { - 'ceph-iscsi_0.processed_application_requests': - '{"app_data": {"cert": "appcert", "key": "appkey"}}', - 'ca': 'ca'}) - expect_calls = [ - call('/etc/ceph/iscsi-gateway.crt', 'w'), - call('/etc/ceph/iscsi-gateway.key', 'w'), - call('/etc/ceph/iscsi-gateway.pem', 'w'), - call('/usr/local/share/ca-certificates/vault_ca_cert.crt', 'w')] - for open_call in expect_calls: - self.assertIn(open_call, _open.call_args_list) - handle = _open() - handle.write.assert_has_calls([ - call('appcert'), - call('appkey'), - call('appcert\nappkey'), - call('ca'), - call('pubkey')]) + self.harness.charm.TLS_CERT_PATH = mock_TLS_CERT_PATH + self.harness.charm.TLS_CA_CERT_PATH = mock_TLS_CA_CERT_PATH + self.harness.charm.TLS_KEY_PATH = mock_TLS_KEY_PATH + self.harness.charm.TLS_KEY_AND_CERT_PATH = mock_KEY_AND_CERT_PATH + self.harness.charm.TLS_PUB_KEY_PATH = mock_TLS_PUB_KEY_PATH + self.harness.add_relation_unit( + rel_id, + 'vault/0') + rel_data = { + 'app_data': { + 'cert': TEST_APP_CERT, + 'key': TEST_APP_KEY}} + self.harness.update_relation_data( + rel_id, + 'vault/0', + { + 'ceph-iscsi_0.processed_application_requests': json.dumps( + rel_data), + 'ca': TEST_CA}) + mock_TLS_CERT_PATH.write_bytes.assert_called_once() + mock_TLS_CA_CERT_PATH.write_bytes.assert_called_once() + mock_TLS_KEY_PATH.write_bytes.assert_called_once() + mock_KEY_AND_CERT_PATH.write_bytes.assert_called_once() + mock_TLS_PUB_KEY_PATH.write_bytes.assert_called_once() self.subprocess.check_call.assert_called_once_with( ['update-ca-certificates']) - self.subprocess.check_output.assert_called_once_with( - ['openssl', 'x509', '-inform', 'pem', '-in', - '/etc/ceph/iscsi-gateway.pem', '-pubkey', '-noout']) self.assertTrue(self.harness.charm.state.enable_tls) def test_custom_status_check(self):