openstack/charm-cinder
Code Issues Proposed changes

synced charm-helpers

Browse Source
This commit is contained in:
Edward Hope-Morley
2014-09-26 13:41:03 +01:00
parent 1a60e944bb
commit 5c363250c2
15 changed files with 265 additions and 90 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
hooks/charmhelpers/contrib/hahelpers/apache.py
View File

@@ -20,20 +20,27 @@ from charmhelpers.core.hookenv import (
)
def get_cert():
def get_cert(cn=None):
# TODO: deal with multiple https endpoints via charm config
cert = config_get('ssl_cert')
key = config_get('ssl_key')
if not (cert and key):
log("Inspecting identity-service relations for SSL certificate.",
level=INFO)
cert = key = None
if cn:
ssl_cert_attr = 'ssl_cert_{}'.format(cn)
ssl_key_attr = 'ssl_key_{}'.format(cn)
else:
ssl_cert_attr = 'ssl_cert'
ssl_key_attr = 'ssl_key'
for r_id in relation_ids('identity-service'):
for unit in relation_list(r_id):
if not cert:
cert = relation_get('ssl_cert',
cert = relation_get(ssl_cert_attr,
rid=r_id, unit=unit)
if not key:
key = relation_get('ssl_key',
key = relation_get(ssl_key_attr,
rid=r_id, unit=unit)
return (cert, key)

3
hooks/charmhelpers/contrib/hahelpers/cluster.py
View File

@@ -139,10 +139,9 @@ def https():
return True
for r_id in relation_ids('identity-service'):
for unit in relation_list(r_id):
# TODO - needs fixing for new helper as ssl_cert/key suffixes with CN
rel_state = [
relation_get('https_keystone', rid=r_id, unit=unit),
relation_get('ssl_cert', rid=r_id, unit=unit),
relation_get('ssl_key', rid=r_id, unit=unit),
relation_get('ca_cert', rid=r_id, unit=unit),
]
# NOTE: works around (LP: #1203241)

68
hooks/charmhelpers/contrib/network/ip.py
View File

@@ -1,4 +1,6 @@
import glob
import re
import subprocess
import sys
from functools import partial
@@ -172,7 +174,8 @@ def format_ipv6_addr(address):
return address
def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=True, exc_list=None):
def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False,
fatal=True, exc_list=None):
"""
Return the assigned IP address for a given interface, if any, or [].
"""
@@ -212,26 +215,67 @@ def get_iface_addr(iface='eth0', inet_type='AF_INET', inc_aliases=False, fatal=T
if 'addr' in entry and entry['addr'] not in exc_list:
addresses.append(entry['addr'])
if fatal and not addresses:
raise Exception("Interface '%s' doesn't have any %s addresses." % (iface, inet_type))
raise Exception("Interface '%s' doesn't have any %s addresses." %
(iface, inet_type))
return addresses
get_ipv4_addr = partial(get_iface_addr, inet_type='AF_INET')
def get_ipv6_addr(iface='eth0', inc_aliases=False, fatal=True, exc_list=None):
"""
Return the assigned IPv6 address for a given interface, if any, or [].
def get_ipv6_addr(iface='eth0', inc_aliases=False, fatal=True, exc_list=None,
dynamic_only=True):
"""Get assigned IPv6 address for a given interface.
Returns list of addresses found. If no address found, returns empty list.
We currently only support scope global IPv6 addresses i.e. non-temporary
addresses. If no global IPv6 address is found, return the first one found
in the ipv6 address list.
"""
addresses = get_iface_addr(iface=iface, inet_type='AF_INET6',
inc_aliases=inc_aliases, fatal=fatal,
exc_list=exc_list)
remotly_addressable = []
for address in addresses:
if not address.startswith('fe80'):
remotly_addressable.append(address)
if fatal and not remotly_addressable:
raise Exception("Interface '%s' doesn't have global ipv6 address." % iface)
return remotly_addressable
if addresses:
global_addrs = []
for addr in addresses:
key_scope_link_local = re.compile("^fe80::..(.+)%(.+)")
m = re.match(key_scope_link_local, addr)
if m:
eui_64_mac = m.group(1)
iface = m.group(2)
else:
global_addrs.append(addr)
if global_addrs:
# Make sure any found global addresses are not temporary
cmd = ['ip', 'addr', 'show', iface]
out = subprocess.check_output(cmd)
if dynamic_only:
key = re.compile("inet6 (.+)/[0-9]+ scope global dynamic.*")
else:
key = re.compile("inet6 (.+)/[0-9]+ scope global.*")
addrs = []
for line in out.split('\n'):
line = line.strip()
m = re.match(key, line)
if m and 'temporary' not in line:
# Return the first valid address we find
for addr in global_addrs:
if m.group(1) == addr:
if not dynamic_only or \
m.group(1).endswith(eui_64_mac):
addrs.append(addr)
if addrs:
return addrs
if fatal:
raise Exception("Interface '%s' doesn't have a scope global "
"non-temporary ipv6 address." % iface)
return []
def get_bridges(vnic_dir='/sys/devices/virtual/net'):

37
hooks/charmhelpers/contrib/openstack/amulet/deployment.py
View File

@@ -1,3 +1,6 @@
from bzrlib.branch import Branch
import os
import re
from charmhelpers.contrib.amulet.deployment import (
AmuletDeployment
)
@@ -16,11 +19,41 @@ class OpenStackAmuletDeployment(AmuletDeployment):
self.openstack = openstack
self.source = source
def _is_dev_branch(self):
"""Determine if branch being tested is a dev (i.e. next) branch."""
branch = Branch.open(os.getcwd())
parent = branch.get_parent()
pattern = re.compile("^.*/next/$")
if (pattern.match(parent)):
return True
else:
return False
def _determine_branch_locations(self, other_services):
"""Determine the branch locations for the other services.
If the branch being tested is a dev branch, then determine the
development branch locations for the other services. Otherwise,
the default charm store branches will be used."""
name = 0
if self._is_dev_branch():
updated_services = []
for svc in other_services:
if svc[name] in ['mysql', 'mongodb', 'rabbitmq-server']:
location = 'lp:charms/{}'.format(svc[name])
else:
temp = 'lp:~openstack-charmers/charms/trusty/{}/next'
location = temp.format(svc[name])
updated_services.append(svc + (location,))
other_services = updated_services
return other_services
def _add_services(self, this_service, other_services):
"""Add services to the deployment and set openstack-origin."""
"""Add services to the deployment and set openstack-origin/source."""
name = 0
other_services = self._determine_branch_locations(other_services)
super(OpenStackAmuletDeployment, self)._add_services(this_service,
other_services)
name = 0
services = other_services
services.append(this_service)
use_source = ['mysql', 'mongodb', 'rabbitmq-server', 'ceph']

9
hooks/charmhelpers/contrib/openstack/amulet/utils.py
View File

@@ -187,15 +187,16 @@ class OpenStackAmuletUtils(AmuletUtils):
f = opener.open("http://download.cirros-cloud.net/version/released")
version = f.read().strip()
cirros_img = "tests/cirros-{}-x86_64-disk.img".format(version)
cirros_img = "cirros-{}-x86_64-disk.img".format(version)
local_path = os.path.join('tests', cirros_img)
if not os.path.exists(cirros_img):
if not os.path.exists(local_path):
cirros_url = "http://{}/{}/{}".format("download.cirros-cloud.net",
version, cirros_img)
opener.retrieve(cirros_url, cirros_img)
opener.retrieve(cirros_url, local_path)
f.close()
with open(cirros_img) as f:
with open(local_path) as f:
image = glance.images.create(name=image_name, is_public=True,
disk_format='qcow2',
container_format='bare', data=f)

101
hooks/charmhelpers/contrib/openstack/context.py
View File

@@ -8,7 +8,6 @@ from subprocess import (
check_call
)
from charmhelpers.fetch import (
apt_install,
filter_installed_packages,
@@ -28,6 +27,11 @@ from charmhelpers.core.hookenv import (
INFO
)
from charmhelpers.core.host import (
mkdir,
write_file
)
from charmhelpers.contrib.hahelpers.cluster import (
determine_apache_port,
determine_api_port,
@@ -38,6 +42,7 @@ from charmhelpers.contrib.hahelpers.cluster import (
from charmhelpers.contrib.hahelpers.apache import (
get_cert,
get_ca_cert,
install_ca_cert,
)
from charmhelpers.contrib.openstack.neutron import (
@@ -47,7 +52,8 @@ from charmhelpers.contrib.openstack.neutron import (
from charmhelpers.contrib.network.ip import (
get_address_in_network,
get_ipv6_addr,
format_ipv6_addr
format_ipv6_addr,
is_address_in_network
)
CA_CERT_PATH = '/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt'
@@ -435,6 +441,11 @@ class HAProxyContext(OSContextGenerator):
'units': cluster_hosts,
}
if config('haproxy-server-timeout'):
ctxt['haproxy-server-timeout'] = config('haproxy-server-timeout')
if config('haproxy-client-timeout'):
ctxt['haproxy-client-timeout'] = config('haproxy-client-timeout')
if config('prefer-ipv6'):
ctxt['local_host'] = 'ip6-localhost'
ctxt['haproxy_host'] = '::'
@@ -504,22 +515,36 @@ class ApacheSSLContext(OSContextGenerator):
cmd = ['a2enmod', 'ssl', 'proxy', 'proxy_http']
check_call(cmd)
def configure_cert(self):
if not os.path.isdir('/etc/apache2/ssl'):
os.mkdir('/etc/apache2/ssl')
def configure_cert(self, cn=None):
ssl_dir = os.path.join('/etc/apache2/ssl/', self.service_namespace)
if not os.path.isdir(ssl_dir):
os.mkdir(ssl_dir)
cert, key = get_cert()
with open(os.path.join(ssl_dir, 'cert'), 'w') as cert_out:
cert_out.write(b64decode(cert))
with open(os.path.join(ssl_dir, 'key'), 'w') as key_out:
key_out.write(b64decode(key))
mkdir(path=ssl_dir)
cert, key = get_cert(cn)
if cn:
cert_filename = 'cert_{}'.format(cn)
key_filename = 'key_{}'.format(cn)
else:
cert_filename = 'cert'
key_filename = 'key'
write_file(path=os.path.join(ssl_dir, cert_filename),
content=b64decode(cert))
write_file(path=os.path.join(ssl_dir, key_filename),
content=b64decode(key))
def configure_ca(self):
ca_cert = get_ca_cert()
if ca_cert:
with open(CA_CERT_PATH, 'w') as ca_out:
ca_out.write(b64decode(ca_cert))
check_call(['update-ca-certificates'])
install_ca_cert(b64decode(ca_cert))
def canonical_names(self):
'''Figure out which canonical names clients will access this service'''
cns = []
for r_id in relation_ids('identity-service'):
for unit in related_units(r_id):
rdata = relation_get(rid=r_id, unit=unit)
for k in rdata:
if k.startswith('ssl_key_'):
cns.append(k.lstrip('ssl_key_'))
return list(set(cns))
def __call__(self):
if isinstance(self.external_ports, basestring):
@@ -527,21 +552,47 @@ class ApacheSSLContext(OSContextGenerator):
if (not self.external_ports or not https()):
return {}
self.configure_cert()
self.configure_ca()
self.enable_modules()
ctxt = {
'namespace': self.service_namespace,
'private_address': unit_get('private-address'),
'endpoints': []
'endpoints': [],
'ext_ports': []
}
if is_clustered():
ctxt['private_address'] = config('vip')
for api_port in self.external_ports:
ext_port = determine_apache_port(api_port)
int_port = determine_api_port(api_port)
portmap = (int(ext_port), int(int_port))
ctxt['endpoints'].append(portmap)
for cn in self.canonical_names():
self.configure_cert(cn)
addresses = []
vips = []
if config('vip'):
vips = config('vip').split()
for network_type in ['os-internal-network',
'os-admin-network',
'os-public-network']:
address = get_address_in_network(config(network_type),
unit_get('private-address'))
if len(vips) > 0 and is_clustered():
for vip in vips:
if is_address_in_network(config(network_type),
vip):
addresses.append((address, vip))
break
elif is_clustered():
addresses.append((address, config('vip')))
else:
addresses.append((address, address))
for address, endpoint in set(addresses):
for api_port in self.external_ports:
ext_port = determine_apache_port(api_port)
int_port = determine_api_port(api_port)
portmap = (address, endpoint, int(ext_port), int(int_port))
ctxt['endpoints'].append(portmap)
ctxt['ext_ports'].append(int(ext_port))
ctxt['ext_ports'] = list(set(ctxt['ext_ports']))
return ctxt

9
hooks/charmhelpers/contrib/openstack/templates/haproxy.cfg
View File

@@ -14,8 +14,17 @@ defaults
retries 3
timeout queue 1000
timeout connect 1000
{% if haproxy-client-timeout -%}
timeout client {{ haproxy-client-timeout }}
{% else -%}
timeout client 30000
{% endif -%}
{% if haproxy-server-timeout -%}
timeout server {{ haproxy-server-timeout }}
{% else -%}
timeout server 30000
{% endif -%}
listen stats {{ stat_port }}
mode http

17
hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend
View File

@@ -1,16 +1,18 @@
{% if endpoints -%}
{% for ext, int in endpoints -%}
Listen {{ ext }}
NameVirtualHost *:{{ ext }}
<VirtualHost *:{{ ext }}>
ServerName {{ private_address }}
{% for ext_port in ext_ports -%}
Listen {{ ext_port }}
{% endfor -%}
{% for address, endpoint, ext, int in endpoints -%}
<VirtualHost {{ address }}:{{ ext }}>
ServerName {{ endpoint }}
SSLEngine on
SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
ProxyPass / http://localhost:{{ int }}/
ProxyPassReverse / http://localhost:{{ int }}/
ProxyPreserveHost on
</VirtualHost>
{% endfor -%}
<Proxy *>
Order deny,allow
Allow from all
@@ -19,5 +21,4 @@ NameVirtualHost *:{{ ext }}
Order allow,deny
Allow from all
</Location>
{% endfor -%}
{% endif -%}

17
hooks/charmhelpers/contrib/openstack/templates/openstack_https_frontend.conf
View File

@@ -1,16 +1,18 @@
{% if endpoints -%}
{% for ext, int in endpoints -%}
Listen {{ ext }}
NameVirtualHost *:{{ ext }}
<VirtualHost *:{{ ext }}>
ServerName {{ private_address }}
{% for ext_port in ext_ports -%}
Listen {{ ext_port }}
{% endfor -%}
{% for address, endpoint, ext, int in endpoints -%}
<VirtualHost {{ address }}:{{ ext }}>
ServerName {{ endpoint }}
SSLEngine on
SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert_{{ endpoint }}
SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key_{{ endpoint }}
ProxyPass / http://localhost:{{ int }}/
ProxyPassReverse / http://localhost:{{ int }}/
ProxyPreserveHost on
</VirtualHost>
{% endfor -%}
<Proxy *>
Order deny,allow
Allow from all
@@ -19,5 +21,4 @@ NameVirtualHost *:{{ ext }}
Order allow,deny
Allow from all
</Location>
{% endfor -%}
{% endif -%}

17
hooks/charmhelpers/contrib/openstack/utils.py
View File

@@ -13,7 +13,9 @@ from charmhelpers.core.hookenv import (
log as juju_log,
charm_dir,
ERROR,
INFO
INFO,
relation_ids,
relation_set
)
from charmhelpers.contrib.storage.linux.lvm import (
@@ -22,6 +24,10 @@ from charmhelpers.contrib.storage.linux.lvm import (
remove_lvm_physical_volume,
)
from charmhelpers.contrib.network.ip import (
get_ipv6_addr
)
from charmhelpers.core.host import lsb_release, mounts, umount
from charmhelpers.fetch import apt_install, apt_cache
from charmhelpers.contrib.storage.linux.utils import is_block_device, zap_disk
@@ -457,3 +463,12 @@ def get_hostname(address, fqdn=True):
return result
else:
return result.split('.')[0]
def sync_db_with_multi_ipv6_addresses():
hosts = get_ipv6_addr(dynamic_only=False)
for rid in relation_ids('shared-db'):
relation_set(relation_id=rid,
database=config('database'),
username=config('database-user'),
hostname=hosts)

10
hooks/charmhelpers/core/hookenv.py
View File

@@ -486,9 +486,10 @@ class Hooks(object):
hooks.execute(sys.argv)
"""
def __init__(self):
def __init__(self, config_save=True):
super(Hooks, self).__init__()
self._hooks = {}
self._config_save = config_save
def register(self, name, function):
"""Register a hook"""
@@ -499,9 +500,10 @@ class Hooks(object):
hook_name = os.path.basename(args[0])
if hook_name in self._hooks:
self._hooks[hook_name]()
cfg = config()
if cfg.implicit_save:
cfg.save()
if self._config_save:
cfg = config()
if cfg.implicit_save:
cfg.save()
else:
raise UnregisteredHookError(hook_name)

12
hooks/charmhelpers/core/host.py
View File

@@ -68,8 +68,8 @@ def service_available(service_name):
"""Determine whether a system service is available"""
try:
subprocess.check_output(['service', service_name, 'status'], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
return False
except subprocess.CalledProcessError as e:
return 'unrecognized service' not in e.output
else:
return True
@@ -229,12 +229,12 @@ def check_hash(path, checksum, hash_type='md5'):
"""
Validate a file using a cryptographic checksum.
:param str checksum: Value of the checksum used to validate the file.
:param str hash_type: Hash algorithm used to generate :param:`checksum`.
Can be any hash alrgorithm supported by :mod:`hashlib`,
such as md5, sha1, sha256, sha512, etc.
:param str hash_type: Hash algorithm used to generate `checksum`.
Can be any hash alrgorithm supported by :mod:`hashlib`,
such as md5, sha1, sha256, sha512, etc.
:raises ChecksumError: If the file fails the checksum
"""
actual_checksum = file_hash(path, hash_type)
if checksum != actual_checksum:

3
hooks/charmhelpers/fetch/__init__.py
View File

@@ -208,7 +208,8 @@ def add_source(source, key=None):
"""Add a package source to this system.
@param source: a URL or sources.list entry, as supported by
add-apt-repository(1). Examples:
add-apt-repository(1). Examples::
ppa:charmers/example
deb https://stub:key@private.example.com/ubuntu trusty main

13
hooks/charmhelpers/fetch/archiveurl.py
View File

@@ -74,18 +74,19 @@ class ArchiveUrlFetchHandler(BaseFetchHandler):
"""
Download and install an archive file, with optional checksum validation.
The checksum can also be given on the :param:`source` URL's fragment.
The checksum can also be given on the `source` URL's fragment.
For example::
handler.install('http://example.com/file.tgz#sha1=deadbeef')
:param str source: URL pointing to an archive file.
:param str dest: Local destination path to install to. If not given,
installs to `$CHARM_DIR/archives/archive_file_name`.
:param str dest: Local destination path to install to. If not given,
installs to `$CHARM_DIR/archives/archive_file_name`.
:param str checksum: If given, validate the archive file after download.
:param str hash_type: Algorithm used to generate :param:`checksum`.
Can be any hash alrgorithm supported by :mod:`hashlib`,
such as md5, sha1, sha256, sha512, etc.
:param str hash_type: Algorithm used to generate `checksum`.
Can be any hash alrgorithm supported by :mod:`hashlib`,
such as md5, sha1, sha256, sha512, etc.
"""
url_parts = self.parse_url(source)
dest_dir = os.path.join(os.environ.get('CHARM_DIR'), 'fetched')

26
unit_tests/test_cinder_contexts.py
View File

@@ -95,17 +95,25 @@ class TestCinderContext(CharmTestCase):
self.assertEquals(contexts.StorageBackendContext()(),
{'backends': 'cinder-ceph,cinder-vmware'})
@patch('charmhelpers.contrib.openstack.context.is_clustered')
@patch('charmhelpers.contrib.openstack.context.determine_apache_port')
@patch('charmhelpers.contrib.openstack.context.determine_api_port')
@patch('charmhelpers.contrib.openstack.context.unit_get')
@patch('charmhelpers.contrib.openstack.context.https')
mod_ch_context = 'charmhelpers.contrib.openstack.context'
@patch('%s.ApacheSSLContext.canonical_names' % (mod_ch_context))
@patch('%s.ApacheSSLContext.configure_ca' % (mod_ch_context))
@patch('%s.config' % (mod_ch_context))
@patch('%s.is_clustered' % (mod_ch_context))
@patch('%s.determine_apache_port' % (mod_ch_context))
@patch('%s.determine_api_port' % (mod_ch_context))
@patch('%s.unit_get' % (mod_ch_context))
@patch('%s.https' % (mod_ch_context))
@patch.object(utils, 'service_enabled')
def test_apache_ssl_context_service_enabled(self, service_enabled,
mock_https, mock_unit_get,
mock_determine_api_port,
mock_determine_apache_port,
mock_is_clustered):
mock_is_clustered,
mock_hookenv,
mock_configure_ca,
mock_cfg_canonical_names):
mock_https.return_value = True
mock_unit_get.return_value = '1.2.3.4'
mock_determine_api_port.return_value = '12'
@@ -120,8 +128,10 @@ class TestCinderContext(CharmTestCase):
self.assertFalse(mock_https.called)
service_enabled.return_value = True
self.assertEquals(ctxt(), {'endpoints': [(34, 12)],
'private_address': '1.2.3.4',
self.assertEquals(ctxt(), {'endpoints': [('1.2.3.4',
'1.2.3.4',
34, 12)],
'ext_ports': [34],
'namespace': 'cinder'})
self.assertTrue(mock_https.called)
mock_unit_get.assert_called_with('private-address')