Refactoring to support shared openstack codebase for:

apache
haproxy
cluster
utils

Better support for HA, cloning of haproxy, https from keystone.

.project

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>swift-proxy</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

.pydevproject

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?><pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/swift-proxy/hooks</path>
+</pydev_pathproperty>
+</pydev_project>

config.yaml

@@ -46,7 +46,25 @@ options:
       zones before the storage ring will be initially balance.  Deployment
       requirements differ based on the zone-assignment policy configured, see
       this charm's README for details.
-  # CA Cert info
+  # User provided SSL cert and key
+  ssl_cert:
+    type: string
+    description: |
+      Base64 encoded SSL certificate to install and use for API ports.
+      .
+         juju set swift-proxy ssl_cert="$(cat cert | base64)" \
+                              ssl_key="$(cat key | base64)"
+      .
+      Setting this value (and ssl_key) will enable reverse proxying, point
+      Swifts's entry in the Keystone catalog to use https, and override
+      any certficiate and key issued by Keystone (if it is configured to
+      do so).
+  ssl_key:
+    type: string
+    description: |
+      Base64 encoded SSL key to use with certificate specified as ssl_cert.
+  # Locally generated CA Cert info (only use without keystone)
+  # These options are deprecated and will be removed sometime
   use-https:
     default: "yes"
     type: string
@@ -67,6 +85,7 @@ options:
     default: CN
     type: string
     description: Common Name
+  # General Swift Proxy configuration
   bind-port:
     default: 8080
     type: int
@@ -105,3 +124,30 @@ options:
   keystone-admin-password:
     type: string
     description: Keystone admin password
+  # HA configuration settings
+  swift-hash:
+    type: string
+    description: Hash to use across all swift-proxy servers - don't loose
+  vip:
+    type: string
+    description: "Virtual IP to use to front swift-proxy in ha configuration"
+  vip_iface:
+    type: string
+    default: eth0
+    description: "Network Interface where to place the Virtual IP"
+  vip_cidr:
+    type: int
+    default: 24
+    description: "Netmask that will be used for the Virtual IP"
+  ha-bindiface:
+    type: string
+    default: eth0
+    description: |
+      Default network interface on which HA cluster will bind to communication
+      with the other members of the HA Cluster.
+  ha-mcastport:
+    type: int
+    default: 5414
+    description: |
+      Default multicast port number that will be used to communicate between
+      HA Cluster nodes.

hooks/cluster-relation-changed

@@ -0,0 +1 @@
+swift_hooks.py

hooks/cluster-relation-joined

@@ -0,0 +1 @@
+swift_hooks.py

hooks/config-changed

@@ -1 +1 @@
-swift-hooks.py
+swift_hooks.py

hooks/ha-relation-changed

@@ -0,0 +1 @@
+swift_hooks.py

hooks/ha-relation-joined

@@ -0,0 +1 @@
+swift_hooks.py

hooks/identity-service-relation-changed

@@ -1 +1 @@
-swift-hooks.py
+swift_hooks.py

hooks/identity-service-relation-joined

@@ -1 +1 @@
-swift-hooks.py
+swift_hooks.py

hooks/install

@@ -1 +1 @@
-swift-hooks.py
+swift_hooks.py

hooks/lib/apache_utils.py

@@ -0,0 +1,193 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  James Page <james.page@ubuntu.com>
+#
+
+from lib.utils import (
+    relation_ids,
+    relation_list,
+    relation_get,
+    render_template,
+    juju_log,
+    config_get,
+    install,
+    get_host_ip,
+    restart
+    )
+from lib.cluster_utils import https
+
+import os
+import subprocess
+from base64 import b64decode
+
+APACHE_SITE_DIR = "/etc/apache2/sites-available"
+SITE_TEMPLATE = "apache2_site.tmpl"
+RELOAD_CHECK = "To activate the new configuration"
+
+
+def get_cert():
+    cert = config_get('ssl_cert')
+    key = config_get('ssl_key')
+    if not (cert and key):
+        juju_log('INFO',
+                 "Inspecting identity-service relations for SSL certificate.")
+        cert = key = None
+        for r_id in relation_ids('identity-service'):
+            for unit in relation_list(r_id):
+                if not cert:
+                    cert = relation_get('ssl_cert',
+                                        rid=r_id, unit=unit)
+                if not key:
+                    key = relation_get('ssl_key',
+                                       rid=r_id, unit=unit)
+    return (cert, key)
+
+
+def get_ca_cert():
+    ca_cert = None
+    juju_log('INFO',
+             "Inspecting identity-service relations for CA SSL certificate.")
+    for r_id in relation_ids('identity-service'):
+        for unit in relation_list(r_id):
+            if not ca_cert:
+                ca_cert = relation_get('ca_cert',
+                                       rid=r_id, unit=unit)
+    return ca_cert
+
+
+def install_ca_cert(ca_cert):
+    if ca_cert:
+        with open('/usr/local/share/ca-certificates/keystone_juju_ca_cert.crt',
+                  'w') as crt:
+            crt.write(ca_cert)
+        subprocess.check_call(['update-ca-certificates', '--fresh'])
+
+
+def enable_https(port_maps, namespace, cert, key, ca_cert=None):
+    '''
+    For a given number of port mappings, configures apache2
+    HTTPs local reverse proxying using certficates and keys provided in
+    either configuration data (preferred) or relation data.  Assumes ports
+    are not in use (calling charm should ensure that).
+
+    port_maps: dict: external to internal port mappings
+    namespace: str: name of charm
+    '''
+    def _write_if_changed(path, new_content):
+        content = None
+        if os.path.exists(path):
+            with open(path, 'r') as f:
+                content = f.read().strip()
+        if content != new_content:
+            with open(path, 'w') as f:
+                f.write(new_content)
+            return True
+        else:
+            return False
+
+    juju_log('INFO', "Enabling HTTPS for port mappings: {}".format(port_maps))
+    http_restart = False
+
+    if cert:
+        cert = b64decode(cert)
+    if key:
+        key = b64decode(key)
+    if ca_cert:
+        ca_cert = b64decode(ca_cert)
+
+    if not cert and not key:
+        juju_log('ERROR',
+                 "Expected but could not find SSL certificate data, not "
+                 "configuring HTTPS!")
+        return False
+
+    install('apache2')
+    if RELOAD_CHECK in subprocess.check_output(['a2enmod', 'ssl',
+                                                'proxy', 'proxy_http']):
+        http_restart = True
+
+    ssl_dir = os.path.join('/etc/apache2/ssl', namespace)
+    if not os.path.exists(ssl_dir):
+        os.makedirs(ssl_dir)
+
+    if (_write_if_changed(os.path.join(ssl_dir, 'cert'), cert)):
+        http_restart = True
+    if (_write_if_changed(os.path.join(ssl_dir, 'key'), key)):
+        http_restart = True
+    os.chmod(os.path.join(ssl_dir, 'key'), 0600)
+
+    install_ca_cert(ca_cert)
+
+    sites_dir = '/etc/apache2/sites-available'
+    for ext_port, int_port in port_maps.items():
+        juju_log('INFO',
+                 'Creating apache2 reverse proxy vhost'
+                 ' for {}:{}'.format(ext_port,
+                                     int_port))
+        site = "{}_{}".format(namespace, ext_port)
+        site_path = os.path.join(sites_dir, site)
+        with open(site_path, 'w') as fsite:
+            context = {
+                "ext": ext_port,
+                "int": int_port,
+                "namespace": namespace,
+                "private_address": get_host_ip()
+                }
+            fsite.write(render_template(SITE_TEMPLATE,
+                                        context))
+
+        if RELOAD_CHECK in subprocess.check_output(['a2ensite', site]):
+            http_restart = True
+
+    if http_restart:
+        restart('apache2')
+
+    return True
+
+
+def disable_https(port_maps, namespace):
+    '''
+    Ensure HTTPS reverse proxying is disables for given port mappings
+
+    port_maps: dict: of ext -> int port mappings
+    namespace: str: name of chamr
+    '''
+    juju_log('INFO', 'Ensuring HTTPS disabled for {}'.format(port_maps))
+
+    if (not os.path.exists('/etc/apache2') or
+        not os.path.exists(os.path.join('/etc/apache2/ssl', namespace))):
+        return
+
+    http_restart = False
+    for ext_port in port_maps.keys():
+        if os.path.exists(os.path.join(APACHE_SITE_DIR,
+                                       "{}_{}".format(namespace,
+                                                      ext_port))):
+            juju_log('INFO',
+                     "Disabling HTTPS reverse proxy"
+                     " for {} {}.".format(namespace,
+                                          ext_port))
+            if (RELOAD_CHECK in
+                subprocess.check_output(['a2dissite',
+                                         '{}_{}'.format(namespace,
+                                                        ext_port)])):
+                http_restart = True
+
+    if http_restart:
+        restart(['apache2'])
+
+
+def setup_https(port_maps, namespace, cert, key, ca_cert=None):
+    '''
+    Ensures HTTPS is either enabled or disabled for given port
+    mapping.
+
+    port_maps: dict: of ext -> int port mappings
+    namespace: str: name of charm
+    '''
+    if not https:
+        disable_https(port_maps, namespace)
+    else:
+        enable_https(port_maps, namespace, cert, key, ca_cert)

hooks/lib/cluster_utils.py

@@ -0,0 +1,127 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  James Page <james.page@ubuntu.com>
+#
+
+from lib.utils import (
+    juju_log,
+    relation_ids,
+    relation_list,
+    relation_get,
+    get_unit_hostname,
+    config_get
+    )
+import subprocess
+import os
+
+
+def is_clustered():
+    for r_id in (relation_ids('ha') or []):
+        for unit in (relation_list(r_id) or []):
+            clustered = relation_get('clustered',
+                                     rid=r_id,
+                                     unit=unit)
+            if clustered:
+                return True
+    return False
+
+
+def is_leader(resource):
+    cmd = [
+        "crm", "resource",
+        "show", resource
+        ]
+    try:
+        status = subprocess.check_output(cmd)
+    except subprocess.CalledProcessError:
+        return False
+    else:
+        if get_unit_hostname() in status:
+            return True
+        else:
+            return False
+
+
+def peer_units():
+    peers = []
+    for r_id in (relation_ids('cluster') or []):
+        for unit in (relation_list(r_id) or []):
+            peers.append(unit)
+    return peers
+
+
+def oldest_peer(peers):
+    local_unit_no = os.getenv('JUJU_UNIT_NAME').split('/')[1]
+    for peer in peers:
+        remote_unit_no = peer.split('/')[1]
+        if remote_unit_no < local_unit_no:
+            return False
+    return True
+
+
+def eligible_leader(resource):
+    if is_clustered():
+        if not is_leader(resource):
+            juju_log('INFO', 'Deferring action to CRM leader.')
+            return False
+    else:
+        peers = peer_units()
+        if peers and not oldest_peer(peers):
+            juju_log('INFO', 'Deferring action to oldest service unit.')
+            return False
+    return True
+
+
+def https():
+    '''
+    Determines whether enough data has been provided in configuration
+    or relation data to configure HTTPS
+    .
+    returns: boolean
+    '''
+    if config_get('use-https') == "yes":
+        return True
+    if config_get('ssl_cert') and config_get('ssl_key'):
+        return True
+    for r_id in relation_ids('identity-service'):
+        for unit in relation_list(r_id):
+            if (relation_get('https_keystone', rid=r_id, unit=unit) and
+                relation_get('ssl_cert', rid=r_id, unit=unit) and
+                relation_get('ssl_key', rid=r_id, unit=unit) and
+                relation_get('ca_cert', rid=r_id, unit=unit)):
+                return True
+    return False
+
+
+def determine_api_port(public_port):
+    '''
+    Determine correct API server listening port based on
+    existence of HTTPS reverse proxy and/or haproxy.
+
+    public_port: int: standard public port for given service
+
+    returns: int: the correct listening port for the API service
+    '''
+    i = 0
+    if len(peer_units()) > 0 or is_clustered():
+        i += 1
+    if https():
+        i += 1
+    return public_port - (i * 10)
+
+
+def determine_haproxy_port(public_port):
+    '''
+    Description: Determine correct proxy listening port based on public IP +
+    existence of HTTPS reverse proxy.
+
+    public_port: int: standard public port for given service
+
+    returns: int: the correct listening port for the HAProxy service
+    '''
+    i = 0
+    if https():
+        i += 1
+    return public_port - (i * 10)

hooks/lib/haproxy_utils.py

@@ -0,0 +1,52 @@
+#
+# Copyright 2012 Canonical Ltd.
+#
+# Authors:
+#  James Page <james.page@ubuntu.com>
+#
+
+from lib.utils import (
+    relation_ids,
+    relation_list,
+    relation_get,
+    unit_get,
+    reload,
+    render_template
+    )
+import os
+
+HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
+HAPROXY_DEFAULT = '/etc/default/haproxy'
+
+
+def configure_haproxy(service_ports):
+    '''
+    Configure HAProxy based on the current peers in the service
+    cluster using the provided port map:
+
+        "swift": [ 8080, 8070 ]
+
+    HAproxy will also be reloaded/started if required
+
+    service_ports: dict: dict of lists of [ frontend, backend ]
+    '''
+    cluster_hosts = {}
+    cluster_hosts[os.getenv('JUJU_UNIT_NAME').replace('/', '-')] = \
+        unit_get('private-address')
+    for r_id in relation_ids('cluster'):
+        for unit in relation_list(r_id):
+            cluster_hosts[unit.replace('/', '-')] = \
+                relation_get(attribute='private-address',
+                             rid=r_id,
+                             unit=unit)
+    context = {
+        'units': cluster_hosts,
+        'service_ports': service_ports
+        }
+    with open(HAPROXY_CONF, 'w') as f:
+        f.write(render_template(os.path.basename(HAPROXY_CONF),
+                                context))
+    with open(HAPROXY_DEFAULT, 'w') as f:
+        f.write('ENABLED=1')
+
+    reload('haproxy')

hooks/lib/openstack_common.py

@@ -12,7 +12,7 @@ ubuntu_openstack_release = {
     'oneiric': 'diablo',
     'precise': 'essex',
     'quantal': 'folsom',
-    'raring' : 'grizzly'
+    'raring': 'grizzly'
 }
 
 
@@ -20,7 +20,8 @@ openstack_codenames = {
     '2011.2': 'diablo',
     '2012.1': 'essex',
     '2012.2': 'folsom',
-    '2013.1': 'grizzly'
+    '2013.1': 'grizzly',
+    '2013.2': 'havana'
 }
 
 # The ugly duckling
@@ -32,6 +33,7 @@ swift_codenames = {
     '1.7.7': 'grizzly'
 }
 
+
 def juju_log(msg):
     subprocess.check_call(['juju-log', msg])
 
@@ -70,12 +72,13 @@ def get_os_codename_install_source(src):
         ca_rel = ca_rel.split('%s-' % ubuntu_rel)[1].split('/')[0]
         return ca_rel
 
-    # Best guess match based on deb string provided
+    # Best guess match based on deb or ppa provided strings
     if src.startswith('deb') or src.startswith('ppa'):
         for k, v in openstack_codenames.iteritems():
             if v in src:
                 return v
 
+
 def get_os_codename_version(vers):
     '''Determine OpenStack codename from version number.'''
     try:
@@ -115,7 +118,7 @@ def get_os_codename_package(pkg):
         return clean
 
     vers = None
-    for l in output.split('\n'):
+    for l in str(output).split('\n'):
         if l.startswith('ii'):
             l = _clean(l)
             if l[1] == pkg:
@@ -153,16 +156,17 @@ def get_os_version_package(pkg):
     e = "Could not determine OpenStack version for package: %s" % pkg
     error_out(e)
 
+
 def configure_installation_source(rel):
     '''Configure apt installation source.'''
 
-    def _import_key(id):
+    def _import_key(keyid):
         cmd = "apt-key adv --keyserver keyserver.ubuntu.com " \
-              "--recv-keys %s" % id
+              "--recv-keys %s" % keyid
         try:
             subprocess.check_call(cmd.split(' '))
-        except:
-            error_out("Error importing repo key %s" % id)
+        except subprocess.CalledProcessError:
+            error_out("Error importing repo key %s" % keyid)
 
     if rel == 'distro':
         return
@@ -171,7 +175,7 @@ def configure_installation_source(rel):
         subprocess.check_call(["add-apt-repository", "-y", src])
     elif rel[:3] == "deb":
         l = len(rel.split('|'))
-        if l ==  2:
+        if l == 2:
             src, key = rel.split('|')
             juju_log("Importing PPA key from keyserver for %s" % src)
             _import_key(key)
@@ -225,26 +229,6 @@ def configure_installation_source(rel):
     else:
         error_out("Invalid openstack-release specified: %s" % rel)
 
-HAPROXY_CONF = '/etc/haproxy/haproxy.cfg'
-HAPROXY_DEFAULT = '/etc/default/haproxy'
-
-def configure_haproxy(units, service_ports, template_dir=None):
-    template_dir = template_dir or 'templates'
-    import jinja2
-    context = {
-        'units': units,
-        'service_ports': service_ports
-        }
-    templates = jinja2.Environment(
-                    loader=jinja2.FileSystemLoader(template_dir)
-                    )
-    template = templates.get_template(
-                    os.path.basename(HAPROXY_CONF)
-                    )
-    with open(HAPROXY_CONF, 'w') as f:
-        f.write(template.render(context))
-    with open(HAPROXY_DEFAULT, 'w') as f:
-        f.write('ENABLED=1')
 
 def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     """
@@ -255,7 +239,7 @@ def save_script_rc(script_path="scripts/scriptrc", **env_vars):
     service changes.
     """
     unit_name = os.getenv('JUJU_UNIT_NAME').replace('/', '-')
-    juju_rc_path="/var/lib/juju/units/%s/charm/%s" % (unit_name, script_path)
+    juju_rc_path = "/var/lib/juju/units/%s/charm/%s" % (unit_name, script_path)
     with open(juju_rc_path, 'wb') as rc_script:
         rc_script.write(
             "#!/bin/bash\n")

hooks/lib/utils.py

@@ -18,10 +18,12 @@ def do_hooks(hooks):
     hook = os.path.basename(sys.argv[0])
 
     try:
-        hooks[hook]()
+        hook_func = hooks[hook]
     except KeyError:
         juju_log('INFO',
                  "This charm doesn't know how to handle '{}'.".format(hook))
+    else:
+        hook_func()
 
 
 def install(*pkgs):
@@ -34,7 +36,7 @@ def install(*pkgs):
         cmd.append(pkg)
     subprocess.check_call(cmd)
 
-TEMPLATES_DIR = 'hooks/templates'
+TEMPLATES_DIR = 'templates'
 
 try:
     import jinja2
@@ -44,11 +46,9 @@ except ImportError:
 
 try:
     import dns.resolver
-    import dns.ipv4
 except ImportError:
     install('python-dnspython')
     import dns.resolver
-    import dns.ipv4
 
 
 def render_template(template_name, context, template_dir=TEMPLATES_DIR):
@@ -66,7 +66,10 @@ deb http://ubuntu-cloud.archive.canonical.com/ubuntu {} main
 CLOUD_ARCHIVE_POCKETS = {
     'folsom': 'precise-updates/folsom',
     'folsom/updates': 'precise-updates/folsom',
-    'folsom/proposed': 'precise-proposed/folsom'
+    'folsom/proposed': 'precise-proposed/folsom',
+    'grizzly': 'precise-updates/grizzly',
+    'grizzly/updates': 'precise-updates/grizzly',
+    'grizzly/proposed': 'precise-proposed/grizzly'
     }
 
 
@@ -133,7 +136,11 @@ def relation_ids(relation):
         'relation-ids',
         relation
         ]
-    return subprocess.check_output(cmd).split()  # IGNORE:E1103
+    result = str(subprocess.check_output(cmd)).split()
+    if result == "":
+        return None
+    else:
+        return result
 
 
 def relation_list(rid):
@@ -141,7 +148,11 @@ def relation_list(rid):
         'relation-list',
         '-r', rid,
         ]
-    return subprocess.check_output(cmd).split()  # IGNORE:E1103
+    result = str(subprocess.check_output(cmd)).split()
+    if result == "":
+        return None
+    else:
+        return result
 
 
 def relation_get(attribute, unit=None, rid=None):
@@ -203,35 +214,47 @@ def config_get(attribute):
     except KeyError:
         return None
 
+
 def get_unit_hostname():
     return socket.gethostname()
 
 
 def get_host_ip(hostname=unit_get('private-address')):
     try:
-      # Test to see if already an IPv4 address
-      socket.inet_aton(hostname)
-      return hostname
+        # Test to see if already an IPv4 address
+        socket.inet_aton(hostname)
+        return hostname
     except socket.error:
-      try:
         answers = dns.resolver.query(hostname, 'A')
         if answers:
-          return answers[0].address
-      except dns.resolver.NXDOMAIN:
-        pass
-      return None
+            return answers[0].address
+    return None
+
+
+def _svc_control(service, action):
+    subprocess.check_call(['service', service, action])
 
 
 def restart(*services):
     for service in services:
-        subprocess.check_call(['service', service, 'restart'])
+        _svc_control(service, 'restart')
 
 
 def stop(*services):
     for service in services:
-        subprocess.check_call(['service', service, 'stop'])
+        _svc_control(service, 'stop')
 
 
 def start(*services):
     for service in services:
-        subprocess.check_call(['service', service, 'start'])
+        _svc_control(service, 'start')
+
+
+def reload(*services):
+    for service in services:
+        try:
+            _svc_control(service, 'reload')
+        except subprocess.CalledProcessError:
+            # Reload failed - either service does not support reload
+            # or it was not running - restart will fixup most things
+            _svc_control(service, 'restart')

hooks/object-store-relation-joined

@@ -1 +1 @@
-swift-hooks.py
+swift_hooks.py

hooks/swift-storage-relation-broken

@@ -1 +1 @@
-swift-hooks.py
+swift_hooks.py

hooks/swift-storage-relation-changed

@@ -1 +1 @@
-swift-hooks.py
+swift_hooks.py

hooks/swift_hooks.py

@@ -1,15 +1,22 @@
 #!/usr/bin/python
 
 import os
-import utils
 import sys
 import shutil
 import uuid
 from subprocess import check_call
 
 import lib.openstack_common as openstack
+import lib.utils as utils
+import lib.cluster_utils as cluster
 import swift_utils as swift
 
+extra_pkgs = [
+    "haproxy",
+    "python-jinja2"
+    ]
+
+
 def install():
     src = utils.config_get('openstack-origin')
     if src != 'distro':
@@ -19,6 +26,7 @@ def install():
 
     pkgs = swift.determine_packages(rel)
     utils.install(*pkgs)
+    utils.install(*extra_pkgs)
 
     swift.ensure_swift_dir()
 
@@ -34,13 +42,10 @@ def install():
     swift.write_proxy_config()
 
     # memcached.conf
-    ctxt = { 'proxy_ip': utils.get_host_ip() }
+    ctxt = {'proxy_ip': utils.get_host_ip()}
     with open(swift.MEMCACHED_CONF, 'w') as conf:
         conf.write(swift.render_config(swift.MEMCACHED_CONF, ctxt))
 
-    # generate or setup SSL certificate
-    swift.configure_ssl()
-
     # initialize new storage rings.
     for ring in swift.SWIFT_RINGS.iteritems():
         swift.initialize_ring(ring[1],
@@ -54,13 +59,18 @@ def install():
     uid, gid = swift.swift_user()
     os.chown(swift.WWW_DIR, uid, gid)
     swift.write_apache_config()
+    swift.configure_https()
 
 
 def keystone_joined(relid=None):
-    hostname = utils.unit_get('private-address')
+    if not cluster.eligible_leader(swift.SWIFT_HA_RES):
+        return
+    if cluster.is_clustered():
+        hostname = utils.config_get('vip')
+    else:
+        hostname = utils.unit_get('private-address')
     port = utils.config_get('bind-port')
-    ssl = utils.config_get('use-https')
-    if ssl == 'yes':
+    if cluster.https():
         proto = 'https'
     else:
         proto = 'http'
@@ -76,6 +86,10 @@ def keystone_joined(relid=None):
 
 def keystone_changed():
     swift.write_proxy_config()
+    swift.configure_https()
+    # Re-fire keystone hooks to ripple back the HTTPS service entry
+    for relid in utils.relation_ids('identity-service'):
+        keystone_joined(relid=relid)
 
 
 def balance_rings():
@@ -93,23 +107,22 @@ def balance_rings():
         shutil.copyfile(os.path.join(swift.SWIFT_CONF_DIR, f),
                         os.path.join(swift.WWW_DIR, f))
 
-    msg = 'Broadcasting notification to all storage nodes that new '\
-          'ring is ready for consumption.'
-    utils.juju_log('INFO', msg)
+    if cluster.eligible_leader(swift.SWIFT_HA_RES):
+        msg = 'Broadcasting notification to all storage nodes that new '\
+              'ring is ready for consumption.'
+        utils.juju_log('INFO', msg)
+        www_dir = swift.WWW_DIR.split('/var/www/')[1]
+        trigger = uuid.uuid4()
+        swift_hash = swift.get_swift_hash()
+        # notify storage nodes that there is a new ring to fetch.
+        for relid in utils.relation_ids('swift-storage'):
+            utils.relation_set(rid=relid, swift_hash=swift_hash,
+                               www_dir=www_dir, trigger=trigger)
 
-    www_dir = swift.WWW_DIR.split('/var/www/')[1]
-    trigger = uuid.uuid4()
-    swift_hash = swift.get_swift_hash()
-    # notify storage nodes that there is a new ring to fetch.
-    for relid in utils.relation_ids('swift-storage'):
-        utils.relation_set(rid=relid, swift_hash=swift_hash,
-                           www_dir=www_dir, trigger=trigger)
     swift.proxy_control('restart')
 
+
 def storage_changed():
-    account_port = utils.config_get('account-ring-port')
-    object_port = utils.config_get('object-ring-port')
-    container_port = utils.config_get('container-ring-port')
     zone = swift.get_zone(utils.config_get('zone-assignment'))
     node_settings = {
         'ip': utils.get_host_ip(utils.relation_get('private-address')),
@@ -139,15 +152,73 @@ def storage_changed():
     if swift.should_balance([r for r in swift.SWIFT_RINGS.itervalues()]):
         balance_rings()
 
+
 def storage_broken():
     swift.write_apache_config()
 
+
 def config_changed():
     relids = utils.relation_ids('identity-service')
     if relids:
         for relid in relids:
             keystone_joined(relid)
     swift.write_proxy_config()
+    swift.configure_https()
+
+
+def cluster_changed():
+    swift.configure_haproxy()
+
+
+def ha_relation_changed():
+    clustered = utils.relation_get('clustered')
+    if clustered and cluster.is_leader(swift.SWIFT_HA_RES):
+        utils.juju_log('INFO',
+                       'Cluster configured, notifying other services and'
+                       'updating keystone endpoint configuration')
+        # Tell all related services to start using
+        # the VIP instead
+        for r_id in utils.relation_ids('identity-service'):
+            keystone_joined(relid=r_id)
+
+
+def ha_relation_joined():
+    # Obtain the config values necessary for the cluster config. These
+    # include multicast port and interface to bind to.
+    corosync_bindiface = utils.config_get('ha-bindiface')
+    corosync_mcastport = utils.config_get('ha-mcastport')
+    vip = utils.config_get('vip')
+    vip_cidr = utils.config_get('vip_cidr')
+    vip_iface = utils.config_get('vip_iface')
+    if not vip:
+        utils.juju_log('ERROR',
+                       'Unable to configure hacluster as vip not provided')
+        sys.exit(1)
+
+    # Obtain resources
+    resources = {
+            'res_swift_vip': 'ocf:heartbeat:IPaddr2',
+            'res_swift_haproxy': 'lsb:haproxy'
+        }
+    resource_params = {
+            'res_swift_vip': 'params ip="%s" cidr_netmask="%s" nic="%s"' % \
+                              (vip, vip_cidr, vip_iface),
+            'res_swift_haproxy': 'op monitor interval="5s"'
+        }
+    init_services = {
+            'res_swift_haproxy': 'haproxy'
+        }
+    clones = {
+            'cl_swift_haproxy': 'res_swift_haproxy'
+        }
+
+    utils.relation_set(init_services=init_services,
+                       corosync_bindiface=corosync_bindiface,
+                       corosync_mcastport=corosync_mcastport,
+                       resources=resources,
+                       resource_params=resource_params,
+                       clones=clones)
+
 
 hooks = {
     'install': install,
@@ -156,6 +227,10 @@ hooks = {
     'identity-service-relation-changed': keystone_changed,
     'swift-storage-relation-changed': storage_changed,
     'swift-storage-relation-broken': storage_broken,
+    "cluster-relation-joined": cluster_changed,
+    "cluster-relation-changed": cluster_changed,
+    "ha-relation-joined": ha_relation_joined,
+    "ha-relation-changed": ha_relation_changed
 }
 
 utils.do_hooks(hooks)

hooks/swift_utils.py

@@ -2,10 +2,16 @@ import os
 import pwd
 import subprocess
 import lib.openstack_common as openstack
-import utils
+import lib.utils as utils
+import lib.haproxy_utils as haproxy
+import lib.apache_utils as apache
+import lib.cluster_utils as cluster
+import sys
+from base64 import b64encode
+
 
 # Various config files that are managed via templating.
-SWIFT_HASH_FILE='/var/lib/juju/swift-hash-path.conf'
+SWIFT_HASH_FILE = '/var/lib/juju/swift-hash-path.conf'
 SWIFT_CONF = '/etc/swift/swift.conf'
 SWIFT_PROXY_CONF = '/etc/swift/proxy-server.conf'
 SWIFT_CONF_DIR = os.path.dirname(SWIFT_CONF)
@@ -32,9 +38,12 @@ BASE_PACKAGES = [
     'python-keystone',
 ]
 
+SWIFT_HA_RES = 'res_swift_vip'
+
 # Folsom-specific packages
 FOLSOM_PACKAGES = BASE_PACKAGES + ['swift-plugin-s3']
 
+
 def proxy_control(action):
     '''utility to work around swift-init's bad RCs.'''
     def _cmd(action):
@@ -49,8 +58,9 @@ def proxy_control(action):
         elif status == 0:
             return subprocess.check_call(_cmd('stop'))
 
-    # the proxy will not start unless there are balanced rings, gzip'd in /etc/swift
-    missing=False
+    # the proxy will not start unless there are balanced rings
+    # gzip'd in /etc/swift
+    missing = False
     for k in SWIFT_RINGS.keys():
         if not os.path.exists(os.path.join(SWIFT_CONF_DIR, '%s.ring.gz' % k)):
             missing = True
@@ -69,8 +79,9 @@ def proxy_control(action):
         elif status == 1:
             return subprocess.check_call(_cmd('start'))
 
+
 def swift_user(username='swift'):
-    user = pwd.getpwnam('swift')
+    user = pwd.getpwnam(username)
     return (user.pw_uid, user.pw_gid)
 
 
@@ -105,6 +116,10 @@ def get_swift_hash():
     if os.path.isfile(SWIFT_HASH_FILE):
         with open(SWIFT_HASH_FILE, 'r') as hashfile:
             swift_hash = hashfile.read().strip()
+    elif utils.config_get('swift-hash'):
+        swift_hash = utils.config_get('swift-hash')
+        with open(SWIFT_HASH_FILE, 'w') as hashfile:
+            hashfile.write(swift_hash)
     else:
         cmd = ['od', '-t', 'x8', '-N', '8', '-A', 'n']
         rand = open('/dev/random', 'r')
@@ -148,11 +163,16 @@ def get_keystone_auth():
                 'keystone_host': utils.relation_get('auth_host',
                                                     unit, relid),
                 'auth_port': utils.relation_get('auth_port', unit, relid),
-                'service_user': utils.relation_get('service_username', unit, relid),
-                'service_password': utils.relation_get('service_password', unit, relid),
-                'service_tenant': utils.relation_get('service_tenant', unit, relid),
-                'service_port': utils.relation_get('service_port', unit, relid),
-                'admin_token': utils.relation_get('admin_token', unit, relid),
+                'service_user': utils.relation_get('service_username',
+                                                   unit, relid),
+                'service_password': utils.relation_get('service_password',
+                                                       unit, relid),
+                'service_tenant': utils.relation_get('service_tenant',
+                                                     unit, relid),
+                'service_port': utils.relation_get('service_port',
+                                                   unit, relid),
+                'admin_token': utils.relation_get('admin_token',
+                                                  unit, relid),
             }
             if None not in ks_auth.itervalues():
                 return ks_auth
@@ -174,17 +194,12 @@ def write_proxy_config():
 
     ctxt = {
         'proxy_ip': utils.get_host_ip(),
-        'bind_port': bind_port,
+        'bind_port': cluster.determine_api_port(bind_port),
         'workers': workers,
         'operator_roles': utils.config_get('operator-roles')
     }
 
-    if utils.config_get('use-https') == 'no':
-        ctxt['ssl'] = False
-    else:
-        ctxt['ssl'] = True
-        ctxt['ssl_cert'] = SSL_CERT
-        ctxt['ssl_key'] = SSL_KEY
+    ctxt['ssl'] = False
 
     ks_auth = get_keystone_auth()
     if ks_auth:
@@ -198,23 +213,10 @@ def write_proxy_config():
     proxy_control('restart')
     subprocess.check_call(['open-port', str(bind_port)])
 
-def configure_ssl():
-    # this should be expanded to cover setting up user-specified certificates
-    if (utils.config_get('use-https') == 'yes' and
-        not os.path.isfile(SSL_CERT) and
-        not os.path.isfile(SSL_KEY)):
-        subj = '/C=%s/ST=%s/L=%s/CN=%s' %\
-               (utils.config_get('country'), utils.config_get('state'),
-                utils.config_get('locale'), utils.config_get('common-name'))
-        cmd = ['openssl', 'req', '-new', '-x509', '-nodes',
-               '-out', SSL_CERT, '-keyout', SSL_KEY,
-               '-subj', subj]
-        subprocess.check_call(cmd)
-
 
 def _load_builder(path):
     # lifted straight from /usr/bin/swift-ring-builder
-    from swift.common.ring import RingBuilder, Ring
+    from swift.common.ring import RingBuilder
     import cPickle as pickle
     try:
         builder = pickle.load(open(path, 'rb'))
@@ -223,10 +225,8 @@ def _load_builder(path):
             builder = RingBuilder(1, 1, 1)
             builder.copy_from(builder_dict)
     except ImportError:  # Happens with really old builder pickles
-        modules['swift.ring_builder'] = \
-            modules['swift.common.ring.builder']
         builder = RingBuilder(1, 1, 1)
-        builder.copy_from(pickle.load(open(argv[1], 'rb')))
+        builder.copy_from(pickle.load(open(path, 'rb')))
     for dev in builder.devs:
         if dev and 'meta' not in dev:
             dev['meta'] = ''
@@ -238,8 +238,6 @@ def _write_ring(ring, ring_path):
     pickle.dump(ring.to_dict(), open(ring_path, 'wb'), protocol=2)
 
 
-
-
 def ring_port(ring_path, node):
     '''determine correct port from relation settings for a given ring file.'''
     for name in ['account', 'object', 'container']:
@@ -253,8 +251,8 @@ def initialize_ring(path, part_power, replicas, min_hours):
     ring = RingBuilder(part_power, replicas, min_hours)
     _write_ring(ring, path)
 
+
 def exists_in_ring(ring_path, node):
-    from swift.common.ring import RingBuilder, Ring
     ring = _load_builder(ring_path).to_dict()
     node['port'] = ring_port(ring_path, node)
 
@@ -271,7 +269,6 @@ def exists_in_ring(ring_path, node):
 
 
 def add_to_ring(ring_path, node):
-    from swift.common.ring import RingBuilder, Ring
     ring = _load_builder(ring_path)
     port = ring_port(ring_path, node)
 
@@ -291,8 +288,9 @@ def add_to_ring(ring_path, node):
     }
     ring.add_dev(new_dev)
     _write_ring(ring, ring_path)
-    msg = 'Added new device to ring %s: %s' % (ring_path,
-                                               [k for k in new_dev.iteritems()])
+    msg = 'Added new device to ring %s: %s' %\
+         (ring_path,
+          [k for k in new_dev.iteritems()])
     utils.juju_log('INFO', msg)
 
 
@@ -337,8 +335,8 @@ def get_zone(assignment_policy):
             potential_zones.append(_get_zone(builder))
         return set(potential_zones).pop()
     else:
-        utils.juju_log('Invalid zone assignment policy: %s' %\
-                       assignemnt_policy)
+        utils.juju_log('ERROR', 'Invalid zone assignment policy: %s' %\
+                       assignment_policy)
         sys.exit(1)
 
 
@@ -356,9 +354,10 @@ def balance_ring(ring_path):
         # swift-ring-builder returns 1 on WARNING (ring didn't require balance)
         return False
     else:
-        utils.juju_log('balance_ring: %s returned %s' % (cmd, rc))
+        utils.juju_log('ERROR', 'balance_ring: %s returned %s' % (cmd, rc))
         sys.exit(1)
 
+
 def should_balance(rings):
     '''Based on zones vs min. replicas, determine whether or not the rings
        should be balanaced during initial configuration.'''
@@ -384,8 +383,68 @@ def write_apache_config():
             host = utils.relation_get('private-address', unit, relid)
             allowed_hosts.append(utils.get_host_ip(host))
 
-    ctxt = { 'www_dir': WWW_DIR, 'allowed_hosts': allowed_hosts }
+    ctxt = {
+        'www_dir': WWW_DIR,
+        'allowed_hosts': allowed_hosts
+        }
     with open(APACHE_CONF, 'w') as conf:
         conf.write(render_config(APACHE_CONF, ctxt))
-    subprocess.check_call(['service', 'apache2', 'reload'])
+    utils.reload('apache2')
 
+
+def generate_cert():
+    '''
+    Generates a self signed certificate and key using the
+    provided charm configuration data.
+
+    returns: tuple of (cert, key)
+    '''
+    CERT = '/etc/swift/ssl.cert'
+    KEY = '/etc/swift/ssl.key'
+    if (not os.path.exists(CERT) and
+        not os.path.exists(KEY)):
+        subj = '/C=%s/ST=%s/L=%s/CN=%s' %\
+            (utils.config_get('country'), utils.config_get('state'),
+             utils.config_get('locale'), utils.config_get('common-name'))
+        cmd = ['openssl', 'req', '-new', '-x509', '-nodes',
+               '-out', CERT, '-keyout', KEY,
+               '-subj', subj]
+        subprocess.check_call(cmd)
+        os.chmod(KEY, 0600)
+    # Slurp as base64 encoded - makes handling easier up the stack
+    with open(CERT, 'r') as cfile:
+        ssl_cert = b64encode(cfile.read())
+    with open(KEY, 'r') as kfile:
+        ssl_key = b64encode(kfile.read())
+    return (ssl_cert, ssl_key)
+
+
+def configure_haproxy():
+    api_port = utils.config_get('bind-port')
+    service_ports = {
+        "swift": [
+            cluster.determine_haproxy_port(api_port),
+            cluster.determine_api_port(api_port)
+            ]
+        }
+    write_proxy_config()
+    haproxy.configure_haproxy(service_ports)
+
+
+def configure_https():
+    if cluster.https():
+        api_port = utils.config_get('bind-port')
+        if (len(cluster.peer_units()) > 0 or
+            cluster.is_clustered()):
+            target_port = cluster.determine_haproxy_port(api_port)
+            configure_haproxy()
+        else:
+            target_port = cluster.determine_api_port(api_port)
+            write_proxy_config()
+        cert, key = apache.get_cert()
+        if None in (cert, key):
+            cert, key = generate_cert()
+        ca_cert = apache.get_ca_cert()
+        apache.setup_https(namespace="swift",
+                           port_maps={api_port: target_port},
+                           cert=cert, key=key, ca_cert=ca_cert)

metadata.yaml

@@ -12,3 +12,9 @@ requires:
     interface: swift
   identity-service:
     interface: keystone
+  ha:
+    interface: hacluster
+    scope: container
+peers:
+  cluster:
+    interface: swift-ha

revision

@@ -1 +1 @@
-110
+121

templates/apache2_site.tmpl

@@ -0,0 +1,19 @@
+Listen {{ ext }}
+NameVirtualHost *:{{ ext }}
+<VirtualHost *:{{ ext }}>
+    ServerName {{ private_address }}
+    SSLEngine on
+    SSLCertificateFile /etc/apache2/ssl/{{ namespace }}/cert
+    SSLCertificateKeyFile /etc/apache2/ssl/{{ namespace }}/key
+    ProxyPass / http://localhost:{{ int }}/
+    ProxyPassReverse / http://localhost:{{ int }}/
+    ProxyPreserveHost on
+</VirtualHost>
+<Proxy *>
+    Order deny,allow
+    Allow from all
+</Proxy>
+<Location />
+    Order allow,deny
+    Allow from all
+</Location>

templates/haproxy.cfg

@@ -0,0 +1,35 @@
+global
+    log 127.0.0.1 local0
+    log 127.0.0.1 local1 notice
+    maxconn 20000
+    user haproxy
+    group haproxy
+    spread-checks 0
+
+defaults
+    log global
+    mode http
+    option httplog
+    option dontlognull
+    retries 3
+    timeout queue 1000
+    timeout connect 1000
+    timeout client 30000
+    timeout server 30000
+
+listen stats :8888
+    mode http
+    stats enable
+    stats hide-version
+    stats realm Haproxy\ Statistics
+    stats uri /
+    stats auth admin:password
+
+{% for service, ports in service_ports.iteritems() -%}
+listen {{ service }} 0.0.0.0:{{ ports[0] }}
+    balance roundrobin
+    option tcplog
+    {% for unit, address in units.iteritems() -%}
+    server {{ unit }} {{ address }}:{{ ports[1] }} check
+    {% endfor %}
+{% endfor %}