Add NRPE Check for Octavia Certificates

Change-Id: Ie4718b2c145e977fd11fff06ed3e854d41715eb8
Closes-Bug: 1885815

src/config.yaml

@@ -182,3 +182,17 @@ options:
       Octavia supports multiple provider drivers. The reference Amphora
       provider driver is distributed as part of the Octavia software, and is
       enabled by default, unless you set this configuration option to 'False'.
+  tls_warn_days:
+    default: 30
+    type: int
+    description: |
+      Number of days left for the Octavia certificates, used for secure
+      communication between the controller and the amphora instances,
+      to expire before raising a Warning alert.
+  tls_crit_days:
+    default: 14
+    type: int
+    description: |
+      Number of days left for the Octavia certificates, used for secure
+      communication between the controller and the amphora instances,
+      to expire before raising a Critical alert.

src/files/nrpe/check_cert.py

@@ -0,0 +1,84 @@
+#!/usr/bin/python3
+# Copyright 2022 Canonical Ltd
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+from datetime import datetime, timedelta
+import nagios_plugin3
+from OpenSSL import crypto
+
+
+class Certificate:
+    DATE_FMT = '%Y%m%d%H%M%SZ'
+
+    def __init__(self, fp_obj):
+        self.cert = crypto.load_certificate(crypto.FILETYPE_PEM, fp_obj.read())
+
+    @property
+    def end_date(self):
+        """Read the expiration date from the cert."""
+        date = self.cert.get_notAfter().decode('utf-8')
+        return datetime.strptime(date, self.DATE_FMT)
+
+
+def run_check(args):
+    """
+    Run the nrpe check on a certificate.
+
+    :params args: parse args namespace
+    """
+    file_name = args.certificate.name
+    try:
+        cert = Certificate(args.certificate)
+    except crypto.Error:
+        raise nagios_plugin3.UnknownError(
+            "UNKNOWN: Couldn't open certificate file %s" % file_name
+        )
+
+    ages_str = args.ages.split(",")
+    if len(ages_str) < 1 or len(ages_str) > 2:
+        raise nagios_plugin3.UnknownError(
+            "UNKNOWN: Invalid check option -C %s" % ages_str
+        )
+
+    ages = []
+    for each in ages_str:
+        ages.append(int(each))
+
+    now = datetime.now()
+    for idx, days in enumerate(ages):
+        if (now + timedelta(days=days)) > cert.end_date:
+            days_left = (cert.end_date - now).days
+            ends = cert.end_date.isoformat(' ')
+            msg = "Certificate expires in %s day(s) (%s)" % (days_left, ends)
+            if idx == 0:
+                raise nagios_plugin3.CriticalError("CRITICAL: " + msg)
+            elif idx == 1:
+                raise nagios_plugin3.WarnError("WARNING: " + msg)
+    print("OK: Certificate is valid")
+
+
+def main():
+    """Parse arguments from stdin, and run check."""
+    parser = argparse.ArgumentParser(description="Validate Certificate files")
+    parser.add_argument('certificate', type=argparse.FileType('r'),
+                        help="Certificate PEM File")
+    parser.add_argument('-C', dest='ages', help='crit_age[,warn_age]',
+                        default='14')
+    args = parser.parse_args()
+    run_check(args)
+
+
+if __name__ == '__main__':
+    nagios_plugin3.try_check(main)

src/lib/charm/openstack/octavia.py

@@ -27,6 +27,7 @@ import charms.leadership as leadership
 import charms.reactive as reactive
 
 import charmhelpers.core as ch_core
+import charmhelpers.contrib.charmsupport.nrpe as ch_nrpe
 import charmhelpers.contrib.network.ip as ch_net_ip
 
 OCTAVIA_DIR = '/etc/octavia'
@@ -54,6 +55,8 @@ OCTAVIA_ROLES = [
     'load-balancer_admin',
 ]
 
+NAGIOS_PLUGINS = '/usr/local/lib/nagios/plugins'
+
 # config.changed is needed to get the policyd override clean-up to work when
 # setting use-policyd-override=false
 charms_openstack.charm.use_defaults('charm.default-select-release',
@@ -218,7 +221,9 @@ def issuing_ca_private_key(cls):
     if config:
         return cls.charm_instance.decode_and_write_cert(
             'issuing_ca_key.pem',
-            config)
+            config,
+            check=False
+        )
 
 
 @charms_openstack.adapters.config_property
@@ -451,15 +456,19 @@ class BaseOctaviaCharm(ch_plugins.PolicydOverridePlugin,
                 ch_core.host.service_reload('apache2',
                                             restart_on_failure=True)
 
-    def decode_and_write_cert(self, filename, encoded_data):
+    def decode_and_write_cert(self, filename, encoded_data, check=True):
         """Write certificate data to disk.
 
+        Side effect of writing a certificate is that an nrpe check is added
+        to check validity and expiration of the certificate if `check=True` is
+        passed.
+
         :param filename: Name of file
         :type filename: str
-        :param group: Group ownership
-        :type group: str
         :param encoded_data: Base64 encoded data
         :type encoded_data: str
+        :param check: Install the nrpe check
+        :type check: bool
         :returns: Full path to file
         :rtype: str
         """
@@ -468,6 +477,22 @@ class BaseOctaviaCharm(ch_plugins.PolicydOverridePlugin,
                            perms=0o750)
         ch_core.host.write_file(filename, base64.b64decode(encoded_data),
                                 group=self.group, perms=0o440)
+        if check:
+            check_cmd = '{} -C {},{} {}'.format(
+                os.path.join(NAGIOS_PLUGINS, 'check_cert.py'),
+                ch_core.hookenv.config('tls_crit_days'),
+                ch_core.hookenv.config('tls_warn_days'),
+                filename
+            )
+            description = 'Check Certificate %s' % os.path.basename(filename)
+
+            nrpe = ch_nrpe.NRPE()
+            nrpe.add_check(
+                shortname='cert_%s' % os.path.basename(filename).split(".")[0],
+                description=description,
+                check_cmd=check_cmd
+            )
+            nrpe.write()
         return filename
 
     @property

src/reactive/octavia_handlers.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 import json
+import os
 import uuid
 
 import charms.reactive as reactive
@@ -301,8 +302,12 @@ def update_nagios():
     with charm.provide_charm_instance() as charm_instance:
         services = charm_instance.full_service_list
     nrpe_instance = nrpe.NRPE()
+    files_dir = os.path.join(os.getenv('CHARM_DIR', '.'), "files", "nrpe")
+    nrpe.copy_nrpe_checks(nrpe_files_dir=files_dir)
     nrpe.add_init_service_checks(nrpe_instance, services, current_unit)
     nrpe_instance.write()
+    if ch_core.host.group_exists("octavia"):
+        ch_core.host.add_user_to_group("nagios", "octavia")
     reactive.set_state('octavia.nrpe.configured')
     ch_core.hookenv.status_set('active', 'Nagios checks configured')
 

unit_tests/__init__.py

@@ -33,6 +33,11 @@ class _fake_decorator(object):
         return f
 
 
+import charmhelpers.contrib
+sys.modules['charmhelpers.contrib.charmsupport'] = \
+    charmhelpers.contrib.charmsupport
+sys.modules['charmhelpers.contrib.charmsupport.nrpe'] = \
+    charmhelpers.contrib.charmsupport.nrpe
 charms = mock.MagicMock()
 sys.modules['charms'] = charms
 charms.leadership = mock.MagicMock()

unit_tests/test_octavia_handlers.py

@@ -251,11 +251,14 @@ class TestOctaviaHandlers(test_utils.PatchHelper):
         nrpe_instance = mock.MagicMock()
         self.patch_object(handlers.nrpe, 'NRPE', return_value=nrpe_instance)
         self.patch_object(handlers.nrpe, 'add_init_service_checks')
+        self.patch_object(handlers.nrpe, 'copy_nrpe_checks')
         self.patch('charms.reactive.set_state')
         handlers.update_nagios()
         self.add_init_service_checks.assert_called_once_with(
             nrpe_instance, self.octavia_charm.full_service_list,
             mock.sentinel.unit_name)
+        self.copy_nrpe_checks.assert_called_once_with(
+            nrpe_files_dir="./files/nrpe")
         nrpe_instance.write.assert_called_once()
         self.set_state.assert_called_once_with('octavia.nrpe.configured')