Bypass charm going into blocked due to a bug

After Octopus, the reporting about image states fails because of a
permission issue in Ceph. This change disables that status reporting
to allow a deployment to be healthy even when some tools cannot query
status.

Also modernize build and func testing: remove python 3.9 and xena from
tests

Related-Bug: #1879749
func-test-pr: https://github.com/openstack-charmers/zaza-openstack-tests/pull/1063
Change-Id: Id86fc043495b89609cf6873ec58aee1e2e388578

bindep.txt

@@ -0,0 +1,3 @@
+libffi-dev [platform:dpkg]
+libxml2-dev [platform:dpkg]
+libxslt1-dev [platform:dpkg]

charmcraft.yaml

@@ -2,24 +2,25 @@ type: charm
 
 parts:
   charm:
+    source: src/
+    plugin: reactive
+    build-snaps:
+      - charm
     build-packages:
       - tox
       - git
       - python3-dev
-    override-build: |
+    build-environment:
-      apt-get install ca-certificates -y
+      - CHARM_INTERFACES_DIR: /root/project/interfaces/
-      tox -e build-reactive
+      - CHARM_LAYERS_DIR: /root/project/layers/
-    override-stage: |
-      echo "Copying charm to staging area: $CHARMCRAFT_STAGE"
-      NAME=$(ls $CHARMCRAFT_PART_BUILD/build/builds)
-      cp -r $CHARMCRAFT_PART_BUILD/build/builds/$NAME/* $CHARMCRAFT_STAGE/
-    override-prime: |
-      # For some reason, the normal priming chokes on the fact that there's a
-      # hooks directory.
-      cp -r $CHARMCRAFT_STAGE/* .
 
 bases:
-  - name: ubuntu
+  - build-on:
-    channel: "20.04"
+      - name: ubuntu
-    architectures:
+        channel: "22.04"
-      - amd64
+        architectures:
+          - amd64
+    run-on:
+      - name: ubuntu
+        channel: "22.04"
+        architectures: [amd64, s390x, ppc64el, arm64]

osci.yaml

@@ -1,17 +1,14 @@
 - project:
     templates:
       - charm-unit-jobs-py38
-      - charm-unit-jobs-py39
+      - charm-unit-jobs-py310
     check:
       jobs:
-      - focal-xena
+      - focal-yoga
-      - focal-yoga:
-          voting: false
-      - impish-xena:
-          voting: false
       - jammy-yoga:
           voting: false
     vars:
       needs_charm_build: true
       charm_build_name: ceph-rbd-mirror
       build_type: charmcraft
+      charmcraft_channel: 2.1/stable

requirements.txt

@@ -8,16 +8,13 @@
 # requirements.txt
 setuptools<50.0.0  # https://github.com/pypa/setuptools/commit/04e3df22df840c6bb244e9b27bc56750c44b7c85
 
-# Build requirements
+# NOTE: newer versions of cryptography require a Rust compiler to build,
-cffi==1.14.6; python_version < '3.6'  # cffi 1.15.0 drops support for py35.
+# see
-charm-tools==2.8.3
+# * https://github.com/openstack-charmers/zaza/issues/421
+# * https://mail.python.org/pipermail/cryptography-dev/2021-January/001003.html
+#
+cryptography<3.4
+
+git+https://github.com/juju/charm-tools.git
 
 simplejson
-
-# Newer versions use keywords that didn't exist in python 3.5 yet (e.g.
-# "ModuleNotFoundError")
-# NOTE(lourot): This might look like a duplication of test-requirements.txt but
-# some tox targets use only test-requirements.txt whereas charm-build uses only
-# requirements.txt
-importlib-metadata<3.0.0; python_version < '3.6'
-importlib-resources<3.0.0; python_version < '3.6'

src/lib/charm/openstack/ceph_rbd_mirror.py

@@ -97,8 +97,12 @@ class CephRBDMirrorCharm(charms_openstack.plugins.CephCharm):
                 if not pool_msg:
                     pool_msg = 'Pools '
                 pool_msg += '{} ({}) '.format(health, count)
-                if health != 'OK':
+
-                    status = 'blocked'
+                # Disabling blocked state until
+                # https://bugs.launchpad.net/charm-ceph-rbd-mirror/+bug/1879749
+                # is resolved
+                # if health != 'OK':
+                #     status = 'blocked'
             for state, count in stats['image_states'].items():
                 if not image_msg:
                     image_msg = 'Images '

src/metadata.yaml

@@ -16,7 +16,7 @@ tags:
 - misc
 series:
 - focal
-- impish
+- jammy
 extra-bindings:
   public:
   cluster:

src/tests/bundles/focal-xena.yaml

@@ -1,187 +0,0 @@
-variables:
-  openstack-origin: &openstack-origin cloud:focal-xena
-
-local_overlay_enabled: False
-
-series: &series focal
-
-machines:
-  '0':
-    constraints: "mem=3072M"
-  '1':
-    constraints: "mem=3072M"
-  '2':
-    constraints: "mem=3072M"
-
-applications:
-
-  keystone-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-  glance-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-  cinder-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-
-  mysql-innodb-cluster:
-    charm: ch:mysql-innodb-cluster
-    num_units: 3
-    options:
-      source: *openstack-origin
-    to:
-      - '0'
-      - '1'
-      - '2'
-    channel: latest/edge
-
-  keystone:
-    charm: ch:keystone
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    channel: yoga/edge
-
-  rabbitmq-server:
-    charm: ch:rabbitmq-server
-    num_units: 1
-    options:
-      source: *openstack-origin
-    channel: latest/edge
-
-  cinder:
-    charm: ch:cinder
-    num_units: 1
-    options:
-      block-device: None
-      glance-api-version: 2
-      openstack-origin: *openstack-origin
-    channel: yoga/edge
-
-  cinder-ceph:
-    charm: ch:cinder-ceph
-    num_units: 0
-    channel: yoga/edge
-
-  glance:
-    charm: ch:glance
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    channel: yoga/edge
-
-  nova-compute:
-    charm: ch:nova-compute
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    channel: yoga/edge
-
-  ceph-mon:
-    charm: ch:ceph-mon
-    num_units: 3
-    options:
-      expected-osd-count: 3
-      source: *openstack-origin
-    channel: quincy/edge
-
-  ceph-osd:
-    charm: ch:ceph-osd
-    num_units: 3
-    storage:
-      osd-devices: 'cinder,10G'
-    options:
-      source: *openstack-origin
-      bluestore: False
-      use-direct-io: False
-      osd-devices: '/dev/test-non-existent'
-    channel: quincy/edge
-
-  ceph-rbd-mirror:
-    series: *series
-    charm: ../../../ceph-rbd-mirror.charm
-    num_units: 1
-    options:
-      source: *openstack-origin
-
-  ceph-mon-b:
-    charm: ch:ceph-mon
-    num_units: 3
-    options:
-      expected-osd-count: 3
-      source: *openstack-origin
-    channel: quincy/edge
-
-  ceph-osd-b:
-    charm: ch:ceph-osd
-    num_units: 3
-    storage:
-      osd-devices: 'cinder,10G'
-    options:
-      source: *openstack-origin
-      bluestore: False
-      use-direct-io: False
-      osd-devices: '/dev/test-non-existent'
-    channel: quincy/edge
-
-  ceph-rbd-mirror-b:
-    series: *series
-    charm: ../../../ceph-rbd-mirror.charm
-    num_units: 1
-    options:
-      source: *openstack-origin
-
-relations:
-
-- - keystone:shared-db
-  - keystone-mysql-router:shared-db
-- - keystone-mysql-router:db-router
-  - mysql-innodb-cluster:db-router
-
-- - glance:shared-db
-  - glance-mysql-router:shared-db
-- - glance-mysql-router:db-router
-  - mysql-innodb-cluster:db-router
-
-- - cinder:shared-db
-  - cinder-mysql-router:shared-db
-- - cinder-mysql-router:db-router
-  - mysql-innodb-cluster:db-router
-
-- - rabbitmq-server
-  - cinder
-
-- - keystone
-  - cinder
-- - keystone
-  - glance
-
-- - cinder
-  - cinder-ceph
-- - cinder-ceph:ceph
-  - ceph-mon:client
-
-- - nova-compute:ceph-access
-  - cinder-ceph:ceph-access
-- - nova-compute:amqp
-  - rabbitmq-server:amqp
-
-- - glance:image-service
-  - nova-compute:image-service
-- - glance
-  - ceph-mon
-
-- - ceph-mon:osd
-  - ceph-osd:mon
-- - ceph-mon
-  - ceph-rbd-mirror:ceph-local
-- - ceph-mon
-  - ceph-rbd-mirror-b:ceph-remote
-
-- - ceph-mon-b:osd
-  - ceph-osd-b:mon
-- - ceph-mon-b
-  - ceph-rbd-mirror-b:ceph-local
-- - ceph-mon-b
-  - ceph-rbd-mirror:ceph-remote

src/tests/bundles/focal-yoga.yaml

@@ -152,9 +152,9 @@ relations:
 - - rabbitmq-server
   - cinder
 
-- - keystone
+- - 'keystone:identity-service'
   - cinder
-- - keystone
+- - 'keystone:identity-service'
   - glance
 
 - - cinder

src/tests/bundles/impish-xena.yaml

@@ -1,186 +0,0 @@
-variables:
-  openstack-origin: &openstack-origin distro
-
-local_overlay_enabled: False
-
-series: &series impish
-
-machines:
-  '0':
-    constraints: "mem=3072M"
-  '1':
-    constraints: "mem=3072M"
-  '2':
-    constraints: "mem=3072M"
-
-applications:
-
-  keystone-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-  glance-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-  cinder-mysql-router:
-    charm: ch:mysql-router
-    channel: latest/edge
-
-  mysql-innodb-cluster:
-    charm: ch:mysql-innodb-cluster
-    num_units: 3
-    options:
-      source: *openstack-origin
-    to:
-      - '0'
-      - '1'
-      - '2'
-    channel: latest/edge
-
-  keystone:
-    charm: ch:keystone
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    channel: yoga/edge
-
-  rabbitmq-server:
-    charm: ch:rabbitmq-server
-    num_units: 1
-    options:
-      source: *openstack-origin
-    channel: latest/edge
-
-  cinder:
-    charm: ch:cinder
-    num_units: 1
-    options:
-      block-device: None
-      glance-api-version: 2
-    channel: yoga/edge
-
-  cinder-ceph:
-    charm: ch:cinder-ceph
-    num_units: 0
-    channel: yoga/edge
-
-  glance:
-    charm: ch:glance
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    channel: yoga/edge
-
-  nova-compute:
-    charm: ch:nova-compute
-    num_units: 1
-    options:
-      openstack-origin: *openstack-origin
-    channel: yoga/edge
-
-  ceph-mon:
-    charm: ch:ceph-mon
-    num_units: 3
-    options:
-      expected-osd-count: 3
-      source: *openstack-origin
-    channel: quincy/edge
-
-  ceph-osd:
-    charm: ch:ceph-osd
-    num_units: 3
-    storage:
-      osd-devices: 'cinder,10G'
-    options:
-      source: *openstack-origin
-      bluestore: False
-      use-direct-io: False
-      osd-devices: '/dev/test-non-existent'
-    channel: quincy/edge
-
-  ceph-rbd-mirror:
-    series: *series
-    charm: ../../../ceph-rbd-mirror.charm
-    num_units: 1
-    options:
-      source: *openstack-origin
-
-  ceph-mon-b:
-    charm: ch:ceph-mon
-    num_units: 3
-    options:
-      expected-osd-count: 3
-      source: *openstack-origin
-    channel: quincy/edge
-
-  ceph-osd-b:
-    charm: ch:ceph-osd
-    num_units: 3
-    storage:
-      osd-devices: 'cinder,10G'
-    options:
-      source: *openstack-origin
-      bluestore: False
-      use-direct-io: False
-      osd-devices: '/dev/test-non-existent'
-    channel: quincy/edge
-
-  ceph-rbd-mirror-b:
-    series: *series
-    charm: ../../../ceph-rbd-mirror.charm
-    num_units: 1
-    options:
-      source: *openstack-origin
-
-relations:
-
-- - keystone:shared-db
-  - keystone-mysql-router:shared-db
-- - keystone-mysql-router:db-router
-  - mysql-innodb-cluster:db-router
-
-- - glance:shared-db
-  - glance-mysql-router:shared-db
-- - glance-mysql-router:db-router
-  - mysql-innodb-cluster:db-router
-
-- - cinder:shared-db
-  - cinder-mysql-router:shared-db
-- - cinder-mysql-router:db-router
-  - mysql-innodb-cluster:db-router
-
-- - rabbitmq-server
-  - cinder
-
-- - keystone
-  - cinder
-- - keystone
-  - glance
-
-- - cinder
-  - cinder-ceph
-- - cinder-ceph:ceph
-  - ceph-mon:client
-
-- - nova-compute:ceph-access
-  - cinder-ceph:ceph-access
-- - nova-compute:amqp
-  - rabbitmq-server:amqp
-
-- - glance:image-service
-  - nova-compute:image-service
-- - glance
-  - ceph-mon
-
-- - ceph-mon:osd
-  - ceph-osd:mon
-- - ceph-mon
-  - ceph-rbd-mirror:ceph-local
-- - ceph-mon
-  - ceph-rbd-mirror-b:ceph-remote
-
-- - ceph-mon-b:osd
-  - ceph-osd-b:mon
-- - ceph-mon-b
-  - ceph-rbd-mirror-b:ceph-local
-- - ceph-mon-b
-  - ceph-rbd-mirror:ceph-remote

src/tests/bundles/jammy-yoga.yaml

@@ -151,9 +151,9 @@ relations:
 - - rabbitmq-server
   - cinder
 
-- - keystone
+- - 'keystone:identity-service'
   - cinder
-- - keystone
+- - 'keystone:identity-service'
   - glance
 
 - - cinder

src/tox.ini

@@ -6,32 +6,26 @@
 
 [tox]
 envlist = pep8
-skipsdist = True
 # NOTE: Avoid build/test env pollution by not enabling sitepackages.
 sitepackages = False
 # NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires = pip < 20.3
-           virtualenv < 20.0
-# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
-minversion = 3.18.0
 
 [testenv]
+# We use tox mainly for virtual environment management for test requirements
+# and do not install the charm code as a Python package into that environment.
+# Ref: https://tox.wiki/en/latest/config.html#skip_install
+skip_install = True
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
 allowlist_externals = juju
-passenv = HOME TERM CS_* OS_* TEST_*
+passenv =
+    HOME
+    TERM
+    CS_*
+    OS_*
+    TEST_*
 deps = -r{toxinidir}/test-requirements.txt
-install_command =
-  pip install {opts} {packages}
 
 [testenv:pep8]
 basepython = python3

tox.ini

@@ -5,53 +5,51 @@
 #     https://github.com/openstack-charmers/release-tools
 
 [tox]
-skipsdist = True
 envlist = pep8,py3
 # NOTE: Avoid build/test env pollution by not enabling sitepackages.
 sitepackages = False
 # NOTE: Avoid false positives by not skipping missing interpreters.
 skip_missing_interpreters = False
-# NOTES:
-# * We avoid the new dependency resolver by pinning pip < 20.3, see
-#   https://github.com/pypa/pip/issues/9187
-# * Pinning dependencies requires tox >= 3.2.0, see
-#   https://tox.readthedocs.io/en/latest/config.html#conf-requires
-# * It is also necessary to pin virtualenv as a newer virtualenv would still
-#   lead to fetching the latest pip in the func* tox targets, see
-#   https://stackoverflow.com/a/38133283
-requires =
-  pip < 20.3
-  virtualenv < 20.0
-  setuptools<50.0.0
-
-# NOTE: https://wiki.canonical.com/engineering/OpenStack/InstallLatestToxOnOsci
-minversion = 3.18.0
 
 [testenv]
+# We use tox mainly for virtual environment management for test requirements
+# and do not install the charm code as a Python package into that environment.
+# Ref: https://tox.wiki/en/latest/config.html#skip_install
+skip_install = True
 setenv = VIRTUAL_ENV={envdir}
          PYTHONHASHSEED=0
          TERM=linux
          LAYER_PATH={toxinidir}/layers
          INTERFACE_PATH={toxinidir}/interfaces
          JUJU_REPOSITORY={toxinidir}/build
-passenv = http_proxy https_proxy INTERFACE_PATH LAYER_PATH JUJU_REPOSITORY
+         CHARM_LAYERS_DIR={toxinidir}/layers
-install_command =
+         CHARM_INTERFACES_DIR={toxinidir}/interfaces
-  {toxinidir}/pip.sh install {opts} {packages}
+passenv =
+    no_proxy
+    http_proxy
+    https_proxy
+    CHARM_INTERFACES_DIR
+    CHARM_LAYERS_DIR
+    JUJU_REPOSITORY
 allowlist_externals =
     charmcraft
     bash
     tox
-    rename.sh
+    {toxinidir}/rename.sh
 deps =
     -r{toxinidir}/requirements.txt
 
 [testenv:build]
 basepython = python3
-deps = -r{toxinidir}/build-requirements.txt
+# charmcraft clean is done to ensure that
+# `tox -e build` always performs a clean, repeatable build.
+# For faster rebuilds during development,
+# directly run `charmcraft -v pack && ./rename.sh`.
 commands =
     charmcraft clean
-    charmcraft -v build
+    charmcraft -v pack
     {toxinidir}/rename.sh
+    charmcraft clean
 
 [testenv:build-reactive]
 basepython = python3
@@ -68,11 +66,6 @@ basepython = python3
 deps = -r{toxinidir}/test-requirements.txt
 commands = stestr run --slowest {posargs}
 
-[testenv:py35]
-basepython = python3.5
-deps = -r{toxinidir}/test-requirements.txt
-commands = stestr run --slowest {posargs}
-
 [testenv:py36]
 basepython = python3.6
 deps = -r{toxinidir}/test-requirements.txt
@@ -88,29 +81,12 @@ basepython = python3.8
 deps = -r{toxinidir}/test-requirements.txt
 commands = stestr run --slowest {posargs}
 
-[testenv:py39]
-basepython = python3.9
-deps = -r{toxinidir}/test-requirements.txt
-commands = stestr run --slowest {posargs}
-
 [testenv:pep8]
 basepython = python3
 deps = flake8==3.9.2
-       charm-tools==2.8.3
+       charm-tools==2.8.4
 commands = flake8 {posargs} src unit_tests
 
-[testenv:func-target]
-# Hack to get functional tests working in the charmcraft
-# world. We should fix this.
-basepython = python3
-passenv = HOME TERM CS_* OS_* TEST_*
-deps = -r{toxinidir}/src/test-requirements.txt
-changedir = {toxinidir}/src
-commands =
-  bash -c "if [ ! -f ../*.charm ]; then echo 'Charm does not exist. Run tox -e build'; exit 1; fi"
-  tox --version
-  tox -e func-target {posargs}
-
 [testenv:cover]
 # Technique based heavily upon
 # https://github.com/openstack/nova/blob/master/tox.ini

unit_tests/test_lib_charm_openstack_ceph_rbd_mirror.py

@@ -51,7 +51,10 @@ class TestCephRBDMirrorCharm(Helper):
                 {'stopped': 2, 'replaying': 2}),
         })
         result = crmc.custom_assess_status_check()
-        self.assertTrue('blocked' in result[0])
+        # Disabling blocked state until
+        # https://bugs.launchpad.net/charm-ceph-rbd-mirror/+bug/1879749
+        # is resolved
+        # self.assertTrue('blocked' in result[0])
         # the order of which the statuses appear in the string is undefined
         self.assertTrue('OK (1)' in result[1])
         self.assertTrue('WARN (1)' in result[1])