Add storage_policy attribute for Swift containers

This exposes a Swift container's storage policy as a metadata
attribute in Ceilometer samples (via the storage.containers.objects
and storage.containers.objects.size meters) and the swift_account
Gnocchi resources, named storage_policy.

The storage policy determines the location and replication
parameters for a Swift container, making it a useful metric
for billing alongside the size of the container. The storage
policy for a container cannot be changed unless the container
is recreated, meaning that it does not change often (making it
suitable to be stored in Gnocchi as a metadata attribute).

The reason why this was not possible before was that Swift
did not return the storage policy for each container in the
GET account request, but a new proposed change to Swift [1]
adds it to the response as the storage_policy attribute.
This change for Ceilometer takes advantage of this and adds
it to Ceilometer samples and the Gnocchi resource type.

For this to work, the Swift change needs to be accepted
and merged.

[1]: https://review.opendev.org/c/openstack/swift/+/940601

Depends-On: I52b37cfa49cac8675f5087bcbcfe18db0b46d887
Change-Id: I25f74df61ef0a6a4b50f1357ddf86d5acc6d71e7

ceilometer/gnocchi_client.py

@@ -209,6 +209,15 @@ resources_update_operations = [
          "value": {"type": "string", "min_length": 0, "max_length": 255,
                    "required": False}
      }]},
+    {"desc": "add storage_policy to swift_account",
+     "type": "update_attribute_type",
+     "resource_type": "swift_account",
+     "data": [{
+         "op": "add",
+         "path": "/attributes/storage_policy",
+         "value": {"type": "string", "min_length": 0, "max_length": 255,
+                   "required": False}  # Only containers have a storage policy
+     }]},
 ]
 
 

ceilometer/objectstore/swift.py

@@ -111,6 +111,14 @@ class _Base(plugin_base.PollsterBase):
                                 'v1/' + reseller_prefix + tenant_id)
 
 
+class _ContainersBase(_Base):
+    FIELDS = ("storage_policy",)
+
+    def _get_resource_metadata(self, container):
+        # NOTE(callumdickinson): Sets value to None if a field is not found.
+        return {f: container.get(f) for f in self.FIELDS}
+
+
 class ObjectsPollster(_Base):
     """Collect the total objects count for each project"""
     def get_samples(self, manager, cache, resources):
@@ -165,7 +173,7 @@ class ObjectsContainersPollster(_Base):
             )
 
 
-class ContainersObjectsPollster(_Base):
+class ContainersObjectsPollster(_ContainersBase):
     """Collect the objects count per container for each project"""
 
     METHOD = 'get'
@@ -184,11 +192,11 @@ class ContainersObjectsPollster(_Base):
                     user_id=None,
                     project_id=tenant,
                     resource_id=tenant + '/' + container['name'],
-                    resource_metadata=None,
+                    resource_metadata=self._get_resource_metadata(container),
                 )
 
 
-class ContainersSizePollster(_Base):
+class ContainersSizePollster(_ContainersBase):
     """Collect the total objects size per container for each project"""
 
     METHOD = 'get'
@@ -207,5 +215,5 @@ class ContainersSizePollster(_Base):
                     user_id=None,
                     project_id=tenant,
                     resource_id=tenant + '/' + container['name'],
-                    resource_metadata=None,
+                    resource_metadata=self._get_resource_metadata(container),
                 )

ceilometer/publisher/data/gnocchi_resources.yaml

@@ -226,6 +226,8 @@ resources:
       storage.objects.containers:
       storage.containers.objects:
       storage.containers.objects.size:
+    attributes:
+      storage_policy: resource_metadata.storage_policy
 
   - resource_type: volume
     metrics:

ceilometer/tests/unit/objectstore/test_swift.py

@@ -13,6 +13,7 @@
 # under the License.
 
 import collections
+import itertools
 from unittest import mock
 
 import fixtures
@@ -44,10 +45,15 @@ GET_ACCOUNTS = [('tenant-000', ({'x-account-object-count': 10,
                                  },
                                 [{'count': 10,
                                   'bytes': 123123,
-                                  'name': 'my_container'},
+                                  'name': 'my_container',
+                                  'storage_policy': 'Policy-0',
+                                  },
                                  {'count': 0,
                                   'bytes': 0,
-                                  'name': 'new_container'
+                                  'name': 'new_container',
+                                  # NOTE(callumdickinson): No storage policy,
+                                  # to test backwards compatibility with older
+                                  # versions of Swift.
                                   }])),
                 ('tenant-001', ({'x-account-object-count': 0,
                                  'x-account-bytes-used': 0,
@@ -81,15 +87,27 @@ class TestSwiftPollster(testscenarios.testcase.WithScenarios,
     # pollsters.
     scenarios = [
         ('storage.objects',
-         {'factory': swift.ObjectsPollster}),
+         {'factory': swift.ObjectsPollster, 'resources': {}}),
         ('storage.objects.size',
-         {'factory': swift.ObjectsSizePollster}),
+         {'factory': swift.ObjectsSizePollster, 'resources': {}}),
         ('storage.objects.containers',
-         {'factory': swift.ObjectsContainersPollster}),
+         {'factory': swift.ObjectsContainersPollster, 'resources': {}}),
         ('storage.containers.objects',
-         {'factory': swift.ContainersObjectsPollster}),
+         {'factory': swift.ContainersObjectsPollster,
+          'resources': {
+              f"{project_id}/{container['name']}": container
+              for project_id, container in itertools.chain.from_iterable(
+                  itertools.product([acc[0]], acc[1][1])
+                  for acc in GET_ACCOUNTS)
+          }}),
         ('storage.containers.objects.size',
-         {'factory': swift.ContainersSizePollster}),
+         {'factory': swift.ContainersSizePollster,
+          'resources': {
+              f"{project_id}/{container['name']}": container
+              for project_id, container in itertools.chain.from_iterable(
+                  itertools.product([acc[0]], acc[1][1])
+                  for acc in GET_ACCOUNTS)
+          }}),
     ]
 
     @staticmethod
@@ -174,6 +192,16 @@ class TestSwiftPollster(testscenarios.testcase.WithScenarios,
                                                      ASSIGNED_TENANTS))
 
         self.assertEqual(2, len(samples), self.pollster.__class__)
+        for resource_id, resource in self.resources.items():
+            for field in getattr(self.pollster, 'FIELDS', []):
+                with self.subTest(f'{resource_id}-{field}'):
+                    sample = next(s for s in samples
+                                  if s.resource_id == resource_id)
+                    if field in resource:
+                        self.assertEqual(resource[field],
+                                         sample.resource_metadata[field])
+                    else:
+                        self.assertIsNone(sample.resource_metadata[field])
 
     def test_get_meter_names(self):
         with fixtures.MockPatchObject(self.factory, '_iter_accounts',

releasenotes/notes/add-swift-storage_policy-attribute-322fbb5716c5bb10.yaml

@@ -0,0 +1,22 @@
+---
+features:
+  - |
+    The ``storage_policy`` resource metadata attribute has been added to the
+    ``swift.containers.objects`` and ``swift.containers.objects.size`` meters,
+    populated from already performed Swift account ``GET`` requests.
+    This functionality requires using a new version of Swift that adds the
+    ``storage_policy`` attribute when listing containers in an account.
+    Ceilometer is backwards compatible with Swift versions that do not
+    provide this functionality, but ``storage_policy`` will be set to
+    ``None`` in samples and Gnocchi resources.
+  - |
+    An optional ``storage_policy`` attribute has been added to the
+    ``swift_account`` Gnocchi resource type, to store the storage policy for
+    Swift containers in Gnocchi. For Swift accounts, ``storage_policy`` will
+    be set to ``None``.
+upgrade:
+  - |
+    To publish the ``storage_policy`` attribute for Swift containers,
+    ``gnocchi_resources.yaml`` will need to be updated to the latest version.
+    Swift in the target OpenStack cloud will also need upgrading to add
+    support for providing the storage policy when listing containers.