Add Ceph Object Storage Usage service

This patch add the service "radosgw.usage" to collectors
in order to allow rating on ceph object storage usage per project.
The service unit is set to GB.

The following work has been done:

- Add radosgw.usage transformer for ceilometer
- Add radosgw.usage collect function to ceilometer and gnocchi (new
  and deprecated format)
- update doc

Change-Id: Ia49c76a285950475ed0a9f15d35b2f23b11e5ef3

cloudkitty/collector/ceilometer.py

@@ -90,6 +90,7 @@ class CeilometerCollector(collector.BaseCollector):
         'network.bw.out': 'MB',
         'network.bw.in': 'MB',
         'network.floating': 'ip',
+        'radosgw.usage': 'GB'
     }
 
     def __init__(self, transformers, **kwargs):
@@ -430,3 +431,27 @@ class CeilometerCollector(collector.BaseCollector):
                                             'network.floating')
         return self.t_cloudkitty.format_service('network.floating',
                                                 floating_data)
+
+    def get_radosgw_usage(self,
+                          start,
+                          end=None,
+                          project_id=None,
+                          q_filter=None):
+        active_rgw_stats = self.resources_stats('radosgw.objects.size', start,
+                                                end, project_id, q_filter)
+        rgw_data = []
+        for rgw_stats in active_rgw_stats:
+            rgw_id = rgw_stats.groupby['resource_id']
+            if not self._cacher.has_resource_detail('radosgw.usage', rgw_id):
+                raw_resource = self._conn.resources.get(rgw_id)
+                rgw = self.t_ceilometer.strip_resource_data('radosgw.usage',
+                                                            raw_resource)
+                self._cacher.add_resource_detail('radosgw.usage', rgw_id, rgw)
+            rgw = self._cacher.get_resource_detail('radosgw.usage', rgw_id)
+            rgw_size = decimal.Decimal(rgw_stats.max) / units.Gi
+            rgw_data.append(self.t_cloudkitty.format_item(
+                rgw, self.units_mappings["radosgw.usage"], rgw_size))
+        if not rgw_data:
+            raise collector.NoDataCollected(self.collector_name,
+                                            'radosgw.usage')
+        return self.t_cloudkitty.format_service('radosgw.usage', rgw_data)

cloudkitty/collector/gnocchi.py

@@ -53,6 +53,7 @@ class GnocchiCollector(collector.BaseCollector):
         'network.bw.out': 'instance_network_interface',
         'network.bw.in': 'instance_network_interface',
         'network.floating': 'network',
+        'radosgw.usage': 'ceph_account',
     }
     metrics_mappings = {
         'compute': [
@@ -73,6 +74,8 @@ class GnocchiCollector(collector.BaseCollector):
             ('network.incoming.bytes', 'max')],
         'network.floating': [
             ('ip.floating', 'max')],
+        'radosgw.usage': [
+            ('radosgw.objects.size', 'max')],
     }
     units_mappings = {
         'compute': (1, 'instance'),
@@ -81,6 +84,7 @@ class GnocchiCollector(collector.BaseCollector):
         'network.bw.out': ('network.outgoing.bytes', 'MB'),
         'network.bw.in': ('network.incoming.bytes', 'MB'),
         'network.floating': (1, 'ip'),
+        'radosgw.usage': ('radosgw.objects.size', 'GB')
     }
     default_unit = (1, 'unknown')
 
@@ -293,6 +297,9 @@ class GnocchiCollector(collector.BaseCollector):
             elif resource.get('type') == 'image':
                 resource_data[qty] = (
                     decimal.Decimal(resource_data[qty]) / units.Mi)
+            elif resource.get('type') == 'ceph_account':
+                resource_data[qty] = (
+                    decimal.Decimal(resource_data[qty]) / units.Gi)
             data = self.t_cloudkitty.format_item(
                 resource_data, unit,
                 decimal.Decimal(

cloudkitty/transformer/ceilometer.py

@@ -65,6 +65,8 @@ class CeilometerTransformer(transformer.BaseTransformer):
         'port_id': ['port_id'],
     }
 
+    radosgw_usage_map = {}
+
     metadata_item = 'metadata'
 
     def _strip_compute(self, data):
@@ -105,6 +107,13 @@ class CeilometerTransformer(transformer.BaseTransformer):
         res_data['floatingip_id'] = data.resource_id
         return res_data
 
+    def _strip_radosgw_usage(self, data):
+        res_data = self.generic_strip('radosgw_usage_size', data)
+        res_data['radosgw_id'] = data.resource_id
+        res_data['user_id'] = data.user_id
+        res_data['project_id'] = data.project_id
+        return res_data
+
     def get_metadata(self, res_type):
         """Return list of metadata available after transformation for given
 

doc/source/admin/rating/hashmap.rst

@@ -34,6 +34,7 @@ the following services are available:
 * network.bw.in
 * network.bw.out
 * network.floating
+* radosgw.usage
 
 Enabled services are defined in the configuration file. By default, only the
 compute service is enabled.
@@ -97,6 +98,7 @@ For an active resource on a collection period, quantity is defined as follow:
 * network.bw.in: ingoing network usage (unit: MB)
 * network.bw.out: outgoing network usage (unit: MB)
 * network.floating: 1 (unit: ip)
+* radosgw.usage: Ceph object storage usage (unit: GB)
 
 Example
 =======

etc/cloudkitty/metrics.yml

@@ -20,6 +20,7 @@
     network.bw.in: instance_network_interface
     network.floating: network
     image: image
+    radosgw.usage: ceph_account
 
   services_metrics:
     compute:
@@ -40,6 +41,8 @@
       - image.size: max
       - image.download: max
       - image.serve: max
+    radosgw.usage:
+      - radosgw.objects.size: max
 
   services_units:
     compute:
@@ -54,5 +57,7 @@
       1: ip
     image:
       image.size: MB
+    radosgw.usage:
+      radosgw.objects.size: GB
     default_unit:
       1: unknown