Add a v2 API endpoint to retrieve DataFrame objects

A new endpoint has been made available to admin users or
scope owners on ``GET /v2/dataframes``. This will allow
end users to retrieve DataFrames in the form of JSON objects
from the CloudKitty storage backend.

Documentation and unit tests are included in this commit.

Change-Id: I784da5565d040f565945fa93d86dcef11faa72f9

cloudkitty/api/v2/dataframes/dataframes.py

@@ -20,6 +20,7 @@ from cloudkitty.api.v2 import base
 from cloudkitty.api.v2 import utils as api_utils
 from cloudkitty.common import policy
 from cloudkitty import dataframe
+from cloudkitty import tzutils
 
 
 class DataFrameList(base.BaseResource):
@@ -40,3 +41,50 @@ class DataFrameList(base.BaseResource):
         self._storage.push(dataframes)
 
         return {}, 204
+
+    @api_utils.paginated
+    @api_utils.add_input_schema('query', {
+        voluptuous.Optional('begin'):
+            api_utils.SingleQueryParam(tzutils.dt_from_iso),
+        voluptuous.Optional('end'):
+            api_utils.SingleQueryParam(tzutils.dt_from_iso),
+        voluptuous.Optional('filters'):
+            api_utils.SingleDictQueryParam(str, str),
+    })
+    @api_utils.add_output_schema({
+        voluptuous.Required('total'): int,
+        voluptuous.Required('dataframes'):
+            [dataframe.DataFrame.as_dict],
+    })
+    def get(self,
+            offset=0,
+            limit=100,
+            begin=None,
+            end=None,
+            filters={}):
+
+        policy.authorize(
+            flask.request.context,
+            'dataframes:get',
+            {'tenant_id': flask.request.context.project_id},
+        )
+
+        begin = begin or tzutils.get_month_start()
+        end = end or tzutils.get_next_month()
+
+        metric_types = [filters.pop('type')] if 'type' in filters else None
+        results = self._storage.retrieve(
+            begin=begin, end=end,
+            filters=filters,
+            metric_types=metric_types,
+            offset=offset, limit=limit,
+        )
+
+        if results['total'] < 1:
+            raise http_exceptions.NotFound(
+                "No resource found for provided filters.")
+
+        return {
+            'total': results['total'],
+            'dataframes': results['dataframes'],
+        }

cloudkitty/common/policies/v2/dataframes.py

@@ -24,6 +24,12 @@ dataframes_policies = [
         description='Add one or several DataFrames',
         operations=[{'path': '/v2/dataframes',
                      'method': 'POST'}]),
+    policy.DocumentedRuleDefault(
+        name='dataframes:get',
+        check_str=base.RULE_ADMIN_OR_OWNER,
+        description='Get DataFrames',
+        operations=[{'path': '/v2/dataframes',
+                     'method': 'GET'}]),
 ]
 
 

cloudkitty/tests/gabbi/fixtures.py

@@ -447,7 +447,33 @@ class MetricsConfFixture(fixture.GabbiFixture):
         ck_utils.load_conf = self._original_function
 
 
-class InfluxStorageDataFixture(NowStorageDataFixture):
+class NowInfluxStorageDataFixture(NowStorageDataFixture):
+
+    def start_fixture(self):
+        cli = influx_utils.FakeInfluxClient()
+        st = storage.get_storage()
+        st._conn = cli
+
+        self._get_storage_patch = mock.patch(
+            'cloudkitty.storage.get_storage',
+            new=lambda **kw: st,
+        )
+        self._get_storage_patch.start()
+
+        super(NowInfluxStorageDataFixture, self).start_fixture()
+
+    def initialize_data(self):
+        data = test_utils.generate_v2_storage_data(
+            start=tzutils.get_month_start(),
+            end=tzutils.localized_now().replace(hour=0),
+        )
+        self.storage.push([data])
+
+    def stop_fixture(self):
+        self._get_storage_patch.stop()
+
+
+class InfluxStorageDataFixture(StorageDataFixture):
 
     def start_fixture(self):
         cli = influx_utils.FakeInfluxClient()
@@ -462,17 +488,20 @@ class InfluxStorageDataFixture(NowStorageDataFixture):
 
         super(InfluxStorageDataFixture, self).start_fixture()
 
-    def initialize_data(self):
-        data = test_utils.generate_v2_storage_data(
-            start=tzutils.get_month_start(),
-            end=tzutils.localized_now().replace(hour=0),
-        )
-        self.storage.push([data])
-
     def stop_fixture(self):
         self._get_storage_patch.stop()
 
 
+class UTCFixture(fixture.GabbiFixture):
+    """Set the local timezone to UTC"""
+    def start_fixture(self):
+        self._tzmock = mock.patch('cloudkitty.tzutils._LOCAL_TZ', tz.UTC)
+        self._tzmock.start()
+
+    def stop_fixture(self):
+        self._tzmock.stop()
+
+
 def setup_app():
     messaging.setup()
     # FIXME(sheeprine): Extension fixtures are interacting with transformers

cloudkitty/tests/gabbi/gabbits/v2-dataframes.yaml

@@ -1,6 +1,7 @@
 fixtures:
   - ConfigFixtureStorageV2
   - InfluxStorageDataFixture
+  - UTCFixture
 
 tests:
   - name: Push dataframes
@@ -184,3 +185,402 @@ tests:
                 metadata:
                   attr_one: one
                   attr_two: two
+
+  - name: fetch period with no data
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2014-01-01T00:00:00"
+      end: "2015-01-04T00:00:00"
+    status: 404
+    response_strings:
+      - "No resource found for provided filters."
+
+  - name: fetch period with no data filtering on tenant_id
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-01T00:00:00"
+      end: "2015-01-04T00:00:00"
+      filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
+    status: 404
+    response_strings:
+      - "No resource found for provided filters."
+
+  - name: fetch data for the first tenant without begin time
+    url: /v2/dataframes
+    query_parameters:
+      end: "2015-01-04T00:00:00"
+      filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
+    status: 404
+    response_strings:
+      - "No resource found for provided filters."
+
+  - name: fetch data for the first tenant without end time
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-04T00:00:00"
+      filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
+    status: 200
+    response_json_paths:
+      $.dataframes.`len`: 56
+
+  - name: fetch data for the first tenant without begin and end time
+    url: /v2/dataframes
+    query_parameters:
+      filters: "project_id:3d9a1b33-482f-42fd-aef9-b575a3da9369"
+    status: 404
+    response_strings:
+      - "No resource found for provided filters."
+
+  - name: fetch data for the first tenant when begin time bigger than end time
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-04T14:00:00"
+      end: "2015-01-04T13:00:00"
+      filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
+    status: 404
+    response_strings:
+      - "No resource found for provided filters."
+
+  - name: fetch data for the first tenant
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-04T13:00:00"
+      end: "2015-01-04T14:00:00"
+      filters: "project_id:8f82cc70-e50c-466e-8624-24bdea811375"
+    status: 200
+    response_json_paths:
+      $:
+        total: 4
+        dataframes:
+          - usage:
+              image.size:
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 8f82cc70-e50c-466e-8624-24bdea811375
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 8f82cc70-e50c-466e-8624-24bdea811375
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              cpu:
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 8f82cc70-e50c-466e-8624-24bdea811375
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 8f82cc70-e50c-466e-8624-24bdea811375
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+            period:
+              begin: '2015-01-04T13:00:00+00:00'
+              end: '2015-01-04T14:00:00+00:00'
+
+  - name: fetch data for the second tenant
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-04T13:00:00"
+      end: "2015-01-04T14:00:00"
+      filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
+    status: 200
+    response_json_paths:
+      $:
+        total: 4
+        dataframes:
+          - usage:
+              image.size:
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              cpu:
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+            period:
+              begin: '2015-01-04T13:00:00+00:00'
+              end: '2015-01-04T14:00:00+00:00'
+
+  - name: fetch data for multiple tenants
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-04T13:00:00"
+      end: "2015-01-04T14:00:00"
+    status: 200
+    response_json_paths:
+      $:
+        total: 8
+        dataframes:
+          - usage:
+              image.size:
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 8f82cc70-e50c-466e-8624-24bdea811375
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 8f82cc70-e50c-466e-8624-24bdea811375
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              cpu:
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 8f82cc70-e50c-466e-8624-24bdea811375
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 8f82cc70-e50c-466e-8624-24bdea811375
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+            period:
+              begin: '2015-01-04T13:00:00+00:00'
+              end: '2015-01-04T14:00:00+00:00'
+
+  - name: fetch data filtering on cpu service and tenant
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-04T13:00:00"
+      end: "2015-01-04T14:00:00"
+      filters: "type:cpu"
+      filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
+    status: 200
+    response_json_paths:
+      $:
+        total: 4
+        dataframes:
+          - usage:
+              image.size:
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 0.121
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              cpu:
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+              - vol:
+                  unit: nothing
+                  qty: 1
+                rating:
+                  price: 1.337
+                groupby:
+                  project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                  fake_meta: 1
+                metadata:
+                  dummy: true
+            period:
+              begin: '2015-01-04T13:00:00+00:00'
+              end: '2015-01-04T14:00:00+00:00'
+
+
+  - name: fetch data filtering on image service and tenant
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-04T13:00:00"
+      end: "2015-01-04T14:00:00"
+      filters: "type:image.size"
+      filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
+    status: 200
+    response_json_paths:
+      $:
+        total: 4
+        dataframes:
+        - usage:
+            image.size:
+            - vol:
+                unit: nothing
+                qty: 1
+              rating:
+                price: 0.121
+              groupby:
+                project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                fake_meta: 1
+              metadata:
+                dummy: true
+            - vol:
+                unit: nothing
+                qty: 1
+              rating:
+                price: 0.121
+              groupby:
+                project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                fake_meta: 1
+              metadata:
+                dummy: true
+            cpu:
+            - vol:
+                unit: nothing
+                qty: 1
+              rating:
+                price: 1.337
+              groupby:
+                project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                fake_meta: 1
+              metadata:
+                dummy: true
+            - vol:
+                unit: nothing
+                qty: 1
+              rating:
+                price: 1.337
+              groupby:
+                project_id: 7606a24a-b8ad-4ae0-be6c-3d7a41334a2e
+                fake_meta: 1
+              metadata:
+                dummy: true
+          period:
+            begin: '2015-01-04T13:00:00+00:00'
+            end: '2015-01-04T14:00:00+00:00'
+
+
+  - name: fetch data filtering on service with no data and tenant
+    url: /v2/dataframes
+    query_parameters:
+      begin: "2015-01-04T13:00:00"
+      end: "2015-01-04T14:00:00"
+      filters: "type:volume"
+      filters: "project_id:7606a24a-b8ad-4ae0-be6c-3d7a41334a2e"
+    status: 200
+    response_json_paths:
+      $.dataframes.`len`: 1

cloudkitty/tests/gabbi/gabbits/v2-summary.yaml

@@ -1,6 +1,6 @@
 fixtures:
   - ConfigFixtureStorageV2
-  - InfluxStorageDataFixture
+  - NowInfluxStorageDataFixture
 
 tests:
   - name: Get a summary

doc/source/_static/cloudkitty.policy.yaml.sample

@@ -88,6 +88,10 @@
 # POST  /v2/dataframes
 #"dataframes:add": "role:admin"
 
+# Get DataFrames
+# GET  /v2/dataframes
+#"dataframes:get": "rule:admin_or_owner"
+
 # Get the state of one or several scopes
 # GET  /v2/scope
 #"scope:get_state": "role:admin"

doc/source/api-reference/v2/api_samples/dataframes/dataframes_get.json

@@ -0,0 +1,78 @@
+{
+    "total": 3,
+    "dataframes": [
+        {
+            "usage": {
+                "metric_one": [
+                    {
+                        "vol": {
+                            "unit": "GiB",
+                            "qty": 1.2
+                        },
+                        "rating": {
+                            "price": 0.04
+                        },
+                        "groupby": {
+                            "group_one": "one",
+                            "group_two": "two"
+                        },
+                        "metadata": {
+                            "attr_one": "one",
+                            "attr_two": "two"
+                        }
+                    }
+                ],
+                "metric_two": [
+                    {
+                        "vol": {
+                            "unit": "GiB",
+                            "qty": 1.2
+                        },
+                        "rating": {
+                            "price": 0.04
+                        },
+                        "groupby": {
+                            "group_one": "one",
+                            "group_two": "two"
+                        },
+                        "metadata": {
+                            "attr_one": "one",
+                            "attr_two": "two"
+                        }
+                    }
+                ]
+            },
+            "period": {
+                "begin": "2019-07-23T12:28:10+00:00",
+                "end": "2019-07-23T13:28:10+00:00"
+            }
+        },
+        {
+            "usage": {
+                "volume.size": [
+                    {
+                        "vol": {
+                            "unit": "GiB",
+                            "qty": 1.9
+                        },
+                        "rating": {
+                            "price": 3.8
+                        },
+                        "groupby": {
+                            "project_id": "8ace6f139a1742548e09f1e446bc9737",
+                            "user_id": "b28fd3f448c34c17bf70e32886900eed",
+                            "id": "be966c6d-78a0-42cf-bab9-e833ed996dee"
+                        },
+                        "metadata": {
+                            "volume_type": ""
+                        }
+                    }
+                ]
+            },
+            "period": {
+                "begin": "2019-08-01T01:00:00+00:00",
+                "end": "2019-08-01T02:00:00+00:00"
+            }
+        }
+    ]
+}

doc/source/api-reference/v2/dataframes/dataframes.inc

@@ -39,3 +39,47 @@ Response
 --------
 
 No content is to be returned.
+
+Get dataframes from the storage backend
+============================================
+
+Get dataframes from the storage backend.
+
+.. rest_method::  GET /v2/dataframes
+
+.. rest_parameters:: dataframes/dataframes_parameters.yml
+
+   - limit: limit
+   - offset: offset
+   - begin: begin
+   - end: end
+   - filters: filters
+
+Status codes
+------------
+
+.. rest_status_code:: success http_status.yml
+
+   - 200
+
+.. rest_status_code:: error http_status.yml
+
+   - 400
+   - 401
+   - 403
+   - 405
+
+Response
+--------
+
+.. rest_parameters:: dataframes/dataframes_parameters.yml
+
+   - total: total_resp
+   - dataframes: dataframes_resp
+
+Response Example
+----------------
+
+.. literalinclude:: ./api_samples/dataframes/dataframes_get.json
+   :language: javascript
+

doc/source/api-reference/v2/dataframes/dataframes_parameters.yml

@@ -1,6 +1,55 @@
 dataframes_body:
   in: body
   description: |
-    List of dataframes to add
+    List of dataframes to add.
   type: list
   required: true
+
+dataframes_resp:
+  in: body
+  description: |
+    List of dataframes matching the query parameters.
+  type: list
+  required: true
+
+total_resp:
+  in: body
+  description: |
+    Total of datapoints matching the query parameters.
+  type: int
+  required: true
+
+limit:
+  in: query
+  description: |
+    For pagination. The maximum number of results to return.
+  type: int
+  required: false
+
+offset:
+  in: query
+  description: |
+    For pagination. The index of the first element that should be returned.
+  type: int
+  required: false
+
+filters:
+  in: query
+  description: |
+    Optional filters.
+  type: dict
+  required: false
+
+begin:
+  in: query
+  description: |
+    Begin of the period for which the dataframes are required.
+  type: iso8601 timestamp
+  required: false
+
+end:
+  in: query
+  description: |
+    End of the period for which the dataframes are required.
+  type: iso8601 timestamp
+  required: false

releasenotes/notes/get-dataframes-v2-api-endpoint-3a4625c6008a5fca.yaml

@@ -0,0 +1,7 @@
+---
+features:
+  - |
+    Added a v2 API endpoint allowing to retrieve dataframes from
+    the CloudKitty storage. This endpoint is available via a ``GET``
+    request on ``/v2/dataframes``. Being the owner of the scope or
+    having admin privileges are required to use this endpoint.