diff --git a/cloudkitty/collector/__init__.py b/cloudkitty/collector/__init__.py index 17650285..193ecac5 100644 --- a/cloudkitty/collector/__init__.py +++ b/cloudkitty/collector/__init__.py @@ -16,11 +16,10 @@ # @author: Stéphane Albert # import abc -import datetime import six -import cloudkitty.utils as utils +import cloudkitty.utils as ck_utils class TransformerDependencyError(Exception): @@ -72,19 +71,16 @@ class BaseCollector(object): @staticmethod def last_month(): - now = datetime.datetime.now() - month_end = (datetime.datetime(now.year, now.month, 1) - - datetime.timedelta(days=1)) - month_start = month_end.replace(day=1) - start_ts = utils.dt2ts(month_start) - end_ts = utils.dt2ts(month_end) + month_start = ck_utils.get_month_start() + month_end = ck_utils.get_month_end() + start_ts = ck_utils.dt2ts(month_start) + end_ts = ck_utils.dt2ts(month_end) return start_ts, end_ts @staticmethod def current_month(): - now = datetime.now() - month_start = datetime(now.year, now.month, 1) - return utils.dt2ts(month_start) + month_start = ck_utils.get_month_start() + return ck_utils.dt2ts(month_start) def retrieve(self, resource, start, end=None, project_id=None, q_filter=None): diff --git a/cloudkitty/collector/ceilometer.py b/cloudkitty/collector/ceilometer.py index f8e4669e..0caa3a5f 100644 --- a/cloudkitty/collector/ceilometer.py +++ b/cloudkitty/collector/ceilometer.py @@ -15,11 +15,10 @@ # # @author: Stéphane Albert # -import datetime - from ceilometerclient import client as cclient from cloudkitty import collector +from cloudkitty import utils as ck_utils class ResourceNotFound(Exception): @@ -111,12 +110,12 @@ class CeilometerCollector(collector.BaseCollector): def get_active_instances(self, start, end=None, project_id=None, q_filter=None): """Instance that were active during the timespan.""" - start_iso = datetime.datetime.fromtimestamp(start).isoformat() + start_iso = ck_utils.ts2iso(start) req_filter = self.gen_filter(op='ge', timestamp=start_iso) if project_id: req_filter.extend(self.gen_filter(project=project_id)) if end: - end_iso = datetime.datetime.fromtimestamp(end).isoformat() + end_iso = ck_utils.ts2iso(end) req_filter.extend(self.gen_filter(op='le', timestamp=end_iso)) if isinstance(q_filter, list): req_filter.extend(q_filter) diff --git a/cloudkitty/orchestrator.py b/cloudkitty/orchestrator.py index 70db0c34..2250f642 100644 --- a/cloudkitty/orchestrator.py +++ b/cloudkitty/orchestrator.py @@ -16,8 +16,6 @@ # # @author: Stéphane Albert # -import time - import eventlet from keystoneclient.v2_0 import client as kclient from oslo.config import cfg @@ -152,9 +150,10 @@ class Orchestrator(object): def _check_state(self): timestamp = self.storage.get_state() if not timestamp: - return ck_utils.get_this_month_timestamp() + month_start = ck_utils.get_month_start() + return ck_utils.dt2ts(month_start) - now = int(time.time() + time.timezone) + now = ck_utils.utcnow_ts() next_timestamp = timestamp + CONF.collect.period wait_time = CONF.collect.wait_periods * CONF.collect.period if next_timestamp + wait_time < now: diff --git a/cloudkitty/storage/__init__.py b/cloudkitty/storage/__init__.py index 39a78e04..d1e15713 100644 --- a/cloudkitty/storage/__init__.py +++ b/cloudkitty/storage/__init__.py @@ -16,12 +16,13 @@ # @author: Stéphane Albert # import abc -import datetime from oslo.config import cfg import six from stevedore import driver +from cloudkitty import utils as ck_utils + STORAGES_NAMESPACE = 'cloudkitty.storage.backends' storage_opts = [ cfg.StrOpt('backend', @@ -153,10 +154,8 @@ class BaseStorage(object): if self.usage_start is None: self.usage_start = usage_start self.usage_end = usage_start + self._period - self.usage_start_dt = ( - datetime.datetime.fromtimestamp(self.usage_start)) - self.usage_end_dt = ( - datetime.datetime.fromtimestamp(self.usage_end)) + self.usage_start_dt = ck_utils.ts2dt(self.usage_start) + self.usage_end_dt = ck_utils.ts2dt(self.usage_end) self._dispatch(data) diff --git a/cloudkitty/storage/sqlalchemy/__init__.py b/cloudkitty/storage/sqlalchemy/__init__.py index 644dbe8c..957cbc4c 100644 --- a/cloudkitty/storage/sqlalchemy/__init__.py +++ b/cloudkitty/storage/sqlalchemy/__init__.py @@ -15,7 +15,6 @@ # # @author: Stéphane Albert # -import datetime import json from oslo.db.sqlalchemy import utils @@ -70,7 +69,7 @@ class SQLAlchemyStorage(storage.BaseStorage): model = models.RatedDataFrame # Boundary calculation - month_start = ck_utils.get_this_month() + month_start = ck_utils.get_month_start() month_end = ck_utils.get_next_month() session = db.get_session() @@ -96,8 +95,8 @@ class SQLAlchemyStorage(storage.BaseStorage): model, session ).filter( - model.begin >= datetime.datetime.fromtimestamp(begin), - model.end <= datetime.datetime.fromtimestamp(end) + model.begin >= ck_utils.ts2dt(begin), + model.end <= ck_utils.ts2dt(end) ) for cur_filter in filters: q = q.filter(getattr(model, cur_filter) == filters[cur_filter]) diff --git a/cloudkitty/tests/test_utils.py b/cloudkitty/tests/test_utils.py new file mode 100644 index 00000000..9c5f87fc --- /dev/null +++ b/cloudkitty/tests/test_utils.py @@ -0,0 +1,138 @@ +# -*- coding: utf-8 -*- +# Copyright 2014 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Stéphane Albert +# +import datetime +import unittest + +import mock +from oslo.utils import timeutils + +from cloudkitty import utils as ck_utils + + +def iso2dt(iso_str): + return timeutils.parse_isotime(iso_str) + + +class UtilsTimeCalculationsTest(unittest.TestCase): + def setUp(self): + self.date_ts = 1416219015 + self.date_iso = '2014-11-17T10:10:15Z' + self.date_params = {'year': 2014, + 'month': 11, + 'day': 17, + 'hour': 10, + 'minute': 10, + 'second': 15} + self.date_tz_params = {'year': 2014, + 'month': 10, + 'day': 26, + 'hour': 2, + 'minute': 00, + 'second': 00} + + def test_dt2ts(self): + date = datetime.datetime(**self.date_params) + trans_ts = ck_utils.dt2ts(date) + self.assertEqual(self.date_ts, trans_ts) + + def test_iso2dt(self): + date = datetime.datetime(**self.date_params) + trans_dt = ck_utils.iso2dt(self.date_iso) + self.assertEqual(date, trans_dt) + + def test_ts2iso(self): + trans_iso = ck_utils.ts2iso(self.date_ts) + self.assertEqual(self.date_iso, trans_iso) + + def test_dt2iso(self): + date = datetime.datetime(**self.date_params) + trans_iso = ck_utils.dt2iso(date) + self.assertEqual(self.date_iso, trans_iso) + + @mock.patch.object(ck_utils, 'utcnow', + return_value=iso2dt('2014-01-31T00:00:00Z')) + def test_month_start_without_dt(self, patch_utcnow_mock): + date = datetime.datetime(2014, 1, 1) + trans_dt = ck_utils.get_month_start() + self.assertEqual(date, trans_dt) + patch_utcnow_mock.assert_called_once() + + @mock.patch.object(ck_utils, 'utcnow', + return_value=iso2dt('2014-01-15T00:00:00Z')) + def test_month_end_without_dt(self, patch_utcnow_mock): + date = datetime.datetime(2014, 1, 31) + trans_dt = ck_utils.get_month_end() + self.assertEqual(date, trans_dt) + patch_utcnow_mock.assert_called_once() + + @mock.patch.object(ck_utils, 'utcnow', + return_value=iso2dt('2014-01-31T00:00:00Z')) + def test_get_last_month_without_dt(self, patch_utcnow_mock): + date = datetime.datetime(2013, 12, 1) + trans_dt = ck_utils.get_last_month() + self.assertEqual(date, trans_dt) + patch_utcnow_mock.assert_called_once() + + @mock.patch.object(ck_utils, 'utcnow', + return_value=iso2dt('2014-01-31T00:00:00Z')) + def test_get_next_month_without_dt(self, patch_utcnow_mock): + date = datetime.datetime(2014, 2, 1) + trans_dt = ck_utils.get_next_month() + self.assertEqual(date, trans_dt) + patch_utcnow_mock.assert_called_once() + + def test_get_last_month_leap(self): + base_date = datetime.datetime(2016, 3, 31) + date = datetime.datetime(2016, 2, 1) + trans_dt = ck_utils.get_last_month(base_date) + self.assertEqual(date, trans_dt) + + def test_get_next_month_leap(self): + base_date = datetime.datetime(2016, 1, 31) + date = datetime.datetime(2016, 2, 1) + trans_dt = ck_utils.get_next_month(base_date) + self.assertEqual(date, trans_dt) + + def test_add_month_leap(self): + base_date = datetime.datetime(2016, 1, 31) + date = datetime.datetime(2016, 3, 3) + trans_dt = ck_utils.add_month(base_date, False) + self.assertEqual(date, trans_dt) + + def test_add_month_keep_leap(self): + base_date = datetime.datetime(2016, 1, 31) + date = datetime.datetime(2016, 2, 29) + trans_dt = ck_utils.add_month(base_date) + self.assertEqual(date, trans_dt) + + def test_sub_month_leap(self): + base_date = datetime.datetime(2016, 3, 31) + date = datetime.datetime(2016, 3, 3) + trans_dt = ck_utils.sub_month(base_date, False) + self.assertEqual(date, trans_dt) + + def test_sub_month_keep_leap(self): + base_date = datetime.datetime(2016, 3, 31) + date = datetime.datetime(2016, 2, 29) + trans_dt = ck_utils.sub_month(base_date) + self.assertEqual(date, trans_dt) + + def test_load_timestamp(self): + calc_dt = ck_utils.iso2dt(self.date_iso) + check_dt = ck_utils.ts2dt(self.date_ts) + self.assertEqual(calc_dt, check_dt) diff --git a/cloudkitty/utils.py b/cloudkitty/utils.py index 323db438..05a5de6f 100644 --- a/cloudkitty/utils.py +++ b/cloudkitty/utils.py @@ -15,36 +15,111 @@ # # @author: Stéphane Albert # +""" +Time calculations functions + +We're mostly using oslo.utils for time calculations but we're encapsulating it +to ease maintenance in case of library modifications. +""" import calendar import datetime -import time -import iso8601 +from oslo.utils import timeutils def dt2ts(orig_dt): - return int(time.mktime(orig_dt.timetuple())) + """Translate a datetime into a timestamp.""" + return calendar.timegm(orig_dt.timetuple()) def iso2dt(iso_date): - return iso8601.parse_date(iso_date) + """iso8601 format to datetime.""" + iso_dt = timeutils.parse_isotime(iso_date) + trans_dt = timeutils.normalize_time(iso_dt) + return trans_dt -def get_this_month(): - now = datetime.datetime.utcnow() - month_start = datetime.datetime(now.year, now.month, 1) +def ts2dt(timestamp): + """timestamp to datetime format.""" + if not isinstance(timestamp, float): + timestamp = float(timestamp) + return datetime.datetime.utcfromtimestamp(timestamp) + + +def ts2iso(timestamp): + """timestamp to is8601 format.""" + if not isinstance(timestamp, float): + timestamp = float(timestamp) + return timeutils.iso8601_from_timestamp(timestamp) + + +def dt2iso(orig_dt): + """datetime to is8601 format.""" + return timeutils.isotime(orig_dt) + + +def utcnow(): + """Returns a datetime for the current utc time.""" + return timeutils.utcnow() + + +def utcnow_ts(): + """Returns a timestamp for the current utc time.""" + return timeutils.utcnow_ts() + + +def get_month_days(dt): + return calendar.monthrange(dt.year, dt.month)[1] + + +def add_days(base_dt, days, stay_on_month=True): + if stay_on_month: + max_days = get_month_days(base_dt) + if days > max_days: + return get_month_end(base_dt) + return base_dt + datetime.timedelta(days=days) + + +def add_month(dt, stay_on_month=True): + next_month = get_next_month(dt) + return add_days(next_month, dt.day, stay_on_month) + + +def sub_month(dt, stay_on_month=True): + prev_month = get_last_month(dt) + return add_days(prev_month, dt.day, stay_on_month) + + +def get_month_start(dt=None): + if not dt: + dt = utcnow() + month_start = datetime.datetime(dt.year, dt.month, 1) return month_start -def get_this_month_timestamp(): - return dt2ts(get_this_month()) +def get_month_start_timestamp(dt=None): + return dt2ts(get_month_start(dt)) -def get_next_month(): - start_dt = get_this_month() - next_dt = start_dt + datetime.timedelta(calendar.mdays[start_dt.month]) - return next_dt +def get_month_end(dt=None): + month_start = get_month_start(dt) + days_of_month = get_month_days(month_start) + month_end = month_start.replace(day=days_of_month) + return month_end -def get_next_month_timestamp(): - return dt2ts(get_next_month()) +def get_last_month(dt=None): + if not dt: + dt = utcnow() + month_end = get_month_start(dt) - datetime.timedelta(days=1) + return get_month_start(month_end) + + +def get_next_month(dt=None): + month_end = get_month_end(dt) + next_month = month_end + datetime.timedelta(days=1) + return next_month + + +def get_next_month_timestamp(dt=None): + return dt2ts(get_next_month(dt)) diff --git a/cloudkitty/write_orchestrator.py b/cloudkitty/write_orchestrator.py index 14c94f75..07998a28 100644 --- a/cloudkitty/write_orchestrator.py +++ b/cloudkitty/write_orchestrator.py @@ -123,7 +123,8 @@ class WriteOrchestrator(object): def restart_month(self): self._load_state_manager_data() - self.usage_end = ck_utils.get_this_month_timestamp() + month_start = ck_utils.get_month_start() + self.usage_end = ck_utils.dt2ts(month_start) self._update_state_manager_data() def process(self): diff --git a/cloudkitty/writer/__init__.py b/cloudkitty/writer/__init__.py index 872b6ceb..aef0c083 100644 --- a/cloudkitty/writer/__init__.py +++ b/cloudkitty/writer/__init__.py @@ -16,11 +16,11 @@ # @author: Stéphane Albert # import abc -import datetime import six from cloudkitty import state +from cloudkitty import utils as ck_utils @six.add_metaclass(abc.ABCMeta) @@ -80,9 +80,9 @@ class BaseReportWriter(object): def _get_state_manager_timeframe(self): timeframe = self._sm.get_state() self.usage_start = timeframe - self.usage_start_dt = datetime.datetime.fromtimestamp(timeframe) + self.usage_start_dt = ck_utils.ts2dt(timeframe) self.usage_end = timeframe + self._period - self.usage_end_dt = datetime.datetime.fromtimestamp(self.usage_end) + self.usage_end_dt = ck_utils.ts2dt(self.usage_end) metadata = self._sm.get_metadata() self.total = metadata.get('total', 0) @@ -150,10 +150,8 @@ class BaseReportWriter(object): if self.usage_start is None: self.usage_start = start self.usage_end = start + self._period - self.usage_start_dt = datetime.datetime.fromtimestamp( - self.usage_start) - self.usage_end_dt = datetime.datetime.fromtimestamp( - self.usage_end) + self.usage_start_dt = ck_utils.ts2dt(self.usage_start) + self.usage_end_dt = ck_utils.ts2dt(self.usage_end) self._update(data) diff --git a/requirements.txt b/requirements.txt index c7215035..f1d83394 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ oslo.config>=1.2.0 oslo.messaging oslo.db oslo.i18n +oslo.utils sqlalchemy six>=1.7.0 stevedore diff --git a/test-requirements.txt b/test-requirements.txt index 50710ce5..1b688b04 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,5 @@ hacking>=0.9.2,<0.10 +coverage>=3.6 discover testtools testscenarios