From be38ba70f92ca911de31e41be87029ff7ff5432b Mon Sep 17 00:00:00 2001 From: Luka Peschke Date: Thu, 7 Feb 2019 15:01:14 +0100 Subject: [PATCH] Added details to storage state This adds the "scope_key", "collector" and "fetcher" columns to the cloudkitty_storage_states table. Story: 2004957 Task: 29389 Change-Id: I22a1f89175a89267a3c7072437bcda4cee288a72 Depends-On: https://review.openstack.org/#/c/635476/ --- cloudkitty/storage_state/__init__.py | 64 ++++++++-- ...d103dd4dcf_add_state_management_columns.py | 37 ++++++ cloudkitty/storage_state/models.py | 10 +- cloudkitty/tests/test_storage_state.py | 110 ++++++++++++++++++ 4 files changed, 213 insertions(+), 8 deletions(-) create mode 100644 cloudkitty/storage_state/alembic/versions/d9d103dd4dcf_add_state_management_columns.py create mode 100644 cloudkitty/tests/test_storage_state.py diff --git a/cloudkitty/storage_state/__init__.py b/cloudkitty/storage_state/__init__.py index 163661eb..2d52eac6 100644 --- a/cloudkitty/storage_state/__init__.py +++ b/cloudkitty/storage_state/__init__.py @@ -15,7 +15,9 @@ # # @author: Luka Peschke # +from oslo_config import cfg from oslo_db.sqlalchemy import utils +from oslo_log import log from cloudkitty import db from cloudkitty.storage_state import migration @@ -23,21 +25,64 @@ from cloudkitty.storage_state import models from cloudkitty import utils as ck_utils +LOG = log.getLogger(__name__) + + +CONF = cfg.CONF +# NOTE(peschk_l): Required for defaults +CONF.import_opt('backend', 'cloudkitty.fetcher', 'fetcher') +CONF.import_opt('collector', 'cloudkitty.collector', 'collect') +CONF.import_opt('scope_key', 'cloudkitty.collector', 'collect') + + class StateManager(object): """Class allowing state management in CloudKitty""" model = models.IdentifierState - def _get_db_item(self, session, identifier): - q = utils.model_query(self.model, session) - return q.filter(self.model.identifier == identifier).first() + def _get_db_item(self, session, identifier, + fetcher=None, collector=None, scope_key=None): + fetcher = fetcher or CONF.fetcher.backend + collector = collector or CONF.collect.collector + scope_key = scope_key or CONF.collect.scope_key - def set_state(self, identifier, state): + q = utils.model_query(self.model, session) + r = q.filter(self.model.identifier == identifier). \ + filter(self.model.scope_key == scope_key). \ + filter(self.model.fetcher == fetcher). \ + filter(self.model.collector == collector). \ + first() + + # In case the identifier exists with empty columns, update them + if not r: + # NOTE(peschk_l): We must use == instead of 'is' because sqlachmey + # overloads this operator + r = q.filter(self.model.identifier == identifier). \ + filter(self.model.scope_key == None). \ + filter(self.model.fetcher == None). \ + filter(self.model.collector == None). \ + first() # noqa + if r: + r.scope_key = scope_key + r.collector = collector + r.fetcher = fetcher + LOG.info('Updating identifier "{i}" with scope_key "{sk}", ' + 'collector "{c}" and fetcher "{f}"'.format( + i=identifier, + sk=scope_key, + c=collector, + f=fetcher)) + session.commit() + return r + + def set_state(self, identifier, state, + fetcher=None, collector=None, scope_key=None): if isinstance(state, int): state = ck_utils.ts2dt(state) session = db.get_session() session.begin() - r = self._get_db_item(session, identifier) + r = self._get_db_item( + session, identifier, fetcher, collector, scope_key) if r and r.state != state: r.state = state session.commit() @@ -45,15 +90,20 @@ class StateManager(object): state_object = self.model( identifier=identifier, state=state, + fetcher=fetcher, + collector=collector, + scope_key=scope_key, ) session.add(state_object) session.commit() session.close() - def get_state(self, identifier): + def get_state(self, identifier, + fetcher=None, collector=None, scope_key=None): session = db.get_session() session.begin() - r = self._get_db_item(session, identifier) + r = self._get_db_item( + session, identifier, fetcher, collector, scope_key) session.close() return ck_utils.dt2ts(r.state) if r else None diff --git a/cloudkitty/storage_state/alembic/versions/d9d103dd4dcf_add_state_management_columns.py b/cloudkitty/storage_state/alembic/versions/d9d103dd4dcf_add_state_management_columns.py new file mode 100644 index 00000000..c2fb3194 --- /dev/null +++ b/cloudkitty/storage_state/alembic/versions/d9d103dd4dcf_add_state_management_columns.py @@ -0,0 +1,37 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Add details to state management + +Revision ID: d9d103dd4dcf +Revises: c14eea9d3cc1 +Create Date: 2019-02-07 13:59:39.294277 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'd9d103dd4dcf' +down_revision = 'c14eea9d3cc1' +branch_labels = None +depends_on = None + + +def upgrade(): + for column_name in ('scope_key', 'collector', 'fetcher'): + op.add_column( + 'cloudkitty_storage_states', + sa.Column(column_name, sa.String(length=40), nullable=True)) diff --git a/cloudkitty/storage_state/models.py b/cloudkitty/storage_state/models.py index 01727b86..ff422502 100644 --- a/cloudkitty/storage_state/models.py +++ b/cloudkitty/storage_state/models.py @@ -31,9 +31,17 @@ class IdentifierState(Base, models.ModelBase): id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True) - # SHA1 of the identifier identifier = sqlalchemy.Column(sqlalchemy.String(256), nullable=False, unique=True) + scope_key = sqlalchemy.Column(sqlalchemy.String(40), + nullable=True, + unique=False) + fetcher = sqlalchemy.Column(sqlalchemy.String(40), + nullable=True, + unique=False) + collector = sqlalchemy.Column(sqlalchemy.String(40), + nullable=True, + unique=False) state = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False) diff --git a/cloudkitty/tests/test_storage_state.py b/cloudkitty/tests/test_storage_state.py new file mode 100644 index 00000000..f4bf149a --- /dev/null +++ b/cloudkitty/tests/test_storage_state.py @@ -0,0 +1,110 @@ +# Copyright 2019 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +try: + from collections.abc import Iterable +except ImportError: + from collections import Iterable +from datetime import datetime +import itertools + +import mock + +from cloudkitty import storage_state +from cloudkitty import tests + + +class StateManagerTest(tests.TestCase): + + class QueryMock(mock.Mock): + """Mocks an SQLalchemy query. + + ``filter()`` can be called any number of times, followed by first(), + which will cycle over the ``output`` parameter passed to the + constructor. The ``first_called`` attributes + """ + def __init__(self, output, *args, **kwargs): + super(StateManagerTest.QueryMock, self).__init__(*args, **kwargs) + self.first_called = 0 + if not isinstance(output, Iterable): + output = (output, ) + self.output = itertools.cycle(output) + + def filter(self, *args, **kwargs): + return self + + def first(self): + self.first_called += 1 + return next(self.output) + + def setUp(self): + super(StateManagerTest, self).setUp() + self._state = storage_state.StateManager() + self.conf.set_override('backend', 'fetcher1', 'fetcher') + self.conf.set_override('collector', 'collector1', 'collect') + self.conf.set_override('scope_key', 'scope_key', 'collect') + + def _get_query_mock(self, *args): + output = self.QueryMock(args) + return output, mock.Mock(return_value=output) + + @staticmethod + def _get_r_mock(scope_key, collector, fetcher, state): + r_mock = mock.Mock() + r_mock.scope_key = scope_key + r_mock.collector = collector + r_mock.fetcher = fetcher + r_mock.state = state + return r_mock + + def _test_x_state_does_update_columns(self, func): + r_mock = self._get_r_mock(None, None, None, datetime(2042, 1, 1)) + output, query_mock = self._get_query_mock(None, r_mock) + with mock.patch('oslo_db.sqlalchemy.utils.model_query', + new=query_mock): + func('fake_identifier') + + self.assertEqual(output.first_called, 2) + self.assertEqual(r_mock.collector, 'collector1') + self.assertEqual(r_mock.scope_key, 'scope_key') + self.assertEqual(r_mock.fetcher, 'fetcher1') + + def test_get_state_does_update_columns(self): + self._test_x_state_does_update_columns(self._state.get_state) + + def test_set_state_does_update_columns(self): + with mock.patch('cloudkitty.db.get_session'): + self._test_x_state_does_update_columns( + lambda x: self._state.set_state(x, datetime(2042, 1, 1))) + + def _test_x_state_no_column_update(self, func): + r_mock = self._get_r_mock( + 'scope_key', 'collector1', 'fetcher1', datetime(2042, 1, 1)) + output, query_mock = self._get_query_mock(r_mock) + with mock.patch('oslo_db.sqlalchemy.utils.model_query', + new=query_mock): + func('fake_identifier') + + self.assertEqual(output.first_called, 1) + self.assertEqual(r_mock.collector, 'collector1') + self.assertEqual(r_mock.scope_key, 'scope_key') + self.assertEqual(r_mock.fetcher, 'fetcher1') + + def test_get_state_no_column_update(self): + self._test_x_state_no_column_update(self._state.get_state) + + def test_set_state_no_column_update(self): + with mock.patch('cloudkitty.db.get_session'): + self._test_x_state_no_column_update( + lambda x: self._state.set_state(x, datetime(2042, 1, 1)))