Added details to storage state

This adds the "scope_key", "collector" and "fetcher" columns
to the cloudkitty_storage_states table.

Story: 2004957
Task: 29389
Change-Id: I22a1f89175a89267a3c7072437bcda4cee288a72
Depends-On: https://review.openstack.org/#/c/635476/

cloudkitty/storage_state/__init__.py

@@ -15,7 +15,9 @@
 #
 # @author: Luka Peschke
 #
+from oslo_config import cfg
 from oslo_db.sqlalchemy import utils
+from oslo_log import log
 
 from cloudkitty import db
 from cloudkitty.storage_state import migration
@@ -23,21 +25,64 @@ from cloudkitty.storage_state import models
 from cloudkitty import utils as ck_utils
 
 
+LOG = log.getLogger(__name__)
+
+
+CONF = cfg.CONF
+# NOTE(peschk_l): Required for defaults
+CONF.import_opt('backend', 'cloudkitty.fetcher', 'fetcher')
+CONF.import_opt('collector', 'cloudkitty.collector', 'collect')
+CONF.import_opt('scope_key', 'cloudkitty.collector', 'collect')
+
+
 class StateManager(object):
     """Class allowing state management in CloudKitty"""
 
     model = models.IdentifierState
 
-    def _get_db_item(self, session, identifier):
-        q = utils.model_query(self.model, session)
-        return q.filter(self.model.identifier == identifier).first()
+    def _get_db_item(self, session, identifier,
+                     fetcher=None, collector=None, scope_key=None):
+        fetcher = fetcher or CONF.fetcher.backend
+        collector = collector or CONF.collect.collector
+        scope_key = scope_key or CONF.collect.scope_key
 
-    def set_state(self, identifier, state):
+        q = utils.model_query(self.model, session)
+        r = q.filter(self.model.identifier == identifier). \
+            filter(self.model.scope_key == scope_key). \
+            filter(self.model.fetcher == fetcher). \
+            filter(self.model.collector == collector). \
+            first()
+
+        # In case the identifier exists with empty columns, update them
+        if not r:
+            # NOTE(peschk_l): We must use == instead of 'is' because sqlachmey
+            # overloads this operator
+            r = q.filter(self.model.identifier == identifier). \
+                filter(self.model.scope_key == None). \
+                filter(self.model.fetcher == None). \
+                filter(self.model.collector == None). \
+                first()  # noqa
+            if r:
+                r.scope_key = scope_key
+                r.collector = collector
+                r.fetcher = fetcher
+                LOG.info('Updating identifier "{i}" with scope_key "{sk}", '
+                         'collector "{c}" and fetcher "{f}"'.format(
+                             i=identifier,
+                             sk=scope_key,
+                             c=collector,
+                             f=fetcher))
+                session.commit()
+        return r
+
+    def set_state(self, identifier, state,
+                  fetcher=None, collector=None, scope_key=None):
         if isinstance(state, int):
             state = ck_utils.ts2dt(state)
         session = db.get_session()
         session.begin()
-        r = self._get_db_item(session, identifier)
+        r = self._get_db_item(
+            session, identifier, fetcher, collector, scope_key)
         if r and r.state != state:
             r.state = state
             session.commit()
@@ -45,15 +90,20 @@ class StateManager(object):
             state_object = self.model(
                 identifier=identifier,
                 state=state,
+                fetcher=fetcher,
+                collector=collector,
+                scope_key=scope_key,
             )
             session.add(state_object)
             session.commit()
         session.close()
 
-    def get_state(self, identifier):
+    def get_state(self, identifier,
+                  fetcher=None, collector=None, scope_key=None):
         session = db.get_session()
         session.begin()
-        r = self._get_db_item(session, identifier)
+        r = self._get_db_item(
+            session, identifier, fetcher, collector, scope_key)
         session.close()
         return ck_utils.dt2ts(r.state) if r else None
 

cloudkitty/storage_state/alembic/versions/d9d103dd4dcf_add_state_management_columns.py

@@ -0,0 +1,37 @@
+# Copyright 2019 Objectif Libre
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+"""Add details to state management
+
+Revision ID: d9d103dd4dcf
+Revises: c14eea9d3cc1
+Create Date: 2019-02-07 13:59:39.294277
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'd9d103dd4dcf'
+down_revision = 'c14eea9d3cc1'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    for column_name in ('scope_key', 'collector', 'fetcher'):
+        op.add_column(
+            'cloudkitty_storage_states',
+            sa.Column(column_name, sa.String(length=40), nullable=True))

cloudkitty/storage_state/models.py

@@ -31,9 +31,17 @@ class IdentifierState(Base, models.ModelBase):
 
     id = sqlalchemy.Column(sqlalchemy.Integer,
                            primary_key=True)
-    # SHA1 of the identifier
     identifier = sqlalchemy.Column(sqlalchemy.String(256),
                                    nullable=False,
                                    unique=True)
+    scope_key = sqlalchemy.Column(sqlalchemy.String(40),
+                                  nullable=True,
+                                  unique=False)
+    fetcher = sqlalchemy.Column(sqlalchemy.String(40),
+                                nullable=True,
+                                unique=False)
+    collector = sqlalchemy.Column(sqlalchemy.String(40),
+                                  nullable=True,
+                                  unique=False)
     state = sqlalchemy.Column(sqlalchemy.DateTime,
                               nullable=False)

cloudkitty/tests/test_storage_state.py

@@ -0,0 +1,110 @@
+# Copyright 2019 Objectif Libre
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+try:
+    from collections.abc import Iterable
+except ImportError:
+    from collections import Iterable
+from datetime import datetime
+import itertools
+
+import mock
+
+from cloudkitty import storage_state
+from cloudkitty import tests
+
+
+class StateManagerTest(tests.TestCase):
+
+    class QueryMock(mock.Mock):
+        """Mocks an SQLalchemy query.
+
+        ``filter()`` can be called any number of times, followed by first(),
+        which will cycle over the ``output`` parameter passed to the
+        constructor. The ``first_called`` attributes
+        """
+        def __init__(self, output, *args, **kwargs):
+            super(StateManagerTest.QueryMock, self).__init__(*args, **kwargs)
+            self.first_called = 0
+            if not isinstance(output, Iterable):
+                output = (output, )
+            self.output = itertools.cycle(output)
+
+        def filter(self, *args, **kwargs):
+            return self
+
+        def first(self):
+            self.first_called += 1
+            return next(self.output)
+
+    def setUp(self):
+        super(StateManagerTest, self).setUp()
+        self._state = storage_state.StateManager()
+        self.conf.set_override('backend', 'fetcher1', 'fetcher')
+        self.conf.set_override('collector', 'collector1', 'collect')
+        self.conf.set_override('scope_key', 'scope_key', 'collect')
+
+    def _get_query_mock(self, *args):
+        output = self.QueryMock(args)
+        return output, mock.Mock(return_value=output)
+
+    @staticmethod
+    def _get_r_mock(scope_key, collector, fetcher, state):
+        r_mock = mock.Mock()
+        r_mock.scope_key = scope_key
+        r_mock.collector = collector
+        r_mock.fetcher = fetcher
+        r_mock.state = state
+        return r_mock
+
+    def _test_x_state_does_update_columns(self, func):
+        r_mock = self._get_r_mock(None, None, None, datetime(2042, 1, 1))
+        output, query_mock = self._get_query_mock(None, r_mock)
+        with mock.patch('oslo_db.sqlalchemy.utils.model_query',
+                        new=query_mock):
+            func('fake_identifier')
+
+        self.assertEqual(output.first_called, 2)
+        self.assertEqual(r_mock.collector, 'collector1')
+        self.assertEqual(r_mock.scope_key, 'scope_key')
+        self.assertEqual(r_mock.fetcher, 'fetcher1')
+
+    def test_get_state_does_update_columns(self):
+        self._test_x_state_does_update_columns(self._state.get_state)
+
+    def test_set_state_does_update_columns(self):
+        with mock.patch('cloudkitty.db.get_session'):
+            self._test_x_state_does_update_columns(
+                lambda x: self._state.set_state(x, datetime(2042, 1, 1)))
+
+    def _test_x_state_no_column_update(self, func):
+        r_mock = self._get_r_mock(
+            'scope_key', 'collector1', 'fetcher1', datetime(2042, 1, 1))
+        output, query_mock = self._get_query_mock(r_mock)
+        with mock.patch('oslo_db.sqlalchemy.utils.model_query',
+                        new=query_mock):
+            func('fake_identifier')
+
+        self.assertEqual(output.first_called, 1)
+        self.assertEqual(r_mock.collector, 'collector1')
+        self.assertEqual(r_mock.scope_key, 'scope_key')
+        self.assertEqual(r_mock.fetcher, 'fetcher1')
+
+    def test_get_state_no_column_update(self):
+        self._test_x_state_no_column_update(self._state.get_state)
+
+    def test_set_state_no_column_update(self):
+        with mock.patch('cloudkitty.db.get_session'):
+            self._test_x_state_no_column_update(
+                lambda x: self._state.set_state(x, datetime(2042, 1, 1)))