introspection data backend: implements db

Configurable introspection data storage backend [1] is
proposed to provide flexible extension of introspection
data storage instead of the single support of Swift storage
backend.

This patch adds database support for using ironic-inspector
database as the storage backend.

A table named ``introspection_data`` is created to serve as
the storage for introspected data.

[1] http://specs.openstack.org/openstack/ironic-inspector-specs/specs/configurable-introspection-data-backends.html

Change-Id: I8b29b7b86d90823d29b921ebf64acddbcd2d8d0d
Story: 1726713
Task: 11373
This commit is contained in:
Kaifeng Wang 2018-07-19 19:14:15 +08:00
parent f7079e9acc
commit a8c1d06bd0
6 changed files with 138 additions and 1 deletions

View File

@ -134,6 +134,14 @@ class RuleAction(Base):
return res
class IntrospectionData(Base):
__tablename__ = 'introspection_data'
uuid = Column(String(36), ForeignKey('nodes.uuid'), primary_key=True)
processed = Column(Boolean, default=False)
data = Column(db_types.JsonEncodedDict(mysql_as_long=True),
nullable=True)
def init():
"""Initialize the database.

View File

@ -0,0 +1,42 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add_introspection_data_table
Revision ID: bf8dec16023c
Revises: 2970d2d44edc
Create Date: 2018-07-19 18:51:38.124614
"""
from alembic import op
from oslo_db.sqlalchemy import types as db_types
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = 'bf8dec16023c'
down_revision = '2970d2d44edc'
branch_labels = None
depends_on = None
def upgrade():
op.create_table(
'introspection_data',
sa.Column('uuid', sa.String(36), sa.ForeignKey('nodes.uuid'),
primary_key=True),
sa.Column('processed', sa.Boolean, default=False),
sa.Column('data', db_types.JsonEncodedDict(mysql_as_long=True).impl,
nullable=True),
mysql_ENGINE='InnoDB',
mysql_DEFAULT_CHARSET='UTF8'
)

View File

@ -755,7 +755,7 @@ def _delete_node(uuid, session=None):
with db.ensure_transaction(session) as session:
db.model_query(db.Attribute, session=session).filter_by(
node_uuid=uuid).delete()
for model in (db.Option, db.Node):
for model in (db.Option, db.IntrospectionData, db.Node):
db.model_query(model,
session=session).filter_by(uuid=uuid).delete()
@ -979,3 +979,44 @@ def get_node_list(ironic=None, marker=None, limit=None):
('started_at', 'uuid'),
marker=marker, sort_dir='desc')
return [NodeInfo.from_row(row, ironic=ironic) for row in rows]
def store_introspection_data(node_id, introspection_data, processed=True):
"""Store introspection data for this node.
:param node_id: node UUID.
:param introspection_data: A dictionary of introspection data
:param processed: Specify the type of introspected data, set to False
indicates the data is unprocessed.
"""
with db.ensure_transaction() as session:
record = db.model_query(db.IntrospectionData,
session=session).filter_by(
uuid=node_id, processed=processed).first()
if record is None:
row = db.IntrospectionData()
row.update({'uuid': node_id, 'processed': processed,
'data': introspection_data})
session.add(row)
else:
record.update({'data': introspection_data})
session.flush()
def get_introspection_data(node_id, processed=True):
"""Get introspection data for this node.
:param node_id: node UUID.
:param processed: Specify the type of introspected data, set to False
indicates retrieving the unprocessed data.
:return: A dictionary representation of intropsected data
"""
try:
ref = db.model_query(db.IntrospectionData).filter_by(
uuid=node_id, processed=processed).one()
return ref['data']
except orm_errors.NoResultFound:
msg = _('Introspection data not found for node %(node)s, '
'processed=%(processed)s') % {'node': node_id,
'processed': processed}
raise utils.IntrospectionDataNotFound(msg)

View File

@ -441,6 +441,19 @@ class MigrationCheckersMixin(object):
n = nodes.select(nodes.c.uuid == 'abcd').execute().first()
self.assertIsNone(n['manage_boot'])
def _check_bf8dec16023c(self, engine, data):
introspection_data = db_utils.get_table(engine, 'introspection_data')
col_names = [column.name for column in introspection_data.c]
self.assertIn('uuid', col_names)
self.assertIn('processed', col_names)
self.assertIn('data', col_names)
self.assertIsInstance(introspection_data.c.uuid.type,
sqlalchemy.types.String)
self.assertIsInstance(introspection_data.c.processed.type,
sqlalchemy.types.Boolean)
self.assertIsInstance(introspection_data.c.data.type,
sqlalchemy.types.Text)
def test_upgrade_and_version(self):
with patch_with_engine(self.engine):
self.migration_ext.upgrade('head')

View File

@ -331,6 +331,8 @@ class TestNodeCacheCleanUp(test_base.NodeTest):
value=v, node_uuid=self.uuid).save(session)
db.Option(uuid=self.uuid, name='foo', value='bar').save(
session)
db.IntrospectionData(uuid=self.uuid, processed=False,
data={'fake': 'data'}).save(session)
def test_no_timeout(self):
CONF.set_override('timeout', 0)
@ -358,6 +360,7 @@ class TestNodeCacheCleanUp(test_base.NodeTest):
self.assertEqual(len(self.macs),
db.model_query(db.Attribute).count())
self.assertEqual(1, db.model_query(db.Option).count())
self.assertEqual(1, db.model_query(db.IntrospectionData).count())
self.assertFalse(get_lock_mock.called)
@mock.patch.object(node_cache, '_get_lock', autospec=True)
@ -1256,3 +1259,29 @@ class TestStartIntrospection(test_base.NodeTest):
node_cache.start_introspection,
self.node_info.uuid)
self.assertFalse(add_node_mock.called)
class TestIntrospectionDataDbStore(test_base.NodeTest):
def setUp(self):
super(TestIntrospectionDataDbStore, self).setUp()
node_cache.add_node(self.node.uuid,
istate.States.processing,
bmc_address='1.2.3.4')
def _test_store_and_get(self, processed=False):
node_cache.store_introspection_data(self.node.uuid,
copy.deepcopy(self.data),
processed=processed)
stored_data = node_cache.get_introspection_data(self.node.uuid,
processed=processed)
self.assertEqual(stored_data, self.data)
def test_store_and_get_unprocessed(self):
self._test_store_and_get(processed=False)
def test_store_and_get_processed(self):
self._test_store_and_get(processed=True)
def test_get_no_data_available(self):
self.assertRaises(utils.IntrospectionDataNotFound,
node_cache.get_introspection_data, self.node.uuid)

View File

@ -140,6 +140,10 @@ class NodeStateInvalidEvent(Error):
"""Invalid event attempted."""
class IntrospectionDataNotFound(NotFoundInCacheError):
"""Introspection data not found."""
def executor():
"""Return the current futures executor."""
global _EXECUTOR