From 2f1f372edefc0f341458dd2cda120a90eba72617 Mon Sep 17 00:00:00 2001 From: wangzh21 Date: Sat, 12 Jan 2019 16:06:34 +0800 Subject: [PATCH] Add ExtArq and ARQ object Change-Id: I5089b6d7e25d9134376abfbcc1ed4cdc3eaf45e6 Story: 2004248 Task: 27770 --- cyborg/common/constants.py | 3 + cyborg/common/exception.py | 8 ++ cyborg/db/api.py | 21 ++++ cyborg/db/sqlalchemy/api.py | 57 +++++++++ cyborg/objects/__init__.py | 2 + cyborg/objects/arq.py | 74 +++++++++++ cyborg/objects/ext_arq.py | 152 +++++++++++++++++++++++ cyborg/objects/fields.py | 13 ++ cyborg/tests/unit/db/utils.py | 37 ++++++ cyborg/tests/unit/objects/test_extarq.py | 90 ++++++++++++++ 10 files changed, 457 insertions(+) create mode 100644 cyborg/objects/arq.py create mode 100644 cyborg/objects/ext_arq.py create mode 100644 cyborg/tests/unit/objects/test_extarq.py diff --git a/cyborg/common/constants.py b/cyborg/common/constants.py index c9c7f985..aa5aaaea 100644 --- a/cyborg/common/constants.py +++ b/cyborg/common/constants.py @@ -16,3 +16,6 @@ CONDUCTOR_TOPIC = 'cyborg-conductor' AGENT_TOPIC = 'cyborg-agent' + +ARQ_STATES = (ARQINITIAL, ARQBOUND, ARQUNBOUND, ARQBINDFAILED) = \ + ('Initial', 'Bound', 'Unbound', 'BindFailed') diff --git a/cyborg/common/exception.py b/cyborg/common/exception.py index 5fe90e71..52e35832 100644 --- a/cyborg/common/exception.py +++ b/cyborg/common/exception.py @@ -98,6 +98,10 @@ class DeployableAlreadyExists(CyborgException): _msg_fmt = _("Deployable with uuid %(uuid)s already exists.") +class ExtArqAlreadyExists(CyborgException): + _msg_fmt = _("ExtArq with uuid %(uuid)s already exists.") + + class Invalid(CyborgException): _msg_fmt = _("Invalid parameters.") code = http_client.BAD_REQUEST @@ -156,6 +160,10 @@ class DeployableNotFound(NotFound): _msg_fmt = _("Deployable %(uuid)s could not be found.") +class ExtArqNotFound(NotFound): + _msg_fmt = _("ExtArq %(uuid)s could not be found.") + + class InvalidDeployType(CyborgException): _msg_fmt = _("Deployable have an invalid type") diff --git a/cyborg/db/api.py b/cyborg/db/api.py index 784730bb..7c9c3e6e 100644 --- a/cyborg/db/api.py +++ b/cyborg/db/api.py @@ -132,3 +132,24 @@ class Connection(object): @abc.abstractmethod def reservation_commit(self, context, reservations, project_id=None): """Check quotas and create appropriate reservations.""" + + # extarq + @abc.abstractmethod + def extarq_create(self, context, values): + """Create a new extarq.""" + + @abc.abstractmethod + def extarq_delete(self, context, uuid): + """Delete an extarq.""" + + @abc.abstractmethod + def extarq_update(self, context, uuid, values): + """Update an extarq.""" + + @abc.abstractmethod + def extarq_list(self, context, limit, marker, sort_key, sort_dir): + """Get requested list of extarqs.""" + + @abc.abstractmethod + def extarq_get(self, context, uuid): + """Get requested extarq.""" diff --git a/cyborg/db/sqlalchemy/api.py b/cyborg/db/sqlalchemy/api.py index 118d5b78..8dbde8f0 100644 --- a/cyborg/db/sqlalchemy/api.py +++ b/cyborg/db/sqlalchemy/api.py @@ -499,6 +499,63 @@ class Connection(api.Connection): if count != 1: raise exception.AttributeNotFound(uuid=uuid) + def extarq_create(self, context, values): + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + if values.get('id'): + values.pop('id', None) + extarq = models.ExtArq() + extarq.update(values) + + with _session_for_write() as session: + try: + session.add(extarq) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.ExtArqAlreadyExists(uuid=values['uuid']) + return extarq + + @oslo_db_api.retry_on_deadlock + def extarq_delete(self, context, uuid): + with _session_for_write(): + query = model_query(context, models.ExtArq) + query = add_identity_filter(query, uuid) + count = query.delete() + if count != 1: + raise exception.ExtArqNotFound(uuid=uuid) + + def extarq_update(self, context, uuid, values): + if 'uuid' in values: + msg = _("Cannot overwrite UUID for an existing ExtArq.") + raise exception.InvalidParameterValue(err=msg) + return self._do_update_extarq(context, uuid, values) + + @oslo_db_api.retry_on_deadlock + def _do_update_extarq(self, context, uuid, values): + with _session_for_write(): + query = model_query(context, models.ExtArq) + query = query.filter_by(uuid=uuid) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.ExtArqNotFound(uuid=uuid) + ref.update(values) + return ref + + def extarq_list(self, context, limit, marker, sort_key, sort_dir): + query = model_query(context, models.ExtArq) + return _paginate_query(context, models.Device, limit, marker, + sort_key, sort_dir, query) + + def extarq_get(self, context, uuid): + query = model_query( + context, + models.ExtArq).filter_by(uuid=uuid) + try: + return query.one() + except NoResultFound: + raise exception.ExtArqNotFound(uuid=uuid) + def _get_quota_usages(self, context, project_id, resources=None): # Broken out for testability query = model_query(context, models.QuotaUsage,).filter_by( diff --git a/cyborg/objects/__init__.py b/cyborg/objects/__init__.py index d2c15d95..8050976e 100644 --- a/cyborg/objects/__init__.py +++ b/cyborg/objects/__init__.py @@ -28,3 +28,5 @@ def register_all(): __import__('cyborg.objects.accelerator') __import__('cyborg.objects.deployable') __import__('cyborg.objects.attribute') + __import__('cyborg.objects.arq') + __import__('cyborg.objects.ext_arq') diff --git a/cyborg/objects/arq.py b/cyborg/objects/arq.py new file mode 100644 index 00000000..b1e44e35 --- /dev/null +++ b/cyborg/objects/arq.py @@ -0,0 +1,74 @@ +# Copyright 2019 Beijing Lenovo Software Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from oslo_versionedobjects import base as object_base + +from cyborg.db import api as dbapi +from cyborg import objects +from cyborg.objects import base +from cyborg.objects import fields as object_fields + + +LOG = logging.getLogger(__name__) + + +@base.CyborgObjectRegistry.register +class ARQ(base.CyborgObject, object_base.VersionedObjectDictCompat): + # Version 1.0: Initial version + VERSION = '1.0' + + dbapi = dbapi.get_instance() + fields = { + 'id': object_fields.IntegerField(nullable=False), + 'uuid': object_fields.UUIDField(nullable=False), + 'state': object_fields.ARQStateField(nullable=False), + 'device_profile': object_fields.ObjectField('DeviceProfile', + nullable=True), + 'hostname': object_fields.StringField(nullable=True), + 'device_rp_uuid': object_fields.UUIDField(nullable=True), + 'device_instance_uuid': object_fields.UUIDField(nullable=True), + 'attach_handle': object_fields.ObjectField('AttachHandle', + nullable=True), + } + + @staticmethod + def _from_db_object(arq, db_extarq): + """Converts an ARQ to a formal object. + + :param arq: An object of the class ARQ + :param db_extarq: A DB model of the object + :return: The object of the class with the database entity added + """ + device_profile_id = db_extarq.pop('device_profile_id', None) + attach_handle_id = db_extarq.pop('attach_handle_id', None) + + for field in arq.fields: + # if field == 'device_profile': + # arq._load_device_profile(device_profile_id) + # if field == 'attach_handle': + # arq._load_device_profile(attach_handle_id) + arq[field] = db_extarq[field] + + arq.obj_reset_changes() + return arq + + def _load_device_profile(self, device_profile_id): + self.device_profile = objects.DeviceProfile.\ + get_by_id(self._context, device_profile_id) + + def _load_attach_handle(self, attach_handle_id): + self.attach_handle = objects.AttachHandle.\ + get_by_id(self._context, attach_handle_id) diff --git a/cyborg/objects/ext_arq.py b/cyborg/objects/ext_arq.py new file mode 100644 index 00000000..a36ca6e9 --- /dev/null +++ b/cyborg/objects/ext_arq.py @@ -0,0 +1,152 @@ +# Copyright 2019 Beijing Lenovo Software Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from oslo_versionedobjects import base as object_base + +from cyborg.db import api as dbapi +from cyborg import objects +from cyborg.common import constants +from cyborg.common import exception +from cyborg.objects import base +from cyborg.objects import fields as object_fields + + +LOG = logging.getLogger(__name__) + + +@base.CyborgObjectRegistry.register +class ExtARQ(base.CyborgObject, object_base.VersionedObjectDictCompat): + """ ExtARQ is a wrapper around ARQ with Cyborg-private fields. + Each ExtARQ object contains exactly one ARQ object as a field. + But, in the db layer, ExtARQ and ARQ are represented together + as a row in a single table. Both share a single UUID. + + ExtARQ version is bumped up either if any of its fields change + or if the ARQ version changes. + """ + # Version 1.0: Initial version + VERSION = '1.0' + + dbapi = dbapi.get_instance() + + fields = { + 'arq': object_fields.ObjectField('ARQ'), + # Cyborg-private fields + # Left substate open now, fill them out during design/implementation + # later. + 'substate': object_fields.StringField(nullable=True), + } + + def create(self, context, device_profile_id=None): + """Create an ExtARQ record in the DB.""" + if 'device_profile' not in self.arq and not device_profile_id: + raise exception.ObjectActionError( + action='create', + reason='Device profile is required in ARQ') + self.arq.state = constants.ARQINITIAL + self.substate = constants.ARQINITIAL + values = self.obj_get_changes() + arq_obj = values.pop('arq', None) + if arq_obj is not None: + values.update(arq_obj.as_dict()) + + # Pass devprof id to db layer, to avoid repeated queries + if device_profile_id is not None: + values['device_profile_id'] = device_profile_id + + db_extarq = self.dbapi.extarq_create(context, values) + self._from_db_object(self, db_extarq) + return self + + @classmethod + def get(cls, context, uuid): + """Find a DB ExtARQ and return an Obj ExtARQ.""" + db_extarq = cls.dbapi.extarq_get(context, uuid) + obj_arq = objects.ARQ(context) + obj_extarq = ExtARQ(context) + obj_extarq['arq'] = obj_arq + obj_extarq = cls._from_db_object(obj_extarq, db_extarq) + return obj_extarq + + @classmethod + def list(cls, context, limit, marker, sort_key, sort_dir): + """Return a list of ExtARQ objects.""" + db_extarqs = cls.dbapi.extarq_list(context, limit, marker, sort_key, + sort_dir) + obj_extarq_list = cls._from_db_object_list(db_extarqs, context) + return obj_extarq_list + + def save(self, context): + """Update an ExtARQ record in the DB.""" + updates = self.obj_get_changes() + db_extarq = self.dbapi.extarq_update(context, self.arq.uuid, updates) + self._from_db_object(self, db_extarq) + + def destroy(self, context): + """Delete an ExtARQ from the DB.""" + self.dbapi.extarq_delete(context, self.arq.uuid) + self.obj_reset_changes() + + def bind(self, context, host_name, devrp_uuid, instance_uuid): + """ Given a device rp UUID, get the deployable UUID and + an attach handle. + """ + # For the fake device, we just set the state to 'Bound' + # TODO(wangzhh): Move bind logic and unbind logic to the agent later. + arq = self.arq + arq.host_name = host_name + arq.device_rp_uuid = devrp_uuid + arq.instance_uuid = instance_uuid + arq.state = constants.ARQBOUND + + self.save(context) + + def unbind(self, context): + arq = self.arq + arq.host_name = '' + arq.device_rp_uuid = '' + arq.instance_uuid = '' + arq.state = constants.ARQUNBOUND + + self.save(context) + + @staticmethod + def _from_db_object(extarq, db_extarq): + """Converts an ExtARQ to a formal object. + + :param extarq: An object of the class ExtARQ + :param db_extarq: A DB model of the object + :return: The object of the class with the database entity added + """ + for field in extarq.fields: + if field != 'arq': + extarq[field] = db_extarq[field] + extarq.arq = objects.ARQ() + extarq.arq._from_db_object(extarq.arq, db_extarq) + extarq.obj_reset_changes() + return extarq + + def obj_get_changes(self): + """Returns a dict of changed fields and their new values.""" + changes = {} + for key in self.obj_what_changed(): + if key != 'arq': + changes[key] = getattr(self, key) + + for key in self.arq.obj_what_changed(): + changes[key] = getattr(self.arq, key) + + return changes diff --git a/cyborg/objects/fields.py b/cyborg/objects/fields.py index 12c0b0a8..05653b95 100644 --- a/cyborg/objects/fields.py +++ b/cyborg/objects/fields.py @@ -15,6 +15,7 @@ from oslo_versionedobjects import fields as object_fields +from cyborg.common import constants # Import fields from oslo_versionedobjects IntegerField = object_fields.IntegerField @@ -28,3 +29,15 @@ ListOfStringsField = object_fields.ListOfStringsField IPAddressField = object_fields.IPAddressField IPNetworkField = object_fields.IPNetworkField UnspecifiedDefault = object_fields.UnspecifiedDefault +EnumField = object_fields.EnumField + + +class ARQState(object_fields.Enum): + ALL = constants.ARQ_STATES + + def __init__(self): + super(ARQState, self).__init__(valid_values=ARQState.ALL) + + +class ARQStateField(object_fields.BaseEnumField): + AUTO_TYPE = ARQState() diff --git a/cyborg/tests/unit/db/utils.py b/cyborg/tests/unit/db/utils.py index 209f7a09..0da0e7b0 100644 --- a/cyborg/tests/unit/db/utils.py +++ b/cyborg/tests/unit/db/utils.py @@ -49,3 +49,40 @@ def get_test_deployable(**kw): 'availability': 'Available', 'accelerator_id': kw.get('accelerator_id', 1), } + + +def get_test_extarq(**kwargs): + return { + 'uuid': kwargs.get('uuid', '10efe63d-dfea-4a37-ad94-4116fba5098'), + 'id': kwargs.get('id', 1), + 'state': kwargs.get('state', 'bound'), + 'device_profile_id': kwargs.get('id', 1), + 'hostname': kwargs.get('hostname', 'testnode1'), + 'device_rp_uuid': kwargs.get('device_rp_uuid', + 'f2b96c5f-242a-41a0-a736-b6e1fada071b'), + 'device_instance_uuid': + kwargs.get('device_rp_uuid', + '6219e0fb-2935-4db2-a3c7-86a2ac3ac84e'), + 'attach_handle_id': kwargs.get('id', 1), + 'created_at': kwargs.get('created_at', None), + 'updated_at': kwargs.get('updated_at', None) + } + + +def get_test_arq(**kwargs): + return { + 'uuid': kwargs.get('uuid', '10efe63d-dfea-4a37-ad94-4116fba5098'), + 'id': kwargs.get('id', 1), + 'state': kwargs.get('state', 'Initial'), + 'device_profile': kwargs.get('device_profile', None), + 'hostname': kwargs.get('hostname', 'testnode1'), + 'device_rp_uuid': kwargs.get('device_rp_uuid', + 'f2b96c5f-242a-41a0-a736-b6e1fada071b'), + 'device_instance_uuid': + kwargs.get('device_rp_uuid', + '6219e0fb-2935-4db2-a3c7-86a2ac3ac84e'), + 'attach_handle': kwargs.get('attach_handle', None), + 'created_at': kwargs.get('created_at', None), + 'updated_at': kwargs.get('updated_at', None), + 'substate': kwargs.get('substate', 'Initial'), + } diff --git a/cyborg/tests/unit/objects/test_extarq.py b/cyborg/tests/unit/objects/test_extarq.py new file mode 100644 index 00000000..4e323545 --- /dev/null +++ b/cyborg/tests/unit/objects/test_extarq.py @@ -0,0 +1,90 @@ +# Copyright 2019 Beijing Lenovo Software Ltd. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from testtools.matchers import HasLength +from cyborg import objects +from cyborg.tests.unit.db import base +from cyborg.tests.unit.db import utils + + +class TestExtARQObject(base.DbTestCase): + + def setUp(self): + super(TestExtARQObject, self).setUp() + self.fake_arq = utils.get_test_arq() + + def test_get(self): + uuid = self.fake_arq['uuid'] + with mock.patch.object(self.dbapi, 'extarq_get', + autospec=True) as mock_extarq_get: + mock_extarq_get.return_value = self.fake_arq + extarq = objects.ExtARQ.get(self.context, uuid) + mock_extarq_get.assert_called_once_with(self.context, uuid) + self.assertEqual(self.context, extarq._context) + + def test_list(self): + with mock.patch.object(self.dbapi, 'extarq_list', + autospec=True) as mock_get_list: + mock_get_list.return_value = [self.fake_arq] + extarqs = objects.ExtARQ.list(self.context, 1, None, None, None) + self.assertEqual(1, mock_get_list.call_count) + self.assertThat(extarqs, HasLength(1)) + self.assertIsInstance(extarqs[0], objects.ExtARQ) + self.assertEqual(self.context, extarqs[0]._context) + + def test_create(self): + with mock.patch.object(self.dbapi, 'extarq_create', + autospec=True) as mock_extarq_create: + mock_extarq_create.return_value = self.fake_arq + extarq = objects.ExtARQ(self.context, **self.fake_arq) + extarq.arq = objects.ARQ(self.context, **self.fake_arq) + extarq.create(self.context) + mock_extarq_create.assert_called_once_with(self.context, + self.fake_arq) + self.assertEqual(self.context, extarq._context) + + def test_destroy(self): + uuid = self.fake_arq['uuid'] + with mock.patch.object(self.dbapi, 'extarq_get', + autospec=True) as mock_extarq_get: + mock_extarq_get.return_value = self.fake_arq + with mock.patch.object(self.dbapi, 'extarq_delete', + autospec=True) as mock_extarq_delete: + extarq = objects.ExtARQ.get(self.context, uuid) + extarq.destroy(self.context) + mock_extarq_delete.assert_called_once_with(self.context, uuid) + self.assertEqual(self.context, extarq._context) + + def test_save(self): + uuid = self.fake_arq['uuid'] + with mock.patch.object(self.dbapi, 'extarq_get', + autospec=True) as mock_extarq_get: + mock_extarq_get.return_value = self.fake_arq + with mock.patch.object(self.dbapi, 'extarq_update', + autospec=True) as mock_extarq_update: + extarq = objects.ExtARQ.get(self.context, uuid) + arq = extarq.arq + arq.hostname = 'newtestnode1' + fake_arq_updated = self.fake_arq + fake_arq_updated['hostname'] = arq.hostname + mock_extarq_update.return_value = fake_arq_updated + extarq.save(self.context) + mock_extarq_get.assert_called_once_with(self.context, uuid) + mock_extarq_update.assert_called_once_with( + self.context, uuid, + {'hostname': 'newtestnode1'}) + self.assertEqual(self.context, extarq._context)