diff --git a/cyborg/__init__.py b/cyborg/__init__.py index d0f34f07..3056acfc 100644 --- a/cyborg/__init__.py +++ b/cyborg/__init__.py @@ -12,8 +12,11 @@ # License for the specific language governing permissions and limitations # under the License. +import eventlet import pbr.version __version__ = pbr.version.VersionInfo( 'cyborg').version_string() + +eventlet.monkey_patch(os=False) diff --git a/cyborg/api/app.py b/cyborg/api/app.py index 656bf1df..83d14aa5 100644 --- a/cyborg/api/app.py +++ b/cyborg/api/app.py @@ -28,6 +28,8 @@ def get_pecan_config(): def setup_app(pecan_config=None, extra_hooks=None): app_hooks = [hooks.ConfigHook(), + hooks.ConductorAPIHook(), + hooks.ContextHook(pecan_config.app.acl_public_routes), hooks.PublicUrlHook()] if extra_hooks: app_hooks.extend(extra_hooks) diff --git a/cyborg/api/config.py b/cyborg/api/config.py index 895e99f9..32a0d276 100644 --- a/cyborg/api/config.py +++ b/cyborg/api/config.py @@ -28,7 +28,8 @@ app = { 'static_root': '%(confdir)s/public', 'debug': False, 'acl_public_routes': [ - '/' + '/', + '/v1' ] } diff --git a/cyborg/api/controllers/v1/__init__.py b/cyborg/api/controllers/v1/__init__.py index f76283a5..6b3d2085 100644 --- a/cyborg/api/controllers/v1/__init__.py +++ b/cyborg/api/controllers/v1/__init__.py @@ -21,6 +21,7 @@ from wsme import types as wtypes from cyborg.api.controllers import base from cyborg.api.controllers import link +from cyborg.api.controllers.v1 import accelerators from cyborg.api import expose @@ -49,6 +50,8 @@ class V1(base.APIBase): class Controller(rest.RestController): """Version 1 API controller root""" + accelerators = accelerators.AcceleratorsController() + @expose.expose(V1) def get(self): return V1.convert() diff --git a/cyborg/api/controllers/v1/accelerators.py b/cyborg/api/controllers/v1/accelerators.py new file mode 100644 index 00000000..5059455c --- /dev/null +++ b/cyborg/api/controllers/v1/accelerators.py @@ -0,0 +1,85 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan +from pecan import rest +from six.moves import http_client +import wsme +from wsme import types as wtypes + +from cyborg.api.controllers import base +from cyborg.api.controllers import link +from cyborg.api.controllers.v1 import types +from cyborg.api import expose +from cyborg import objects + + +class Accelerator(base.APIBase): + """API representation of a accelerator. + + This class enforces type checking and value constraints, and converts + between the internal object model and the API representation of + a accelerator. + """ + + uuid = types.uuid + name = wtypes.text + description = wtypes.text + device_type = wtypes.text + acc_type = wtypes.text + acc_capability = wtypes.text + vendor_id = wtypes.text + product_id = wtypes.text + remotable = wtypes.IntegerType() + + links = wsme.wsattr([link.Link], readonly=True) + """A list containing a self link""" + + def __init__(self, **kwargs): + self.fields = [] + for field in objects.Accelerator.fields: + self.fields.append(field) + setattr(self, field, kwargs.get(field, wtypes.Unset)) + + @classmethod + def convert_with_links(cls, db_accelerator): + accelerator = Accelerator(**db_accelerator.as_dict()) + url = pecan.request.public_url + accelerator.links = [ + link.Link.make_link('self', url, 'accelerators', + accelerator.uuid), + link.Link.make_link('bookmark', url, 'accelerators', + accelerator.uuid, bookmark=True) + ] + + return accelerator + + +class AcceleratorsController(rest.RestController): + """REST controller for Accelerators.""" + + @expose.expose(Accelerator, body=types.jsontype, + status_code=http_client.CREATED) + def post(self, values): + """Create a new accelerator. + + :param accelerator: an accelerator within the request body. + """ + accelerator = pecan.request.conductor_api.accelerator_create( + pecan.request.context, values) + # Set the HTTP Location Header + pecan.response.location = link.build_url('accelerators', + accelerator.uuid) + return Accelerator.convert_with_links(accelerator) diff --git a/cyborg/api/controllers/v1/types.py b/cyborg/api/controllers/v1/types.py new file mode 100644 index 00000000..6f9af86c --- /dev/null +++ b/cyborg/api/controllers/v1/types.py @@ -0,0 +1,64 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + +from oslo_utils import uuidutils +from wsme import types as wtypes + +from cyborg.common import exception + + +class UUIDType(wtypes.UserType): + """A simple UUID type.""" + + basetype = wtypes.text + name = 'uuid' + + @staticmethod + def validate(value): + if not uuidutils.is_uuid_like(value): + raise exception.InvalidUUID(uuid=value) + return value + + @staticmethod + def frombasetype(value): + if value is None: + return None + return UUIDType.validate(value) + + +class JsonType(wtypes.UserType): + """A simple JSON type.""" + + basetype = wtypes.text + name = 'json' + + @staticmethod + def validate(value): + try: + json.dumps(value) + except TypeError: + raise exception.InvalidJsonType(value=value) + else: + return value + + @staticmethod + def frombasetype(value): + return JsonType.validate(value) + + +uuid = UUIDType() +jsontype = JsonType() diff --git a/cyborg/api/hooks.py b/cyborg/api/hooks.py index d48f7cf1..dfc19188 100644 --- a/cyborg/api/hooks.py +++ b/cyborg/api/hooks.py @@ -14,8 +14,11 @@ # under the License. from oslo_config import cfg +from oslo_context import context from pecan import hooks +from cyborg.conductor import rpcapi + class ConfigHook(hooks.PecanHook): """Attach the config object to the request so controllers can get to it.""" @@ -34,3 +37,24 @@ class PublicUrlHook(hooks.PecanHook): def before(self, state): state.request.public_url = ( cfg.CONF.api.public_endpoint or state.request.host_url) + + +class ConductorAPIHook(hooks.PecanHook): + """Attach the conductor_api object to the request.""" + + def __init__(self): + self.conductor_api = rpcapi.ConductorAPI() + + def before(self, state): + state.request.conductor_api = self.conductor_api + + +class ContextHook(hooks.PecanHook): + """Configures a request context and attaches it to the request.""" + + def __init__(self, public_api_routes): + self.public_api_routes = public_api_routes + super(ContextHook, self).__init__() + + def before(self, state): + state.request.context = context.get_admin_context() diff --git a/cyborg/cmd/conductor.py b/cyborg/cmd/conductor.py new file mode 100644 index 00000000..eb04dc2f --- /dev/null +++ b/cyborg/cmd/conductor.py @@ -0,0 +1,39 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The Cyborg Conductor Service.""" + +import sys + +from oslo_config import cfg +from oslo_service import service + +from cyborg.common import constants +from cyborg.common import service as cyborg_service + + +CONF = cfg.CONF + + +def main(): + # Parse config file and command line options, then start logging + cyborg_service.prepare_service(sys.argv) + + mgr = cyborg_service.RPCService('cyborg.conductor.manager', + 'ConductorManager', + constants.CONDUCTOR_TOPIC) + + launcher = service.launch(CONF, mgr) + launcher.wait() diff --git a/cyborg/cmd/dbsync.py b/cyborg/cmd/dbsync.py new file mode 100644 index 00000000..08facbf0 --- /dev/null +++ b/cyborg/cmd/dbsync.py @@ -0,0 +1,91 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Run storage database migration. +""" + +import sys + +from oslo_config import cfg + +from cyborg.common.i18n import _ +from cyborg.common import service +from cyborg.conf import CONF +from cyborg.db import migration + + +class DBCommand(object): + + def upgrade(self): + migration.upgrade(CONF.command.revision) + + def revision(self): + migration.revision(CONF.command.message, CONF.command.autogenerate) + + def stamp(self): + migration.stamp(CONF.command.revision) + + def version(self): + print(migration.version()) + + def create_schema(self): + migration.create_schema() + + +def add_command_parsers(subparsers): + command_object = DBCommand() + + parser = subparsers.add_parser( + 'upgrade', + help=_("Upgrade the database schema to the latest version. " + "Optionally, use --revision to specify an alembic revision " + "string to upgrade to.")) + parser.set_defaults(func=command_object.upgrade) + parser.add_argument('--revision', nargs='?') + + parser = subparsers.add_parser( + 'revision', + help=_("Create a new alembic revision. " + "Use --message to set the message string.")) + parser.set_defaults(func=command_object.revision) + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + + parser = subparsers.add_parser('stamp') + parser.set_defaults(func=command_object.stamp) + parser.add_argument('--revision', nargs='?') + + parser = subparsers.add_parser( + 'version', + help=_("Print the current version information and exit.")) + parser.set_defaults(func=command_object.version) + + parser = subparsers.add_parser( + 'create_schema', + help=_("Create the database schema.")) + parser.set_defaults(func=command_object.create_schema) + + +def main(): + command_opt = cfg.SubCommandOpt('command', + title='Command', + help=_('Available commands'), + handler=add_command_parsers) + + CONF.register_cli_opt(command_opt) + + service.prepare_service(sys.argv) + CONF.command.func() diff --git a/cyborg/common/constants.py b/cyborg/common/constants.py new file mode 100644 index 00000000..54e03065 --- /dev/null +++ b/cyborg/common/constants.py @@ -0,0 +1,17 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +CONDUCTOR_TOPIC = 'cyborg-conductor' diff --git a/cyborg/common/exception.py b/cyborg/common/exception.py index e077e6a8..e3dc7f21 100644 --- a/cyborg/common/exception.py +++ b/cyborg/common/exception.py @@ -88,3 +88,24 @@ class CyborgException(Exception): class ConfigInvalid(CyborgException): _msg_fmt = _("Invalid configuration file. %(error_msg)s") + + +class AcceleratorAlreadyExists(CyborgException): + _msg_fmt = _("Accelerator with uuid %(uuid)s already exists.") + + +class Invalid(CyborgException): + _msg_fmt = _("Invalid parameters.") + code = http_client.BAD_REQUEST + + +class InvalidIdentity(Invalid): + _msg_fmt = _("Expected a uuid/id but received %(identity)s.") + + +class InvalidUUID(Invalid): + _msg_fmt = _("Expected a uuid but received %(uuid)s.") + + +class InvalidJsonType(Invalid): + _msg_fmt = _("%(value)s is not JSON serializable.") diff --git a/cyborg/common/paths.py b/cyborg/common/paths.py new file mode 100644 index 00000000..38d24113 --- /dev/null +++ b/cyborg/common/paths.py @@ -0,0 +1,48 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from cyborg.conf import CONF + + +def basedir_def(*args): + """Return an uninterpolated path relative to $pybasedir.""" + return os.path.join('$pybasedir', *args) + + +def bindir_def(*args): + """Return an uninterpolated path relative to $bindir.""" + return os.path.join('$bindir', *args) + + +def state_path_def(*args): + """Return an uninterpolated path relative to $state_path.""" + return os.path.join('$state_path', *args) + + +def basedir_rel(*args): + """Return a path relative to $pybasedir.""" + return os.path.join(CONF.pybasedir, *args) + + +def bindir_rel(*args): + """Return a path relative to $bindir.""" + return os.path.join(CONF.bindir, *args) + + +def state_path_rel(*args): + """Return a path relative to $state_path.""" + return os.path.join(CONF.state_path, *args) diff --git a/cyborg/common/service.py b/cyborg/common/service.py index eaa46889..1c501188 100644 --- a/cyborg/common/service.py +++ b/cyborg/common/service.py @@ -14,9 +14,12 @@ # under the License. from oslo_concurrency import processutils +from oslo_context import context from oslo_log import log +import oslo_messaging as messaging from oslo_service import service from oslo_service import wsgi +from oslo_utils import importutils from cyborg.api import app from cyborg.common import config @@ -24,11 +27,56 @@ from cyborg.common import exception from cyborg.common.i18n import _ from cyborg.common import rpc from cyborg.conf import CONF +from cyborg import objects +from cyborg.objects import base as objects_base LOG = log.getLogger(__name__) +class RPCService(service.Service): + def __init__(self, manager_module, manager_class, topic, host=None): + super(RPCService, self).__init__() + self.topic = topic + self.host = host or CONF.host + manager_module = importutils.try_import(manager_module) + manager_class = getattr(manager_module, manager_class) + self.manager = manager_class(self.topic, self.host) + self.rpcserver = None + + def start(self): + super(RPCService, self).start() + + target = messaging.Target(topic=self.topic, server=self.host) + endpoints = [self.manager] + serializer = objects_base.CyborgObjectSerializer() + self.rpcserver = rpc.get_server(target, endpoints, serializer) + self.rpcserver.start() + + admin_context = context.get_admin_context() + self.tg.add_dynamic_timer( + self.manager.periodic_tasks, + periodic_interval_max=CONF.periodic_interval, + context=admin_context) + + LOG.info('Created RPC server for service %(service)s on host ' + '%(host)s.', + {'service': self.topic, 'host': self.host}) + + def stop(self, graceful=True): + try: + self.rpcserver.stop() + self.rpcserver.wait() + except Exception as e: + LOG.exception('Service error occurred when stopping the ' + 'RPC server. Error: %s', e) + + super(RPCService, self).stop(graceful=graceful) + LOG.info('Stopped RPC server for service %(service)s on host ' + '%(host)s.', + {'service': self.topic, 'host': self.host}) + + def prepare_service(argv=None): log.register_options(CONF) log.set_defaults() @@ -39,6 +87,7 @@ def prepare_service(argv=None): log.setup(CONF, 'cyborg') rpc.init(CONF) + objects.register_all() def process_launcher(): diff --git a/cyborg/conductor/conductor.conf b/cyborg/conductor/conductor.conf deleted file mode 100644 index 50144e12..00000000 --- a/cyborg/conductor/conductor.conf +++ /dev/null @@ -1,4 +0,0 @@ -[cyborg] -transport_url= -server_id= -database_url= diff --git a/cyborg/conductor/conductor.py b/cyborg/conductor/conductor.py deleted file mode 100644 index 7c995e56..00000000 --- a/cyborg/conductor/conductor.py +++ /dev/null @@ -1,80 +0,0 @@ -# -*- coding: utf-8 -*- - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import conf -import eventlet -import handlers -from oslo_config import cfg -from oslo_log import log as logging -import oslo_messaging as messaging -import rpcapi -from sqlalchemy import create_engine -import time - -eventlet.monkey_patch() - -CONF = cfg.CONF -conf.register_opts(CONF) - -LOG = logging.getLogger(__name__) -logging.register_options(CONF) -logging.setup(CONF, 'Cyborg.Conductor') - -CONF(['--config-file', 'conductor.conf']) -url = messaging.TransportURL.parse(CONF, url=CONF.cyborg.transport_url) -transport = messaging.get_notification_transport(CONF, url) - -message_endpoints = [ - handlers.NotificationEndpoint -] -message_targets = [ - messaging.Target(topic='info'), - messaging.Target(topic='update'), - messaging.Target(topic='warn'), - messaging.Target(topic='error') -] -rpc_targets = messaging.Target(topic='cyborg_control', - server=CONF.cyborg.server_id) -rpc_endpoints = [ - rpcapi.RPCEndpoint() -] -access_policy = messaging.ExplicitRPCAccessPolicy -rpc_server = messaging.get_rpc_server(transport, - rpc_targets, - rpc_endpoints, - executor='eventlet', - access_policy=access_policy) -pool = "listener-workers" -message_server = messaging.get_notification_listener(transport, - message_targets, - message_endpoints, - executor='eventlet', - allow_requeue=True) - -engine = create_engine(CONF.cyborg.connection, echo=True) -engine.connect() - -try: - message_server.start() - rpc_server.start() - print("Cyborg Conductor running") - while True: - time.sleep(1) -except KeyboardInterrupt: - print("Stopping server") - -message_server.stop() -rpc_server.stop() -message_server.wait() -rpc_server.wait() diff --git a/cyborg/conductor/manager.py b/cyborg/conductor/manager.py new file mode 100644 index 00000000..fd47eba8 --- /dev/null +++ b/cyborg/conductor/manager.py @@ -0,0 +1,40 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import oslo_messaging as messaging + +from cyborg.conf import CONF +from cyborg import objects + + +class ConductorManager(object): + """Cyborg Conductor manager main class.""" + + RPC_API_VERSION = '1.0' + target = messaging.Target(version=RPC_API_VERSION) + + def __init__(self, topic, host=None): + super(ConductorManager, self).__init__() + self.topic = topic + self.host = host or CONF.host + + def periodic_tasks(self, context, raise_on_error=False): + pass + + def accelerator_create(self, context, values): + """Create a new accelerator.""" + accelerator = objects.Accelerator(context, **values) + accelerator.create() + return accelerator diff --git a/cyborg/conductor/rpcapi.py b/cyborg/conductor/rpcapi.py index da7616e1..ae9f6cd0 100644 --- a/cyborg/conductor/rpcapi.py +++ b/cyborg/conductor/rpcapi.py @@ -1,42 +1,53 @@ -# -*- coding: utf-8 -*- - +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. # -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Client side of the conductor RPC API.""" + +from oslo_config import cfg +import oslo_messaging as messaging + +from cyborg.common import constants +from cyborg.common import rpc +from cyborg.objects import base as objects_base -class RPCEndpoint(object): +CONF = cfg.CONF - # Conductor functions exposed for external calls - # Mostly called by the API? - def __init__(self): - pass - # Returns a list of all accelerators managed by Cyborg - def list_accelerators(self, ctxt): - pass +class ConductorAPI(object): + """Client side of the conductor RPC API. - # Returns an accelerator from the DB - def get_accelerator(self, ctxt, accelerator): - pass + API version history: - # Deletes an accelerator from the DB and from the agent that hosts it - def delete_accelerator(self, ctxt, accelerator): - pass + | 1.0 - Initial version. - # Updates an accelerator both in the DB and on the agent that hosts it - def update_accelerator(self, ctxt, accelerator): - pass + """ - # Runs discovery on either a single agent or all agents - def discover_accelerators(self, ctxt, agent_id=None): - pass + RPC_API_VERSION = '1.0' + + def __init__(self, topic=None): + super(ConductorAPI, self).__init__() + self.topic = topic or constants.CONDUCTOR_TOPIC + target = messaging.Target(topic=self.topic, + version='1.0') + serializer = objects_base.CyborgObjectSerializer() + self.client = rpc.get_client(target, + version_cap=self.RPC_API_VERSION, + serializer=serializer) + + def accelerator_create(self, context, values): + """Signal to conductor service to create an accelerator.""" + cctxt = self.client.prepare(topic=self.topic, server=CONF.host) + return cctxt.call(context, 'accelerator_create', values=values) diff --git a/cyborg/conf/__init__.py b/cyborg/conf/__init__.py index 448a3f8a..dadcf641 100644 --- a/cyborg/conf/__init__.py +++ b/cyborg/conf/__init__.py @@ -16,10 +16,12 @@ from oslo_config import cfg from cyborg.conf import api +from cyborg.conf import database from cyborg.conf import default CONF = cfg.CONF api.register_opts(CONF) +database.register_opts(CONF) default.register_opts(CONF) diff --git a/cyborg/conductor/conf.py b/cyborg/conf/database.py similarity index 51% rename from cyborg/conductor/conf.py rename to cyborg/conf/database.py index 3d2ce2ce..be653552 100644 --- a/cyborg/conductor/conf.py +++ b/cyborg/conf/database.py @@ -1,3 +1,5 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -12,22 +14,19 @@ # under the License. from oslo_config import cfg -import uuid -default_opts = [ - cfg.StrOpt('transport_url', - default='', - help='Transport url for messating, copy from transport_url= in \ - your Nova config default section'), - cfg.StrOpt('database_url', - default='', - help='Database url for storage, copy from connection= in your \ - Nova db config section'), - cfg.StrOpt('server_id', - default=uuid.uuid4(), - help='Unique ID for this conductor instance'), +from cyborg.common.i18n import _ + + +opts = [ + cfg.StrOpt('mysql_engine', + default='InnoDB', + help=_('MySQL engine to use.')) ] +opt_group = cfg.OptGroup(name='database', + title='Options for the database service') + def register_opts(conf): - conf.register_opts(default_opts, group='cyborg') + conf.register_opts(opts, group=opt_group) diff --git a/cyborg/conf/default.py b/cyborg/conf/default.py index e248ca5c..5b459daf 100644 --- a/cyborg/conf/default.py +++ b/cyborg/conf/default.py @@ -13,6 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. +import os +import socket + from oslo_config import cfg from cyborg.common.i18n import _ @@ -27,6 +30,39 @@ exc_log_opts = [ 'message.')), ] +service_opts = [ + cfg.HostAddressOpt('host', + default=socket.getfqdn(), + sample_default='localhost', + help=_('Name of this node. This can be an opaque ' + 'identifier. It is not necessarily a hostname, ' + 'FQDN, or IP address. However, the node name ' + 'must be valid within an AMQP key, and if using ' + 'ZeroMQ, a valid hostname, FQDN, or IP address.') + ), + cfg.IntOpt('periodic_interval', + default=60, + help=_('Default interval (in seconds) for running periodic ' + 'tasks.')), +] + +path_opts = [ + cfg.StrOpt('pybasedir', + default=os.path.abspath( + os.path.join(os.path.dirname(__file__), '../')), + sample_default='/usr/lib/python/site-packages/cyborg/cyborg', + help=_('Directory where the cyborg python module is ' + 'installed.')), + cfg.StrOpt('bindir', + default='$pybasedir/bin', + help=_('Directory where cyborg binaries are installed.')), + cfg.StrOpt('state_path', + default='$pybasedir', + help=_("Top-level directory for maintaining cyborg's state.")), +] + def register_opts(conf): conf.register_opts(exc_log_opts) + conf.register_opts(service_opts) + conf.register_opts(path_opts) diff --git a/cyborg/db/__init__.py b/cyborg/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/db/api.py b/cyborg/db/api.py new file mode 100644 index 00000000..6da79c27 --- /dev/null +++ b/cyborg/db/api.py @@ -0,0 +1,47 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Base classes for storage engines.""" + +import abc + +from oslo_config import cfg +from oslo_db import api as db_api +import six + + +_BACKEND_MAPPING = {'sqlalchemy': 'cyborg.db.sqlalchemy.api'} +IMPL = db_api.DBAPI.from_config(cfg.CONF, + backend_mapping=_BACKEND_MAPPING, + lazy=True) + + +def get_instance(): + """Return a DB API instance.""" + return IMPL + + +@six.add_metaclass(abc.ABCMeta) +class Connection(object): + """Base class for storage system connections.""" + + @abc.abstractmethod + def __init__(self): + """Constructor.""" + + # accelerator + @abc.abstractmethod + def accelerator_create(self, context, values): + """Create a new server type.""" diff --git a/cyborg/db/migration.py b/cyborg/db/migration.py new file mode 100644 index 00000000..5c7f580d --- /dev/null +++ b/cyborg/db/migration.py @@ -0,0 +1,52 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Database setup and migration commands.""" + +from oslo_config import cfg +from stevedore import driver + + +_IMPL = None + + +def get_backend(): + global _IMPL + if not _IMPL: + cfg.CONF.import_opt('backend', 'oslo_db.options', group='database') + _IMPL = driver.DriverManager("cyborg.database.migration_backend", + cfg.CONF.database.backend).driver + return _IMPL + + +def upgrade(version=None): + """Migrate the database to `version` or the most recent version.""" + return get_backend().upgrade(version) + + +def version(): + return get_backend().version() + + +def stamp(version): + return get_backend().stamp(version) + + +def revision(message, autogenerate): + return get_backend().revision(message, autogenerate) + + +def create_schema(): + return get_backend().create_schema() diff --git a/cyborg/db/sqlalchemy/__init__.py b/cyborg/db/sqlalchemy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/db/sqlalchemy/alembic.ini b/cyborg/db/sqlalchemy/alembic.ini new file mode 100644 index 00000000..a7689803 --- /dev/null +++ b/cyborg/db/sqlalchemy/alembic.ini @@ -0,0 +1,54 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +#sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/cyborg/db/sqlalchemy/alembic/README b/cyborg/db/sqlalchemy/alembic/README new file mode 100644 index 00000000..9af08b37 --- /dev/null +++ b/cyborg/db/sqlalchemy/alembic/README @@ -0,0 +1,12 @@ +Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation + +To create alembic migrations use: +$ cyborg-dbsync revision --message --autogenerate + +Stamp db with most recent migration version, without actually running migrations +$ cyborg-dbsync stamp --revision head + +Upgrade can be performed by: +$ cyborg-dbsync - for backward compatibility +$ cyborg-dbsync upgrade +# cyborg-dbsync upgrade --revision head diff --git a/cyborg/db/sqlalchemy/alembic/env.py b/cyborg/db/sqlalchemy/alembic/env.py new file mode 100644 index 00000000..982b99b8 --- /dev/null +++ b/cyborg/db/sqlalchemy/alembic/env.py @@ -0,0 +1,61 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from logging import config as log_config + +from alembic import context +from oslo_db.sqlalchemy import enginefacade + +try: + # NOTE(whaom): This is to register the DB2 alembic code which + # is an optional runtime dependency. + from ibm_db_alembic.ibm_db import IbmDbImpl # noqa +except ImportError: + pass + +from cyborg.db.sqlalchemy import models + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +log_config.fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +target_metadata = models.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = enginefacade.get_legacy_facade().get_engine() + with engine.connect() as connection: + context.configure(connection=connection, + target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +run_migrations_online() diff --git a/cyborg/db/sqlalchemy/alembic/script.py.mako b/cyborg/db/sqlalchemy/alembic/script.py.mako new file mode 100644 index 00000000..3b1c960c --- /dev/null +++ b/cyborg/db/sqlalchemy/alembic/script.py.mako @@ -0,0 +1,18 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} diff --git a/cyborg/db/sqlalchemy/alembic/versions/f50980397351_initial_migration.py b/cyborg/db/sqlalchemy/alembic/versions/f50980397351_initial_migration.py new file mode 100644 index 00000000..2b2a2b63 --- /dev/null +++ b/cyborg/db/sqlalchemy/alembic/versions/f50980397351_initial_migration.py @@ -0,0 +1,49 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""initial migration. + +Revision ID: f50980397351 +Revises: None +Create Date: 2017-08-15 08:44:36.010417 + +""" + +# revision identifiers, used by Alembic. +revision = 'f50980397351' +down_revision = None + + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + op.create_table( + 'accelerators', + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('uuid', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('description', sa.Text(), nullable=True), + sa.Column('device_type', sa.Text(), nullable=False), + sa.Column('acc_type', sa.Text(), nullable=False), + sa.Column('acc_capability', sa.Text(), nullable=False), + sa.Column('vendor_id', sa.Text(), nullable=False), + sa.Column('product_id', sa.Text(), nullable=False), + sa.Column('remotable', sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint('id'), + mysql_ENGINE='InnoDB', + mysql_DEFAULT_CHARSET='UTF8' + ) diff --git a/cyborg/db/sqlalchemy/api.py b/cyborg/db/sqlalchemy/api.py new file mode 100644 index 00000000..24c24c0a --- /dev/null +++ b/cyborg/db/sqlalchemy/api.py @@ -0,0 +1,114 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""SQLAlchemy storage backend.""" + +import threading + +from oslo_db import exception as db_exc +from oslo_db.sqlalchemy import enginefacade +from oslo_db.sqlalchemy import utils as sqlalchemyutils +from oslo_log import log +from oslo_utils import strutils +from oslo_utils import uuidutils + +from cyborg.common import exception +from cyborg.db import api +from cyborg.db.sqlalchemy import models + + +_CONTEXT = threading.local() +LOG = log.getLogger(__name__) + + +def get_backend(): + """The backend is this module itself.""" + return Connection() + + +def _session_for_read(): + return enginefacade.reader.using(_CONTEXT) + + +def _session_for_write(): + return enginefacade.writer.using(_CONTEXT) + + +def model_query(context, model, *args, **kwargs): + """Query helper for simpler session usage. + + :param context: Context of the query + :param model: Model to query. Must be a subclass of ModelBase. + :param args: Arguments to query. If None - model is used. + + Keyword arguments: + + :keyword project_only: + If set to True, then will do query filter with context's project_id. + if set to False or absent, then will not do query filter with context's + project_id. + :type project_only: bool + """ + + if kwargs.pop("project_only", False): + kwargs["project_id"] = context.tenant + + with _session_for_read() as session: + query = sqlalchemyutils.model_query( + model, session, args, **kwargs) + return query + + +def add_identity_filter(query, value): + """Adds an identity filter to a query. + + Filters results by ID, if supplied value is a valid integer. + Otherwise attempts to filter results by UUID. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + if strutils.is_int_like(value): + return query.filter_by(id=value) + elif uuidutils.is_uuid_like(value): + return query.filter_by(uuid=value) + else: + raise exception.InvalidIdentity(identity=value) + + +class Connection(api.Connection): + """SqlAlchemy connection.""" + + def __init__(self): + pass + + def accelerator_create(self, context, values): + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + + if not values.get('description'): + values['description'] = '' + + accelerator = models.Accelerator() + accelerator.update(values) + + with _session_for_write() as session: + try: + session.add(accelerator) + session.flush() + except db_exc.DBDuplicateEntry: + raise exception.AcceleratorAlreadyExists(uuid=values['uuid']) + return accelerator diff --git a/cyborg/db/sqlalchemy/migration.py b/cyborg/db/sqlalchemy/migration.py new file mode 100644 index 00000000..2c8aa359 --- /dev/null +++ b/cyborg/db/sqlalchemy/migration.py @@ -0,0 +1,108 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import alembic +from alembic import config as alembic_config +import alembic.migration as alembic_migration +from oslo_db import exception as db_exc +from oslo_db.sqlalchemy import enginefacade + +from cyborg.db.sqlalchemy import models + + +def _alembic_config(): + path = os.path.join(os.path.dirname(__file__), 'alembic.ini') + config = alembic_config.Config(path) + return config + + +def version(config=None, engine=None): + """Current database version. + + :returns: Database version + :rtype: string + """ + if engine is None: + engine = enginefacade.get_legacy_facade().get_engine() + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_revision() + + +def upgrade(revision, config=None): + """Used for upgrading database. + + :param version: Desired database version + :type version: string + """ + revision = revision or 'head' + config = config or _alembic_config() + alembic.command.upgrade(config, revision) + + +def create_schema(config=None, engine=None): + """Create database schema from models description. + + Can be used for initial installation instead of upgrade('head'). + """ + if engine is None: + engine = enginefacade.get_legacy_facade().get_engine() + + if version(engine=engine) is not None: + raise db_exc.DbMigrationError("DB schema is already under version" + " control. Use upgrade() instead") + + models.Base.metadata.create_all(engine) + stamp('head', config=config) + + +def downgrade(revision, config=None): + """Used for downgrading database. + + :param version: Desired database version + :type version: string + """ + revision = revision or 'base' + config = config or _alembic_config() + return alembic.command.downgrade(config, revision) + + +def stamp(revision, config=None): + """Stamps database with provided revision. + + Don't run any migrations. + + :param revision: Should match one from repository or head - to stamp + database with most recent revision + :type revision: string + """ + config = config or _alembic_config() + return alembic.command.stamp(config, revision=revision) + + +def revision(message=None, autogenerate=False, config=None): + """Creates template for migration. + + :param message: Text that will be used for migration title + :type message: string + :param autogenerate: If True - generates diff based on current database + state + :type autogenerate: bool + """ + config = config or _alembic_config() + return alembic.command.revision(config, message=message, + autogenerate=autogenerate) diff --git a/cyborg/db/sqlalchemy/models.py b/cyborg/db/sqlalchemy/models.py new file mode 100644 index 00000000..a8fd9961 --- /dev/null +++ b/cyborg/db/sqlalchemy/models.py @@ -0,0 +1,72 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""SQLAlchemy models for accelerator service.""" + +from oslo_db import options as db_options +from oslo_db.sqlalchemy import models +import six.moves.urllib.parse as urlparse +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Column, String, Integer +from sqlalchemy import schema + +from cyborg.common import paths +from cyborg.conf import CONF + + +_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('cyborg.sqlite') +db_options.set_defaults(CONF, connection=_DEFAULT_SQL_CONNECTION) + + +def table_args(): + engine_name = urlparse.urlparse(CONF.database.connection).scheme + if engine_name == 'mysql': + return {'mysql_engine': CONF.database.mysql_engine, + 'mysql_charset': "utf8"} + return None + + +class CyborgBase(models.TimestampMixin, models.ModelBase): + metadata = None + + def as_dict(self): + d = {} + for c in self.__table__.columns: + d[c.name] = self[c.name] + return d + + +Base = declarative_base(cls=CyborgBase) + + +class Accelerator(Base): + """Represents the accelerators.""" + + __tablename__ = 'accelerators' + __table_args__ = ( + schema.UniqueConstraint('uuid', name='uniq_accelerators0uuid'), + table_args() + ) + + id = Column(Integer, primary_key=True) + uuid = Column(String(36), nullable=False) + name = Column(String(255), nullable=False) + description = Column(String(255), nullable=True) + device_type = Column(String(255), nullable=False) + acc_type = Column(String(255), nullable=False) + acc_capability = Column(String(255), nullable=False) + vendor_id = Column(String(255), nullable=False) + product_id = Column(String(255), nullable=False) + remotable = Column(Integer, nullable=False) diff --git a/cyborg/objects/__init__.py b/cyborg/objects/__init__.py new file mode 100644 index 00000000..56c2dc77 --- /dev/null +++ b/cyborg/objects/__init__.py @@ -0,0 +1,28 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(comstud): You may scratch your head as you see code that imports +# this module and then accesses attributes for objects such as Node, +# etc, yet you do not see these attributes in here. Never fear, there is +# a little bit of magic. When objects are registered, an attribute is set +# on this module automatically, pointing to the newest/latest version of +# the object. + + +def register_all(): + # NOTE(danms): You must make sure your object gets imported in this + # function in order for it to be registered by services that may + # need to receive it via RPC. + __import__('cyborg.objects.accelerator') diff --git a/cyborg/objects/accelerator.py b/cyborg/objects/accelerator.py new file mode 100644 index 00000000..1859bbef --- /dev/null +++ b/cyborg/objects/accelerator.py @@ -0,0 +1,49 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_versionedobjects import base as object_base + +from cyborg.db import api as dbapi +from cyborg.objects import base +from cyborg.objects import fields as object_fields + + +@base.CyborgObjectRegistry.register +class Accelerator(base.CyborgObject, object_base.VersionedObjectDictCompat): + # Version 1.0: Initial version + VERSION = '1.0' + + dbapi = dbapi.get_instance() + + fields = { + 'uuid': object_fields.UUIDField(nullable=False), + 'name': object_fields.StringField(nullable=False), + 'description': object_fields.StringField(nullable=True), + 'device_type': object_fields.StringField(nullable=False), + 'acc_type': object_fields.StringField(nullable=False), + 'acc_capability': object_fields.StringField(nullable=False), + 'vendor_id': object_fields.StringField(nullable=False), + 'product_id': object_fields.StringField(nullable=False), + 'remotable': object_fields.IntegerField(nullable=False), + } + + def __init__(self, *args, **kwargs): + super(Accelerator, self).__init__(*args, **kwargs) + + def create(self, context=None): + """Create an Accelerator record in the DB.""" + values = self.obj_get_changes() + db_accelerator = self.dbapi.accelerator_create(context, values) + self._from_db_object(context, self, db_accelerator) diff --git a/cyborg/objects/base.py b/cyborg/objects/base.py new file mode 100644 index 00000000..5b09f7bf --- /dev/null +++ b/cyborg/objects/base.py @@ -0,0 +1,83 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Cyborg common internal object model""" + +from oslo_utils import versionutils +from oslo_versionedobjects import base as object_base + +from cyborg import objects +from cyborg.objects import fields as object_fields + + +class CyborgObjectRegistry(object_base.VersionedObjectRegistry): + def registration_hook(self, cls, index): + # NOTE(jroll): blatantly stolen from nova + # NOTE(danms): This is called when an object is registered, + # and is responsible for maintaining cyborg.objects.$OBJECT + # as the highest-versioned implementation of a given object. + version = versionutils.convert_version_to_tuple(cls.VERSION) + if not hasattr(objects, cls.obj_name()): + setattr(objects, cls.obj_name(), cls) + else: + cur_version = versionutils.convert_version_to_tuple( + getattr(objects, cls.obj_name()).VERSION) + if version >= cur_version: + setattr(objects, cls.obj_name(), cls) + + +class CyborgObject(object_base.VersionedObject): + """Base class and object factory. + + This forms the base of all objects that can be remoted or instantiated + via RPC. Simply defining a class that inherits from this base class + will make it remotely instantiatable. Objects should implement the + necessary "get" classmethod routines as well as "save" object methods + as appropriate. + """ + + OBJ_SERIAL_NAMESPACE = 'cyborg_object' + OBJ_PROJECT_NAMESPACE = 'cyborg' + + fields = { + 'created_at': object_fields.DateTimeField(nullable=True), + 'updated_at': object_fields.DateTimeField(nullable=True), + } + + def as_dict(self): + return dict((k, getattr(self, k)) + for k in self.fields + if hasattr(self, k)) + + @staticmethod + def _from_db_object(context, obj, db_object): + """Converts a database entity to a formal object. + + :param context: security context + :param obj: An object of the class. + :param db_object: A DB model of the object + :return: The object of the class with the database entity added + """ + + for field in obj.fields: + obj[field] = db_object[field] + + obj.obj_reset_changes() + return obj + + +class CyborgObjectSerializer(object_base.VersionedObjectSerializer): + # Base class to use for object hydration + OBJ_BASE_CLASS = CyborgObject diff --git a/cyborg/objects/fields.py b/cyborg/objects/fields.py new file mode 100644 index 00000000..59400684 --- /dev/null +++ b/cyborg/objects/fields.py @@ -0,0 +1,32 @@ +# Copyright 2017 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_versionedobjects import fields as object_fields + + +class IntegerField(object_fields.IntegerField): + pass + + +class UUIDField(object_fields.UUIDField): + pass + + +class StringField(object_fields.StringField): + pass + + +class DateTimeField(object_fields.DateTimeField): + pass diff --git a/requirements.txt b/requirements.txt index 83679503..21251594 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,3 +15,9 @@ oslo.messaging!=5.25.0,>=5.24.2 # Apache-2.0 oslo.concurrency>=3.8.0 # Apache-2.0 oslo.service>=1.10.0 # Apache-2.0 oslo.serialization!=2.19.1,>=1.10.0 # Apache-2.0 +oslo.db>=4.24.0 # Apache-2.0 +oslo.utils>=3.20.0 # Apache-2.0 +oslo.versionedobjects>=1.17.0 # Apache-2.0 +SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT +alembic>=0.8.10 # MIT +stevedore>=1.20.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 4dab9ec3..5d112f73 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,11 @@ packages = [entry_points] console_scripts = cyborg-api = cyborg.cmd.api:main + cyborg-conductor = cyborg.cmd.conductor:main + cyborg-dbsync = cyborg.cmd.dbsync:main + +cyborg.database.migration_backend = + sqlalchemy = cyborg.db.sqlalchemy.migration [build_sphinx] source-dir = doc/source