add cyborg-conductor & db

1. add conductor rpc
2. add cyborg-conductor command
3. add db and init `accelerators` table
4. add accelerator_create method
5. add cyborg-dbsync command

Change-Id: I07333a4df7a42878dcf950b2b7893a37670da87b
This commit is contained in:
zhuli 2017-08-08 10:37:13 +08:00
parent 55e0b2bcb7
commit 57e4c042ca
38 changed files with 1371 additions and 130 deletions

View File

@ -12,8 +12,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import eventlet
import pbr.version
__version__ = pbr.version.VersionInfo(
'cyborg').version_string()
eventlet.monkey_patch(os=False)

View File

@ -28,6 +28,8 @@ def get_pecan_config():
def setup_app(pecan_config=None, extra_hooks=None):
app_hooks = [hooks.ConfigHook(),
hooks.ConductorAPIHook(),
hooks.ContextHook(pecan_config.app.acl_public_routes),
hooks.PublicUrlHook()]
if extra_hooks:
app_hooks.extend(extra_hooks)

View File

@ -28,7 +28,8 @@ app = {
'static_root': '%(confdir)s/public',
'debug': False,
'acl_public_routes': [
'/'
'/',
'/v1'
]
}

View File

@ -21,6 +21,7 @@ from wsme import types as wtypes
from cyborg.api.controllers import base
from cyborg.api.controllers import link
from cyborg.api.controllers.v1 import accelerators
from cyborg.api import expose
@ -49,6 +50,8 @@ class V1(base.APIBase):
class Controller(rest.RestController):
"""Version 1 API controller root"""
accelerators = accelerators.AcceleratorsController()
@expose.expose(V1)
def get(self):
return V1.convert()

View File

@ -0,0 +1,85 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pecan
from pecan import rest
from six.moves import http_client
import wsme
from wsme import types as wtypes
from cyborg.api.controllers import base
from cyborg.api.controllers import link
from cyborg.api.controllers.v1 import types
from cyborg.api import expose
from cyborg import objects
class Accelerator(base.APIBase):
"""API representation of a accelerator.
This class enforces type checking and value constraints, and converts
between the internal object model and the API representation of
a accelerator.
"""
uuid = types.uuid
name = wtypes.text
description = wtypes.text
device_type = wtypes.text
acc_type = wtypes.text
acc_capability = wtypes.text
vendor_id = wtypes.text
product_id = wtypes.text
remotable = wtypes.IntegerType()
links = wsme.wsattr([link.Link], readonly=True)
"""A list containing a self link"""
def __init__(self, **kwargs):
self.fields = []
for field in objects.Accelerator.fields:
self.fields.append(field)
setattr(self, field, kwargs.get(field, wtypes.Unset))
@classmethod
def convert_with_links(cls, db_accelerator):
accelerator = Accelerator(**db_accelerator.as_dict())
url = pecan.request.public_url
accelerator.links = [
link.Link.make_link('self', url, 'accelerators',
accelerator.uuid),
link.Link.make_link('bookmark', url, 'accelerators',
accelerator.uuid, bookmark=True)
]
return accelerator
class AcceleratorsController(rest.RestController):
"""REST controller for Accelerators."""
@expose.expose(Accelerator, body=types.jsontype,
status_code=http_client.CREATED)
def post(self, values):
"""Create a new accelerator.
:param accelerator: an accelerator within the request body.
"""
accelerator = pecan.request.conductor_api.accelerator_create(
pecan.request.context, values)
# Set the HTTP Location Header
pecan.response.location = link.build_url('accelerators',
accelerator.uuid)
return Accelerator.convert_with_links(accelerator)

View File

@ -0,0 +1,64 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
from oslo_utils import uuidutils
from wsme import types as wtypes
from cyborg.common import exception
class UUIDType(wtypes.UserType):
"""A simple UUID type."""
basetype = wtypes.text
name = 'uuid'
@staticmethod
def validate(value):
if not uuidutils.is_uuid_like(value):
raise exception.InvalidUUID(uuid=value)
return value
@staticmethod
def frombasetype(value):
if value is None:
return None
return UUIDType.validate(value)
class JsonType(wtypes.UserType):
"""A simple JSON type."""
basetype = wtypes.text
name = 'json'
@staticmethod
def validate(value):
try:
json.dumps(value)
except TypeError:
raise exception.InvalidJsonType(value=value)
else:
return value
@staticmethod
def frombasetype(value):
return JsonType.validate(value)
uuid = UUIDType()
jsontype = JsonType()

View File

@ -14,8 +14,11 @@
# under the License.
from oslo_config import cfg
from oslo_context import context
from pecan import hooks
from cyborg.conductor import rpcapi
class ConfigHook(hooks.PecanHook):
"""Attach the config object to the request so controllers can get to it."""
@ -34,3 +37,24 @@ class PublicUrlHook(hooks.PecanHook):
def before(self, state):
state.request.public_url = (
cfg.CONF.api.public_endpoint or state.request.host_url)
class ConductorAPIHook(hooks.PecanHook):
"""Attach the conductor_api object to the request."""
def __init__(self):
self.conductor_api = rpcapi.ConductorAPI()
def before(self, state):
state.request.conductor_api = self.conductor_api
class ContextHook(hooks.PecanHook):
"""Configures a request context and attaches it to the request."""
def __init__(self, public_api_routes):
self.public_api_routes = public_api_routes
super(ContextHook, self).__init__()
def before(self, state):
state.request.context = context.get_admin_context()

39
cyborg/cmd/conductor.py Normal file
View File

@ -0,0 +1,39 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""The Cyborg Conductor Service."""
import sys
from oslo_config import cfg
from oslo_service import service
from cyborg.common import constants
from cyborg.common import service as cyborg_service
CONF = cfg.CONF
def main():
# Parse config file and command line options, then start logging
cyborg_service.prepare_service(sys.argv)
mgr = cyborg_service.RPCService('cyborg.conductor.manager',
'ConductorManager',
constants.CONDUCTOR_TOPIC)
launcher = service.launch(CONF, mgr)
launcher.wait()

91
cyborg/cmd/dbsync.py Normal file
View File

@ -0,0 +1,91 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Run storage database migration.
"""
import sys
from oslo_config import cfg
from cyborg.common.i18n import _
from cyborg.common import service
from cyborg.conf import CONF
from cyborg.db import migration
class DBCommand(object):
def upgrade(self):
migration.upgrade(CONF.command.revision)
def revision(self):
migration.revision(CONF.command.message, CONF.command.autogenerate)
def stamp(self):
migration.stamp(CONF.command.revision)
def version(self):
print(migration.version())
def create_schema(self):
migration.create_schema()
def add_command_parsers(subparsers):
command_object = DBCommand()
parser = subparsers.add_parser(
'upgrade',
help=_("Upgrade the database schema to the latest version. "
"Optionally, use --revision to specify an alembic revision "
"string to upgrade to."))
parser.set_defaults(func=command_object.upgrade)
parser.add_argument('--revision', nargs='?')
parser = subparsers.add_parser(
'revision',
help=_("Create a new alembic revision. "
"Use --message to set the message string."))
parser.set_defaults(func=command_object.revision)
parser.add_argument('-m', '--message')
parser.add_argument('--autogenerate', action='store_true')
parser = subparsers.add_parser('stamp')
parser.set_defaults(func=command_object.stamp)
parser.add_argument('--revision', nargs='?')
parser = subparsers.add_parser(
'version',
help=_("Print the current version information and exit."))
parser.set_defaults(func=command_object.version)
parser = subparsers.add_parser(
'create_schema',
help=_("Create the database schema."))
parser.set_defaults(func=command_object.create_schema)
def main():
command_opt = cfg.SubCommandOpt('command',
title='Command',
help=_('Available commands'),
handler=add_command_parsers)
CONF.register_cli_opt(command_opt)
service.prepare_service(sys.argv)
CONF.command.func()

View File

@ -0,0 +1,17 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
CONDUCTOR_TOPIC = 'cyborg-conductor'

View File

@ -88,3 +88,24 @@ class CyborgException(Exception):
class ConfigInvalid(CyborgException):
_msg_fmt = _("Invalid configuration file. %(error_msg)s")
class AcceleratorAlreadyExists(CyborgException):
_msg_fmt = _("Accelerator with uuid %(uuid)s already exists.")
class Invalid(CyborgException):
_msg_fmt = _("Invalid parameters.")
code = http_client.BAD_REQUEST
class InvalidIdentity(Invalid):
_msg_fmt = _("Expected a uuid/id but received %(identity)s.")
class InvalidUUID(Invalid):
_msg_fmt = _("Expected a uuid but received %(uuid)s.")
class InvalidJsonType(Invalid):
_msg_fmt = _("%(value)s is not JSON serializable.")

48
cyborg/common/paths.py Normal file
View File

@ -0,0 +1,48 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from cyborg.conf import CONF
def basedir_def(*args):
"""Return an uninterpolated path relative to $pybasedir."""
return os.path.join('$pybasedir', *args)
def bindir_def(*args):
"""Return an uninterpolated path relative to $bindir."""
return os.path.join('$bindir', *args)
def state_path_def(*args):
"""Return an uninterpolated path relative to $state_path."""
return os.path.join('$state_path', *args)
def basedir_rel(*args):
"""Return a path relative to $pybasedir."""
return os.path.join(CONF.pybasedir, *args)
def bindir_rel(*args):
"""Return a path relative to $bindir."""
return os.path.join(CONF.bindir, *args)
def state_path_rel(*args):
"""Return a path relative to $state_path."""
return os.path.join(CONF.state_path, *args)

View File

@ -14,9 +14,12 @@
# under the License.
from oslo_concurrency import processutils
from oslo_context import context
from oslo_log import log
import oslo_messaging as messaging
from oslo_service import service
from oslo_service import wsgi
from oslo_utils import importutils
from cyborg.api import app
from cyborg.common import config
@ -24,11 +27,56 @@ from cyborg.common import exception
from cyborg.common.i18n import _
from cyborg.common import rpc
from cyborg.conf import CONF
from cyborg import objects
from cyborg.objects import base as objects_base
LOG = log.getLogger(__name__)
class RPCService(service.Service):
def __init__(self, manager_module, manager_class, topic, host=None):
super(RPCService, self).__init__()
self.topic = topic
self.host = host or CONF.host
manager_module = importutils.try_import(manager_module)
manager_class = getattr(manager_module, manager_class)
self.manager = manager_class(self.topic, self.host)
self.rpcserver = None
def start(self):
super(RPCService, self).start()
target = messaging.Target(topic=self.topic, server=self.host)
endpoints = [self.manager]
serializer = objects_base.CyborgObjectSerializer()
self.rpcserver = rpc.get_server(target, endpoints, serializer)
self.rpcserver.start()
admin_context = context.get_admin_context()
self.tg.add_dynamic_timer(
self.manager.periodic_tasks,
periodic_interval_max=CONF.periodic_interval,
context=admin_context)
LOG.info('Created RPC server for service %(service)s on host '
'%(host)s.',
{'service': self.topic, 'host': self.host})
def stop(self, graceful=True):
try:
self.rpcserver.stop()
self.rpcserver.wait()
except Exception as e:
LOG.exception('Service error occurred when stopping the '
'RPC server. Error: %s', e)
super(RPCService, self).stop(graceful=graceful)
LOG.info('Stopped RPC server for service %(service)s on host '
'%(host)s.',
{'service': self.topic, 'host': self.host})
def prepare_service(argv=None):
log.register_options(CONF)
log.set_defaults()
@ -39,6 +87,7 @@ def prepare_service(argv=None):
log.setup(CONF, 'cyborg')
rpc.init(CONF)
objects.register_all()
def process_launcher():

View File

@ -1,4 +0,0 @@
[cyborg]
transport_url=
server_id=
database_url=

View File

@ -1,80 +0,0 @@
# -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import conf
import eventlet
import handlers
from oslo_config import cfg
from oslo_log import log as logging
import oslo_messaging as messaging
import rpcapi
from sqlalchemy import create_engine
import time
eventlet.monkey_patch()
CONF = cfg.CONF
conf.register_opts(CONF)
LOG = logging.getLogger(__name__)
logging.register_options(CONF)
logging.setup(CONF, 'Cyborg.Conductor')
CONF(['--config-file', 'conductor.conf'])
url = messaging.TransportURL.parse(CONF, url=CONF.cyborg.transport_url)
transport = messaging.get_notification_transport(CONF, url)
message_endpoints = [
handlers.NotificationEndpoint
]
message_targets = [
messaging.Target(topic='info'),
messaging.Target(topic='update'),
messaging.Target(topic='warn'),
messaging.Target(topic='error')
]
rpc_targets = messaging.Target(topic='cyborg_control',
server=CONF.cyborg.server_id)
rpc_endpoints = [
rpcapi.RPCEndpoint()
]
access_policy = messaging.ExplicitRPCAccessPolicy
rpc_server = messaging.get_rpc_server(transport,
rpc_targets,
rpc_endpoints,
executor='eventlet',
access_policy=access_policy)
pool = "listener-workers"
message_server = messaging.get_notification_listener(transport,
message_targets,
message_endpoints,
executor='eventlet',
allow_requeue=True)
engine = create_engine(CONF.cyborg.connection, echo=True)
engine.connect()
try:
message_server.start()
rpc_server.start()
print("Cyborg Conductor running")
while True:
time.sleep(1)
except KeyboardInterrupt:
print("Stopping server")
message_server.stop()
rpc_server.stop()
message_server.wait()
rpc_server.wait()

View File

@ -0,0 +1,40 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import oslo_messaging as messaging
from cyborg.conf import CONF
from cyborg import objects
class ConductorManager(object):
"""Cyborg Conductor manager main class."""
RPC_API_VERSION = '1.0'
target = messaging.Target(version=RPC_API_VERSION)
def __init__(self, topic, host=None):
super(ConductorManager, self).__init__()
self.topic = topic
self.host = host or CONF.host
def periodic_tasks(self, context, raise_on_error=False):
pass
def accelerator_create(self, context, values):
"""Create a new accelerator."""
accelerator = objects.Accelerator(context, **values)
accelerator.create()
return accelerator

View File

@ -1,42 +1,53 @@
# -*- coding: utf-8 -*-
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Client side of the conductor RPC API."""
from oslo_config import cfg
import oslo_messaging as messaging
from cyborg.common import constants
from cyborg.common import rpc
from cyborg.objects import base as objects_base
class RPCEndpoint(object):
CONF = cfg.CONF
# Conductor functions exposed for external calls
# Mostly called by the API?
def __init__(self):
pass
# Returns a list of all accelerators managed by Cyborg
def list_accelerators(self, ctxt):
pass
class ConductorAPI(object):
"""Client side of the conductor RPC API.
# Returns an accelerator from the DB
def get_accelerator(self, ctxt, accelerator):
pass
API version history:
# Deletes an accelerator from the DB and from the agent that hosts it
def delete_accelerator(self, ctxt, accelerator):
pass
| 1.0 - Initial version.
# Updates an accelerator both in the DB and on the agent that hosts it
def update_accelerator(self, ctxt, accelerator):
pass
"""
# Runs discovery on either a single agent or all agents
def discover_accelerators(self, ctxt, agent_id=None):
pass
RPC_API_VERSION = '1.0'
def __init__(self, topic=None):
super(ConductorAPI, self).__init__()
self.topic = topic or constants.CONDUCTOR_TOPIC
target = messaging.Target(topic=self.topic,
version='1.0')
serializer = objects_base.CyborgObjectSerializer()
self.client = rpc.get_client(target,
version_cap=self.RPC_API_VERSION,
serializer=serializer)
def accelerator_create(self, context, values):
"""Signal to conductor service to create an accelerator."""
cctxt = self.client.prepare(topic=self.topic, server=CONF.host)
return cctxt.call(context, 'accelerator_create', values=values)

View File

@ -16,10 +16,12 @@
from oslo_config import cfg
from cyborg.conf import api
from cyborg.conf import database
from cyborg.conf import default
CONF = cfg.CONF
api.register_opts(CONF)
database.register_opts(CONF)
default.register_opts(CONF)

View File

@ -1,3 +1,5 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
@ -12,22 +14,19 @@
# under the License.
from oslo_config import cfg
import uuid
default_opts = [
cfg.StrOpt('transport_url',
default='',
help='Transport url for messating, copy from transport_url= in \
your Nova config default section'),
cfg.StrOpt('database_url',
default='',
help='Database url for storage, copy from connection= in your \
Nova db config section'),
cfg.StrOpt('server_id',
default=uuid.uuid4(),
help='Unique ID for this conductor instance'),
from cyborg.common.i18n import _
opts = [
cfg.StrOpt('mysql_engine',
default='InnoDB',
help=_('MySQL engine to use.'))
]
opt_group = cfg.OptGroup(name='database',
title='Options for the database service')
def register_opts(conf):
conf.register_opts(default_opts, group='cyborg')
conf.register_opts(opts, group=opt_group)

View File

@ -13,6 +13,9 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
import socket
from oslo_config import cfg
from cyborg.common.i18n import _
@ -27,6 +30,39 @@ exc_log_opts = [
'message.')),
]
service_opts = [
cfg.HostAddressOpt('host',
default=socket.getfqdn(),
sample_default='localhost',
help=_('Name of this node. This can be an opaque '
'identifier. It is not necessarily a hostname, '
'FQDN, or IP address. However, the node name '
'must be valid within an AMQP key, and if using '
'ZeroMQ, a valid hostname, FQDN, or IP address.')
),
cfg.IntOpt('periodic_interval',
default=60,
help=_('Default interval (in seconds) for running periodic '
'tasks.')),
]
path_opts = [
cfg.StrOpt('pybasedir',
default=os.path.abspath(
os.path.join(os.path.dirname(__file__), '../')),
sample_default='/usr/lib/python/site-packages/cyborg/cyborg',
help=_('Directory where the cyborg python module is '
'installed.')),
cfg.StrOpt('bindir',
default='$pybasedir/bin',
help=_('Directory where cyborg binaries are installed.')),
cfg.StrOpt('state_path',
default='$pybasedir',
help=_("Top-level directory for maintaining cyborg's state.")),
]
def register_opts(conf):
conf.register_opts(exc_log_opts)
conf.register_opts(service_opts)
conf.register_opts(path_opts)

0
cyborg/db/__init__.py Normal file
View File

47
cyborg/db/api.py Normal file
View File

@ -0,0 +1,47 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Base classes for storage engines."""
import abc
from oslo_config import cfg
from oslo_db import api as db_api
import six
_BACKEND_MAPPING = {'sqlalchemy': 'cyborg.db.sqlalchemy.api'}
IMPL = db_api.DBAPI.from_config(cfg.CONF,
backend_mapping=_BACKEND_MAPPING,
lazy=True)
def get_instance():
"""Return a DB API instance."""
return IMPL
@six.add_metaclass(abc.ABCMeta)
class Connection(object):
"""Base class for storage system connections."""
@abc.abstractmethod
def __init__(self):
"""Constructor."""
# accelerator
@abc.abstractmethod
def accelerator_create(self, context, values):
"""Create a new server type."""

52
cyborg/db/migration.py Normal file
View File

@ -0,0 +1,52 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Database setup and migration commands."""
from oslo_config import cfg
from stevedore import driver
_IMPL = None
def get_backend():
global _IMPL
if not _IMPL:
cfg.CONF.import_opt('backend', 'oslo_db.options', group='database')
_IMPL = driver.DriverManager("cyborg.database.migration_backend",
cfg.CONF.database.backend).driver
return _IMPL
def upgrade(version=None):
"""Migrate the database to `version` or the most recent version."""
return get_backend().upgrade(version)
def version():
return get_backend().version()
def stamp(version):
return get_backend().stamp(version)
def revision(message, autogenerate):
return get_backend().revision(message, autogenerate)
def create_schema():
return get_backend().create_schema()

View File

View File

@ -0,0 +1,54 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = %(here)s/alembic
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
#sqlalchemy.url = driver://user:pass@localhost/dbname
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

View File

@ -0,0 +1,12 @@
Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation
To create alembic migrations use:
$ cyborg-dbsync revision --message --autogenerate
Stamp db with most recent migration version, without actually running migrations
$ cyborg-dbsync stamp --revision head
Upgrade can be performed by:
$ cyborg-dbsync - for backward compatibility
$ cyborg-dbsync upgrade
# cyborg-dbsync upgrade --revision head

View File

@ -0,0 +1,61 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from logging import config as log_config
from alembic import context
from oslo_db.sqlalchemy import enginefacade
try:
# NOTE(whaom): This is to register the DB2 alembic code which
# is an optional runtime dependency.
from ibm_db_alembic.ibm_db import IbmDbImpl # noqa
except ImportError:
pass
from cyborg.db.sqlalchemy import models
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
log_config.fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
target_metadata = models.Base.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = enginefacade.get_legacy_facade().get_engine()
with engine.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
run_migrations_online()

View File

@ -0,0 +1,18 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}

View File

@ -0,0 +1,49 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""initial migration.
Revision ID: f50980397351
Revises: None
Create Date: 2017-08-15 08:44:36.010417
"""
# revision identifiers, used by Alembic.
revision = 'f50980397351'
down_revision = None
from alembic import op
import sqlalchemy as sa
def upgrade():
op.create_table(
'accelerators',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=36), nullable=False),
sa.Column('name', sa.String(length=255), nullable=False),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('device_type', sa.Text(), nullable=False),
sa.Column('acc_type', sa.Text(), nullable=False),
sa.Column('acc_capability', sa.Text(), nullable=False),
sa.Column('vendor_id', sa.Text(), nullable=False),
sa.Column('product_id', sa.Text(), nullable=False),
sa.Column('remotable', sa.Integer(), nullable=False),
sa.PrimaryKeyConstraint('id'),
mysql_ENGINE='InnoDB',
mysql_DEFAULT_CHARSET='UTF8'
)

114
cyborg/db/sqlalchemy/api.py Normal file
View File

@ -0,0 +1,114 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""SQLAlchemy storage backend."""
import threading
from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import enginefacade
from oslo_db.sqlalchemy import utils as sqlalchemyutils
from oslo_log import log
from oslo_utils import strutils
from oslo_utils import uuidutils
from cyborg.common import exception
from cyborg.db import api
from cyborg.db.sqlalchemy import models
_CONTEXT = threading.local()
LOG = log.getLogger(__name__)
def get_backend():
"""The backend is this module itself."""
return Connection()
def _session_for_read():
return enginefacade.reader.using(_CONTEXT)
def _session_for_write():
return enginefacade.writer.using(_CONTEXT)
def model_query(context, model, *args, **kwargs):
"""Query helper for simpler session usage.
:param context: Context of the query
:param model: Model to query. Must be a subclass of ModelBase.
:param args: Arguments to query. If None - model is used.
Keyword arguments:
:keyword project_only:
If set to True, then will do query filter with context's project_id.
if set to False or absent, then will not do query filter with context's
project_id.
:type project_only: bool
"""
if kwargs.pop("project_only", False):
kwargs["project_id"] = context.tenant
with _session_for_read() as session:
query = sqlalchemyutils.model_query(
model, session, args, **kwargs)
return query
def add_identity_filter(query, value):
"""Adds an identity filter to a query.
Filters results by ID, if supplied value is a valid integer.
Otherwise attempts to filter results by UUID.
:param query: Initial query to add filter to.
:param value: Value for filtering results by.
:return: Modified query.
"""
if strutils.is_int_like(value):
return query.filter_by(id=value)
elif uuidutils.is_uuid_like(value):
return query.filter_by(uuid=value)
else:
raise exception.InvalidIdentity(identity=value)
class Connection(api.Connection):
"""SqlAlchemy connection."""
def __init__(self):
pass
def accelerator_create(self, context, values):
if not values.get('uuid'):
values['uuid'] = uuidutils.generate_uuid()
if not values.get('description'):
values['description'] = ''
accelerator = models.Accelerator()
accelerator.update(values)
with _session_for_write() as session:
try:
session.add(accelerator)
session.flush()
except db_exc.DBDuplicateEntry:
raise exception.AcceleratorAlreadyExists(uuid=values['uuid'])
return accelerator

View File

@ -0,0 +1,108 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import alembic
from alembic import config as alembic_config
import alembic.migration as alembic_migration
from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import enginefacade
from cyborg.db.sqlalchemy import models
def _alembic_config():
path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
config = alembic_config.Config(path)
return config
def version(config=None, engine=None):
"""Current database version.
:returns: Database version
:rtype: string
"""
if engine is None:
engine = enginefacade.get_legacy_facade().get_engine()
with engine.connect() as conn:
context = alembic_migration.MigrationContext.configure(conn)
return context.get_current_revision()
def upgrade(revision, config=None):
"""Used for upgrading database.
:param version: Desired database version
:type version: string
"""
revision = revision or 'head'
config = config or _alembic_config()
alembic.command.upgrade(config, revision)
def create_schema(config=None, engine=None):
"""Create database schema from models description.
Can be used for initial installation instead of upgrade('head').
"""
if engine is None:
engine = enginefacade.get_legacy_facade().get_engine()
if version(engine=engine) is not None:
raise db_exc.DbMigrationError("DB schema is already under version"
" control. Use upgrade() instead")
models.Base.metadata.create_all(engine)
stamp('head', config=config)
def downgrade(revision, config=None):
"""Used for downgrading database.
:param version: Desired database version
:type version: string
"""
revision = revision or 'base'
config = config or _alembic_config()
return alembic.command.downgrade(config, revision)
def stamp(revision, config=None):
"""Stamps database with provided revision.
Don't run any migrations.
:param revision: Should match one from repository or head - to stamp
database with most recent revision
:type revision: string
"""
config = config or _alembic_config()
return alembic.command.stamp(config, revision=revision)
def revision(message=None, autogenerate=False, config=None):
"""Creates template for migration.
:param message: Text that will be used for migration title
:type message: string
:param autogenerate: If True - generates diff based on current database
state
:type autogenerate: bool
"""
config = config or _alembic_config()
return alembic.command.revision(config, message=message,
autogenerate=autogenerate)

View File

@ -0,0 +1,72 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""SQLAlchemy models for accelerator service."""
from oslo_db import options as db_options
from oslo_db.sqlalchemy import models
import six.moves.urllib.parse as urlparse
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, String, Integer
from sqlalchemy import schema
from cyborg.common import paths
from cyborg.conf import CONF
_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('cyborg.sqlite')
db_options.set_defaults(CONF, connection=_DEFAULT_SQL_CONNECTION)
def table_args():
engine_name = urlparse.urlparse(CONF.database.connection).scheme
if engine_name == 'mysql':
return {'mysql_engine': CONF.database.mysql_engine,
'mysql_charset': "utf8"}
return None
class CyborgBase(models.TimestampMixin, models.ModelBase):
metadata = None
def as_dict(self):
d = {}
for c in self.__table__.columns:
d[c.name] = self[c.name]
return d
Base = declarative_base(cls=CyborgBase)
class Accelerator(Base):
"""Represents the accelerators."""
__tablename__ = 'accelerators'
__table_args__ = (
schema.UniqueConstraint('uuid', name='uniq_accelerators0uuid'),
table_args()
)
id = Column(Integer, primary_key=True)
uuid = Column(String(36), nullable=False)
name = Column(String(255), nullable=False)
description = Column(String(255), nullable=True)
device_type = Column(String(255), nullable=False)
acc_type = Column(String(255), nullable=False)
acc_capability = Column(String(255), nullable=False)
vendor_id = Column(String(255), nullable=False)
product_id = Column(String(255), nullable=False)
remotable = Column(Integer, nullable=False)

View File

@ -0,0 +1,28 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# NOTE(comstud): You may scratch your head as you see code that imports
# this module and then accesses attributes for objects such as Node,
# etc, yet you do not see these attributes in here. Never fear, there is
# a little bit of magic. When objects are registered, an attribute is set
# on this module automatically, pointing to the newest/latest version of
# the object.
def register_all():
# NOTE(danms): You must make sure your object gets imported in this
# function in order for it to be registered by services that may
# need to receive it via RPC.
__import__('cyborg.objects.accelerator')

View File

@ -0,0 +1,49 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_versionedobjects import base as object_base
from cyborg.db import api as dbapi
from cyborg.objects import base
from cyborg.objects import fields as object_fields
@base.CyborgObjectRegistry.register
class Accelerator(base.CyborgObject, object_base.VersionedObjectDictCompat):
# Version 1.0: Initial version
VERSION = '1.0'
dbapi = dbapi.get_instance()
fields = {
'uuid': object_fields.UUIDField(nullable=False),
'name': object_fields.StringField(nullable=False),
'description': object_fields.StringField(nullable=True),
'device_type': object_fields.StringField(nullable=False),
'acc_type': object_fields.StringField(nullable=False),
'acc_capability': object_fields.StringField(nullable=False),
'vendor_id': object_fields.StringField(nullable=False),
'product_id': object_fields.StringField(nullable=False),
'remotable': object_fields.IntegerField(nullable=False),
}
def __init__(self, *args, **kwargs):
super(Accelerator, self).__init__(*args, **kwargs)
def create(self, context=None):
"""Create an Accelerator record in the DB."""
values = self.obj_get_changes()
db_accelerator = self.dbapi.accelerator_create(context, values)
self._from_db_object(context, self, db_accelerator)

83
cyborg/objects/base.py Normal file
View File

@ -0,0 +1,83 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Cyborg common internal object model"""
from oslo_utils import versionutils
from oslo_versionedobjects import base as object_base
from cyborg import objects
from cyborg.objects import fields as object_fields
class CyborgObjectRegistry(object_base.VersionedObjectRegistry):
def registration_hook(self, cls, index):
# NOTE(jroll): blatantly stolen from nova
# NOTE(danms): This is called when an object is registered,
# and is responsible for maintaining cyborg.objects.$OBJECT
# as the highest-versioned implementation of a given object.
version = versionutils.convert_version_to_tuple(cls.VERSION)
if not hasattr(objects, cls.obj_name()):
setattr(objects, cls.obj_name(), cls)
else:
cur_version = versionutils.convert_version_to_tuple(
getattr(objects, cls.obj_name()).VERSION)
if version >= cur_version:
setattr(objects, cls.obj_name(), cls)
class CyborgObject(object_base.VersionedObject):
"""Base class and object factory.
This forms the base of all objects that can be remoted or instantiated
via RPC. Simply defining a class that inherits from this base class
will make it remotely instantiatable. Objects should implement the
necessary "get" classmethod routines as well as "save" object methods
as appropriate.
"""
OBJ_SERIAL_NAMESPACE = 'cyborg_object'
OBJ_PROJECT_NAMESPACE = 'cyborg'
fields = {
'created_at': object_fields.DateTimeField(nullable=True),
'updated_at': object_fields.DateTimeField(nullable=True),
}
def as_dict(self):
return dict((k, getattr(self, k))
for k in self.fields
if hasattr(self, k))
@staticmethod
def _from_db_object(context, obj, db_object):
"""Converts a database entity to a formal object.
:param context: security context
:param obj: An object of the class.
:param db_object: A DB model of the object
:return: The object of the class with the database entity added
"""
for field in obj.fields:
obj[field] = db_object[field]
obj.obj_reset_changes()
return obj
class CyborgObjectSerializer(object_base.VersionedObjectSerializer):
# Base class to use for object hydration
OBJ_BASE_CLASS = CyborgObject

32
cyborg/objects/fields.py Normal file
View File

@ -0,0 +1,32 @@
# Copyright 2017 Huawei Technologies Co.,LTD.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_versionedobjects import fields as object_fields
class IntegerField(object_fields.IntegerField):
pass
class UUIDField(object_fields.UUIDField):
pass
class StringField(object_fields.StringField):
pass
class DateTimeField(object_fields.DateTimeField):
pass

View File

@ -15,3 +15,9 @@ oslo.messaging!=5.25.0,>=5.24.2 # Apache-2.0
oslo.concurrency>=3.8.0 # Apache-2.0
oslo.service>=1.10.0 # Apache-2.0
oslo.serialization!=2.19.1,>=1.10.0 # Apache-2.0
oslo.db>=4.24.0 # Apache-2.0
oslo.utils>=3.20.0 # Apache-2.0
oslo.versionedobjects>=1.17.0 # Apache-2.0
SQLAlchemy!=1.1.5,!=1.1.6,!=1.1.7,!=1.1.8,>=1.0.10 # MIT
alembic>=0.8.10 # MIT
stevedore>=1.20.0 # Apache-2.0

View File

@ -26,6 +26,11 @@ packages =
[entry_points]
console_scripts =
cyborg-api = cyborg.cmd.api:main
cyborg-conductor = cyborg.cmd.conductor:main
cyborg-dbsync = cyborg.cmd.dbsync:main
cyborg.database.migration_backend =
sqlalchemy = cyborg.db.sqlalchemy.migration
[build_sphinx]
source-dir = doc/source