From a3225f937746d89a0a412be7607497d653462860 Mon Sep 17 00:00:00 2001 From: Graham Hayes Date: Tue, 15 Dec 2015 16:46:40 +0000 Subject: [PATCH] Add Basic utils / requirements Change-Id: Ifb921c92223454c9d0831293c8fe13dd321edc03 --- .gitignore | 2 +- etc/kosmos/kosmos.conf | 8 ++ etc/kosmos/paste.ini | 10 ++ etc/kosmos/policy.json | 3 + kosmos/__init__.py | 2 +- kosmos/api/__init__.py | 29 +++++ kosmos/api/service.py | 28 ++++ kosmos/api/v0/__init__.py | 32 +++++ kosmos/api/v0/app.py | 41 ++++++ kosmos/api/v0/controllers/__init__.py | 0 kosmos/api/v0/controllers/errors.py | 28 ++++ kosmos/api/v0/controllers/gslbs.py | 25 ++++ kosmos/api/v0/controllers/monitors.py | 25 ++++ kosmos/api/v0/controllers/pools.py | 25 ++++ kosmos/api/v0/controllers/root.py | 33 +++++ kosmos/api/versions.py | 48 +++++++ kosmos/cmd/__init__.py | 0 kosmos/cmd/all.py | 53 ++++++++ kosmos/cmd/api.py | 43 +++++++ kosmos/cmd/conductor.py | 35 +++++ kosmos/cmd/engine.py | 36 ++++++ kosmos/common/config.py | 25 +++- kosmos/common/context.py | 63 +++++++++ kosmos/common/exceptions.py | 58 +++++++++ kosmos/common/policy.py | 102 +++++++++++++++ kosmos/common/rpc.py | 179 ++++++++++++++++++++++++++ kosmos/common/utils.py | 1 - kosmos/conductor/__init__.py | 25 ++++ kosmos/conductor/service.py | 37 ++++++ kosmos/engine/__init__.py | 25 ++++ kosmos/engine/service.py | 37 ++++++ kosmos/service.py | 121 +++++++++++++++++ requirements.txt | 1 + setup.cfg | 12 ++ 34 files changed, 1185 insertions(+), 7 deletions(-) create mode 100644 etc/kosmos/kosmos.conf create mode 100644 etc/kosmos/paste.ini create mode 100644 etc/kosmos/policy.json create mode 100644 kosmos/api/__init__.py create mode 100644 kosmos/api/service.py create mode 100644 kosmos/api/v0/__init__.py create mode 100644 kosmos/api/v0/app.py create mode 100644 kosmos/api/v0/controllers/__init__.py create mode 100644 kosmos/api/v0/controllers/errors.py create mode 100644 kosmos/api/v0/controllers/gslbs.py create mode 100644 kosmos/api/v0/controllers/monitors.py create mode 100644 kosmos/api/v0/controllers/pools.py create mode 100644 kosmos/api/v0/controllers/root.py create mode 100644 kosmos/api/versions.py create mode 100644 kosmos/cmd/__init__.py create mode 100644 kosmos/cmd/all.py create mode 100644 kosmos/cmd/api.py create mode 100644 kosmos/cmd/conductor.py create mode 100644 kosmos/cmd/engine.py create mode 100644 kosmos/common/context.py create mode 100644 kosmos/common/exceptions.py create mode 100644 kosmos/common/policy.py create mode 100644 kosmos/common/rpc.py create mode 100644 kosmos/conductor/__init__.py create mode 100644 kosmos/conductor/service.py create mode 100644 kosmos/engine/__init__.py create mode 100644 kosmos/engine/service.py create mode 100644 kosmos/service.py diff --git a/.gitignore b/.gitignore index 4121058..d11634a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ *.py[cod] - +etc/kosmos/kosmos.conf # C extensions *.so diff --git a/etc/kosmos/kosmos.conf b/etc/kosmos/kosmos.conf new file mode 100644 index 0000000..a9499c7 --- /dev/null +++ b/etc/kosmos/kosmos.conf @@ -0,0 +1,8 @@ +[DEFAULT] + +[service:conductor] + +[service:engine] + +[service:api] + diff --git a/etc/kosmos/paste.ini b/etc/kosmos/paste.ini new file mode 100644 index 0000000..bca3b57 --- /dev/null +++ b/etc/kosmos/paste.ini @@ -0,0 +1,10 @@ +[composite:kosmos] +use = egg:Paste#urlmap +/ = kosmos_versions +/v0 = kosmos_v0 + +[app:kosmos_versions] +paste.app_factory = kosmos.api.versions:factory + +[app:kosmos_v0] +paste.app_factory = kosmos.api.v0:factory diff --git a/etc/kosmos/policy.json b/etc/kosmos/policy.json new file mode 100644 index 0000000..8a28d02 --- /dev/null +++ b/etc/kosmos/policy.json @@ -0,0 +1,3 @@ +{ + "default": "@" +} diff --git a/kosmos/__init__.py b/kosmos/__init__.py index 39cec1e..3bca9d5 100644 --- a/kosmos/__init__.py +++ b/kosmos/__init__.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# Copyright 2015 Hewlett Packard Enterprise Development LP # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,6 +15,5 @@ import pbr.version - __version__ = pbr.version.VersionInfo( 'kosmos').version_string() diff --git a/kosmos/api/__init__.py b/kosmos/api/__init__.py new file mode 100644 index 0000000..757eb65 --- /dev/null +++ b/kosmos/api/__init__.py @@ -0,0 +1,29 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +cfg.CONF.register_group(cfg.OptGroup( + name='service:api', title="Configuration for Engine Service" +)) + +cfg.CONF.register_opts([ + cfg.IntOpt('workers', default=3, + help='Number of central worker processes to spawn'), + cfg.IntOpt('threads', default=1000, + help='Number of central greenthreads to spawn'), + cfg.IPOpt('bind_host', default="0.0.0.0", + help='IP address for the API to listen to'), + cfg.PortOpt('bind_port', default=9100, + help='Port for the API to listen to') +], group='service:api') diff --git a/kosmos/api/service.py b/kosmos/api/service.py new file mode 100644 index 0000000..aaf9fc3 --- /dev/null +++ b/kosmos/api/service.py @@ -0,0 +1,28 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg +from oslo_log import log as logging + +from kosmos import service +from oslo_service.wsgi import Server + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class Service(Server, service.Service): + + @property + def service_name(self): + return 'api' diff --git a/kosmos/api/v0/__init__.py b/kosmos/api/v0/__init__.py new file mode 100644 index 0000000..d50dc20 --- /dev/null +++ b/kosmos/api/v0/__init__.py @@ -0,0 +1,32 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from pecan import deploy + + +def factory(global_config, **local_conf): + conf = { + 'app': { + 'root': 'kosmos.api.v0.controllers.root.RootController', + 'modules': ['kosmos.api.v0'], + 'errors': { + 404: '/errors/not_found', + 405: '/errors/method_not_allowed', + '__force_dict__': True + } + } + } + + app = deploy.deploy(conf) + + return app diff --git a/kosmos/api/v0/app.py b/kosmos/api/v0/app.py new file mode 100644 index 0000000..4eceea4 --- /dev/null +++ b/kosmos/api/v0/app.py @@ -0,0 +1,41 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import pecan +import pecan.deploy +from oslo_config import cfg +from oslo_log import log as logging + + +LOG = logging.getLogger(__name__) + +cfg.CONF.register_opts([ + cfg.BoolOpt('pecan_debug', default=False, + help='Pecan HTML Debug Interface'), +], group='service:api') + + +def setup_app(pecan_config): + config = dict(pecan_config) + + config['app']['debug'] = cfg.CONF['service:api'].pecan_debug + + pecan.configuration.set_config(config, overwrite=True) + + app = pecan.make_app( + pecan_config.app.root, + debug=getattr(pecan_config.app, 'debug', False), + force_canonical=getattr(pecan_config.app, 'force_canonical', True) + ) + + return app diff --git a/kosmos/api/v0/controllers/__init__.py b/kosmos/api/v0/controllers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kosmos/api/v0/controllers/errors.py b/kosmos/api/v0/controllers/errors.py new file mode 100644 index 0000000..bed1339 --- /dev/null +++ b/kosmos/api/v0/controllers/errors.py @@ -0,0 +1,28 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from pecan import expose +from oslo_log import log as logging + +LOG = logging.getLogger(__name__) + + +class ErrorsController(object): + + @expose('json') + def not_found(self): + return dict(status=404, message="not_found") + + @expose('json') + def method_not_allowed(self): + return dict(status=404, message="method_not_allowed") diff --git a/kosmos/api/v0/controllers/gslbs.py b/kosmos/api/v0/controllers/gslbs.py new file mode 100644 index 0000000..ce49190 --- /dev/null +++ b/kosmos/api/v0/controllers/gslbs.py @@ -0,0 +1,25 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_log import log as logging + +from pecan import expose + +LOG = logging.getLogger(__name__) + + +class GSLBSController(object): + + @expose(generic=True) + def index(self): + return dict() diff --git a/kosmos/api/v0/controllers/monitors.py b/kosmos/api/v0/controllers/monitors.py new file mode 100644 index 0000000..4a06f0f --- /dev/null +++ b/kosmos/api/v0/controllers/monitors.py @@ -0,0 +1,25 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_log import log as logging + +from pecan import expose + +LOG = logging.getLogger(__name__) + + +class MonitorsController(object): + + @expose(generic=True) + def index(self): + return dict() diff --git a/kosmos/api/v0/controllers/pools.py b/kosmos/api/v0/controllers/pools.py new file mode 100644 index 0000000..daf84e7 --- /dev/null +++ b/kosmos/api/v0/controllers/pools.py @@ -0,0 +1,25 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_log import log as logging + +from pecan import expose + +LOG = logging.getLogger(__name__) + + +class PoolsController(object): + + @expose(generic=True) + def index(self): + return dict() diff --git a/kosmos/api/v0/controllers/root.py b/kosmos/api/v0/controllers/root.py new file mode 100644 index 0000000..d60548b --- /dev/null +++ b/kosmos/api/v0/controllers/root.py @@ -0,0 +1,33 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_log import log as logging +from pecan import expose +from kosmos.api.v0.controllers import errors +from kosmos.api.v0.controllers import gslbs +from kosmos.api.v0.controllers import pools +from kosmos.api.v0.controllers import monitors + +LOG = logging.getLogger(__name__) + + +class RootController(object): + + @expose(generic=True) + def index(self): + return dict() + + gslbs = gslbs.GSLBSController() + monitors = monitors.MonitorsController() + pools = pools.PoolsController() + errors = errors.ErrorsController() diff --git a/kosmos/api/versions.py b/kosmos/api/versions.py new file mode 100644 index 0000000..068ee23 --- /dev/null +++ b/kosmos/api/versions.py @@ -0,0 +1,48 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import json + +from oslo_log import log as logging + + +LOG = logging.getLogger(__name__) + + +def factory(global_config, **local_conf): + + def versioned_app(environ, response): + + versions = { + 'versions': [ + { + 'id': 'v0', + 'status': 'EXPERIMENTAL', + 'links': [ + { + 'rel': 'self', + 'href': 'http://127.0.0.1:9100/v0' + } + ] + } + ] + } + + versions_str = json.dumps(versions) + + status = '200 OK' + content_type = ('Content-Type', 'application/json') + response(status, [content_type]) + return versions_str + + return versioned_app diff --git a/kosmos/cmd/__init__.py b/kosmos/cmd/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/kosmos/cmd/all.py b/kosmos/cmd/all.py new file mode 100644 index 0000000..ed6b9af --- /dev/null +++ b/kosmos/cmd/all.py @@ -0,0 +1,53 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import sys + +from oslo_config import cfg +from oslo_service import service +from oslo_service.wsgi import Loader + +from kosmos.engine import service as engine +from kosmos.conductor import service as conductor +from kosmos.api import service as api +from kosmos.common import config + +CONF = cfg.CONF + + +def main(): + + config.setup_logging(CONF) + config.init(sys.argv) + + process_launcher = service.ProcessLauncher(CONF) + process_launcher.launch_service( + engine.Service(threads=CONF['service:engine'].threads), + workers=CONF['service:engine'].workers + ) + process_launcher.launch_service( + conductor.Service(threads=CONF['service:conductor'].threads), + workers=CONF['service:conductor'].workers) + + process_launcher.launch_service( + api.Service( + CONF, + 'API', + Loader(CONF).load_app('kosmos'), + host=CONF['service:api'].bind_host, + port=CONF['service:api'].bind_port + ), + + workers=CONF['service:api'].workers) + + process_launcher.wait() diff --git a/kosmos/cmd/api.py b/kosmos/cmd/api.py new file mode 100644 index 0000000..52bfaa7 --- /dev/null +++ b/kosmos/cmd/api.py @@ -0,0 +1,43 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import sys + +from oslo_config import cfg +from oslo_service import service +from oslo_service.wsgi import Loader + +from kosmos.api import service as api +from kosmos.common import config + +CONF = cfg.CONF + + +def main(): + + config.setup_logging(CONF) + config.init(sys.argv) + + process_launcher = service.ProcessLauncher(CONF) + process_launcher.launch_service( + api.Service( + CONF, + 'API', + Loader(CONF).load_app('kosmos'), + host=CONF['service:api'].bind_host, + port=CONF['service:api'].bind_port + ), + + workers=CONF['service:api'].workers) + + process_launcher.wait() diff --git a/kosmos/cmd/conductor.py b/kosmos/cmd/conductor.py new file mode 100644 index 0000000..d91b686 --- /dev/null +++ b/kosmos/cmd/conductor.py @@ -0,0 +1,35 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import sys + +from oslo_config import cfg +from oslo_service import service + +from kosmos.conductor import service as conductor +from kosmos.common import config + +CONF = cfg.CONF + + +def main(): + + config.setup_logging(CONF) + config.init(sys.argv) + + process_launcher = service.ProcessLauncher(CONF) + process_launcher.launch_service( + conductor.Service(threads=CONF['service:conductor'].threads), + workers=CONF['service:conductor'].workers) + + process_launcher.wait() diff --git a/kosmos/cmd/engine.py b/kosmos/cmd/engine.py new file mode 100644 index 0000000..a42f0fc --- /dev/null +++ b/kosmos/cmd/engine.py @@ -0,0 +1,36 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import sys + +from oslo_config import cfg +from oslo_service import service + +from kosmos.engine import service as engine +from kosmos.common import config + +CONF = cfg.CONF + + +def main(): + + config.setup_logging(CONF) + config.init(sys.argv) + + process_launcher = service.ProcessLauncher(CONF) + process_launcher.launch_service( + engine.Service(threads=CONF['service:engine'].threads), + workers=CONF['service:engine'].workers + ) + + process_launcher.wait() diff --git a/kosmos/common/config.py b/kosmos/common/config.py index 1feb3e3..d24f5fc 100644 --- a/kosmos/common/config.py +++ b/kosmos/common/config.py @@ -1,4 +1,6 @@ # Copyright 2011 VMware, Inc., 2014 A10 Networks +# Copyright 2015 Hewlett Packard Enterprise Development LP +# # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -16,6 +18,7 @@ """ Routines for configuring Kosmos """ +import os from oslo_config import cfg from oslo_db import options as db_options @@ -44,6 +47,16 @@ core_cli_opts = [] cfg.CONF.register_opts(core_opts) cfg.CONF.register_cli_opts(core_cli_opts) +cfg.CONF.register_opts([ + cfg.StrOpt( + 'pybasedir', + default=os.path.abspath(os.path.join(os.path.dirname(__file__), + '../')), + help='Directory where the kosmos python module is installed' + ), + cfg.StrOpt('state-path', default='/var/lib/kosmos', + help='Top-level directory for maintaining kosmos\'s state'), +]) # Ensure that the control exchange is set correctly messaging.set_transport_defaults(control_exchange='kosmos') @@ -58,10 +71,9 @@ db_options.set_defaults(cfg.CONF, logging.register_options(cfg.CONF) -def init(args, **kwargs): - cfg.CONF(args=args, project='kosmos', - version='%%prog %s' % version.version_info.release_string(), - **kwargs) +def init(args): + cfg.CONF(args=args[1:], project='kosmos', + version='%%prog %s' % version.version_info.release_string()) def setup_logging(conf): @@ -72,3 +84,8 @@ def setup_logging(conf): product_name = "kosmos" logging.setup(conf, product_name) LOG.info(_LI("Logging enabled!")) + + +def read_config(prog, argv): + logging.register_options(cfg.CONF) + cfg.CONF(argv[1:], project='kosmos', prog=prog) diff --git a/kosmos/common/context.py b/kosmos/common/context.py new file mode 100644 index 0000000..726de55 --- /dev/null +++ b/kosmos/common/context.py @@ -0,0 +1,63 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import copy + +from oslo_context import context +from oslo_log import log as logging + +LOG = logging.getLogger(__name__) + + +class KosmosContext(context.RequestContext): + + def __init__(self, auth_token=None, user=None, tenant=None, domain=None, + user_domain=None, project_domain=None, is_admin=False, + read_only=False, show_deleted=False, request_id=None, + resource_uuid=None, overwrite=True, roles=None, + service_catalog=None): + + # NOTE: user_identity may be passed in, but will be silently dropped as + # it is a generated field based on several others. + super(KosmosContext, self).__init__( + auth_token=auth_token, + user=user, + tenant=tenant, + domain=domain, + user_domain=user_domain, + project_domain=project_domain, + is_admin=is_admin, + read_only=read_only, + show_deleted=show_deleted, + request_id=request_id, + resource_uuid=resource_uuid, + overwrite=overwrite) + + self.roles = roles or [] + self.service_catalog = service_catalog + + def deepcopy(self): + d = self.to_dict() + return self.from_dict(d) + + def to_dict(self): + d = super(KosmosContext, self).to_dict() + return copy.deepcopy(d) + + @classmethod + def from_dict(cls, values): + return cls(**values) + + +def get_current(): + return context.get_current() diff --git a/kosmos/common/exceptions.py b/kosmos/common/exceptions.py new file mode 100644 index 0000000..eb55eeb --- /dev/null +++ b/kosmos/common/exceptions.py @@ -0,0 +1,58 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import six + + +class Base(Exception): + error_code = 500 + error_type = None + error_message = None + errors = None + + def __init__(self, *args, **kwargs): + self.errors = kwargs.pop('errors', None) + self.object = kwargs.pop('object', None) + + super(Base, self).__init__(*args, **kwargs) + + if len(args) > 0 and isinstance(args[0], six.string_types): + self.error_message = args[0] + + +# 500 Errors +class NotImplemented(Base, NotImplementedError): + pass + + +class ConfigurationError(Base): + error_type = 'configuration_error' + + +# 400 Errors +class BadRequest(Base): + error_code = 400 + error_type = 'bad_request' + expected = True + + +class Forbidden(Base): + error_code = 403 + error_type = 'forbidden' + expected = True + + +class NotFound(Base): + error_code = 404 + error_type = 'not_found' + expected = True diff --git a/kosmos/common/policy.py b/kosmos/common/policy.py new file mode 100644 index 0000000..a9e4310 --- /dev/null +++ b/kosmos/common/policy.py @@ -0,0 +1,102 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from kosmos._i18n import _ +from kosmos._i18n import _LI +from kosmos.common import config +from kosmos.common import exceptions +from oslo_config import cfg +from oslo_log import log as logging +from oslo_policy import opts +from oslo_policy import policy + +CONF = cfg.CONF + +# Add the default policy opts +opts.set_defaults(CONF) + +LOG = logging.getLogger(__name__) + + +_ENFORCER = None + + +def reset(): + global _ENFORCER + if _ENFORCER: + _ENFORCER.clear() + _ENFORCER = None + + +def set_rules(data, default_rule=None, overwrite=True): + default_rule = default_rule or cfg.CONF.policy_default_rule + if not _ENFORCER: + LOG.debug("Enforcer not present, recreating at rules stage.") + init() + + if default_rule: + _ENFORCER.default_rule = default_rule + + msg = "Loading rules %s, default: %s, overwrite: %s" + LOG.debug(msg, data, default_rule, overwrite) + + if isinstance(data, dict): + rules = policy.Rules.from_dict(data, default_rule) + else: + rules = policy.Rules.load_json(data, default_rule) + + _ENFORCER.set_rules(rules, overwrite=overwrite) + + +def init(default_rule=None): + policy_files = config.find_config(CONF['oslo_policy'].policy_file) + + if len(policy_files) == 0: + msg = 'Unable to determine appropriate policy json file' + raise exceptions.ConfigurationError(msg) + + LOG.info(_LI('Using policy_file found at: %s'), policy_files[0]) + + with open(policy_files[0]) as fh: + policy_string = fh.read() + rules = policy.Rules.load_json(policy_string, default_rule=default_rule) + + global _ENFORCER + if not _ENFORCER: + LOG.debug("Enforcer is not present, recreating.") + _ENFORCER = policy.Enforcer(CONF) + + _ENFORCER.set_rules(rules) + + +def check(rule, ctxt, target=None, do_raise=True, exc=exceptions.Forbidden): + creds = ctxt.to_dict() + target = target or {} + try: + result = _ENFORCER.enforce(rule, target, creds, do_raise, exc) + except Exception: + result = False + raise + else: + return result + finally: + extra = {'policy': {'rule': rule, 'target': target}} + + if result: + LOG.info(_("Policy check succeeded for rule '%(rule)s' " + "on target %(target)s"), + {'rule': rule, 'target': repr(target)}, extra=extra) + else: + LOG.info(_("Policy check failed for rule '%(rule)s' " + "on target %(target)s"), + {'rule': rule, 'target': repr(target)}, extra=extra) diff --git a/kosmos/common/rpc.py b/kosmos/common/rpc.py new file mode 100644 index 0000000..ef5bcf6 --- /dev/null +++ b/kosmos/common/rpc.py @@ -0,0 +1,179 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'init', + 'cleanup', + 'set_defaults', + 'add_extra_exmods', + 'clear_extra_exmods', + 'get_allowed_exmods', + 'RequestContextSerializer', + 'get_client', + 'get_server', + 'get_notifier', +] + +import kosmos.common.context +import kosmos.common.exceptions +import oslo_messaging as messaging +from kosmos.objects.base import KosmosObject +from oslo_config import cfg +from oslo_messaging import server as msg_server +from oslo_messaging.rpc import dispatcher as rpc_dispatcher +from oslo_serialization import jsonutils +from oslo_versionedobjects.base import VersionedObjectSerializer + +CONF = cfg.CONF +TRANSPORT = None +NOTIFIER = None + + +# NOTE: Additional entries to kosmos.exceptions goes here. +CONF.register_opts([ + cfg.ListOpt( + 'allowed_remote_exmods', + default=[], + help="Additional modules that contains allowed RPC exceptions.", + deprecated_name='allowed_rpc_exception_modules') +]) +ALLOWED_EXMODS = [ + kosmos.common.exceptions.__name__, +] +EXTRA_EXMODS = [] + + +def init(conf): + global TRANSPORT, NOTIFIER + exmods = get_allowed_exmods() + TRANSPORT = messaging.get_transport(conf, + allowed_remote_exmods=exmods) + + serializer = RequestContextSerializer(JsonPayloadSerializer()) + NOTIFIER = messaging.Notifier(TRANSPORT, serializer=serializer) + + +def initialized(): + return None not in [TRANSPORT, NOTIFIER] + + +def cleanup(): + global TRANSPORT, NOTIFIER + assert TRANSPORT is not None + assert NOTIFIER is not None + TRANSPORT.cleanup() + TRANSPORT = NOTIFIER = None + + +def set_defaults(control_exchange): + messaging.set_transport_defaults(control_exchange) + + +def add_extra_exmods(*args): + EXTRA_EXMODS.extend(args) + + +def clear_extra_exmods(): + del EXTRA_EXMODS[:] + + +def get_allowed_exmods(): + return ALLOWED_EXMODS + EXTRA_EXMODS + CONF.allowed_remote_exmods + + +class JsonPayloadSerializer(messaging.NoOpSerializer): + @staticmethod + def serialize_entity(context, entity): + return jsonutils.to_primitive(entity, convert_instances=True) + + +class KosmosObjectSerializer(VersionedObjectSerializer): + + OBJ_BASE_CLASS = KosmosObject + + +class RequestContextSerializer(messaging.Serializer): + + def __init__(self, base): + self._base = base + + def serialize_entity(self, context, entity): + if not self._base: + return entity + return self._base.serialize_entity(context, entity) + + def deserialize_entity(self, context, entity): + if not self._base: + return entity + return self._base.deserialize_entity(context, entity) + + def serialize_context(self, context): + return context.to_dict() + + def deserialize_context(self, context): + return kosmos.common.context.KosmosContext.from_dict(context) + + +class RPCDispatcher(rpc_dispatcher.RPCDispatcher): + def _dispatch(self, *args, **kwds): + try: + return super(RPCDispatcher, self)._dispatch(*args, **kwds) + except Exception as e: + if getattr(e, 'expected', False): + raise rpc_dispatcher.ExpectedException() + else: + raise + + +def get_transport_url(url_str=None): + return messaging.TransportURL.parse(CONF, url_str) + + +def get_client(target, version_cap=None, serializer=None): + assert TRANSPORT is not None + if serializer is None: + serializer = KosmosObjectSerializer() + serializer = RequestContextSerializer(serializer) + return messaging.RPCClient(TRANSPORT, + target, + version_cap=version_cap, + serializer=serializer) + + +def get_server(target, endpoints, serializer=None): + assert TRANSPORT is not None + if serializer is None: + serializer = KosmosObjectSerializer() + serializer = RequestContextSerializer(serializer) + + dispatcher = RPCDispatcher(target, endpoints, serializer) + return msg_server.MessageHandlingServer(TRANSPORT, dispatcher, 'eventlet') + + +def get_listener(targets, endpoints, serializer=None): + assert TRANSPORT is not None + if serializer is None: + serializer = JsonPayloadSerializer() + return messaging.get_notification_listener(TRANSPORT, + targets, + endpoints, + executor='eventlet', + serializer=serializer) + + +def get_notifier(service=None, host=None, publisher_id=None): + assert NOTIFIER is not None + if not publisher_id: + publisher_id = "%s.%s" % (service, host or CONF.host) + return NOTIFIER.prepare(publisher_id=publisher_id) diff --git a/kosmos/common/utils.py b/kosmos/common/utils.py index c322757..a479980 100644 --- a/kosmos/common/utils.py +++ b/kosmos/common/utils.py @@ -23,7 +23,6 @@ import hashlib import random import socket - from oslo_log import log as logging from oslo_utils import excutils diff --git a/kosmos/conductor/__init__.py b/kosmos/conductor/__init__.py new file mode 100644 index 0000000..0302251 --- /dev/null +++ b/kosmos/conductor/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +cfg.CONF.register_group(cfg.OptGroup( + name='service:conductor', title="Configuration for Engine Service" +)) + +cfg.CONF.register_opts([ + cfg.IntOpt('workers', default=3, + help='Number of central worker processes to spawn'), + cfg.IntOpt('threads', default=1000, + help='Number of central greenthreads to spawn'), +], group='service:conductor') diff --git a/kosmos/conductor/service.py b/kosmos/conductor/service.py new file mode 100644 index 0000000..d25de80 --- /dev/null +++ b/kosmos/conductor/service.py @@ -0,0 +1,37 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg +import oslo_messaging as messaging +from oslo_log import log as logging + +from kosmos import service + + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class Service(service.RPCService, service.Service): + """ + API version history: + + 1.0 - Initial version + """ + RPC_API_VERSION = '1.0' + + target = messaging.Target(version=RPC_API_VERSION) + + @property + def service_name(self): + return 'conductor' diff --git a/kosmos/engine/__init__.py b/kosmos/engine/__init__.py new file mode 100644 index 0000000..1bb0186 --- /dev/null +++ b/kosmos/engine/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg + +cfg.CONF.register_group(cfg.OptGroup( + name='service:engine', title="Configuration for Engine Service" +)) + +cfg.CONF.register_opts([ + cfg.IntOpt('workers', default=3, + help='Number of central worker processes to spawn'), + cfg.IntOpt('threads', default=1000, + help='Number of central greenthreads to spawn'), +], group='service:engine') diff --git a/kosmos/engine/service.py b/kosmos/engine/service.py new file mode 100644 index 0000000..a00dd9e --- /dev/null +++ b/kosmos/engine/service.py @@ -0,0 +1,37 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from oslo_config import cfg +import oslo_messaging as messaging +from oslo_log import log as logging + +from kosmos import service + + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + + +class Service(service.RPCService, service.Service): + """ + API version history: + + 1.0 - Initial version + """ + RPC_API_VERSION = '1.0' + + target = messaging.Target(version=RPC_API_VERSION) + + @property + def service_name(self): + return 'engine' diff --git a/kosmos/service.py b/kosmos/service.py new file mode 100644 index 0000000..fdf5eb2 --- /dev/null +++ b/kosmos/service.py @@ -0,0 +1,121 @@ +# Copyright 2015 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import abc +import oslo_messaging as messaging +import six +from kosmos import policy +from kosmos import version +from kosmos._i18n import _ +from kosmos.common import rpc +from oslo_config import cfg +from oslo_log import log as logging +from oslo_service import service + + +CONF = cfg.CONF + +LOG = logging.getLogger(__name__) + + +@six.add_metaclass(abc.ABCMeta) +class Service(service.Service): + """ + Service class to be shared inside Kosmos + """ + def __init__(self, threads=None): + threads = threads or 1000 + + super(Service, self).__init__(threads) + + self._host = CONF.host + self._service_config = CONF['service:%s' % self.service_name] + + policy.init() + + if not rpc.initialized(): + rpc.init(CONF) + + @abc.abstractproperty + def service_name(self): + pass + + def start(self): + super(Service, self).start() + + LOG.info(_('Starting %(name)s service (version: %(version)s)'), + {'name': self.service_name, + 'version': version.version_info.version_string()}) + + def stop(self): + LOG.info(_('Stopping %(name)s service'), {'name': self.service_name}) + + super(Service, self).stop() + + +class RPCService(object): + """ + RPC Service mixin used by all Kosmos RPC Servers + """ + def __init__(self, *args, **kwargs): + super(RPCService, self).__init__(*args, **kwargs) + + LOG.debug("Creating RPC Server on topic '%s'" % self._rpc_topic) + self._rpc_server = rpc.get_server( + messaging.Target(topic=self._rpc_topic, server=self._host), + self._rpc_endpoints) + + @property + def _rpc_endpoints(self): + return [self] + + @property + def _rpc_topic(self): + return self.service_name + + def start(self): + super(RPCService, self).start() + + LOG.debug("Starting RPC server on topic '%s'" % self._rpc_topic) + self._rpc_server.start() + + # TODO(kiall): This probably belongs somewhere else, maybe the base + # Service class? + self.notifier = rpc.get_notifier(self.service_name) + + for e in self._rpc_endpoints: + if e != self and hasattr(e, 'start'): + e.start() + + def stop(self): + LOG.debug("Stopping RPC server on topic '%s'" % self._rpc_topic) + + for e in self._rpc_endpoints: + if e != self and hasattr(e, 'stop'): + e.stop() + + # Try to shut the connection down, but if we get any sort of + # errors, go ahead and ignore them.. as we're shutting down anyway + try: + self._rpc_server.stop() + except Exception: + pass + + super(RPCService, self).stop() + + def wait(self): + for e in self._rpc_endpoints: + if e != self and hasattr(e, 'wait'): + e.wait() + + super(RPCService, self).wait() diff --git a/requirements.txt b/requirements.txt index bdf6896..a27ee9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ oslo.serialization>=1.10.0 # Apache-2.0 oslo.service>=1.0.0 # Apache-2.0 oslo.utils>=3.4.0 # Apache-2.0 keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0 +pecan>=1.0.0 # BSD diff --git a/setup.cfg b/setup.cfg index ee5c0b2..913f0b5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,6 +22,10 @@ classifier = packages = kosmos +data_files = + etc/kosmos = + etc/kosmos.conf + [build_sphinx] source-dir = doc/source build-dir = doc/build @@ -43,3 +47,11 @@ input_file = kosmos/locale/kosmos.pot keywords = _ gettext ngettext l_ lazy_gettext mapping_file = babel.cfg output_file = kosmos/locale/kosmos.pot + +[entry_points] +console_scripts = + kosmos-all = kosmos.cmd.all:main + kosmos-api = kosmos.cmd.api:main + kosmos-engine = kosmos.cmd.engine:main + kosmos-conductor = kosmos.cmd.conductor:main +