diff --git a/devstack/README.rst b/devstack/README.rst new file mode 100644 index 0000000..9f86a1a --- /dev/null +++ b/devstack/README.rst @@ -0,0 +1,9 @@ +==================== +DevStack Integration +==================== + +This directory contains the files necessary to integrate gyan with devstack. + +Refer the quickstart guide at +https://docs.openstack.org/gyan/latest/contributor/quickstart.html +for more information on using devstack and gyan. diff --git a/devstack/lib/gyan b/devstack/lib/gyan new file mode 100644 index 0000000..caabb7e --- /dev/null +++ b/devstack/lib/gyan @@ -0,0 +1,325 @@ +#!/bin/bash +# +# lib/gyan +# Functions to control the configuration and operation of the **gyan** service + +# Dependencies: +# +# - ``functions`` file +# - ``DEST``, ``DATA_DIR``, ``STACK_USER`` must be defined +# - ``SERVICE_{TENANT_NAME|PASSWORD}`` must be defined + +# ``stack.sh`` calls the entry points in this order: +# +# - install_gyan +# - configure_gyan +# - create_gyan_conf +# - create_gyan_accounts +# - init_gyan +# - start_gyan +# - stop_gyan +# - cleanup_gyan + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set +o xtrace + + +# Defaults +# -------- + +# Set up default directories +GYAN_REPO=${GYAN_REPO:-${GIT_BASE}/openstack/gyan.git} +GYAN_BRANCH=${GYAN_BRANCH:-master} +GYAN_DIR=$DEST/gyan + +GITREPO["python-gyanclient"]=${GYANCLIENT_REPO:-${GIT_BASE}/openstack/python-gyanclient.git} +GITBRANCH["python-gyanclient"]=${GYANCLIENT_BRANCH:-master} +GITDIR["python-gyanclient"]=$DEST/python-gyanclient + +GYAN_STATE_PATH=${GYAN_STATE_PATH:=$DATA_DIR/gyan} +GYAN_AUTH_CACHE_DIR=${GYAN_AUTH_CACHE_DIR:-/var/cache/gyan} + +GYAN_CONF_DIR=/etc/gyan +GYAN_CONF=$GYAN_CONF_DIR/gyan.conf +GYAN_API_PASTE=$GYAN_CONF_DIR/api-paste.ini + +if is_ssl_enabled_service "gyan" || is_service_enabled tls-proxy; then + GYAN_SERVICE_PROTOCOL="https" +fi + +# Toggle for deploying GYAN-API under a wsgi server +GYAN_USE_UWSGI=${GYAN_USE_UWSGI:-True} + + +# Public facing bits +GYAN_SERVICE_HOST=${GYAN_SERVICE_HOST:-$SERVICE_HOST} +GYAN_SERVICE_PORT=${GYAN_SERVICE_PORT:-8517} +GYAN_SERVICE_PORT_INT=${GYAN_SERVICE_PORT_INT:-18517} +GYAN_SERVICE_PROTOCOL=${GYAN_SERVICE_PROTOCOL:-$SERVICE_PROTOCOL} + +GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD=${GYAN_TRUSTEE_DOMAIN_ADMIN_PASSWORD:-secret} + +# Support entry points installation of console scripts +if [[ -d $GYAN_DIR/bin ]]; then + GYAN_BIN_DIR=$GYAN_DIR/bin +else + GYAN_BIN_DIR=$(get_python_exec_prefix) +fi + +GYAN_UWSGI=$GYAN_BIN_DIR/gyan-api-wsgi +GYAN_UWSGI_CONF=$GYAN_CONF_DIR/gyan-api-uwsgi.ini + +GYAN_DB_TYPE=${GYAN_DB_TYPE:-sql} + +if is_ubuntu; then + UBUNTU_RELEASE_BASE_NUM=`lsb_release -r | awk '{print $2}' | cut -d '.' -f 1` +fi + +# Functions +# --------- + +# cleanup_gyan() - Remove residual data files, anything left over from previous +# runs that a clean run would need to clean up +function cleanup_gyan { + sudo rm -rf $GYAN_STATE_PATH $GYAN_AUTH_CACHE_DIR + + remove_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI" +} + +# configure_gyan() - Set config files, create data dirs, etc +function configure_gyan { + # Put config files in ``/etc/gyan`` for everyone to find + if [[ ! -d $GYAN_CONF_DIR ]]; then + sudo mkdir -p $GYAN_CONF_DIR + sudo chown $STACK_USER $GYAN_CONF_DIR + fi + + configure_rootwrap gyan + + # Rebuild the config file from scratch + create_gyan_conf + + create_api_paste_conf + + write_uwsgi_config "$GYAN_UWSGI_CONF" "$GYAN_UWSGI" "/ml-infra" + + if [[ "$USE_PYTHON3" = "True" ]]; then + # Switch off glance->swift communication as swift fails under py3.x + iniset /etc/glance/glance-api.conf glance_store default_store file + fi +} + + +# create_gyan_accounts() - Set up common required GYAN accounts +# +# Project User Roles +# ------------------------------------------------------------------ +# SERVICE_PROJECT_NAME gyan service +function create_gyan_accounts { + + create_service_user "gyan" "admin" + + if is_service_enabled gyan-api; then + + local gyan_api_url + if [[ "$GYAN_USE_UWSGI" == "True" ]]; then + gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST/ml-infra" + else + gyan_api_url="$GYAN_SERVICE_PROTOCOL://$GYAN_SERVICE_HOST:$GYAN_SERVICE_PORT" + fi + + local gyan_service=$(get_or_create_service "gyan" \ + "ml-infra" "ML Infra As Service") + get_or_create_endpoint $gyan_service \ + "$REGION_NAME" \ + "$gyan_api_url/v1" \ + "$gyan_api_url/v1" \ + "$gyan_api_url/v1" + fi + +} + +# create_gyan_conf() - Create a new gyan.conf file +function create_gyan_conf { + + # (Re)create ``gyan.conf`` + rm -f $GYAN_CONF + if [[ ${GYAN_DRIVER} == "tensorflow" ]]; then + iniset $GYAN_CONF DEFAULT ml_model_driver "ml_model.driver.TensorflowDriver" + fi + if [[ ${GYAN_DB_TYPE} == "sql" ]]; then + iniset $GYAN_CONF DEFAULT db_type sql + fi + iniset $GYAN_CONF DEFAULT debug "$ENABLE_DEBUG_LOG_LEVEL" + iniset $GYAN_CONF DEFAULT my_ip "$HOST_IP" + iniset $GYAN_CONF DEFAULT host "$HOST_IP" + iniset $GYAN_CONF oslo_messaging_rabbit rabbit_userid $RABBIT_USERID + iniset $GYAN_CONF oslo_messaging_rabbit rabbit_password $RABBIT_PASSWORD + iniset $GYAN_CONF oslo_messaging_rabbit rabbit_host $RABBIT_HOST + iniset $GYAN_CONF database connection `database_connection_url gyan` + iniset $GYAN_CONF api host_ip "$GYAN_SERVICE_HOST" + iniset $GYAN_CONF api port "$GYAN_SERVICE_PORT" + + iniset $GYAN_CONF keystone_auth auth_type password + iniset $GYAN_CONF keystone_auth username gyan + iniset $GYAN_CONF keystone_auth password $SERVICE_PASSWORD + iniset $GYAN_CONF keystone_auth project_name $SERVICE_PROJECT_NAME + iniset $GYAN_CONF keystone_auth project_domain_id default + iniset $GYAN_CONF keystone_auth user_domain_id default + + # FIXME(pauloewerton): keystone_authtoken section is deprecated. Remove it + # after deprecation period. + iniset $GYAN_CONF keystone_authtoken admin_user gyan + iniset $GYAN_CONF keystone_authtoken admin_password $SERVICE_PASSWORD + iniset $GYAN_CONF keystone_authtoken admin_tenant_name $SERVICE_PROJECT_NAME + + configure_auth_token_middleware $GYAN_CONF gyan $GYAN_AUTH_CACHE_DIR + + iniset $GYAN_CONF keystone_auth auth_url $KEYSTONE_AUTH_URI_V3 + iniset $GYAN_CONF keystone_authtoken www_authenticate_uri $KEYSTONE_SERVICE_URI_V3 + iniset $GYAN_CONF keystone_authtoken auth_url $KEYSTONE_AUTH_URI_V3 + iniset $GYAN_CONF keystone_authtoken auth_version v3 + + + if is_fedora || is_suse; then + # gyan defaults to /usr/local/bin, but fedora and suse pip like to + # install things in /usr/bin + iniset $GYAN_CONF DEFAULT bindir "/usr/bin" + fi + + if [ -n "$GYAN_STATE_PATH" ]; then + iniset $GYAN_CONF DEFAULT state_path "$GYAN_STATE_PATH" + iniset $GYAN_CONF oslo_concurrency lock_path "$GYAN_STATE_PATH" + fi + + if [ "$SYSLOG" != "False" ]; then + iniset $GYAN_CONF DEFAULT use_syslog "True" + fi + + # Format logging + if [ "$LOG_COLOR" == "True" ] && [ "$SYSLOG" == "False" ]; then + setup_colorized_logging $GYAN_CONF DEFAULT + else + # Show user_name and project_name instead of user_id and project_id + iniset $GYAN_CONF DEFAULT logging_context_format_string "%(asctime)s.%(msecs)03d %(levelname)s %(name)s [%(request_id)s %(user_name)s %(project_name)s] %(instance)s%(message)s" + fi + + # Register SSL certificates if provided + if is_ssl_enabled_service gyan; then + ensure_certificates gyan + + iniset $GYAN_CONF DEFAULT ssl_cert_file "$GYAN_SSL_CERT" + iniset $GYAN_CONF DEFAULT ssl_key_file "$GYAN_SSL_KEY" + + iniset $GYAN_CONF DEFAULT enabled_ssl_apis "$GYAN_ENABLED_APIS" + fi +} + +function create_api_paste_conf { + # copy api_paste.ini + cp $GYAN_DIR/etc/gyan/api-paste.ini $GYAN_API_PASTE +} + +# create_gyan_cache_dir() - Part of the init_GYAN() process +function create_gyan_cache_dir { + # Create cache dir + sudo mkdir -p $GYAN_AUTH_CACHE_DIR + sudo chown $STACK_USER $GYAN_AUTH_CACHE_DIR + rm -f $GYAN_AUTH_CACHE_DIR/* +} + + +# init_gyan() - Initialize databases, etc. +function init_gyan { + # Only do this step once on the API node for an entire cluster. + if is_service_enabled gyan-api; then + if is_service_enabled $DATABASE_BACKENDS; then + # (Re)create gyan database + recreate_database gyan + + # Migrate gyan database + $GYAN_BIN_DIR/gyan-db-manage upgrade + fi + + if is_service_enabled gyan-etcd; then + install_etcd_server + fi + create_gyan_cache_dir + fi +} + +# install_gyanclient() - Collect source and prepare +function install_gyanclient { + if use_library_from_git "python-gyanclient"; then + git_clone_by_name "python-gyanclient" + setup_dev_lib "python-gyanclient" + sudo install -D -m 0644 -o $STACK_USER {${GITDIR["python-gyanclient"]}/tools/,/etc/bash_completion.d/}gyan.bash_completion + fi +} + +# install_gyan() - Collect source and prepare +function install_gyan { + git_clone $GYAN_REPO $GYAN_DIR $GYAN_BRANCH + setup_develop $GYAN_DIR +} + +# start_gyan_api() - Start the API process ahead of other things +function start_gyan_api { + # Get right service port for testing + local service_port=$GYAN_SERVICE_PORT + local service_protocol=$GYAN_SERVICE_PROTOCOL + if is_service_enabled tls-proxy; then + service_port=$GYAN_SERVICE_PORT_INT + service_protocol="http" + fi + + local gyan_url + if [ "$GYAN_USE_UWSGI" == "True" ]; then + run_process gyan-api "$GYAN_BIN_DIR/uwsgi --procname-prefix gyan-api --ini $GYAN_UWSGI_CONF" + gyan_url=$service_protocol://$GYAN_SERVICE_HOST/ml-infra + else + run_process gyan-api "$GYAN_BIN_DIR/gyan-api" + gyan_url=$service_protocol://$GYAN_SERVICE_HOST:$service_port + fi + + echo "Waiting for gyan-api to start..." + if ! wait_for_service $SERVICE_TIMEOUT $gyan_url; then + die $LINENO "gyan-api did not start" + fi + + # Start proxies if enabled + if is_service_enabled tls-proxy; then + start_tls_proxy '*' $GYAN_SERVICE_PORT $GYAN_SERVICE_HOST $GYAN_SERVICE_PORT_INT & + fi +} + +# start_gyan_compute() - Start Gyan compute agent +function start_gyan_compute { + echo "Start gyan compute..." + run_process gyan-compute "$GYAN_BIN_DIR/gyan-compute" +} + + +# start_gyan() - Start running processes, including screen +function start_gyan { + + # ``run_process`` checks ``is_service_enabled``, it is not needed here + start_gyan_api + start_gyan_compute +} + +# stop_gyan() - Stop running processes (non-screen) +function stop_gyan { + + if [ "$GYAN_USE_UWSGI" == "True" ]; then + disable_apache_site gyan + restart_apache_server + else + stop_process gyan-api + fi + stop_process gyan-compute +} + +# Restore xtrace +$XTRACE diff --git a/devstack/local.conf.sample b/devstack/local.conf.sample new file mode 100644 index 0000000..3a8cd11 --- /dev/null +++ b/devstack/local.conf.sample @@ -0,0 +1,13 @@ +[[local|localrc]] +HOST_IP=10.0.0.11 # change this to your IP address +DATABASE_PASSWORD=password +RABBIT_PASSWORD=password +SERVICE_TOKEN=password +SERVICE_PASSWORD=password +ADMIN_PASSWORD=password +enable_plugin gyan https://git.openstack.org/openstack/gyan + +# install python-gyanclient from git +LIBS_FROM_GIT="python-gyanclient" + + diff --git a/devstack/local.conf.subnode.sample b/devstack/local.conf.subnode.sample new file mode 100644 index 0000000..491ba4a --- /dev/null +++ b/devstack/local.conf.subnode.sample @@ -0,0 +1,18 @@ +[[local|localrc]] +HOST_IP=10.0.0.31 # change this to your IP address +DATABASE_PASSWORD=password +RABBIT_PASSWORD=password +SERVICE_TOKEN=password +SERVICE_PASSWORD=password +ADMIN_PASSWORD=password +enable_plugin gyan https://git.openstack.org/openstack/gyan + + +# Following is for multi host settings +MULTI_HOST=True +SERVICE_HOST=10.0.0.11 # change this to controller's IP address +DATABASE_TYPE=mysql +MYSQL_HOST=$SERVICE_HOST +RABBIT_HOST=$SERVICE_HOST + +ENABLED_SERVICES=gyan-compute diff --git a/devstack/plugin.sh b/devstack/plugin.sh new file mode 100755 index 0000000..58fa0ea --- /dev/null +++ b/devstack/plugin.sh @@ -0,0 +1,47 @@ +# gyan - Devstack extras script to install gyan + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set -o xtrace + +echo_summary "gyan's plugin.sh was called..." +source $DEST/gyan/devstack/lib/gyan +(set -o posix; set) + +if is_service_enabled gyan-api gyan-compute; then + if [[ "$1" == "stack" && "$2" == "install" ]]; then + echo_summary "Installing gyan" + install_gyan + + install_gyanclient + cleanup_gyan + + elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then + echo_summary "Configuring gyan" + configure_gyan + + if is_service_enabled key; then + create_gyan_accounts + fi + + elif [[ "$1" == "stack" && "$2" == "extra" ]]; then + # Initialize gyan + init_gyan + + # Start the gyan API and gyan compute + echo_summary "Starting gyan" + start_gyan + + fi + + if [[ "$1" == "unstack" ]]; then + stop_gyan + fi + + if [[ "$1" == "clean" ]]; then + cleanup_gyan + fi +fi + +# Restore xtrace +$XTRACE diff --git a/devstack/settings b/devstack/settings new file mode 100644 index 0000000..e4706f6 --- /dev/null +++ b/devstack/settings @@ -0,0 +1,24 @@ +# Devstack settings + +## Modify to your environment +# FLOATING_RANGE=192.168.1.224/27 +# PUBLIC_NETWORK_GATEWAY=192.168.1.225 +# PUBLIC_INTERFACE=em1 +# FIXED_RANGE=10.0.0.0/24 +## Log all output to files +# LOGFILE=$HOME/devstack.log +## Neutron settings +# Q_USE_SECGROUP=True +# ENABLE_TENANT_VLANS=True +# TENANT_VLAN_RANGE= +# PHYSICAL_NETWORK=public +# OVS_PHYSICAL_BRIDGE=br-ex + +# Enable Gyan services +if [[ ${HOST_IP} == ${SERVICE_HOST} ]]; then + enable_service gyan-api + enable_service gyan-compute +else + enable_service gyan-compute +fi + diff --git a/etc/apache2/gyan.conf.template b/etc/apache2/gyan.conf.template new file mode 100644 index 0000000..194d6b1 --- /dev/null +++ b/etc/apache2/gyan.conf.template @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is an example Apache2 configuration file for using the +# gyan API through mod_wsgi. + +# Note: If you are using a Debian-based system then the paths +# "/var/log/httpd" and "/var/run/httpd" will use "apache2" instead +# of "httpd". +# +# The number of processes and threads is an example only and should +# be adjusted according to local requirements. + +Listen %PUBLICPORT% +LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-agent}i\" %D(us)" gyan_combined + + + WSGIDaemonProcess gyan-api user=%USER% processes=5 threads=1 display-name=%{GROUP} + WSGIScriptAlias / %PUBLICWSGI% + WSGIProcessGroup gyan-api + ErrorLogFormat "%M" + ErrorLog /var/log/%APACHE_NAME%/gyan_api.log + LogLevel info + CustomLog /var/log/%APACHE_NAME%/gyan_access.log gyan_combined + + + WSGIProcessGroup gyan-api + WSGIApplicationGroup %{GLOBAL} + AllowOverride All + Require all granted + + diff --git a/etc/gyan/api-paste.ini b/etc/gyan/api-paste.ini new file mode 100644 index 0000000..101de01 --- /dev/null +++ b/etc/gyan/api-paste.ini @@ -0,0 +1,19 @@ +[pipeline:main] +pipeline = cors request_id osprofiler authtoken api_v1 + +[app:api_v1] +paste.app_factory = gyan.api.app:app_factory + +[filter:authtoken] +acl_public_routes = /, /v1 +paste.filter_factory = gyan.api.middleware.auth_token:AuthTokenMiddleware.factory + +[filter:osprofiler] +paste.filter_factory = gyan.common.profiler:WsgiMiddleware.factory + +[filter:request_id] +paste.filter_factory = oslo_middleware:RequestId.factory + +[filter:cors] +paste.filter_factory = oslo_middleware.cors:filter_factory +oslo_config_project = gyan diff --git a/etc/gyan/gyan-config-generator.conf b/etc/gyan/gyan-config-generator.conf new file mode 100644 index 0000000..0868fbc --- /dev/null +++ b/etc/gyan/gyan-config-generator.conf @@ -0,0 +1,14 @@ +[DEFAULT] +output_file = etc/gyan/gyan.conf.sample +wrap_width = 79 + +namespace = gyan.conf +namespace = keystonemiddleware.auth_token +namespace = oslo.concurrency +namespace = oslo.db +namespace = oslo.log +namespace = oslo.messaging +namespace = oslo.middleware.cors +namespace = oslo.policy +namespace = oslo.service.periodic_task +namespace = oslo.service.service diff --git a/etc/gyan/gyan-policy-generator.conf b/etc/gyan/gyan-policy-generator.conf new file mode 100644 index 0000000..e3c4830 --- /dev/null +++ b/etc/gyan/gyan-policy-generator.conf @@ -0,0 +1,3 @@ +[DEFAULT] +output_file = etc/gyan/policy.yaml.sample +namespace = gyan diff --git a/etc/gyan/rootwrap.conf b/etc/gyan/rootwrap.conf new file mode 100644 index 0000000..6153b84 --- /dev/null +++ b/etc/gyan/rootwrap.conf @@ -0,0 +1,27 @@ +# Configuration for gyan-rootwrap +# This file should be owned by (and only-writable by) the root user + +[DEFAULT] +# List of directories to load filter definitions from (separated by ','). +# These directories MUST all be only writable by root ! +filters_path=/etc/gyan/rootwrap.d + +# List of directories to search executables in, in case filters do not +# explicitely specify a full path (separated by ',') +# If not specified, defaults to system PATH environment variable. +# These directories MUST all be only writable by root ! +exec_dirs=/sbin,/usr/sbin,/bin,/usr/bin,/usr/local/bin,/usr/local/sbin + +# Enable logging to syslog +# Default value is False +use_syslog=False + +# Which syslog facility to use. +# Valid values include auth, authpriv, syslog, local0, local1... +# Default value is 'syslog' +syslog_log_facility=syslog + +# Which messages to log. +# INFO means log all usage +# ERROR means only log unsuccessful attempts +syslog_log_level=ERROR diff --git a/etc/gyan/rootwrap.d/gyan.filters b/etc/gyan/rootwrap.d/gyan.filters new file mode 100644 index 0000000..880f0eb --- /dev/null +++ b/etc/gyan/rootwrap.d/gyan.filters @@ -0,0 +1,8 @@ +# gyan command filters +# This file should be owned by (and only-writeable by) the root user + +[Filters] +# privileged/__init__.py: priv_context.PrivContext(default) +# This line ties the superuser privs with the config files, context name, +# and (implicitly) the actual python code invoked. +privsep-rootwrap: RegExpFilter, privsep-helper, root, privsep-helper, --config-file, /etc/(?!\.\.).*, --privsep_context, os_brick.privileged.default, --privsep_sock_path, /tmp/.* diff --git a/gyan/MANIFEST.in b/gyan/MANIFEST.in new file mode 100644 index 0000000..f2f925c --- /dev/null +++ b/gyan/MANIFEST.in @@ -0,0 +1 @@ +recursive-include public * \ No newline at end of file diff --git a/gyan/api/__init__.py b/gyan/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/api/app.py b/gyan/api/app.py new file mode 100644 index 0000000..bd33fe8 --- /dev/null +++ b/gyan/api/app.py @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from oslo_config import cfg +from oslo_log import log +from paste import deploy +import pecan + +from gyan.api import config as api_config +from gyan.api import middleware +from gyan.common import config as common_config +import gyan.conf + +CONF = gyan.conf.CONF +LOG = log.getLogger(__name__) + + +def get_pecan_config(): + # Set up the pecan configuration + filename = api_config.__file__.replace('.pyc', '.py') + return pecan.configuration.conf_from_file(filename) + + +def setup_app(config=None): + if not config: + config = get_pecan_config() + + app_conf = dict(config.app) + common_config.set_config_defaults() + + app = pecan.make_app( + app_conf.pop('root'), + logging=getattr(config, 'logging', {}), + wrap_app=middleware.ParsableErrorMiddleware, + **app_conf + ) + + return app + + +def load_app(): + cfg_file = None + cfg_path = CONF.api.api_paste_config + if not os.path.isabs(cfg_path): + cfg_file = CONF.find_file(cfg_path) + elif os.path.exists(cfg_path): + cfg_file = cfg_path + + if not cfg_file: + raise cfg.ConfigFilesNotFoundError([CONF.api.api_paste_config]) + LOG.info("Full WSGI config used: %s", cfg_file) + return deploy.loadapp("config:" + cfg_file) + + +def app_factory(global_config, **local_conf): + return setup_app() diff --git a/gyan/api/app.wsgi b/gyan/api/app.wsgi new file mode 100644 index 0000000..a6733bd --- /dev/null +++ b/gyan/api/app.wsgi @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Use this file for deploying the API under mod_wsgi. +See https://pecan.readthedocs.org/en/latest/deployment.html for details. +""" + +from gyan.api import wsgi + +application = wsgi.init_application() \ No newline at end of file diff --git a/gyan/api/config.py b/gyan/api/config.py new file mode 100644 index 0000000..51dde2f --- /dev/null +++ b/gyan/api/config.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gyan.api import hooks + +# Pecan Application Configurations +app = { + 'root': 'gyan.api.controllers.root.RootController', + 'modules': ['gyan'], + 'hooks': [ + hooks.ContextHook(), + hooks.NoExceptionTracebackHook(), + hooks.RPCHook(), + ], + 'debug': True, +} \ No newline at end of file diff --git a/gyan/api/controllers/__init__.py b/gyan/api/controllers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/api/controllers/base.py b/gyan/api/controllers/base.py new file mode 100644 index 0000000..e5b4a9a --- /dev/null +++ b/gyan/api/controllers/base.py @@ -0,0 +1,233 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import operator +import six + +import pecan +from pecan import rest +from webob import exc +from gyan.api.controllers import versions +from gyan.api import versioned_method +from gyan.common import exception +from gyan.common.i18n import _ + + +# name of attribute to keep version method information +VER_METHOD_ATTR = 'versioned_methods' + + +class APIBase(object): + + def __init__(self, **kwargs): + for field in self.fields: + if field in kwargs: + value = kwargs[field] + setattr(self, field, value) + + def __setattr__(self, field, value): + super(APIBase, self).__setattr__(field, value) + + def as_dict(self): + """Render this object as a dict of its fields.""" + return {f: getattr(self, f) + for f in self.fields + if hasattr(self, f)} + + def __json__(self): + return self.as_dict() + + def unset_fields_except(self, except_list=None): + """Unset fields so they don't appear in the message body. + + :param except_list: A list of fields that won't be touched. + + """ + if except_list is None: + except_list = [] + + for k in self.as_dict(): + if k not in except_list: + setattr(self, k, None) + + +class ControllerMetaclass(type): + """Controller metaclass. + + This metaclass automates the task of assembling a dictionary + mapping action keys to method names. + """ + + def __new__(mcs, name, bases, cls_dict): + """Adds version function dictionary to the class.""" + + versioned_methods = None + + for base in bases: + if base.__name__ == "Controller": + # NOTE(cyeoh): This resets the VER_METHOD_ATTR attribute + # between API controller class creations. This allows us + # to use a class decorator on the API methods that doesn't + # require naming explicitly what method is being versioned as + # it can be implicit based on the method decorated. It is a bit + # ugly. + if VER_METHOD_ATTR in base.__dict__: + versioned_methods = getattr(base, VER_METHOD_ATTR) + delattr(base, VER_METHOD_ATTR) + + if versioned_methods: + cls_dict[VER_METHOD_ATTR] = versioned_methods + + return super(ControllerMetaclass, mcs).__new__(mcs, name, bases, + cls_dict) + + +@six.add_metaclass(ControllerMetaclass) +class Controller(rest.RestController): + """Base Rest Controller""" + + @pecan.expose('json') + def _no_version_match(self, *args, **kwargs): + from pecan import request + + raise exc.HTTPNotAcceptable(_( + "Version %(ver)s was requested but the requested API is not " + "supported for this version.") % {'ver': request.version}) + + def __getattribute__(self, key): + + def version_select(): + """Select the correct method based on version + + @return: Returns the correct versioned method + @raises: HTTPNotAcceptable if there is no method which + matches the name and version constraints + """ + + from pecan import request + ver = request.version + + func_list = self.versioned_methods[key] + for func in func_list: + if ver.matches(func.start_version, func.end_version): + return func.func + + return self._no_version_match + + try: + version_meth_dict = object.__getattribute__(self, VER_METHOD_ATTR) + except AttributeError: + # No versioning on this class + return object.__getattribute__(self, key) + if version_meth_dict and key in version_meth_dict: + return version_select().__get__(self, self.__class__) + + return object.__getattribute__(self, key) + + # NOTE: This decorator MUST appear first (the outermost + # decorator) on an API method for it to work correctly + @classmethod + def api_version(cls, min_ver, max_ver=None): + """Decorator for versioning api methods. + + Add the decorator to any pecan method that has been exposed. + This decorator will store the method, min version, and max + version in a list for each api. It will check that there is no + overlap between versions and methods. When the api is called the + controller will use the list for each api to determine which + method to call. + + Example: + @base.Controller.api_version("1.1", "1.2") + def get_one(self, ml_model_id): + {...code for versions 1.1 to 1.2...} + + @base.Controller.api_version("1.3") + def get_one(self, ml_model_id): + {...code for versions 1.3 to latest} + + @min_ver: string representing minimum version + @max_ver: optional string representing maximum version + @raises: ApiVersionsIntersect if an version overlap is found between + method versions. + """ + + def decorator(f): + obj_min_ver = versions.Version('', '', '', min_ver) + if max_ver: + obj_max_ver = versions.Version('', '', '', max_ver) + else: + obj_max_ver = versions.Version('', '', '', + versions.CURRENT_MAX_VER) + + # Add to list of versioned methods registered + func_name = f.__name__ + new_func = versioned_method.VersionedMethod( + func_name, obj_min_ver, obj_max_ver, f) + + func_dict = getattr(cls, VER_METHOD_ATTR, {}) + if not func_dict: + setattr(cls, VER_METHOD_ATTR, func_dict) + + func_list = func_dict.get(func_name, []) + if not func_list: + func_dict[func_name] = func_list + func_list.append(new_func) + + is_intersect = Controller.check_for_versions_intersection( + func_list) + + if is_intersect: + raise exception.ApiVersionsIntersect( + name=new_func.name, + min_ver=new_func.start_version, + max_ver=new_func.end_version + ) + + # Ensure the list is sorted by minimum version (reversed) + # so later when we work through the list in order we find + # the method which has the latest version which supports + # the version requested. + func_list.sort(key=lambda f: f.start_version, reverse=True) + + return f + + return decorator + + @staticmethod + def check_for_versions_intersection(func_list): + """Determines whether function list intersections + + General algorithm: + https://en.wikipedia.org/wiki/Intersection_algorithm + + :param func_list: list of VersionedMethod objects + :return: boolean + """ + + pairs = [] + counter = 0 + + for f in func_list: + pairs.append((f.start_version, 1)) + pairs.append((f.end_version, -1)) + + pairs.sort(key=operator.itemgetter(1), reverse=True) + pairs.sort(key=operator.itemgetter(0)) + + for p in pairs: + counter += p[1] + + if counter > 1: + return True + + return False diff --git a/gyan/api/controllers/link.py b/gyan/api/controllers/link.py new file mode 100644 index 0000000..ea29527 --- /dev/null +++ b/gyan/api/controllers/link.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan + + +def build_url(resource, resource_args, bookmark=False, base_url=None): + if base_url is None: + base_url = pecan.request.host_url + + template = '%(url)s/%(res)s' if bookmark else '%(url)s/v1/%(res)s' + # FIXME(tbh): I'm getting a 404 when doing a GET on + # a nested resource that the URL ends with a '/'. + # https://groups.google.com/forum/#!topic/pecan-dev/QfSeviLg5qs + template += '%(args)s' if resource_args.startswith('?') else '/%(args)s' + return template % {'url': base_url, 'res': resource, 'args': resource_args} + + +def make_link(rel_name, url, resource, resource_args, + bookmark=False, type=None): + href = build_url(resource, resource_args, + bookmark=bookmark, base_url=url) + if type is None: + return {'href': href, 'rel': rel_name} + else: + return {'href': href, 'rel': rel_name, 'type': type} diff --git a/gyan/api/controllers/root.py b/gyan/api/controllers/root.py new file mode 100644 index 0000000..6d01e94 --- /dev/null +++ b/gyan/api/controllers/root.py @@ -0,0 +1,97 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pecan +from pecan import rest + +from gyan.api.controllers import base +from gyan.api.controllers import link +from gyan.api.controllers import v1 +from gyan.api.controllers import versions + + +class Version(base.APIBase): + """An API version representation.""" + + fields = ( + 'id', + 'links', + 'status', + 'max_version', + 'min_version' + ) + + @staticmethod + def convert(id, status, max, min): + version = Version() + version.id = id + version.links = [link.make_link('self', pecan.request.host_url, + id, '', bookmark=True)] + version.status = status + version.max_version = max + version.min_version = min + return version + + +class Root(base.APIBase): + + fields = ( + 'name', + 'description', + 'versions', + 'default_version', + ) + + @staticmethod + def convert(): + root = Root() + root.name = "OpenStack Gyan API" + root.description = ("Gyan is an OpenStack project which aims to " + "provide ML infra service.") + + root.versions = [Version.convert('v1', "CURRENT", + versions.CURRENT_MAX_VER, + versions.BASE_VER)] + root.default_version = Version.convert('v1', "CURRENT", + versions.CURRENT_MAX_VER, + versions.BASE_VER) + return root + + +class RootController(rest.RestController): + + _versions = ['v1'] + """All supported API versions""" + + _default_version = 'v1' + """The default API version""" + + v1 = v1.Controller() + + @pecan.expose('json') + def get(self): + # NOTE: The reason why convert() it's being called for every + # request is because we need to get the host url from + # the request object to make the links. + return Root.convert() + + @pecan.expose() + def _route(self, args): + """Overrides the default routing behavior. + + It redirects the request to the default version of the gyan API + if the version number is not specified in the url. + """ + + if args[0] and args[0] not in self._versions: + args = [self._default_version] + args + return super(RootController, self)._route(args) diff --git a/gyan/api/controllers/v1/__init__.py b/gyan/api/controllers/v1/__init__.py new file mode 100644 index 0000000..02f32ec --- /dev/null +++ b/gyan/api/controllers/v1/__init__.py @@ -0,0 +1,155 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Version 1 of the Gyan API + +NOTE: IN PROGRESS AND NOT FULLY IMPLEMENTED. +""" + +from oslo_log import log as logging +import pecan + +from gyan.api.controllers import base as controllers_base +from gyan.api.controllers import link +from gyan.api.controllers.v1 import hosts as host_controller +from gyan.api.controllers.v1 import ml_models as ml_model_controller +from gyan.api.controllers import versions as ver +from gyan.api import http_error +from gyan.common.i18n import _ + +LOG = logging.getLogger(__name__) + + +BASE_VERSION = 1 + +MIN_VER_STR = '%s %s' % (ver.Version.service_string, ver.BASE_VER) + +MAX_VER_STR = '%s %s' % (ver.Version.service_string, ver.CURRENT_MAX_VER) + +MIN_VER = ver.Version({ver.Version.string: MIN_VER_STR}, + MIN_VER_STR, MAX_VER_STR) +MAX_VER = ver.Version({ver.Version.string: MAX_VER_STR}, + MIN_VER_STR, MAX_VER_STR) + + +class MediaType(controllers_base.APIBase): + """A media type representation.""" + + fields = ( + 'base', + 'type', + ) + + +class V1(controllers_base.APIBase): + """The representation of the version 1 of the API.""" + + fields = ( + 'id', + 'media_types', + 'links', + 'hosts', + 'ml_models' + ) + + @staticmethod + def convert(): + v1 = V1() + v1.id = "v1" + v1.links = [link.make_link('self', pecan.request.host_url, + 'v1', '', bookmark=True), + link.make_link('describedby', + 'https://docs.openstack.org', + 'developer/gyan/dev', + 'api-spec-v1.html', + bookmark=True, type='text/html')] + v1.media_types = [MediaType(base='application/json', + type='application/vnd.openstack.gyan.v1+json')] + v1.hosts = [link.make_link('self', pecan.request.host_url, + 'hosts', ''), + link.make_link('bookmark', + pecan.request.host_url, + 'hosts', '', + bookmark=True)] + v1.ml_models = [link.make_link('self', pecan.request.host_url, + 'ml_models', ''), + link.make_link('bookmark', + pecan.request.host_url, + 'ml_models', '', + bookmark=True)] + return v1 + + +class Controller(controllers_base.Controller): + """Version 1 API controller root.""" + + hosts = host_controller.HostController() + ml_models = ml_model_controller.MLModelController() + + @pecan.expose('json') + def get(self): + return V1.convert() + + def _check_version(self, version, headers=None): + if headers is None: + headers = {} + # ensure that major version in the URL matches the header + if version.major != BASE_VERSION: + raise http_error.HTTPNotAcceptableAPIVersion(_( + "Mutually exclusive versions requested. Version %(ver)s " + "requested but not supported by this service. " + "The supported version range is: " + "[%(min)s, %(max)s].") % {'ver': version, + 'min': MIN_VER_STR, + 'max': MAX_VER_STR}, + headers=headers, + max_version=str(MAX_VER), + min_version=str(MIN_VER)) + # ensure the minor version is within the supported range + if version < MIN_VER or version > MAX_VER: + raise http_error.HTTPNotAcceptableAPIVersion(_( + "Version %(ver)s was requested but the minor version is not " + "supported by this service. The supported version range is: " + "[%(min)s, %(max)s].") % {'ver': version, 'min': MIN_VER_STR, + 'max': MAX_VER_STR}, + headers=headers, + max_version=str(MAX_VER), + min_version=str(MIN_VER)) + + @pecan.expose() + def _route(self, args): + version = ver.Version( + pecan.request.headers, MIN_VER_STR, MAX_VER_STR) + + # Always set the basic version headers + pecan.response.headers[ver.Version.min_string] = MIN_VER_STR + pecan.response.headers[ver.Version.max_string] = MAX_VER_STR + pecan.response.headers[ver.Version.string] = " ".join( + [ver.Version.service_string, str(version)]) + pecan.response.headers["vary"] = ver.Version.string + + # assert that requested version is supported + self._check_version(version, pecan.response.headers) + pecan.request.version = version + if pecan.request.body: + msg = ("Processing request: url: %(url)s, %(method)s, " + "body: %(body)s" % + {'url': pecan.request.url, + 'method': pecan.request.method, + 'body': pecan.request.body}) + LOG.debug(msg) + + return super(Controller, self)._route(args) + + +__all__ = ('Controller',) \ No newline at end of file diff --git a/gyan/api/controllers/v1/collection.py b/gyan/api/controllers/v1/collection.py new file mode 100644 index 0000000..b6567ac --- /dev/null +++ b/gyan/api/controllers/v1/collection.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan + +from gyan.api.controllers import base +from gyan.api.controllers import link + + +class Collection(base.APIBase): + + @property + def collection(self): + return getattr(self, self._type) + + def has_next(self, limit): + """Return whether collection has more items.""" + return len(self.collection) and len(self.collection) == limit + + def get_next(self, limit, url=None, **kwargs): + """Return a link to the next subset of the collection.""" + if not self.has_next(limit): + return None + + resource_url = url or self._type + q_args = ''.join(['%s=%s&' % (key, kwargs[key]) for key in kwargs]) + next_args = '?%(args)slimit=%(limit)d&marker=%(marker)s' % { + 'args': q_args, 'limit': limit, + 'marker': self.collection[-1]['uuid']} + + return link.make_link('next', pecan.request.host_url, + resource_url, next_args)['href'] diff --git a/gyan/api/controllers/v1/hosts.py b/gyan/api/controllers/v1/hosts.py new file mode 100644 index 0000000..5264771 --- /dev/null +++ b/gyan/api/controllers/v1/hosts.py @@ -0,0 +1,109 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pecan + +from gyan.api.controllers import base +from gyan.api.controllers.v1 import collection +from gyan.api.controllers.v1.views import hosts_view as view +from gyan.api import utils as api_utils +from gyan.common import exception +from gyan.common import policy +from gyan import objects + + +def _get_host(host_ident): + host = api_utils.get_resource('ComputeHost', host_ident) + if not host: + pecan.abort(404, ('Not found; the host you requested ' + 'does not exist.')) + + return host + + +def check_policy_on_host(host, action): + context = pecan.request.context + policy.enforce(context, action, host, action=action) + + +class HostCollection(collection.Collection): + """API representation of a collection of hosts.""" + + fields = { + 'hosts', + 'next' + } + + """A list containing compute host objects""" + + def __init__(self, **kwargs): + super(HostCollection, self).__init__(**kwargs) + self._type = 'hosts' + + @staticmethod + def convert_with_links(hosts, limit, url=None, + expand=False, **kwargs): + collection = HostCollection() + collection.hosts = [view.format_host(url, p) for p in hosts] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +class HostController(base.Controller): + """Host info controller""" + + @pecan.expose('json') + @base.Controller.api_version("1.0") + @exception.wrap_pecan_controller_exception + def get_all(self, **kwargs): + """Retrieve a list of hosts""" + context = pecan.request.context + policy.enforce(context, "host:get_all", + action="host:get_all") + return self._get_host_collection(**kwargs) + + def _get_host_collection(self, **kwargs): + context = pecan.request.context + limit = api_utils.validate_limit(kwargs.get('limit')) + sort_dir = api_utils.validate_sort_dir(kwargs.get('sort_dir', 'asc')) + sort_key = kwargs.get('sort_key', 'hostname') + expand = kwargs.get('expand') + filters = None + marker_obj = None + resource_url = kwargs.get('resource_url') + marker = kwargs.get('marker') + if marker: + marker_obj = objects.ComputeHost.get_by_uuid(context, marker) + hosts = objects.ComputeHost.list(context, + limit, + marker_obj, + sort_key, + sort_dir, + filters=filters) + return HostCollection.convert_with_links(hosts, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @pecan.expose('json') + @base.Controller.api_version("1.0") + @exception.wrap_pecan_controller_exception + def get_one(self, host_ident): + """Retrieve information about the given host. + + :param host_ident: UUID or name of a host. + """ + context = pecan.request.context + policy.enforce(context, "host:get", action="host:get") + host = _get_host(host_ident) + return view.format_host(pecan.request.host_url, host) diff --git a/gyan/api/controllers/v1/ml_models.py b/gyan/api/controllers/v1/ml_models.py new file mode 100644 index 0000000..9e0bf3e --- /dev/null +++ b/gyan/api/controllers/v1/ml_models.py @@ -0,0 +1,289 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import shlex + +from oslo_log import log as logging +from oslo_utils import strutils +from oslo_utils import uuidutils +import pecan +import six + +from gyan.api.controllers import base +from gyan.api.controllers import link +from gyan.api.controllers.v1 import collection +from gyan.api.controllers.v1.schemas import ml_models as schema +from gyan.api.controllers.v1.views import ml_models_view as view +from gyan.api import utils as api_utils +from gyan.api import validation +from gyan.common import consts +from gyan.common import context as gyan_context +from gyan.common import exception +from gyan.common.i18n import _ +from gyan.common.policies import ml_model as policies +from gyan.common import policy +from gyan.common import utils +import gyan.conf +from gyan import objects + +CONF = gyan.conf.CONF +LOG = logging.getLogger(__name__) + + +def check_policy_on_ml_model(ml_model, action): + context = pecan.request.context + policy.enforce(context, action, ml_model, action=action) + + +class MLModelCollection(collection.Collection): + """API representation of a collection of ml models.""" + + fields = { + 'ml_models', + 'next' + } + + """A list containing ml models objects""" + + def __init__(self, **kwargs): + super(MLModelCollection, self).__init__(**kwargs) + self._type = 'ml_models' + + @staticmethod + def convert_with_links(rpc_ml_models, limit, url=None, + expand=False, **kwargs): + context = pecan.request.context + collection = MLModelCollection() + collection.ml_models = \ + [view.format_ml_model(context, url, p.as_dict()) + for p in rpc_ml_models] + collection.next = collection.get_next(limit, url=url, **kwargs) + return collection + + +class MLModelController(base.Controller): + """Controller for MLModels.""" + + _custom_actions = { + 'train': ['POST'], + 'deploy': ['GET'], + 'undeploy': ['GET'] + } + + + @pecan.expose('json') + @exception.wrap_pecan_controller_exception + def get_all(self, **kwargs): + """Retrieve a list of ml models. + + """ + context = pecan.request.context + policy.enforce(context, "ml_model:get_all", + action="ml_model:get_all") + return self._get_ml_models_collection(**kwargs) + + def _get_ml_models_collection(self, **kwargs): + context = pecan.request.context + if utils.is_all_projects(kwargs): + policy.enforce(context, "ml_model:get_all_all_projects", + action="ml_model:get_all_all_projects") + context.all_projects = True + kwargs.pop('all_projects', None) + limit = api_utils.validate_limit(kwargs.pop('limit', None)) + sort_dir = api_utils.validate_sort_dir(kwargs.pop('sort_dir', 'asc')) + sort_key = kwargs.pop('sort_key', 'id') + resource_url = kwargs.pop('resource_url', None) + expand = kwargs.pop('expand', None) + + ml_model_allowed_filters = ['name', 'status', 'project_id', 'user_id', + 'type'] + filters = {} + for filter_key in ml_model_allowed_filters: + if filter_key in kwargs: + policy_action = policies.MLMODEL % ('get_one:' + filter_key) + context.can(policy_action, might_not_exist=True) + filter_value = kwargs.pop(filter_key) + filters[filter_key] = filter_value + marker_obj = None + marker = kwargs.pop('marker', None) + if marker: + marker_obj = objects.ML_Model.get_by_uuid(context, + marker) + if kwargs: + unknown_params = [str(k) for k in kwargs] + msg = _("Unknown parameters: %s") % ", ".join(unknown_params) + raise exception.InvalidValue(msg) + + ml_models = objects.ML_Model.list(context, + limit, + marker_obj, + sort_key, + sort_dir, + filters=filters) + return MLModelCollection.convert_with_links(ml_models, limit, + url=resource_url, + expand=expand, + sort_key=sort_key, + sort_dir=sort_dir) + + @pecan.expose('json') + @exception.wrap_pecan_controller_exception + def get_one(self, ml_model_ident, **kwargs): + """Retrieve information about the given ml_model. + + :param ml_model_ident: UUID or name of a ml_model. + """ + context = pecan.request.context + if utils.is_all_projects(kwargs): + policy.enforce(context, "ml_model:get_one_all_projects", + action="ml_model:get_one_all_projects") + context.all_projects = True + ml_model = utils.get_ml_model(ml_model_ident) + check_policy_on_ml_model(ml_model.as_dict(), "ml_model:get_one") + if ml_model.node: + compute_api = pecan.request.compute_api + try: + ml_model = compute_api.ml_model_show(context, ml_model) + except exception.MLModelHostNotUp: + raise exception.ServerNotUsable + + return view.format_ml_model(context, pecan.request.host_url, + ml_model.as_dict()) + + @base.Controller.api_version("1.0") + @pecan.expose('json') + @api_utils.enforce_content_types(['application/json']) + @exception.wrap_pecan_controller_exception + @validation.validate_query_param(pecan.request, schema.query_param_create) + @validation.validated(schema.ml_model_create) + def post(self, **ml_model_dict): + return self._do_post(**ml_model_dict) + + + def _do_post(self, **ml_model_dict): + """Create or run a new ml model. + + :param ml_model_dict: a ml_model within the request body. + """ + context = pecan.request.context + compute_api = pecan.request.compute_api + policy.enforce(context, "ml_model:create", + action="ml_model:create") + + ml_model_dict['project_id'] = context.project_id + ml_model_dict['user_id'] = context.user_id + name = ml_model_dict.get('name') + ml_model_dict['name'] = name + + ml_model_dict['status'] = consts.CREATING + extra_spec = {} + extra_spec['hints'] = ml_model_dict.get('hints', None) + new_ml_model = objects.ML_Model(context, **ml_model_dict) + new_ml_model.create(context) + + compute_api.ml_model_create(context, new_ml_model, **kwargs) + # Set the HTTP Location Header + pecan.response.location = link.build_url('ml_models', + new_ml_model.uuid) + pecan.response.status = 202 + return view.format_ml_model(context, pecan.request.node_url, + new_ml_model.as_dict()) + + + @pecan.expose('json') + @exception.wrap_pecan_controller_exception + @validation.validated(schema.ml_model_update) + def patch(self, ml_model_ident, **patch): + """Update an existing ml model. + + :param ml_model_ident: UUID or name of a ml model. + :param patch: a json PATCH document to apply to this ml model. + """ + ml_model = utils.get_ml_model(ml_model_ident) + check_policy_on_ml_model(ml_model.as_dict(), "ml_model:update") + utils.validate_ml_model_state(ml_model, 'update') + context = pecan.request.context + compute_api = pecan.request.compute_api + ml_model = compute_api.ml_model_update(context, ml_model, patch) + return view.format_ml_model(context, pecan.request.node_url, + ml_model.as_dict()) + + + @pecan.expose('json') + @exception.wrap_pecan_controller_exception + @validation.validate_query_param(pecan.request, schema.query_param_delete) + def delete(self, ml_model_ident, force=False, **kwargs): + """Delete a ML Model. + + :param ml_model_ident: UUID or Name of a ML Model. + :param force: If True, allow to force delete the ML Model. + """ + context = pecan.request.context + ml_model = utils.get_ml_model(ml_model_ident) + check_policy_on_ml_model(ml_model.as_dict(), "ml_model:delete") + try: + force = strutils.bool_from_string(force, strict=True) + except ValueError: + bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS) + raise exception.InvalidValue(_('Valid force values are: %s') + % bools) + stop = kwargs.pop('stop', False) + try: + stop = strutils.bool_from_string(stop, strict=True) + except ValueError: + bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS) + raise exception.InvalidValue(_('Valid stop values are: %s') + % bools) + compute_api = pecan.request.compute_api + if not force: + utils.validate_ml_model_state(ml_model, 'delete') + ml_model.status = consts.DELETING + if ml_model.node: + compute_api.ml_model_delete(context, ml_model, force) + else: + ml_model.destroy(context) + pecan.response.status = 204 + + + @pecan.expose('json') + @exception.wrap_pecan_controller_exception + def deploy(self, ml_model_ident, **kwargs): + """Deploy ML Model. + + :param ml_model_ident: UUID or Name of a ML Model. + """ + ml_model = utils.get_ml_model(ml_model_ident) + check_policy_on_ml_model(ml_model.as_dict(), "ml_model:deploy") + utils.validate_ml_model_state(ml_model, 'deploy') + LOG.debug('Calling compute.ml_model_deploy with %s', + ml_model.uuid) + context = pecan.request.context + compute_api = pecan.request.compute_api + compute_api.ml_model_deploy(context, ml_model) + pecan.response.status = 202 + + @pecan.expose('json') + @exception.wrap_pecan_controller_exception + def undeploy(self, ml_model_ident, **kwargs): + """Undeploy ML Model. + + :param ml_model_ident: UUID or Name of a ML Model. + """ + ml_model = utils.get_ml_model(ml_model_ident) + check_policy_on_ml_model(ml_model.as_dict(), "ml_model:deploy") + utils.validate_ml_model_state(ml_model, 'undeploy') + LOG.debug('Calling compute.ml_model_deploy with %s', + ml_model.uuid) + context = pecan.request.context + compute_api = pecan.request.compute_api + compute_api.ml_model_undeploy(context, ml_model) + pecan.response.status = 202 diff --git a/gyan/api/controllers/v1/schemas/__init__.py b/gyan/api/controllers/v1/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/api/controllers/v1/schemas/ml_models.py b/gyan/api/controllers/v1/schemas/ml_models.py new file mode 100644 index 0000000..07c2eb5 --- /dev/null +++ b/gyan/api/controllers/v1/schemas/ml_models.py @@ -0,0 +1,49 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +from gyan.api.controllers.v1.schemas import parameter_types + +_ml_model_properties = {} + +ml_model_create = { + 'type': 'object', + 'properties': _ml_model_properties, + 'required': ['name'], + 'additionalProperties': False +} + + +query_param_create = { + 'type': 'object', + 'properties': { + 'run': parameter_types.boolean_extended + }, + 'additionalProperties': False +} + +ml_model_update = { + 'type': 'object', + 'properties': {}, + 'additionalProperties': False +} + +query_param_delete = { + 'type': 'object', + 'properties': { + 'force': parameter_types.boolean_extended, + 'all_projects': parameter_types.boolean_extended, + 'stop': parameter_types.boolean_extended + }, + 'additionalProperties': False +} \ No newline at end of file diff --git a/gyan/api/controllers/v1/schemas/parameter_types.py b/gyan/api/controllers/v1/schemas/parameter_types.py new file mode 100644 index 0000000..4a2ecab --- /dev/null +++ b/gyan/api/controllers/v1/schemas/parameter_types.py @@ -0,0 +1,97 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy +import signal +import sys +import gyan.conf + +CONF = gyan.conf.CONF + +non_negative_integer = { + 'type': ['integer', 'string'], + 'pattern': '^[0-9]*$', 'minimum': 0 +} + +positive_integer = { + 'type': ['integer', 'string'], + 'pattern': '^[0-9]*$', 'minimum': 1 +} + +boolean_extended = { + 'type': ['boolean', 'string'], + 'enum': [True, 'True', 'TRUE', 'true', '1', 'ON', 'On', 'on', + 'YES', 'Yes', 'yes', + False, 'False', 'FALSE', 'false', '0', 'OFF', 'Off', 'off', + 'NO', 'No', 'no'], +} + +boolean = { + 'type': ['boolean', 'string'], + 'enum': [True, 'True', 'true', False, 'False', 'false'], +} + +ml_model_name = { + 'type': ['string', 'null'], + 'minLength': 2, + 'maxLength': 255, + 'pattern': '^[a-zA-Z0-9][a-zA-Z0-9_.-]+$' +} + +hex_uuid = { + 'type': 'string', + 'maxLength': 32, + 'minLength': 32, + 'pattern': '^[a-fA-F0-9]*$' +} + + +labels = { + 'type': ['object', 'null'] +} + +hints = { + 'type': ['object', 'null'] +} +hostname = { + 'type': ['string', 'null'], + 'minLength': 2, + 'maxLength': 63 +} + +repo = { + 'type': 'string', + 'minLength': 2, + 'maxLength': 255, + 'pattern': '[a-zA-Z0-9][a-zA-Z0-9_.-]' +} + + + +string_ps_args = { + 'type': ['string'], + 'pattern': '[a-zA-Z- ,+]*' +} + +str_and_int = { + 'type': ['string', 'integer', 'null'], +} + +hostname = { + 'type': 'string', 'minLength': 1, 'maxLength': 255, + # NOTE: 'host' is defined in "services" table, and that + # means a hostname. The hostname grammar in RFC952 does + # not allow for underscores in hostnames. However, this + # schema allows them, because it sometimes occurs in + # real systems. + 'pattern': '^[a-zA-Z0-9-._]*$', +} diff --git a/gyan/api/controllers/v1/schemas/services.py b/gyan/api/controllers/v1/schemas/services.py new file mode 100644 index 0000000..c68d945 --- /dev/null +++ b/gyan/api/controllers/v1/schemas/services.py @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gyan.api.controllers.v1.schemas import parameter_types + +query_param_enable = { + 'type': 'object', + 'properties': { + 'host': parameter_types.hostname, + 'binary': { + 'type': 'string', 'minLength': 1, 'maxLength': 255, + }, + }, + 'additionalProperties': False +} + +query_param_disable = { + 'type': 'object', + 'properties': { + 'host': parameter_types.hostname, + 'binary': { + 'type': 'string', 'minLength': 1, 'maxLength': 255, + }, + 'disabled_reason': { + 'type': 'string', 'minLength': 1, 'maxLength': 255, + }, + }, + 'additionalProperties': False +} + +query_param_force_down = { + 'type': 'object', + 'properties': { + 'host': parameter_types.hostname, + 'binary': { + 'type': 'string', 'minLength': 1, 'maxLength': 255, + }, + 'forced_down': parameter_types.boolean + }, + 'additionalProperties': False +} diff --git a/gyan/api/controllers/v1/views/__init__.py b/gyan/api/controllers/v1/views/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/api/controllers/v1/views/hosts_view.py b/gyan/api/controllers/v1/views/hosts_view.py new file mode 100644 index 0000000..1643804 --- /dev/null +++ b/gyan/api/controllers/v1/views/hosts_view.py @@ -0,0 +1,43 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +from gyan.api.controllers import link + + +_basic_keys = ( + 'id', + 'hostname', + 'type', + 'status' +) + + +def format_host(url, host): + def transform(key, value): + if key not in _basic_keys: + return + if key == 'id': + yield ('id', value) + yield ('links', [link.make_link( + 'self', url, 'hosts', value), + link.make_link( + 'bookmark', url, + 'hosts', value, + bookmark=True)]) + else: + yield (key, value) + + return dict(itertools.chain.from_iterable( + transform(k, v) for k, v in host.as_dict().items())) diff --git a/gyan/api/controllers/v1/views/ml_models_view.py b/gyan/api/controllers/v1/views/ml_models_view.py new file mode 100644 index 0000000..3721fec --- /dev/null +++ b/gyan/api/controllers/v1/views/ml_models_view.py @@ -0,0 +1,55 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +from gyan.api.controllers import link +from gyan.common.policies import ml_model as policies + +_basic_keys = ( + 'uuid', + 'user_id', + 'project_id', + 'name', + 'url', + 'status', + 'status_reason', + 'task_state', + 'labels', + 'host', + 'status_detail' +) + + +def format_ml_model(context, url, ml_model): + def transform(key, value): + if key not in _basic_keys: + return + # strip the key if it is not allowed by policy + policy_action = policies.ML_MODEL % ('get_one:%s' % key) + if not context.can(policy_action, fatal=False, might_not_exist=True): + return + if key == 'uuid': + yield ('uuid', value) + if url: + yield ('links', [link.make_link( + 'self', url, 'ml_models', value), + link.make_link( + 'bookmark', url, + 'ml_models', value, + bookmark=True)]) + else: + yield (key, value) + + return dict(itertools.chain.from_iterable( + transform(k, v) for k, v in ml_model.items())) diff --git a/gyan/api/controllers/versions.py b/gyan/api/controllers/versions.py new file mode 100644 index 0000000..9207d9e --- /dev/null +++ b/gyan/api/controllers/versions.py @@ -0,0 +1,145 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from webob import exc + +from gyan.common.i18n import _ + +# NOTE(tbh): v1.0 is reserved to indicate Ocata's API, but is not presently +# supported by the API service. All changes between Ocata and the +# point where we added microversioning are considered backwards- +# compatible, but are not specifically discoverable at this time. +# +# The v1.1 version indicates this "initial" version as being +# different from Ocata (v1.0), and includes the following changes: +# +# Add details of new api versions here: + +# +# For each newly added microversion change, update the API version history +# string below with a one or two line description. Also update +# rest_api_version_history.rst for extra information on microversion. +REST_API_VERSION_HISTORY = """REST API Version History: + + * 1.1 - Initial version +""" + +BASE_VER = '1.1' +CURRENT_MAX_VER = '1.1' + + +class Version(object): + """API Version object.""" + + string = 'OpenStack-API-Version' + """HTTP Header string carrying the requested version""" + + min_string = 'OpenStack-API-Minimum-Version' + """HTTP response header""" + + max_string = 'OpenStack-API-Maximum-Version' + """HTTP response header""" + + service_string = 'ml' + + def __init__(self, headers, default_version, latest_version, + from_string=None): + """Create an API Version object from the supplied headers. + + :param headers: webob headers + :param default_version: version to use if not specified in headers + :param latest_version: version to use if latest is requested + :param from_string: create the version from string not headers + :raises: webob.HTTPNotAcceptable + """ + if from_string: + (self.major, self.minor) = tuple(int(i) + for i in from_string.split('.')) + + else: + (self.major, self.minor) = Version.parse_headers(headers, + default_version, + latest_version) + + def __repr__(self): + return '%s.%s' % (self.major, self.minor) + + @staticmethod + def parse_headers(headers, default_version, latest_version): + """Determine the API version requested based on the headers supplied. + + :param headers: webob headers + :param default_version: version to use if not specified in headers + :param latest_version: version to use if latest is requested + :returns: a tuple of (major, minor) version numbers + :raises: webob.HTTPNotAcceptable + """ + + version_hdr = headers.get(Version.string, default_version) + + try: + version_service, version_str = version_hdr.split() + except ValueError: + raise exc.HTTPNotAcceptable(_( + "Invalid service type for %s header") % Version.string) + + if version_str.lower() == 'latest': + version_service, version_str = latest_version.split() + + if version_service != Version.service_string: + raise exc.HTTPNotAcceptable(_( + "Invalid service type for %s header") % Version.string) + try: + version = tuple(int(i) for i in version_str.split('.')) + except ValueError: + version = () + + if len(version) != 2: + raise exc.HTTPNotAcceptable(_( + "Invalid value for %s header") % Version.string) + return version + + def is_null(self): + return self.major == 0 and self.minor == 0 + + def matches(self, start_version, end_version): + if self.is_null(): + raise ValueError + + return start_version <= self <= end_version + + def __lt__(self, other): + if self.major < other.major: + return True + if self.major == other.major and self.minor < other.minor: + return True + return False + + def __gt__(self, other): + if self.major > other.major: + return True + if self.major == other.major and self.minor > other.minor: + return True + return False + + def __eq__(self, other): + return self.major == other.major and self.minor == other.minor + + def __le__(self, other): + return self < other or self == other + + def __ne__(self, other): + return not self.__eq__(other) + + def __ge__(self, other): + return self > other or self == other diff --git a/gyan/api/hooks.py b/gyan/api/hooks.py new file mode 100644 index 0000000..1f6c2da --- /dev/null +++ b/gyan/api/hooks.py @@ -0,0 +1,114 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from pecan import hooks + +from gyan.common import context +from gyan.compute import api as compute_api +import gyan.conf + +CONF = gyan.conf.CONF + + +class ContextHook(hooks.PecanHook): + """Configures a request context and attaches it to the request. + + The following HTTP request headers are used: + + X-User-Name: + Used for context.user_name. + + X-User-Id: + Used for context.user_id. + + X-Project-Name: + Used for context.project. + + X-Project-Id: + Used for context.project_id. + + X-Auth-Token: + Used for context.auth_token. + + X-Roles: + Used for context.roles. + """ + + def before(self, state): + headers = state.request.headers + user_name = headers.get('X-User-Name') + user_id = headers.get('X-User-Id') + project = headers.get('X-Project-Name') + project_id = headers.get('X-Project-Id') + domain_id = headers.get('X-User-Domain-Id') + domain_name = headers.get('X-User-Domain-Name') + auth_token = headers.get('X-Auth-Token') + roles = headers.get('X-Roles', '').split(',') + auth_token_info = state.request.environ.get('keystone.token_info') + + state.request.context = context.make_context( + auth_token=auth_token, + auth_token_info=auth_token_info, + user_name=user_name, + user_id=user_id, + project_name=project, + project_id=project_id, + domain_id=domain_id, + domain_name=domain_name, + roles=roles) + + +class RPCHook(hooks.PecanHook): + """Attach the rpcapi object to the request so controllers can get to it.""" + + def before(self, state): + context = state.request.context + state.request.compute_api = compute_api.API(context) + + +class NoExceptionTracebackHook(hooks.PecanHook): + """Workaround rpc.common: deserialize_remote_exception. + + deserialize_remote_exception builds rpc exception traceback into error + message which is then sent to the client. Such behavior is a security + concern so this hook is aimed to cut-off traceback from the error message. + """ + # NOTE(tbh): 'after' hook used instead of 'on_error' because + # 'on_error' never fired for wsme+pecan pair. wsme @wsexpose decorator + # catches and handles all the errors, so 'on_error' dedicated for unhandled + # exceptions never fired. + def after(self, state): + # Omit empty body. Some errors may not have body at this level yet. + if not state.response.body: + return + + # Do nothing if there is no error. + if 200 <= state.response.status_int < 400: + return + + json_body = state.response.json + # Do not remove traceback when server in debug mode (except 'Server' + # errors when 'debuginfo' will be used for traces). + if CONF.debug and json_body.get('faultcode') != 'Server': + return + + title = json_body.get('title') + traceback_marker = 'Traceback (most recent call last):' + if title and (traceback_marker in title): + # Cut-off traceback. + title = title.split(traceback_marker, 1)[0] + # Remove trailing newlines and spaces if any. + json_body['title'] = title.rstrip() + # Replace the whole json. Cannot change original one beacause it's + # generated on the fly. + state.response.json = json_body \ No newline at end of file diff --git a/gyan/api/http_error.py b/gyan/api/http_error.py new file mode 100644 index 0000000..b6a5b51 --- /dev/null +++ b/gyan/api/http_error.py @@ -0,0 +1,69 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six + +from oslo_serialization import jsonutils as json +from webob import exc + + +class HTTPNotAcceptableAPIVersion(exc.HTTPNotAcceptable): + # subclass of :class:`~HTTPNotAcceptable` + # + # This indicates the resource identified by the request is only + # capable of generating response entities which have content + # characteristics not acceptable according to the accept headers + # sent in the request. + # + # code: 406, title: Not Acceptable + # + # differences from webob.exc.HTTPNotAcceptable: + # + # - additional max and min version parameters + # - additional error info for code, title, and links + code = 406 + title = 'Not Acceptable' + max_version = '' + min_version = '' + + def __init__(self, detail=None, headers=None, comment=None, + body_template=None, max_version='', min_version='', **kwargs): + + super(HTTPNotAcceptableAPIVersion, self).__init__( + detail=detail, headers=headers, comment=comment, + body_template=body_template, **kwargs) + + self.max_version = max_version + self.min_version = min_version + + def __call__(self, environ, start_response): + for err_str in self.app_iter: + err = {} + try: + err = json.loads(err_str.decode('utf-8')) + except ValueError: + pass + + links = {'rel': 'help', 'href': 'https://developer.openstack.org' + '/api-guide/compute/microversions.html'} + + err['max_version'] = self.max_version + err['min_version'] = self.min_version + err['code'] = "gyan.microversion-unsupported" + err['links'] = [links] + err['title'] = "Requested microversion is unsupported" + + self.app_iter = [six.b(json.dump_as_bytes(err))] + self.headers['Content-Length'] = str(len(self.app_iter[0])) + + return super(HTTPNotAcceptableAPIVersion, self).__call__( + environ, start_response) \ No newline at end of file diff --git a/gyan/api/middleware/__init__.py b/gyan/api/middleware/__init__.py new file mode 100644 index 0000000..764df76 --- /dev/null +++ b/gyan/api/middleware/__init__.py @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gyan.api.middleware import auth_token +from gyan.api.middleware import parsable_error + + +AuthTokenMiddleware = auth_token.AuthTokenMiddleware +ParsableErrorMiddleware = parsable_error.ParsableErrorMiddleware + +__all__ = ('AuthTokenMiddleware', + 'ParsableErrorMiddleware') diff --git a/gyan/api/middleware/auth_token.py b/gyan/api/middleware/auth_token.py new file mode 100644 index 0000000..7e6b18d --- /dev/null +++ b/gyan/api/middleware/auth_token.py @@ -0,0 +1,71 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re + +from keystonemiddleware import auth_token +from oslo_log import log + +from gyan.common import exception +from gyan.common.i18n import _ +from gyan.common import utils + +LOG = log.getLogger(__name__) + + +class AuthTokenMiddleware(auth_token.AuthProtocol): + """A wrapper on Keystone auth_token middleware. + + Does not perform verification of authentication tokens + for public routes in the API. + + """ + + def __init__(self, app, conf, public_api_routes=None): + if public_api_routes is None: + public_api_routes = [] + route_pattern_tpl = '%s(\.json)?$' + + try: + self.public_api_routes = [re.compile(route_pattern_tpl % route_tpl) + for route_tpl in public_api_routes] + except re.error as e: + msg = _('Cannot compile public API routes: %s') % e + + LOG.error(msg) + raise exception.ConfigInvalid(error_msg=msg) + + super(AuthTokenMiddleware, self).__init__(app, conf) + + def __call__(self, env, start_response): + path = utils.safe_rstrip(env.get('PATH_INFO'), '/') + + # The information whether the API call is being performed against the + # public API is required for some other components. Saving it to the + # WSGI environment is reasonable thereby. + env['is_public_api'] = any([re.match(pattern, path) + for pattern in self.public_api_routes]) + + if env['is_public_api']: + return self._app(env, start_response) + + return super(AuthTokenMiddleware, self).__call__(env, start_response) + + @classmethod + def factory(cls, global_config, **local_conf): + public_routes = local_conf.get('acl_public_routes', '') + public_api_routes = [path.strip() for path in public_routes.split(',')] + + def _factory(app): + return cls(app, global_config, public_api_routes=public_api_routes) + + return _factory diff --git a/gyan/api/middleware/parsable_error.py b/gyan/api/middleware/parsable_error.py new file mode 100644 index 0000000..63e61a4 --- /dev/null +++ b/gyan/api/middleware/parsable_error.py @@ -0,0 +1,99 @@ +# Copyright ? 2012 New Dream Network, LLC (DreamHost) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Middleware to replace the plain text message body of an error +response with one formatted so the client can parse it. + +Based on pecan.middleware.errordocument +""" + +import six + +from oslo_serialization import jsonutils as json +from gyan.common.i18n import _ + + +class ParsableErrorMiddleware(object): + """Replace error body with something the client can parse.""" + + def __init__(self, app): + self.app = app + + def __call__(self, environ, start_response): + # Request for this state, modified by replace_start_response() + # and used when an error is being reported. + state = {} + + def replacement_start_response(status, headers, exc_info=None): + """Overrides the default response to make errors parsable.""" + try: + status_code = int(status.split(' ')[0]) + state['status_code'] = status_code + except (ValueError, TypeError): # pragma: nocover + raise Exception(_( + 'ErrorDocumentMiddleware received an invalid ' + 'status %s') % status) + else: + if (state['status_code'] // 100) not in (2, 3): + # Remove some headers so we can replace them later + # when we have the full error message and can + # compute the length. + headers = [(h, v) + for (h, v) in headers + if h not in ('Content-Length', 'Content-Type') + ] + # Save the headers in case we need to modify them. + state['headers'] = headers + return start_response(status, headers, exc_info) + + app_iter = self.app(environ, replacement_start_response) + + if (state['status_code'] // 100) not in (2, 3): + errs = [] + for err_str in app_iter: + err = {} + try: + err = json.loads(err_str.decode('utf-8')) + except ValueError: + pass + + if 'title' in err and 'description' in err: + title = err['title'] + desc = err['description'] + else: + title = '' + desc = '' + + error_code = err['faultstring'].lower() \ + if 'faultstring' in err else '' + # 'ml-infra' is the service-name. The general form of the + # code is service-name.error-code. + code = '.'.join(['ml-infra', error_code]) + + errs.append({ + 'request_id': '', + 'code': code, + 'status': state['status_code'], + 'title': title, + 'detail': desc, + 'links': [] + }) + + body = [six.b(json.dumps({'errors': errs}))] + + state['headers'].append(('Content-Type', 'application/json')) + state['headers'].append(('Content-Length', str(len(body[0])))) + else: + body = app_iter + return body diff --git a/gyan/api/servicegroup.py b/gyan/api/servicegroup.py new file mode 100644 index 0000000..247b089 --- /dev/null +++ b/gyan/api/servicegroup.py @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_utils import timeutils + +import gyan.conf +from gyan import objects + +CONF = gyan.conf.CONF + + +class ServiceGroup(object): + def __init__(self): + self.service_down_time = CONF.service_down_time + + def service_is_up(self, member): + if not isinstance(member, objects.GyanService): + raise TypeError + if member.forced_down: + return False + + last_heartbeat = (member.last_seen_up or + member.updated_at or member.created_at) + now = timeutils.utcnow() + elapsed = timeutils.delta_seconds(last_heartbeat, now) + is_up = abs(elapsed) <= self.service_down_time + return is_up diff --git a/gyan/api/utils.py b/gyan/api/utils.py new file mode 100644 index 0000000..d48aa8b --- /dev/null +++ b/gyan/api/utils.py @@ -0,0 +1,116 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools +from oslo_utils import uuidutils +import pecan + +from gyan.api.controllers import versions +from gyan.common import exception +from gyan.common.i18n import _ +import gyan.conf +from gyan import objects + +CONF = gyan.conf.CONF + + +def string_or_none(value): + if value in [None, 'None']: + return None + else: + return value + + +def validate_limit(limit): + try: + if limit is not None and int(limit) <= 0: + raise exception.InvalidValue(_("Limit must be positive integer")) + except ValueError: + raise exception.InvalidValue(_("Limit must be positive integer")) + + if limit is not None: + return min(CONF.api.max_limit, int(limit)) + else: + return CONF.api.max_limit + + +def validate_sort_dir(sort_dir): + if sort_dir not in ['asc', 'desc']: + raise exception.InvalidValue(_("Invalid sort direction: %s. " + "Acceptable values are " + "'asc' or 'desc'") % sort_dir) + return sort_dir + + +def get_resource(resource, resource_ident): + """Get the resource from the uuid or logical name. + + :param resource: the resource type. + :param resource_ident: the UUID or logical name of the resource. + + :returns: The resource. + """ + resource = getattr(objects, resource) + context = pecan.request.context + if context.is_admin: + context.all_projects = True + if uuidutils.is_uuid_like(resource_ident): + return resource.get_by_uuid(context, resource_ident) + + return resource.get_by_name(context, resource_ident) + + +def _do_enforce_content_types(pecan_req, valid_content_types): + """Content type enforcement + + Check to see that content type in the request is one of the valid + types passed in by our caller. + """ + if pecan_req.content_type not in valid_content_types: + m = ( + "Unexpected content type: {type}. Expected content types " + "are: {expected}" + ).format( + type=pecan_req.content_type.decode('utf-8'), + expected=valid_content_types + ) + pecan.abort(415, m) + + +def enforce_content_types(valid_content_types): + """Decorator handling content type enforcement on behalf of REST verbs.""" + + def content_types_decorator(fn): + + @functools.wraps(fn) + def content_types_enforcer(inst, *args, **kwargs): + _do_enforce_content_types(pecan.request, valid_content_types) + return fn(inst, *args, **kwargs) + + return content_types_enforcer + + return content_types_decorator + + +def version_check(action, version): + """Check whether the current version supports the operation. + + :param action: Operations to be executed. + :param version: The minimum version required to perform the operation. + + """ + req_version = pecan.request.version + min_version = versions.Version('', '', '', version) + if req_version < min_version: + raise exception.InvalidParamInVersion(param=action, + req_version=req_version, + min_version=min_version) \ No newline at end of file diff --git a/gyan/api/validation/__init__.py b/gyan/api/validation/__init__.py new file mode 100644 index 0000000..5c999df --- /dev/null +++ b/gyan/api/validation/__init__.py @@ -0,0 +1,57 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools + +from gyan.api.validation import validators + + +def validated(request_body_schema): + """Register a schema to validate a resource reference. + + Registered schema will be used for validating a request body just before + API method execution. + + :param request_body_schema: a schema to validate the resource reference + """ + schema_validator = validators.SchemaValidator(request_body_schema, + is_body=True) + + def add_validator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + schema_validator.validate(kwargs) + return func(*args, **kwargs) + return wrapper + return add_validator + + +def validate_query_param(req, query_param_schema): + """Register a schema to validate a resource reference. + + Registered schema will be used for validating a request query params + just before API method execution. + + :param req: the request object + :param query_param_schema: a schema to validate the resource reference + """ + + schema_validator = validators.SchemaValidator(query_param_schema, + is_body=False) + + def add_validator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + schema_validator.validate(req.params.mixed()) + return func(*args, **kwargs) + return wrapper + return add_validator diff --git a/gyan/api/validation/validators.py b/gyan/api/validation/validators.py new file mode 100644 index 0000000..aedf9c2 --- /dev/null +++ b/gyan/api/validation/validators.py @@ -0,0 +1,80 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import jsonschema +import six + +from gyan.common import exception +from gyan.common.i18n import _ + + +class SchemaValidator(object): + """Resource reference validator class.""" + + validator_org = jsonschema.Draft4Validator + + def __init__(self, schema, is_body=True): + self.is_body = is_body + validators = { + 'minimum': self._validate_minimum, + 'maximum': self._validate_maximum + } + validator_cls = jsonschema.validators.extend(self.validator_org, + validators) + fc = jsonschema.FormatChecker() + self.validator = validator_cls(schema, format_checker=fc) + + def validate(self, *args, **kwargs): + try: + self.validator.validate(*args, **kwargs) + except jsonschema.ValidationError as ex: + if len(ex.path) > 0: + if self.is_body: + detail = _("Invalid input for field '%(path)s'." + "Value: '%(value)s'. %(message)s") + else: + detail = _("Invalid input for query parameters " + "'%(path)s'. Value: '%(value)s'. %(message)s") + detail = detail % { + 'path': ex.path.pop(), 'value': ex.instance, + 'message': six.text_type(ex) + } + else: + detail = six.text_type(ex) + raise exception.SchemaValidationError(detail=detail) + + def _number_from_str(self, instance): + if isinstance(instance, float) or isinstance(instance, int): + return instance + + try: + value = int(instance) + except (ValueError, TypeError): + try: + value = float(instance) + except (ValueError, TypeError): + return None + return value + + def _validate_minimum(self, validator, minimum, instance, schema): + instance = self._number_from_str(instance) + if instance is None: + return + return self.validator_org.VALIDATORS['minimum'](validator, minimum, + instance, schema) + + def _validate_maximum(self, validator, maximum, instance, schema): + instance = self._number_from_str(instance) + if instance is None: + return + return self.validator_org.VALIDATORS['maximum'](validator, maximum, + instance, schema) diff --git a/gyan/api/versioned_method.py b/gyan/api/versioned_method.py new file mode 100644 index 0000000..53478f2 --- /dev/null +++ b/gyan/api/versioned_method.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class VersionedMethod(object): + + def __init__(self, name, start_version, end_version, func): + """Versioning information for a single method + + @name: Name of the method + @start_version: Minimum acceptable version + @end_version: Maximum acceptable_version + @func: Method to call + + Minimum and maximums are inclusive + """ + self.name = name + self.start_version = start_version + self.end_version = end_version + self.func = func + + def __str__(self): + return ("Version Method %s: min: %s, max: %s" + % (self.name, self.start_version, self.end_version)) \ No newline at end of file diff --git a/gyan/api/wsgi.py b/gyan/api/wsgi.py new file mode 100644 index 0000000..9e62b9c --- /dev/null +++ b/gyan/api/wsgi.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from oslo_log import log + +from gyan.api import app +from gyan.common import profiler +from gyan.common import service +import gyan.conf + + +CONF = gyan.conf.CONF +LOG = log.getLogger(__name__) + + +def init_application(): + # Initialize the oslo configuration library and logging + service.prepare_service(sys.argv) + # NOTE:(tbh) change localhost to CONF.host + profiler.setup('gyan-api', "localhost") + + LOG.debug("Configuration:") + CONF.log_opt_values(LOG, log.DEBUG) + + return app.load_app() diff --git a/gyan/cmd/__init__.py b/gyan/cmd/__init__.py new file mode 100644 index 0000000..c45022f --- /dev/null +++ b/gyan/cmd/__init__.py @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# NOTE(tbh): we monkey patch all eventlet services for easier tracking/debug + +import eventlet + +eventlet.monkey_patch() diff --git a/gyan/cmd/api.py b/gyan/cmd/api.py new file mode 100644 index 0000000..ae473d8 --- /dev/null +++ b/gyan/cmd/api.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The gyan Service API.""" + +import sys + +from gyan.common import profiler +from gyan.common import service as gyan_service +import gyan.conf + +CONF = gyan.conf.CONF + + +def main(): + # Parse config file and command line options, then start logging + gyan_service.prepare_service(sys.argv) + + # Enable object backporting via the conductor + # TODO(tbh): Uncomment after rpc services are implemented + # base.gyanObject.indirection_api = base.gyanObjectIndirectionAPI() + + # Setup OSprofiler for WSGI service + profiler.setup('gyan-api', CONF.api.host_ip) + + # Build and start the WSGI app + launcher = gyan_service.process_launcher() + server = gyan_service.WSGIService( + 'gyan_api', + CONF.api.enable_ssl_api + ) + launcher.launch_service(server, workers=server.workers) + launcher.wait() + +if __name__ == '__main__': + sys.exit(main()) diff --git a/gyan/cmd/compute.py b/gyan/cmd/compute.py new file mode 100644 index 0000000..4edfcc8 --- /dev/null +++ b/gyan/cmd/compute.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import shlex +import sys + +from oslo_log import log as logging +from oslo_privsep import priv_context +from oslo_service import service + +from gyan.common import rpc_service +from gyan.common import service as gyan_service +from gyan.common import utils +import gyan.conf + +CONF = gyan.conf.CONF +LOG = logging.getLogger(__name__) + + +def main(): + gyan_service.prepare_service(sys.argv) + + LOG.info('Starting server in PID %s', os.getpid()) + CONF.log_opt_values(LOG, logging.DEBUG) + + CONF.import_opt('topic', 'gyan.conf.compute', group='compute') + CONF.import_opt('host', 'gyan.conf.compute', group='compute') + + + from gyan.compute import manager as compute_manager + endpoints = [ + compute_manager.Manager(), + ] + server = rpc_service.Service.create(CONF.compute.topic, CONF.compute.host, + endpoints, binary='gyan-compute') + launcher = service.launch(CONF, server, restart_method='mutate') + launcher.wait() diff --git a/gyan/cmd/db_manage.py b/gyan/cmd/db_manage.py new file mode 100644 index 0000000..28001fa --- /dev/null +++ b/gyan/cmd/db_manage.py @@ -0,0 +1,67 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Starter script for gyan-db-manage.""" + +from oslo_config import cfg + +from gyan.db import migration + + +CONF = cfg.CONF + + +def do_version(): + print('Current DB revision is %s' % migration.version()) + + +def do_upgrade(): + migration.upgrade(CONF.command.revision) + + +def do_stamp(): + migration.stamp(CONF.command.revision) + + +def do_revision(): + migration.revision(message=CONF.command.message, + autogenerate=CONF.command.autogenerate) + + +def add_command_parsers(subparsers): + parser = subparsers.add_parser('version') + parser.set_defaults(func=do_version) + + parser = subparsers.add_parser('upgrade') + parser.add_argument('revision', nargs='?') + parser.set_defaults(func=do_upgrade) + + parser = subparsers.add_parser('stamp') + parser.add_argument('revision') + parser.set_defaults(func=do_stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.set_defaults(func=do_revision) + + +def main(): + command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + CONF.register_cli_opt(command_opt) + + CONF(project='gyan') + CONF.command.func() diff --git a/gyan/common/__init__.py b/gyan/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/common/config.py b/gyan/common/config.py new file mode 100644 index 0000000..ffd1856 --- /dev/null +++ b/gyan/common/config.py @@ -0,0 +1,54 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_middleware import cors + +from gyan.common import rpc +import gyan.conf +from gyan import version + + +def parse_args(argv, default_config_files=None): + rpc.set_defaults(control_exchange='gyan') + gyan.conf.CONF(argv[1:], + project='gyan', + version=version.version_info.release_string(), + default_config_files=default_config_files) + rpc.init(gyan.conf.CONF) + + +def set_config_defaults(): + """This method updates all configuration default values.""" + set_cors_middleware_defaults() + + +def set_cors_middleware_defaults(): + """Update default configuration options for oslo.middleware.""" + cors.set_defaults( + allow_headers=['X-Auth-Token', + 'X-Identity-Status', + 'X-Roles', + 'X-Service-Catalog', + 'X-User-Id', + 'X-Project-Id', + 'X-OpenStack-Request-ID', + 'X-Server-Management-Url'], + expose_headers=['X-Auth-Token', + 'X-Subject-Token', + 'X-Service-Token', + 'X-OpenStack-Request-ID', + 'X-Server-Management-Url'], + allow_methods=['GET', + 'PUT', + 'POST', + 'DELETE', + 'PATCH']) diff --git a/gyan/common/consts.py b/gyan/common/consts.py new file mode 100644 index 0000000..d45edaa --- /dev/null +++ b/gyan/common/consts.py @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ALLOCATED = 'allocated' +CREATED = 'created' +UNDEPLOYED = 'undeployed' +DEPLOYED = 'deployed' \ No newline at end of file diff --git a/gyan/common/context.py b/gyan/common/context.py new file mode 100644 index 0000000..d7af07a --- /dev/null +++ b/gyan/common/context.py @@ -0,0 +1,180 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools + +import copy +from oslo_context import context +from oslo_utils import timeutils +import six + +from gyan.common import exception +from gyan.common import policy + + +class RequestContext(context.RequestContext): + """Extends security contexts from the OpenStack common library.""" + + def __init__(self, auth_token=None, domain_id=None, + domain_name=None, user_name=None, user_id=None, + user_domain_name=None, user_domain_id=None, + project_name=None, project_id=None, roles=None, + is_admin=None, read_only=False, show_deleted=False, + request_id=None, trust_id=None, auth_token_info=None, + all_projects=False, password=None, timestamp=None, **kwargs): + """Stores several additional request parameters: + + :param domain_id: The ID of the domain. + :param domain_name: The name of the domain. + :param user_domain_id: The ID of the domain to + authenticate a user against. + :param user_domain_name: The name of the domain to + authenticate a user against. + + """ + super(RequestContext, self).__init__(auth_token=auth_token, + user_id=user_name, + project_id=project_name, + is_admin=is_admin, + read_only=read_only, + show_deleted=show_deleted, + request_id=request_id, + roles=roles) + + self.user_name = user_name + self.user_id = user_id + self.project_name = project_name + self.project_id = project_id + self.domain_id = domain_id + self.domain_name = domain_name + self.user_domain_id = user_domain_id + self.user_domain_name = user_domain_name + self.auth_token_info = auth_token_info + self.trust_id = trust_id + self.all_projects = all_projects + self.password = password + if is_admin is None: + self.is_admin = policy.check_is_admin(self) + else: + self.is_admin = is_admin + + if not timestamp: + timestamp = timeutils.utcnow() + if isinstance(timestamp, six.string_types): + timestamp = timeutils.parse_strtime(timestamp) + self.timestamp = timestamp + + def to_dict(self): + value = super(RequestContext, self).to_dict() + value.update({'auth_token': self.auth_token, + 'domain_id': self.domain_id, + 'domain_name': self.domain_name, + 'user_domain_id': self.user_domain_id, + 'user_domain_name': self.user_domain_name, + 'user_name': self.user_name, + 'user_id': self.user_id, + 'project_name': self.project_name, + 'project_id': self.project_id, + 'is_admin': self.is_admin, + 'read_only': self.read_only, + 'roles': self.roles, + 'show_deleted': self.show_deleted, + 'request_id': self.request_id, + 'trust_id': self.trust_id, + 'auth_token_info': self.auth_token_info, + 'password': self.password, + 'all_projects': self.all_projects, + 'timestamp': timeutils.strtime(self.timestamp) if + hasattr(self, 'timestamp') else None + }) + return value + + def to_policy_values(self): + policy = super(RequestContext, self).to_policy_values() + policy['is_admin'] = self.is_admin + return policy + + @classmethod + def from_dict(cls, values): + return cls(**values) + + def elevated(self): + """Return a version of this context with admin flag set.""" + context = copy.copy(self) + # context.roles must be deepcopied to leave original roles + # without changes + context.roles = copy.deepcopy(self.roles) + context.is_admin = True + + if 'admin' not in context.roles: + context.roles.append('admin') + + return context + + def can(self, action, target=None, fatal=True, might_not_exist=False): + """Verifies that the given action is valid on the target in this context. + + :param action: string representing the action to be checked. + :param target: dictionary representing the object of the action + for object creation this should be a dictionary representing the + location of the object e.g. ``{'project_id': context.project_id}``. + If None, then this default target will be considered: + {'project_id': self.project_id, 'user_id': self.user_id} + :param fatal: if False, will return False when an + exception.NotAuthorized occurs. + :param might_not_exist: If True the policy check is skipped (and the + function returns True) if the specified policy does not exist. + Defaults to false. + + :raises gyan.common.exception.NotAuthorized: if verification fails and + fatal is True. + + :return: returns a non-False value (not necessarily "True") if + authorized and False if not authorized and fatal is False. + """ + if target is None: + target = {'project_id': self.project_id, + 'user_id': self.user_id} + + try: + return policy.authorize(self, action, target, + might_not_exist=might_not_exist) + except exception.NotAuthorized: + if fatal: + raise + return False + + +def make_context(*args, **kwargs): + return RequestContext(*args, **kwargs) + + +def get_admin_context(show_deleted=False, all_projects=False): + """Create an administrator context. + + :param show_deleted: if True, will show deleted items when query db + """ + context = RequestContext(user_id=None, + project=None, + is_admin=True, + show_deleted=show_deleted, + all_projects=all_projects) + return context + + +def set_context(func): + @functools.wraps(func) + def handler(self, ctx): + if ctx is None: + ctx = get_admin_context(all_projects=True) + func(self, ctx) + return handler diff --git a/gyan/common/exception.py b/gyan/common/exception.py new file mode 100644 index 0000000..67c1ffc --- /dev/null +++ b/gyan/common/exception.py @@ -0,0 +1,485 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gyan base exception handling. + +Includes decorator for re-raising Gyan-type exceptions. + +""" + +import functools +import inspect +import re +import sys + +from keystoneclient import exceptions as keystone_exceptions +from oslo_config import cfg +from oslo_log import log as logging +from oslo_utils import excutils +from oslo_utils import uuidutils +import pecan +import six +from webob import util as woutil + +from gyan.common.i18n import _ +import gyan.conf + +LOG = logging.getLogger(__name__) + +CONF = gyan.conf.CONF + +try: + CONF.import_opt('fatal_exception_format_errors', + 'oslo_versionedobjects.exception') +except cfg.NoSuchOptError as e: + # Note:work around for gyan run against master branch + # in devstack gate job, as gyan not branched yet + # versionobjects kilo/master different version can + # cause issue here. As it changed import group. So + # add here before branch to prevent gate failure. + # Bug: #1447873 + CONF.import_opt('fatal_exception_format_errors', + 'oslo_versionedobjects.exception', + group='oslo_versionedobjects') + + +def wrap_exception(notifier=None, event_type=None): + """This decorator wraps a method to catch any exceptions. + + It logs the exception as well as optionally sending + it to the notification system. + """ + def inner(f): + def wrapped(self, context, *args, **kwargs): + # Don't store self or context in the payload, it now seems to + # contain confidential information. + try: + return f(self, context, *args, **kwargs) + except Exception as e: + with excutils.save_and_reraise_exception(): + if notifier: + call_dict = inspect.getcallargs(f, self, context, + *args, **kwargs) + payload = dict(exception=e, + private=dict(args=call_dict) + ) + + temp_type = event_type + if not temp_type: + # If f has multiple decorators, they must use + # functools.wraps to ensure the name is + # propagated. + temp_type = f.__name__ + + notifier.error(context, temp_type, payload) + + return functools.wraps(f)(wrapped) + return inner + + +OBFUSCATED_MSG = _('Your request could not be handled ' + 'because of a problem in the server. ' + 'Error Correlation id is: %s') + + +def wrap_controller_exception(func, func_server_error, func_client_error): + """This decorator wraps controllers methods to handle exceptions: + + - if an unhandled Exception or a GyanException with an error code >=500 + is catched, raise a http 5xx ClientSideError and correlates it with a log + message + + - if a GyanException is catched and its error code is <500, raise a http + 4xx and logs the excp in debug mode + + """ + @functools.wraps(func) + def wrapped(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as excp: + if isinstance(excp, GyanException): + http_error_code = excp.code + else: + http_error_code = 500 + + if http_error_code >= 500: + # log the error message with its associated + # correlation id + log_correlation_id = uuidutils.generate_uuid() + LOG.exception("%(correlation_id)s:%(excp)s", + {'correlation_id': log_correlation_id, + 'excp': str(excp)}) + # raise a client error with an obfuscated message + return func_server_error(log_correlation_id, http_error_code) + else: + # raise a client error the original message + LOG.debug(excp) + return func_client_error(excp, http_error_code) + return wrapped + + +def convert_excp_to_err_code(excp_name): + """Convert Exception class name (CamelCase) to error-code (Snake-case)""" + words = re.findall(r'[A-Z]?[a-z]+|[A-Z]{2,}(?=[A-Z][a-z]|\d|\W|$)|\d+', + excp_name) + return '-'.join([str.lower(word) for word in words]) + + +def wrap_pecan_controller_exception(func): + """This decorator wraps pecan controllers to handle exceptions.""" + def _func_server_error(log_correlation_id, status_code): + pecan.response.status = status_code + return { + 'faultcode': 'Server', + 'status_code': status_code, + 'title': woutil.status_reasons[status_code], + 'description': six.text_type(OBFUSCATED_MSG % log_correlation_id), + } + + def _func_client_error(excp, status_code): + pecan.response.status = status_code + return { + 'faultcode': 'Client', + 'faultstring': convert_excp_to_err_code(excp.__class__.__name__), + 'status_code': status_code, + 'title': six.text_type(excp), + 'description': six.text_type(excp), + } + + return wrap_controller_exception(func, + _func_server_error, + _func_client_error) + + +def wrap_keystone_exception(func): + """Wrap keystone exceptions and throw gyan specific exceptions.""" + @functools.wraps(func) + def wrapped(*args, **kwargs): + try: + return func(*args, **kwargs) + except keystone_exceptions.AuthorizationFailure: + raise AuthorizationFailure( + client=func.__name__, message="reason: %s" % sys.exc_info()[1]) + except keystone_exceptions.ClientException: + raise AuthorizationFailure( + client=func.__name__, + message="unexpected keystone client error occurred: %s" + % sys.exc_info()[1]) + return wrapped + + +class GyanException(Exception): + """Base gyan Exception + + To correctly use this class, inherit from it and define + a 'message' property. That message will get printf'd + with the keyword arguments provided to the constructor. + + """ + message = _("An unknown exception occurred.") + code = 500 + + def __init__(self, message=None, **kwargs): + self.kwargs = kwargs + + if 'code' not in self.kwargs and hasattr(self, 'code'): + self.kwargs['code'] = self.code + + if message: + self.message = message + + try: + self.message = str(self.message) % kwargs + except KeyError: + # kwargs doesn't match a variable in the message + # log the issue and the kwargs + LOG.exception('Exception in string format operation, ' + 'kwargs: %s', kwargs) + try: + ferr = CONF.fatal_exception_format_errors + except cfg.NoSuchOptError: + ferr = CONF.oslo_versionedobjects.fatal_exception_format_errors + if ferr: + raise + + super(GyanException, self).__init__(self.message) + + def __str__(self): + if six.PY3: + return self.message + return self.message.encode('utf-8') + + def __unicode__(self): + return self.message + + def format_message(self): + if self.__class__.__name__.endswith('_Remote'): + return self.args[0] + else: + return six.text_type(self) + + +class ObjectNotFound(GyanException): + message = _("The %(name)s %(id)s could not be found.") + + +class ObjectNotUnique(GyanException): + message = _("The %(name)s already exists.") + + +class ObjectActionError(GyanException): + message = _('Object action %(action)s failed because: %(reason)s') + + +class ResourceNotFound(ObjectNotFound): + message = _("The %(name)s resource %(id)s could not be found.") + code = 404 + + +class ResourceExists(ObjectNotUnique): + message = _("The %(name)s resource already exists.") + code = 409 + + +class AuthorizationFailure(GyanException): + message = _("%(client)s connection failed. %(message)s") + + +class UnsupportedObjectError(GyanException): + message = _('Unsupported object type %(objtype)s') + + +class IncompatibleObjectVersion(GyanException): + message = _('Version %(objver)s of %(objname)s is not supported') + + +class OrphanedObjectError(GyanException): + message = _('Cannot call %(method)s on orphaned %(objtype)s object') + + +class Invalid(GyanException): + message = _("Unacceptable parameters.") + code = 400 + + +class InvalidValue(Invalid): + message = _("Received value '%(value)s' is invalid for type %(type)s.") + + +class ValidationError(Invalid): + message = "%(detail)s" + + +class SchemaValidationError(ValidationError): + message = "%(detail)s" + + +class InvalidUUID(Invalid): + message = _("Expected a uuid but received %(uuid)s.") + + +class InvalidName(Invalid): + message = _("Expected a name but received %(uuid)s.") + + +class InvalidDiscoveryURL(Invalid): + message = _("Received invalid discovery URL '%(discovery_url)s' for " + "discovery endpoint '%(discovery_endpoint)s'.") + + +class GetDiscoveryUrlFailed(GyanException): + message = _("Failed to get discovery url from '%(discovery_endpoint)s'.") + + +class InvalidUuidOrName(Invalid): + message = _("Expected a name or uuid but received %(uuid)s.") + + +class InvalidIdentity(Invalid): + message = _("Expected an uuid or int but received %(identity)s.") + + +class InvalidCsr(Invalid): + message = _("Received invalid csr %(csr)s.") + + +class HTTPNotFound(ResourceNotFound): + pass + + +class Conflict(GyanException): + message = _('Conflict.') + code = 409 + + +class ConflictOptions(Conflict): + message = _('Conflicting options.') + + +class InvalidState(Conflict): + message = _("Invalid resource state.") + + +# Cannot be templated as the error syntax varies. +# msg needs to be constructed when raised. +class InvalidParameterValue(Invalid): + message = _("%(err)s") + + +class InvalidParamInVersion(Invalid): + message = _('Invalid param %(param)s because current request ' + 'version is %(req_version)s. %(param)s is only ' + 'supported from version %(min_version)s') + + +class InvalidQuotaValue(Invalid): + message = _("Change would make usage less than 0 for the following " + "resources: %(unders)s") + + +class InvalidQuotaMethodUsage(Invalid): + message = _("Wrong quota method %(method)s used on resource %(res)s") + + +class PatchError(Invalid): + message = _("Couldn't apply patch '%(patch)s'. Reason: %(reason)s") + + +class NotAuthorized(GyanException): + message = _("Not authorized.") + code = 403 + + +class MLModelAlreadyExists(GyanException): + message = _("A ML Model with %(field)s %(value)s already exists.") + + +class MLModelNotFound(GyanException): + message = _("ML Model %(ml_model)s could not be found.") + +class ConfigInvalid(GyanException): + message = _("Invalid configuration file. %(error_msg)s") + + +class PolicyNotAuthorized(NotAuthorized): + message = _("Policy doesn't allow %(action)s to be performed.") + + +class ComputeHostNotFound(HTTPNotFound): + message = _("Compute host %(compute_host)s could not be found.") + + +class GyanServiceNotFound(HTTPNotFound): + message = _("Gyan service %(binary)s on host %(host)s could not be found.") + + +class ResourceProviderNotFound(HTTPNotFound): + message = _("Resource provider %(resource_provider)s could not be found.") + + +class ResourceClassNotFound(HTTPNotFound): + message = _("Resource class %(resource_class)s could not be found.") + + +class ComputeHostAlreadyExists(ResourceExists): + message = _("A compute host with %(field)s %(value)s already exists.") + + +class GyanServiceAlreadyExists(ResourceExists): + message = _("Service %(binary)s on host %(host)s already exists.") + + +class ResourceProviderAlreadyExists(ResourceExists): + message = _("A resource provider with %(field)s %(value)s already exists.") + + +class ResourceClassAlreadyExists(ResourceExists): + message = _("A resource class with %(field)s %(value)s already exists.") + + +class UniqueConstraintViolated(ResourceExists): + message = _("A resource with %(fields)s violates unique constraint.") + + +class InvalidStateException(GyanException): + message = _("Cannot %(action)s ml model %(id)s in %(actual_state)s state") + code = 409 + + +class ServerInError(GyanException): + message = _('Went to status %(resource_status)s due to ' + '"%(status_reason)s"') + + +class ServerUnknownStatus(GyanException): + message = _('%(result)s - Unknown status %(resource_status)s due to ' + '"%(status_reason)s"') + + +class EntityNotFound(GyanException): + message = _("The %(entity)s (%(name)s) could not be found.") + + +class CommandError(GyanException): + message = _("The command: %(cmd)s failed on the system, due to %(error)s") + + +class NoValidHost(GyanException): + message = _("No valid host was found. %(reason)s") + + +class NotFound(GyanException): + message = _("Resource could not be found.") + code = 404 + + +class SchedulerHostFilterNotFound(NotFound): + message = _("Scheduler Host Filter %(filter_name)s could not be found.") + + +class ClassNotFound(NotFound): + message = _("Class %(class_name)s could not be found: %(exception)s") + + +class ApiVersionsIntersect(GyanException): + message = _("Version of %(name)s %(min_ver)s %(max_ver)s intersects " + "with another versions.") + + +class ConnectionFailed(GyanException): + message = _("Failed to connect to remote host") + + +class SocketException(GyanException): + message = _("Socket exceptions") + + +class ResourcesUnavailable(GyanException): + message = _("Insufficient compute resources: %(reason)s.") + + +class FileNotFound(GyanException): + message = _("The expected file not exist") + + +class FailedParseStringToJson(GyanException): + message = _("Failed parse string to json: %(reason)s.") + + +class ServerNotUsable(GyanException): + message = _("gyan server not usable") + code = 404 + + +class Base64Exception(Invalid): + msg_fmt = _("Invalid Base 64 file data") diff --git a/gyan/common/i18n.py b/gyan/common/i18n.py new file mode 100644 index 0000000..a9a63c8 --- /dev/null +++ b/gyan/common/i18n.py @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# It's based on oslo.i18n usage in OpenStack Keystone project and +# recommendations from +# https://docs.openstack.org/oslo.i18n/latest/user/usage.html + +import oslo_i18n + + +_translators = oslo_i18n.TranslatorFactory(domain='gyan') + +# The primary translation function using the well-known name "_" +_ = _translators.primary diff --git a/gyan/common/keystone.py b/gyan/common/keystone.py new file mode 100644 index 0000000..bc56edd --- /dev/null +++ b/gyan/common/keystone.py @@ -0,0 +1,89 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from keystoneauth1.access import access as ka_access +from keystoneauth1.identity import access as ka_access_plugin +from keystoneauth1.identity import v3 as ka_v3 +from keystoneauth1 import loading as ka_loading +from keystoneclient.v3 import client as kc_v3 +from oslo_log import log as logging + +from gyan.common import exception +import gyan.conf +from gyan.conf import keystone as ksconf + + +CONF = gyan.conf.CONF +LOG = logging.getLogger(__name__) + + +class KeystoneClientV3(object): + """Keystone client wrapper so we can encapsulate logic in one place.""" + + def __init__(self, context): + self.context = context + self._client = None + self._session = None + + @property + def auth_url(self): + # FIXME(pauloewerton): auth_url should be retrieved from keystone_auth + # section by default + url = CONF[ksconf.CFG_LEGACY_GROUP].www_authenticate_uri or \ + CONF[ksconf.CFG_LEGACY_GROUP].auth_uri + return url.replace('v2.0', 'v3') + + @property + def auth_token(self): + return self.session.get_token() + + @property + def session(self): + if self._session: + return self._session + auth = self._get_auth() + session = self._get_session(auth) + self._session = session + return session + + def _get_session(self, auth): + session = ka_loading.load_session_from_conf_options( + CONF, ksconf.CFG_GROUP, auth=auth) + return session + + def _get_auth(self): + if self.context.auth_token_info: + access_info = ka_access.create(body=self.context.auth_token_info, + auth_token=self.context.auth_token) + auth = ka_access_plugin.AccessInfoPlugin(access_info) + elif self.context.auth_token: + auth = ka_v3.Token(auth_url=self.auth_url, + token=self.context.auth_token) + elif self.context.is_admin: + auth = ka_loading.load_auth_from_conf_options(CONF, + ksconf.CFG_GROUP) + else: + msg = ('Keystone API connection failed: no password, ' + 'trust_id or token found.') + LOG.error(msg) + raise exception.AuthorizationFailure(client='keystone', + message='reason %s' % msg) + + return auth + + @property + def client(self): + if self._client: + return self._client + client = kc_v3.Client(session=self.session) + self._client = client + return client diff --git a/gyan/common/paths.py b/gyan/common/paths.py new file mode 100644 index 0000000..05acac6 --- /dev/null +++ b/gyan/common/paths.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import gyan.conf + +CONF = gyan.conf.CONF + + +def basedir_def(*args): + """Return an uninterpolated path relative to $pybasedir.""" + return os.path.join('$pybasedir', *args) + + +def bindir_def(*args): + """Return an uninterpolated path relative to $bindir.""" + return os.path.join('$bindir', *args) + + +def state_path_def(*args): + """Return an uninterpolated path relative to $state_path.""" + return os.path.join('$state_path', *args) + + +def basedir_rel(*args): + """Return a path relative to $pybasedir.""" + return os.path.join(CONF.pybasedir, *args) + + +def bindir_rel(*args): + """Return a path relative to $bindir.""" + return os.path.join(CONF.bindir, *args) + + +def state_path_rel(*args): + """Return a path relative to $state_path.""" + return os.path.join(CONF.state_path, *args) diff --git a/gyan/common/policies/__init__.py b/gyan/common/policies/__init__.py new file mode 100644 index 0000000..fd2d970 --- /dev/null +++ b/gyan/common/policies/__init__.py @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +from gyan.common.policies import host +from gyan.common.policies import base +from gyan.common.policies import ml_model + +def list_rules(): + return itertools.chain( + base.list_rules(), + host.list_rules(), + ml_model.list_rules() + ) diff --git a/gyan/common/policies/base.py b/gyan/common/policies/base.py new file mode 100644 index 0000000..2104f33 --- /dev/null +++ b/gyan/common/policies/base.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_policy import policy + +ROLE_ADMIN = 'role:admin' +RULE_ADMIN_OR_OWNER = 'is_admin:True or project_id:%(project_id)s' +RULE_ADMIN_API = 'rule:context_is_admin' +RULE_DENY_EVERYBODY = 'rule:deny_everybody' + +rules = [ + policy.RuleDefault( + name='context_is_admin', + check_str=ROLE_ADMIN + ), + policy.RuleDefault( + name='admin_or_owner', + check_str=RULE_ADMIN_OR_OWNER + ), + policy.RuleDefault( + name='admin_api', + check_str=RULE_ADMIN_API + ), + policy.RuleDefault( + name="deny_everybody", + check_str="!", + description="Default rule for deny everybody."), +] + + +def list_rules(): + return rules diff --git a/gyan/common/policies/host.py b/gyan/common/policies/host.py new file mode 100644 index 0000000..528eb4e --- /dev/null +++ b/gyan/common/policies/host.py @@ -0,0 +1,46 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_policy import policy + +from gyan.common.policies import base + +HOST = 'host:%s' + +rules = [ + policy.DocumentedRuleDefault( + name=HOST % 'get_all', + check_str=base.RULE_ADMIN_API, + description='List all compute hosts.', + operations=[ + { + 'path': '/v1/hosts', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=HOST % 'get', + check_str=base.RULE_ADMIN_API, + description='Show the details of a specific compute host.', + operations=[ + { + 'path': '/v1/hosts/{host_ident}', + 'method': 'GET' + } + ] + ) +] + + +def list_rules(): + return rules diff --git a/gyan/common/policies/ml_model.py b/gyan/common/policies/ml_model.py new file mode 100644 index 0000000..f45b167 --- /dev/null +++ b/gyan/common/policies/ml_model.py @@ -0,0 +1,123 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_policy import policy + +from gyan.common.policies import base + +ML_MODEL = 'ml_model:%s' + +rules = [ + policy.DocumentedRuleDefault( + name=ML_MODEL % 'create', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Create a new ML Model.', + operations=[ + { + 'path': '/v1/ml_models', + 'method': 'POST' + } + ] + ), + policy.DocumentedRuleDefault( + name=ML_MODEL % 'delete', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Delete a ML Model.', + operations=[ + { + 'path': '/v1/ml_models/{ml_model_ident}', + 'method': 'DELETE' + } + ] + ), + policy.DocumentedRuleDefault( + name=ML_MODEL % 'delete_all_projects', + check_str=base.RULE_ADMIN_API, + description='Delete a ml models from all projects.', + operations=[ + { + 'path': '/v1/ml_models/{ml_model_ident}', + 'method': 'DELETE' + } + ] + ), + policy.DocumentedRuleDefault( + name=ML_MODEL % 'delete_force', + check_str=base.RULE_ADMIN_API, + description='Forcibly delete a ML model.', + operations=[ + { + 'path': '/v1/ml_models/{ml_model_ident}', + 'method': 'DELETE' + } + ] + ), + policy.DocumentedRuleDefault( + name=ML_MODEL % 'get_one', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Retrieve the details of a specific ml model.', + operations=[ + { + 'path': '/v1/ml_models/{ml_model_ident}', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=ML_MODEL % 'get_all', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Retrieve the details of all ml models.', + operations=[ + { + 'path': '/v1/ml_models', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=ML_MODEL % 'get_all_all_projects', + check_str=base.RULE_ADMIN_API, + description='Retrieve the details of all ml models across projects.', + operations=[ + { + 'path': '/v1/ml_models', + 'method': 'GET' + } + ] + ), + policy.DocumentedRuleDefault( + name=ML_MODEL % 'update', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Update a ML Model.', + operations=[ + { + 'path': '/v1/ml_models/{ml_model_ident}', + 'method': 'PATCH' + } + ] + ), + policy.DocumentedRuleDefault( + name=ML_MODEL % 'upload', + check_str=base.RULE_ADMIN_OR_OWNER, + description='Upload the trained ML Model', + operations=[ + { + 'path': '/v1/ml_models/{ml_model_ident}/upload', + 'method': 'POST' + } + ] + ), +] + + +def list_rules(): + return rules diff --git a/gyan/common/policy.py b/gyan/common/policy.py new file mode 100644 index 0000000..3f5e058 --- /dev/null +++ b/gyan/common/policy.py @@ -0,0 +1,155 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Policy Engine For Gyan.""" + +from oslo_log import log as logging +from oslo_policy import policy +from oslo_utils import excutils + +from gyan.common import exception +from gyan.common import policies +import gyan.conf + +_ENFORCER = None +CONF = gyan.conf.CONF +LOG = logging.getLogger(__name__) + + +# we can get a policy enforcer by this init. +# oslo policy support change policy rule dynamically. +# at present, policy.enforce will reload the policy rules when it checks +# the policy files have been touched. +def init(policy_file=None, rules=None, + default_rule=None, use_conf=True, overwrite=True): + """Init an Enforcer class. + + :param policy_file: Custom policy file to use, if none is + specified, ``conf.policy_file`` will be + used. + :param rules: Default dictionary / Rules to use. It will be + considered just in the first instantiation. If + :meth:`load_rules` with ``force_reload=True``, + :meth:`clear` or :meth:`set_rules` with + ``overwrite=True`` is called this will be overwritten. + :param default_rule: Default rule to use, conf.default_rule will + be used if none is specified. + :param use_conf: Whether to load rules from cache or config file. + :param overwrite: Whether to overwrite existing rules when reload rules + from config file. + """ + global _ENFORCER + if not _ENFORCER: + # https://docs.openstack.org/oslo.policy/latest/user/usage.html + _ENFORCER = policy.Enforcer(CONF, + policy_file=policy_file, + rules=rules, + default_rule=default_rule, + use_conf=use_conf, + overwrite=overwrite) + register_rules(_ENFORCER) + return _ENFORCER + + +def register_rules(enforcer): + enforcer.register_defaults(policies.list_rules()) + + +def enforce(context, rule=None, target=None, + do_raise=True, exc=None, *args, **kwargs): + + """Checks authorization of a rule against the target and credentials. + + :param dict context: As much information about the user performing the + action as possible. + :param rule: The rule to evaluate. + :param dict target: As much information about the object being operated + on as possible. + :param do_raise: Whether to raise an exception or not if check + fails. + :param exc: Class of the exception to raise if the check fails. + Any remaining arguments passed to :meth:`enforce` (both + positional and keyword arguments) will be passed to + the exception class. If not specified, + :class:`PolicyNotAuthorized` will be used. + + :return: ``False`` if the policy does not allow the action and `exc` is + not provided; otherwise, returns a value that evaluates to + ``True``. Note: for rules using the "case" expression, this + ``True`` value will be the specified string from the + expression. + """ + enforcer = init() + credentials = context.to_policy_values() + if not exc: + exc = exception.PolicyNotAuthorized + if target is None: + target = {'project_id': context.project_id, + 'user_id': context.user_id} + return enforcer.enforce(rule, target, credentials, + do_raise=do_raise, exc=exc, *args, **kwargs) + + +def authorize(context, action, target, do_raise=True, exc=None, + might_not_exist=False): + """Verifies that the action is valid on the target in this context. + + :param context: gyan context + :param action: string representing the action to be checked + this should be colon separated for clarity. + i.e. ``network:attach_external_network`` + :param target: dictionary representing the object of the action + for object creation this should be a dictionary representing the + location of the object e.g. ``{'project_id': context.project_id}`` + :param do_raise: if True (the default), raises PolicyNotAuthorized; + if False, returns False + :param exc: Class of the exception to raise if the check fails. + Any remaining arguments passed to :meth:`authorize` (both + positional and keyword arguments) will be passed to + the exception class. If not specified, + :class:`PolicyNotAuthorized` will be used. + :param might_not_exist: If True the policy check is skipped (and the + function returns True) if the specified policy does not exist. + Defaults to false. + + :raises gyan.common.exception.PolicyNotAuthorized: if verification fails + and do_raise is True. Or if 'exc' is specified it will raise an + exception of that type. + + :return: returns a non-False value (not necessarily "True") if + authorized, and the exact value False if not authorized and + do_raise is False. + """ + credentials = context.to_policy_values() + if not exc: + exc = exception.PolicyNotAuthorized + if might_not_exist and not (_ENFORCER.rules and action in _ENFORCER.rules): + return True + try: + result = _ENFORCER.enforce(action, target, credentials, + do_raise=do_raise, exc=exc, action=action) + except Exception: + with excutils.save_and_reraise_exception(): + LOG.debug('Policy check for %(action)s failed with credentials ' + '%(credentials)s', + {'action': action, 'credentials': credentials}) + return result + + +def check_is_admin(context): + """Whether or not user is admin according to policy setting. + + """ + init() + target = {} + credentials = context.to_policy_values() + return _ENFORCER.enforce('context_is_admin', target, credentials) diff --git a/gyan/common/privileged.py b/gyan/common/privileged.py new file mode 100644 index 0000000..6f686c6 --- /dev/null +++ b/gyan/common/privileged.py @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_privsep import capabilities as c +from oslo_privsep import priv_context + + +default = priv_context.PrivContext( + 'gyan.common', + cfg_section='privsep', + pypath=__name__ + '.default', + capabilities=[c.CAP_SYS_ADMIN], +) diff --git a/gyan/common/profiler.py b/gyan/common/profiler.py new file mode 100644 index 0000000..2c93049 --- /dev/null +++ b/gyan/common/profiler.py @@ -0,0 +1,101 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +### +# This code is taken from nova. Goal is minimal modification. +### + +from oslo_log import log as logging +from oslo_utils import importutils +import webob.dec + +from gyan.common import context +import gyan.conf + +profiler = importutils.try_import("osprofiler.profiler") +profiler_initializer = importutils.try_import("osprofiler.initializer") +profiler_web = importutils.try_import("osprofiler.web") + + +CONF = gyan.conf.CONF + +LOG = logging.getLogger(__name__) + + +class WsgiMiddleware(object): + + def __init__(self, application, **kwargs): + self.application = application + + @classmethod + def factory(cls, global_conf, **local_conf): + if profiler_web: + return profiler_web.WsgiMiddleware.factory(global_conf, + **local_conf) + + def filter_(app): + return cls(app, **local_conf) + + return filter_ + + @webob.dec.wsgify + def __call__(self, request): + return request.get_response(self.application) + + +def setup(binary, host): + if profiler_initializer and CONF.profiler.enabled: + profiler_initializer.init_from_conf( + conf=CONF, + context=context.get_admin_context().to_dict(), + project="gyan", + service=binary, + host=host) + LOG.info('OSProfiler is enabled.') + + +def trace_cls(name, **kwargs): + """Wrap the OSprofiler trace_cls. + + Wrap the OSprofiler trace_cls decorator so that it will not try to + patch the class unless OSprofiler is present. + + :param name: The name of action. For example, wsgi, rpc, db, ... + :param kwargs: Any other keyword args used by profiler.trace_cls + """ + + def decorator(cls): + if profiler and 'profiler' in CONF: + trace_decorator = profiler.trace_cls(name, kwargs) + return trace_decorator(cls) + return cls + + return decorator + + +def trace(name, **kwargs): + """Wrap the OSprofiler trace. + + Wrap the OSprofiler trace decorator so that it will not try to + patch the functions unless OSprofiler is present. + + :param name: The name of action. For example, wsgi, rpc, db, ... + :param kwargs: Any other keyword args used by profiler.trace + """ + + def decorator(f): + if profiler and 'profiler' in CONF: + trace_decorator = profiler.trace(name, kwargs) + return trace_decorator(f) + return f + + return decorator diff --git a/gyan/common/rpc.py b/gyan/common/rpc.py new file mode 100644 index 0000000..95f25d1 --- /dev/null +++ b/gyan/common/rpc.py @@ -0,0 +1,120 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +__all__ = [ + 'init', + 'set_defaults', + 'add_extra_exmods', + 'clear_extra_exmods', + 'get_allowed_exmods', + 'RequestContextSerializer', + 'get_client', +] + +import oslo_messaging as messaging +from oslo_serialization import jsonutils as json +from oslo_utils import importutils + +from gyan.common import context as gyan_context +from gyan.common import exception + +profiler = importutils.try_import("osprofiler.profiler") + +TRANSPORT = None +ALLOWED_EXMODS = [ + exception.__name__, +] +EXTRA_EXMODS = [] + + +def init(conf): + global TRANSPORT + exmods = get_allowed_exmods() + TRANSPORT = messaging.get_rpc_transport( + conf, allowed_remote_exmods=exmods) + + +def set_defaults(control_exchange): + messaging.set_transport_defaults(control_exchange) + + +def add_extra_exmods(*args): + EXTRA_EXMODS.extend(args) + + +def clear_extra_exmods(): + del EXTRA_EXMODS[:] + + +def get_allowed_exmods(): + return ALLOWED_EXMODS + EXTRA_EXMODS + + +class JsonPayloadSerializer(messaging.NoOpSerializer): + @staticmethod + def serialize_entity(context, entity): + return json.to_primitive(entity, convert_instances=True) + + +class RequestContextSerializer(messaging.Serializer): + + def __init__(self, base): + self._base = base + + def serialize_entity(self, context, entity): + if not self._base: + return entity + return self._base.serialize_entity(context, entity) + + def deserialize_entity(self, context, entity): + if not self._base: + return entity + return self._base.deserialize_entity(context, entity) + + def serialize_context(self, context): + return context.to_dict() + + def deserialize_context(self, context): + return gyan_context.RequestContext.from_dict(context) + + +class ProfilerRequestContextSerializer(RequestContextSerializer): + def serialize_context(self, context): + _context = super(ProfilerRequestContextSerializer, + self).serialize_context(context) + + prof = profiler.get() + if prof: + trace_info = { + "hmac_key": prof.hmac_key, + "base_id": prof.get_base_id(), + "parent_id": prof.get_id() + } + _context.update({"trace_info": trace_info}) + + return _context + + def deserialize_context(self, context): + trace_info = context.pop("trace_info", None) + if trace_info: + profiler.init(**trace_info) + + return super(ProfilerRequestContextSerializer, + self).deserialize_context(context) + + +def get_client(target, serializer=None, timeout=None): + assert TRANSPORT is not None + return messaging.RPCClient(TRANSPORT, + target, + serializer=serializer, + timeout=timeout) diff --git a/gyan/common/rpc_service.py b/gyan/common/rpc_service.py new file mode 100644 index 0000000..a2f9992 --- /dev/null +++ b/gyan/common/rpc_service.py @@ -0,0 +1,105 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Common RPC service and API tools for Gyan.""" + +import oslo_messaging as messaging +from oslo_messaging.rpc import dispatcher +from oslo_service import service +from oslo_utils import importutils + +from gyan.common import context +from gyan.common import profiler +from gyan.common import rpc +import gyan.conf +from gyan.objects import base as objects_base +from gyan.servicegroup import gyan_service_periodic as servicegroup + +osprofiler = importutils.try_import("osprofiler.profiler") + +CONF = gyan.conf.CONF + + +def _init_serializer(): + serializer = rpc.RequestContextSerializer( + objects_base.GyanObjectSerializer()) + if osprofiler: + serializer = rpc.ProfilerRequestContextSerializer(serializer) + else: + serializer = rpc.RequestContextSerializer(serializer) + return serializer + + +class Service(service.Service): + + def __init__(self, topic, server, endpoints, binary): + super(Service, self).__init__() + serializer = _init_serializer() + transport = messaging.get_rpc_transport(CONF) + access_policy = dispatcher.DefaultRPCAccessPolicy + # TODO(asalkeld) add support for version='x.y' + target = messaging.Target(topic=topic, server=server) + self.endpoints = endpoints + self._server = messaging.get_rpc_server(transport, target, endpoints, + executor='eventlet', + serializer=serializer, + access_policy=access_policy) + self.binary = binary + profiler.setup(binary, CONF.compute.host) + + def start(self): + servicegroup.setup(CONF, self.binary, self.tg) + for endpoint in self.endpoints: + if hasattr(endpoint, 'init_containers'): + endpoint.init_containers( + context.get_admin_context(all_projects=True)) + self.tg.add_dynamic_timer( + endpoint.run_periodic_tasks, + periodic_interval_max=CONF.periodic_interval_max, + context=context.get_admin_context(all_projects=True) + ) + self._server.start() + + def stop(self): + if self._server: + self._server.stop() + self._server.wait() + super(Service, self).stop() + + @classmethod + def create(cls, topic, server, handlers, binary): + service_obj = cls(topic, server, handlers, binary) + return service_obj + + +class API(object): + def __init__(self, context=None, topic=None, server=None, + timeout=None): + serializer = _init_serializer() + self._context = context + if topic is None: + topic = '' + target = messaging.Target(topic=topic, server=server) + self._client = rpc.get_client(target, + serializer=serializer, + timeout=timeout) + + def _call(self, server, method, *args, **kwargs): + cctxt = self._client.prepare(server=server) + return cctxt.call(self._context, method, *args, **kwargs) + + def _cast(self, server, method, *args, **kwargs): + cctxt = self._client.prepare(server=server) + return cctxt.cast(self._context, method, *args, **kwargs) + + def echo(self, message): + self._cast('echo', message=message) diff --git a/gyan/common/service.py b/gyan/common/service.py new file mode 100644 index 0000000..f0b8e64 --- /dev/null +++ b/gyan/common/service.py @@ -0,0 +1,92 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from oslo_concurrency import processutils +from oslo_log import log +from oslo_service import service +from oslo_service import wsgi + +from gyan.api import app +from gyan.common import config +from gyan.common import exception +from gyan.common.i18n import _ +import gyan.conf + +CONF = gyan.conf.CONF + + +def prepare_service(argv=None): + if argv is None: + argv = [] + log.register_options(CONF) + config.parse_args(argv) + config.set_config_defaults() + log.setup(CONF, 'gyan') + # TODO(yuanying): Uncomment after objects are implemented + # objects.register_all() + + +def process_launcher(): + return service.ProcessLauncher(CONF, restart_method='mutate') + + +class WSGIService(service.ServiceBase): + """Provides ability to launch Gyan API from wsgi app.""" + + def __init__(self, name, use_ssl=False): + """Initialize, but do not start the WSGI server. + + :param name: The name of the WSGI server given to the loader. + :param use_ssl: Wraps the socket in an SSL context if True. + :returns: None + """ + self.name = name + self.app = app.load_app() + self.workers = (CONF.api.workers or processutils.get_worker_count()) + if self.workers and self.workers < 1: + raise exception.ConfigInvalid( + _("api_workers value of %d is invalid, " + "must be greater than 0.") % self.workers) + + self.server = wsgi.Server(CONF, name, self.app, + host=CONF.api.host_ip, + port=CONF.api.port, + use_ssl=use_ssl) + + def start(self): + """Start serving this service using loaded configuration. + + :returns: None + """ + self.server.start() + + def stop(self): + """Stop serving this API. + + :returns: None + """ + self.server.stop() + + def wait(self): + """Wait for the service to stop serving this API. + + :returns: None + """ + self.server.wait() + + def reset(self): + """Reset server greenpool size to default. + + :returns: None + """ + self.server.reset() diff --git a/gyan/common/short_id.py b/gyan/common/short_id.py new file mode 100644 index 0000000..f50d7f1 --- /dev/null +++ b/gyan/common/short_id.py @@ -0,0 +1,63 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Utilities for creating short ID strings based on a random UUID. +The IDs each comprise 12 (lower-case) alphanumeric characters. +""" + +import base64 +from oslo_utils import uuidutils +import uuid + +import six + +from gyan.common.i18n import _ + + +def _to_byte_string(value, num_bits): + """Convert an integer to a big-endian string of bytes with padding. + + Padding is added at the end (i.e. after the least-significant bit) if + required. + """ + + shifts = six.moves.xrange(num_bits - 8, -8, -8) + byte_at = lambda off: (value >> off if off >= 0 else value << -off) & 0xff + return ''.join(six.int2byte(byte_at(offset)) for offset in shifts) + + +def get_id(source_uuid): + """Derive a short (12 character) id from a random UUID. + + The supplied UUID must be a version 4 UUID object. + """ + + if isinstance(source_uuid, six.string_types): + source_uuid = uuid.UUID(source_uuid) + if source_uuid.version != 4: + raise ValueError(_('Invalid UUID version (%d)') % source_uuid.version) + + # The "time" field of a v4 UUID contains 60 random bits + # (see RFC4122, Section 4.4) + random_bytes = _to_byte_string(source_uuid.time, 60) + # The first 12 bytes (= 60 bits) of base32-encoded output is our data + encoded = base64.b32encode(six.b(random_bytes))[:12] + + if six.PY3: + return encoded.lower().decode('utf-8') + else: + return encoded.lower() + + +def generate_id(): + """Generate a short (12 character), random id.""" + return uuidutils.generate_uuid() diff --git a/gyan/common/singleton.py b/gyan/common/singleton.py new file mode 100644 index 0000000..6d606b9 --- /dev/null +++ b/gyan/common/singleton.py @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_concurrency import lockutils + + +class Singleton(type): + _instances = {} + _semaphores = lockutils.Semaphores() + + def __call__(cls, *args, **kwargs): + with lockutils.lock('singleton_lock', semaphores=cls._semaphores): + if cls not in cls._instances: + cls._instances[cls] = super( + Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] diff --git a/gyan/common/utils.py b/gyan/common/utils.py new file mode 100644 index 0000000..f3950a7 --- /dev/null +++ b/gyan/common/utils.py @@ -0,0 +1,255 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# It's based on oslo.i18n usage in OpenStack Keystone project and +# recommendations from +# https://docs.openstack.org/oslo.i18n/latest/user/usage.html + +"""Utilities and helper functions.""" +import base64 +import binascii +import eventlet +import functools +import inspect +import json +import mimetypes + +from oslo_concurrency import processutils +from oslo_context import context as common_context +from oslo_log import log as logging +from oslo_utils import excutils +from oslo_utils import strutils +import pecan +import six + +from gyan.api import utils as api_utils +from gyan.common import consts +from gyan.common import exception +from gyan.common.i18n import _ +from gyan.common import privileged +import gyan.conf +from gyan import objects + +CONF = gyan.conf.CONF +LOG = logging.getLogger(__name__) + +VALID_STATES = { + 'deploy': [consts.CREATED, consts.UNDEPLOYED], + 'undeploy': [consts.DEPLOYED] +} +def safe_rstrip(value, chars=None): + """Removes trailing characters from a string if that does not make it empty + + :param value: A string value that will be stripped. + :param chars: Characters to remove. + :return: Stripped value. + + """ + if not isinstance(value, six.string_types): + LOG.warning( + "Failed to remove trailing character. Returning original object. " + "Supplied object is not a string: %s.", value) + return value + + return value.rstrip(chars) or value + + +def _do_allow_certain_content_types(func, content_types_list): + # Allows you to bypass pecan's content-type restrictions + cfg = pecan.util._cfg(func) + cfg.setdefault('content_types', {}) + cfg['content_types'].update((value, '') + for value in content_types_list) + return func + + +def allow_certain_content_types(*content_types_list): + def _wrapper(func): + return _do_allow_certain_content_types(func, content_types_list) + return _wrapper + + +def allow_all_content_types(f): + return _do_allow_certain_content_types(f, mimetypes.types_map.values()) + + +def spawn_n(func, *args, **kwargs): + """Passthrough method for eventlet.spawn_n. + + This utility exists so that it can be stubbed for testing without + interfering with the service spawns. + + It will also grab the context from the threadlocal store and add it to + the store on the new thread. This allows for continuity in logging the + context when using this method to spawn a new thread. + """ + _context = common_context.get_current() + + @functools.wraps(func) + def context_wrapper(*args, **kwargs): + # NOTE: If update_store is not called after spawn_n it won't be + # available for the logger to pull from threadlocal storage. + if _context is not None: + _context.update_store() + func(*args, **kwargs) + + eventlet.spawn_n(context_wrapper, *args, **kwargs) + + +def translate_exception(function): + """Wraps a method to catch exceptions. + + If the exception is not an instance of GyanException, + translate it into one. + """ + + @functools.wraps(function) + def decorated_function(self, context, *args, **kwargs): + try: + return function(self, context, *args, **kwargs) + except Exception as e: + if not isinstance(e, exception.GyanException): + LOG.exception("Unexpected error: %s", six.text_type(e)) + e = exception.GyanException("Unexpected error: %s" + % six.text_type(e)) + raise e + raise + + return decorated_function + + +def custom_execute(*cmd, **kwargs): + try: + return processutils.execute(*cmd, **kwargs) + except processutils.ProcessExecutionError as e: + sanitized_cmd = strutils.mask_password(' '.join(cmd)) + raise exception.CommandError(cmd=sanitized_cmd, + error=six.text_type(e)) + + +def is_all_projects(search_opts): + all_projects = search_opts.get('all_projects') + if all_projects: + try: + all_projects = strutils.bool_from_string(all_projects, True) + except ValueError: + bools = ', '.join(strutils.TRUE_STRINGS + strutils.FALSE_STRINGS) + raise exception.InvalidValue(_('Valid all_projects values are: %s') + % bools) + else: + all_projects = False + return all_projects + + +def get_ml_model(ml_model_ident): + ml_model = api_utils.get_resource('ML_Model', ml_model_ident) + if not ml_model: + pecan.abort(404, ('Not found; the ml model you requested ' + 'does not exist.')) + + return ml_model + +def validate_ml_model_state(ml_model, action): + if ml_model.status not in VALID_STATES[action]: + raise exception.InvalidStateException( + id=ml_model.uuid, + action=action, + actual_state=ml_model.status) + + +def get_wrapped_function(function): + """Get the method at the bottom of a stack of decorators.""" + if not hasattr(function, '__closure__') or not function.__closure__: + return function + + def _get_wrapped_function(function): + if not hasattr(function, '__closure__') or not function.__closure__: + return None + + for closure in function.__closure__: + func = closure.cell_contents + + deeper_func = _get_wrapped_function(func) + if deeper_func: + return deeper_func + elif hasattr(closure.cell_contents, '__call__'): + return closure.cell_contents + + return function + + return _get_wrapped_function(function) + + +def wrap_ml_model_event(prefix): + """Warps a method to log the event taken on the ml_model, and result. + + This decorator wraps a method to log the start and result of an event, as + part of an action taken on a ml_model. + """ + def helper(function): + + @functools.wraps(function) + def decorated_function(self, context, *args, **kwargs): + wrapped_func = get_wrapped_function(function) + keyed_args = inspect.getcallargs(wrapped_func, self, context, + *args, **kwargs) + ml_model_uuid = keyed_args['ml_model'].uuid + + event_name = '{0}_{1}'.format(prefix, function.__name__) + with EventReporter(context, event_name, ml_model_uuid): + return function(self, context, *args, **kwargs) + return decorated_function + return helper + + +def wrap_exception(): + def helper(function): + + @functools.wraps(function) + def decorated_function(self, context, ml_model, *args, **kwargs): + try: + return function(self, context, ml_model, *args, **kwargs) + except exception.DockerError as e: + with excutils.save_and_reraise_exception(reraise=False): + LOG.error("Error occurred while calling Docker API: %s", + six.text_type(e)) + except Exception as e: + with excutils.save_and_reraise_exception(reraise=False): + LOG.exception("Unexpected exception: %s", six.text_type(e)) + return decorated_function + return helper + + +def is_close(x, y, rel_tol=1e-06, abs_tol=0.0): + return abs(x - y) <= max(rel_tol * max(abs(x), abs(y)), abs_tol) + + +def is_less_than(x, y): + if isinstance(x, int) and isinstance(y, int): + return x < y + if isinstance(x, float) or isinstance(y, float): + return False if (x - y) >= 0 or is_close(x, y) else True + + +def encode_file_data(data): + if six.PY3 and isinstance(data, str): + data = data.encode('utf-8') + return base64.b64encode(data).decode('utf-8') + + +def decode_file_data(data): + # Py3 raises binascii.Error instead of TypeError as in Py27 + try: + return base64.b64decode(data) + except (TypeError, binascii.Error): + raise exception.Base64Exception() diff --git a/gyan/common/yamlutils.py b/gyan/common/yamlutils.py new file mode 100644 index 0000000..527d50c --- /dev/null +++ b/gyan/common/yamlutils.py @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import yaml + + +def load(s): + try: + yml_dict = yaml.safe_load(s) + except yaml.YAMLError as exc: + msg = 'An error occurred during YAML parsing.' + if hasattr(exc, 'problem_mark'): + msg += ' Error position: (%s:%s)' % (exc.problem_mark.line + 1, + exc.problem_mark.column + 1) + raise ValueError(msg) + if not isinstance(yml_dict, dict) and not isinstance(yml_dict, list): + raise ValueError('The source is not a YAML mapping or list.') + if isinstance(yml_dict, dict) and len(yml_dict) < 1: + raise ValueError('Could not find any element in your YAML mapping.') + return yml_dict + + +def dump(s): + return yaml.safe_dump(s) diff --git a/gyan/compute/__init__.py b/gyan/compute/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/compute/api.py b/gyan/compute/api.py new file mode 100644 index 0000000..cf62ed0 --- /dev/null +++ b/gyan/compute/api.py @@ -0,0 +1,63 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Handles all requests relating to compute resources (e.g. ml_models, +and compute hosts on which they run).""" + +from oslo_log import log as logging + +from gyan.common import consts +from gyan.common import exception +from gyan.common.i18n import _ +from gyan.common import profiler +from gyan.compute import rpcapi +import gyan.conf +from gyan import objects + + +CONF = gyan.conf.CONF +LOG = logging.getLogger(__name__) + + +@profiler.trace_cls("rpc") +class API(object): + """API for interacting with the compute manager.""" + + def __init__(self, context): + self.rpcapi = rpcapi.API(context=context) + super(API, self).__init__() + + def ml_model_create(self, context, new_ml_model, extra_spec): + try: + host_state = self._schedule_ml_model(context, ml_model, + extra_spec) + except exception.NoValidHost: + new_ml_model.status = consts.ERROR + new_ml_model.status_reason = _( + "There are not enough hosts available.") + new_ml_model.save(context) + return + except Exception: + new_ml_model.status = consts.ERROR + new_ml_model.status_reason = _("Unexpected exception occurred.") + new_ml_model.save(context) + raise + + self.rpcapi.ml_model_create(context, host_state['host'], + new_ml_model) + + def ml_model_delete(self, context, ml_model, *args): + self._record_action_start(context, ml_model, ml_model_actions.DELETE) + return self.rpcapi.ml_model_delete(context, ml_model, *args) + + def ml_model_show(self, context, ml_model): + return self.rpcapi.ml_model_show(context, ml_model) \ No newline at end of file diff --git a/gyan/compute/compute_host_tracker.py b/gyan/compute/compute_host_tracker.py new file mode 100644 index 0000000..1c15367 --- /dev/null +++ b/gyan/compute/compute_host_tracker.py @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import collections +import copy + +from oslo_log import log as logging + +from gyan.common import exception +from gyan.common import utils +from gyan import objects +from gyan.objects import base as obj_base + +LOG = logging.getLogger(__name__) +COMPUTE_RESOURCE_SEMAPHORE = "compute_resources" + + +class ComputeHostTracker(object): + def __init__(self, host, ml_model_driver): + self.host = host + self.ml_model_driver = ml_model_driver + self.compute_host = None + self.tracked_ml_models = {} + self.old_resources = collections.defaultdict(objects.ComputeHost) + + + def update_available_resources(self, context): + # Check if the compute_host is already registered + host = self._get_compute_host(context) + if not host: + # If not, register it and pass the object to the driver + host = objects.ComputeHost(context) + host.hostname = self.host + host.type = self.ml_model_driver.__class__.__name__ + host.status = "AVAILABLE" + host.create(context) + LOG.info('Host created for :%(host)s', {'host': self.host}) + self.ml_model_driver.get_available_resources(host) + self.compute_host = host + return host + + def _get_compute_host(self, context): + """Returns compute host for the host""" + try: + return objects.ComputeHost.get_by_name(context, self.host) + except exception.ComputeHostNotFound: + LOG.warning("No compute host record for: %(host)s", + {'host': self.host}) + + + def _set_ml_model_host(self, context, ml_model): + """Tag the ml_model as belonging to this host. + + This should be done while the COMPUTE_RESOURCES_SEMAPHORE is held so + the resource claim will not be lost if the audit process starts. + """ + ml_model.host = self.host + ml_model.save(context) \ No newline at end of file diff --git a/gyan/compute/manager.py b/gyan/compute/manager.py new file mode 100644 index 0000000..dee77ed --- /dev/null +++ b/gyan/compute/manager.py @@ -0,0 +1,121 @@ +# Copyright 2016 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools + +import six +import time + +from oslo_log import log as logging +from oslo_service import periodic_task +from oslo_utils import excutils +from oslo_utils import timeutils +from oslo_utils import uuidutils + +from gyan.common import consts +from gyan.common import context +from gyan.common import exception +from gyan.common.i18n import _ +from gyan.common import utils +from gyan.common.utils import translate_exception +from gyan.common.utils import wrap_ml_model_event +from gyan.common.utils import wrap_exception +from gyan.compute import compute_host_tracker +import gyan.conf +from gyan.ml_model import driver +from gyan import objects + +CONF = gyan.conf.CONF +LOG = logging.getLogger(__name__) + + +class Manager(periodic_task.PeriodicTasks): + """Manages the running ml_models.""" + + def __init__(self, ml_model_driver=None): + super(Manager, self).__init__(CONF) + self.driver = driver.load_ml_model_driver(ml_model_driver) + self.host = CONF.compute.host + self._resource_tracker = None + + def ml_model_create(self, context, limits, requested_networks, + requested_volumes, ml_model, run, pci_requests=None): + @utils.synchronized(ml_model.uuid) + def do_ml_model_create(): + created_ml_model = self._do_ml_model_create( + context, ml_model, requested_networks, requested_volumes, + pci_requests, limits) + if run: + self._do_ml_model_start(context, created_ml_model) + + utils.spawn_n(do_ml_model_create) + + @wrap_ml_model_event(prefix='compute') + def _do_ml_model_create(self, context, ml_model, requested_networks, + requested_volumes, pci_requests=None, + limits=None): + LOG.debug('Creating ml_model: %s', ml_model.uuid) + + try: + rt = self._get_resource_tracker() + # As sriov port also need to claim, we need claim pci port before + # create sandbox. + with rt.ml_model_claim(context, ml_model, pci_requests, limits): + sandbox = None + if self.use_sandbox: + sandbox = self._create_sandbox(context, ml_model, + requested_networks) + + created_ml_model = self._do_ml_model_create_base( + context, ml_model, requested_networks, requested_volumes, + sandbox, limits) + return created_ml_model + except exception.ResourcesUnavailable as e: + with excutils.save_and_reraise_exception(): + LOG.exception("ML Model resource claim failed: %s", + six.text_type(e)) + self._fail_ml_model(context, ml_model, six.text_type(e), + unset_host=True) + + @wrap_ml_model_event(prefix='compute') + def _do_ml_model_start(self, context, ml_model): + pass + + @translate_exception + def ml_model_delete(self, context, ml_model, force=False): + pass + + @translate_exception + def ml_model_show(self, context, ml_model): + pass + + @translate_exception + def ml_model_start(self, context, ml_model): + pass + + @translate_exception + def ml_model_update(self, context, ml_model, patch): + pass + + @periodic_task.periodic_task(run_immediately=True) + def inventory_host(self, context): + rt = self._get_resource_tracker() + rt.update_available_resources(context) + + def _get_resource_tracker(self): + if not self._resource_tracker: + rt = compute_host_tracker.ComputeHostTracker(self.host, + self.driver) + self._resource_tracker = rt + return self._resource_tracker \ No newline at end of file diff --git a/gyan/compute/rpcapi.py b/gyan/compute/rpcapi.py new file mode 100644 index 0000000..f8824bb --- /dev/null +++ b/gyan/compute/rpcapi.py @@ -0,0 +1,68 @@ +# Copyright 2016 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools + +from gyan.api import servicegroup +from gyan.common import exception +from gyan.common import profiler +from gyan.common import rpc_service +import gyan.conf +from gyan import objects + + +def check_ml_model_host(func): + """Verify the state of ML Model host""" + @functools.wraps(func) + def wrap(self, context, ml_model, *args, **kwargs): + return func(self, context, ml_model, *args, **kwargs) + return wrap + + +@profiler.trace_cls("rpc") +class API(rpc_service.API): + """Client side of the ml_model compute rpc API. + + API version history: + + * 1.0 - Initial version. + """ + + def __init__(self, transport=None, context=None, topic=None): + if topic is None: + gyan.conf.CONF.import_opt( + 'topic', 'gyan.conf.compute', group='compute') + + super(API, self).__init__( + context, gyan.conf.CONF.compute.topic, transport) + + def ml_model_create(self, context, host, ml_model): + self._cast(host, 'ml_model_create', + ml_model=ml_model) + + @check_ml_model_host + def ml_model_delete(self, context, ml_model, force): + return self._cast(ml_model.host, 'ml_model_delete', + ml_model=ml_model, force=force) + + @check_ml_model_host + def ml_model_show(self, context, ml_model): + return self._call(ml_model.host, 'ml_model_show', + ml_model=ml_model) + + + @check_ml_model_host + def ml_model_update(self, context, ml_model, patch): + return self._call(ml_model.host, 'ml_model_update', + ml_model=ml_model, patch=patch) diff --git a/gyan/conf/__init__.py b/gyan/conf/__init__.py new file mode 100644 index 0000000..43683ad --- /dev/null +++ b/gyan/conf/__init__.py @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from gyan.conf import api +from gyan.conf import compute +from gyan.conf import ml_model_driver +from gyan.conf import database +from gyan.conf import keystone +from gyan.conf import path +from gyan.conf import profiler +from gyan.conf import scheduler +from gyan.conf import services +from gyan.conf import ssl +from gyan.conf import utils +from gyan.conf import gyan_client + +CONF = cfg.CONF + +api.register_opts(CONF) +compute.register_opts(CONF) +ml_model_driver.register_opts(CONF) +database.register_opts(CONF) +keystone.register_opts(CONF) +path.register_opts(CONF) +scheduler.register_opts(CONF) +services.register_opts(CONF) +gyan_client.register_opts(CONF) +ssl.register_opts(CONF) +profiler.register_opts(CONF) +utils.register_opts(CONF) \ No newline at end of file diff --git a/gyan/conf/api.py b/gyan/conf/api.py new file mode 100644 index 0000000..8aeb1d7 --- /dev/null +++ b/gyan/conf/api.py @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +api_service_opts = [ + cfg.PortOpt('port', + default=8517, + help='The port for the gyan API server.'), + cfg.StrOpt('host', + default="localhost", + help='The port for the gyan API server.'), + cfg.IPOpt('host_ip', + default='127.0.0.1', + help="The listen IP for the gyan API server. " + "The default is ``$my_ip``, " + "the IP address of this host."), + cfg.BoolOpt('enable_ssl_api', + default=False, + help="Enable the integrated stand-alone API to service " + "requests via HTTPS instead of HTTP. If there is a " + "front-end service performing HTTPS offloading from " + "the service, this option should be False; note, you " + "will want to change public API endpoint to represent " + "SSL termination URL with 'public_endpoint' option."), + cfg.IntOpt('workers', + help="Number of workers for gyan-api service. " + "The default will be the number of CPUs available."), + cfg.IntOpt('max_limit', + default=1000, + help='The maximum number of items returned in a single ' + 'response from a collection resource.'), + cfg.StrOpt('api_paste_config', + default="api-paste.ini", + help="Configuration file for WSGI definition of API."), + cfg.BoolOpt('enable_image_validation', + default=True, + help="Enable image validation.") +] + + +api_group = cfg.OptGroup(name='api', + title='Options for the gyan-api service') + + +ALL_OPTS = (api_service_opts) + + +def register_opts(conf): + conf.register_group(api_group) + conf.register_opts(ALL_OPTS, api_group) + + +def list_opts(): + return { + api_group: ALL_OPTS + } diff --git a/gyan/conf/compute.py b/gyan/conf/compute.py new file mode 100644 index 0000000..8d0bd63 --- /dev/null +++ b/gyan/conf/compute.py @@ -0,0 +1,40 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +service_opts = [ + cfg.StrOpt( + 'topic', + default='gyan-compute', + help='The queue to add compute tasks to.'), + cfg.StrOpt( + 'host', + default='localhost', + help='hostname'), + +] + +opt_group = cfg.OptGroup( + name='compute', title='Options for the gyan-compute service') + +ALL_OPTS = (service_opts) + + +def register_opts(conf): + conf.register_group(opt_group) + conf.register_opts(ALL_OPTS, opt_group) + + +def list_opts(): + return {opt_group: ALL_OPTS} diff --git a/gyan/conf/database.py b/gyan/conf/database.py new file mode 100644 index 0000000..73dac89 --- /dev/null +++ b/gyan/conf/database.py @@ -0,0 +1,32 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +sql_opts = [ + cfg.StrOpt('mysql_engine', + default='InnoDB', + help='MySQL engine to use.') +] + + + +DEFAULT_OPTS = (sql_opts) + + +def register_opts(conf): + conf.register_opts(sql_opts, 'database') + + +def list_opts(): + return {"DEFAULT": DEFAULT_OPTS} diff --git a/gyan/conf/gyan_client.py b/gyan/conf/gyan_client.py new file mode 100644 index 0000000..3c97cf9 --- /dev/null +++ b/gyan/conf/gyan_client.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +gyan_group = cfg.OptGroup(name='gyan_client', + title='Options for the Gyan client') + +gyan_client_opts = [ + cfg.StrOpt('region_name', + help='Region in Identity service catalog to use for ' + 'communication with the OpenStack service.'), + cfg.StrOpt('endpoint_type', + default='publicURL', + help='Type of endpoint in Identity service catalog to use ' + 'for communication with the OpenStack service.')] + + +common_security_opts = [ + cfg.StrOpt('ca_file', + help='Optional CA cert file to use in SSL connections.'), + cfg.StrOpt('cert_file', + help='Optional PEM-formatted certificate chain file.'), + cfg.StrOpt('key_file', + help='Optional PEM-formatted file that contains the ' + 'private key.'), + cfg.BoolOpt('insecure', + default=False, + help="If set, then the server's certificate will not " + "be verified.")] + +ALL_OPTS = (gyan_client_opts + common_security_opts) + + +def register_opts(conf): + conf.register_group(gyan_group) + conf.register_opts(gyan_client_opts, group=gyan_group) + + +def list_opts(): + return {gyan_group: ALL_OPTS} diff --git a/gyan/conf/keystone.py b/gyan/conf/keystone.py new file mode 100644 index 0000000..434a3d9 --- /dev/null +++ b/gyan/conf/keystone.py @@ -0,0 +1,35 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from keystoneauth1 import loading as ka_loading +from oslo_config import cfg + +CFG_GROUP = 'keystone_auth' +CFG_LEGACY_GROUP = 'keystone_authtoken' + +keystone_auth_group = cfg.OptGroup(name=CFG_GROUP, + title='Options for Keystone in Gyan') + + +def register_opts(conf): + conf.import_group(CFG_LEGACY_GROUP, 'keystonemiddleware.auth_token') + ka_loading.register_auth_conf_options(conf, CFG_GROUP) + ka_loading.register_session_conf_options(conf, CFG_GROUP) + conf.set_default('auth_type', default='password', group=CFG_GROUP) + + +def list_opts(): + keystone_auth_opts = (ka_loading.get_auth_common_conf_options() + + ka_loading.get_auth_plugin_conf_options('password')) + return { + keystone_auth_group: keystone_auth_opts + } diff --git a/gyan/conf/ml_model_driver.py b/gyan/conf/ml_model_driver.py new file mode 100644 index 0000000..db2eb43 --- /dev/null +++ b/gyan/conf/ml_model_driver.py @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + +driver_opts = [ + cfg.StrOpt('ml_model_driver', + default='gyan.ml_model.tensorflow.driver.TensorflowDriver', + help="""Defines which driver to use for controlling ml_model. +Possible values: + +* ``ml_model.driver.TensorflowDriver`` + +Services which consume this: + +* ``gyan-compute`` + +Interdependencies to other options: + +* None +"""), + cfg.IntOpt('default_sleep_time', default=1, + help='Time to sleep (in seconds) during waiting for an event.'), + cfg.IntOpt('default_timeout', default=60 * 10, + help='Maximum time (in seconds) to wait for an event.') +] + + +ALL_OPTS = (driver_opts) + + +def register_opts(conf): + conf.register_opts(ALL_OPTS) + + +def list_opts(): + return {"DEFAULT": ALL_OPTS} diff --git a/gyan/conf/opts.py b/gyan/conf/opts.py new file mode 100644 index 0000000..7ef8943 --- /dev/null +++ b/gyan/conf/opts.py @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +This is the single point of entry to generate the sample configuration +file for Gyan. It collects all the necessary info from the other modules +in this package. It is assumed that: + +* every other module in this package has a 'list_opts' function which + return a dict where + * the keys are strings which are the group names + * the value of each key is a list of config options for that group +* the gyan.conf package doesn't have further packages with config options +* this module is only used in the context of sample file generation +""" + +import collections +import importlib +import os +import pkgutil + +LIST_OPTS_FUNC_NAME = "list_opts" + + +def _tupleize(dct): + """Take the dict of options and convert to the 2-tuple format.""" + return [(key, val) for key, val in dct.items()] + + +def list_opts(): + opts = collections.defaultdict(list) + module_names = _list_module_names() + imported_modules = _import_modules(module_names) + _append_config_options(imported_modules, opts) + return _tupleize(opts) + + +def _list_module_names(): + module_names = [] + package_path = os.path.dirname(os.path.abspath(__file__)) + for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]): + if modname == "opts" or ispkg: + continue + else: + module_names.append(modname) + return module_names + + +def _import_modules(module_names): + imported_modules = [] + for modname in module_names: + mod = importlib.import_module("gyan.conf." + modname) + if not hasattr(mod, LIST_OPTS_FUNC_NAME): + msg = "The module 'gyan.conf.%s' should have a '%s' "\ + "function which returns the config options." % \ + (modname, LIST_OPTS_FUNC_NAME) + raise AttributeError(msg) + else: + imported_modules.append(mod) + return imported_modules + + +def _append_config_options(imported_modules, config_options): + for mod in imported_modules: + configs = mod.list_opts() + for key, val in configs.items(): + config_options[key].extend(val) diff --git a/gyan/conf/path.py b/gyan/conf/path.py new file mode 100644 index 0000000..de73651 --- /dev/null +++ b/gyan/conf/path.py @@ -0,0 +1,42 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from oslo_config import cfg + + +path_opts = [ + cfg.StrOpt('pybasedir', + default=os.path.abspath(os.path.join(os.path.dirname(__file__), + '../')), + help='Directory where the gyan python module is installed.'), + cfg.StrOpt('bindir', + default='$pybasedir/bin', + help='Directory where gyan binaries are installed.'), + cfg.StrOpt('state_path', + default='$pybasedir', + help="Top-level directory for maintaining gyan's state."), +] + + +def state_path_def(*args): + """Return an uninterpolated path relative to $state_path.""" + return os.path.join('$state_path', *args) + + +def register_opts(conf): + conf.register_opts(path_opts) + + +def list_opts(): + return {"DEFAULT": path_opts} diff --git a/gyan/conf/profiler.py b/gyan/conf/profiler.py new file mode 100644 index 0000000..8041f94 --- /dev/null +++ b/gyan/conf/profiler.py @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_utils import importutils + + +profiler_opts = importutils.try_import('osprofiler.opts') + + +def register_opts(conf): + if profiler_opts: + profiler_opts.set_defaults(conf) + + +def list_opts(): + if not profiler_opts: + return {} + return { + profiler_opts._profiler_opt_group: profiler_opts._PROFILER_OPTS + } diff --git a/gyan/conf/scheduler.py b/gyan/conf/scheduler.py new file mode 100644 index 0000000..554f614 --- /dev/null +++ b/gyan/conf/scheduler.py @@ -0,0 +1,104 @@ +# Copyright 2015 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +scheduler_group = cfg.OptGroup(name="scheduler", + title="Scheduler configuration") + +scheduler_opts = [ + cfg.StrOpt("driver", + default="filter_scheduler", + choices=("chance_scheduler", "fake_scheduler", + "filter_scheduler"), + help=""" +The class of the driver used by the scheduler. + +The options are chosen from the entry points under the namespace +'gyan.scheduler.driver' in 'setup.cfg'. + +Possible values: + +* A string, where the string corresponds to the class name of a scheduler + driver. There are a number of options available: +** 'chance_scheduler', which simply picks a host at random +** A custom scheduler driver. In this case, you will be responsible for + creating and maintaining the entry point in your 'setup.cfg' file +"""), + cfg.MultiStrOpt("available_filters", + default=["gyan.scheduler.filters.all_filters"], + help=""" +Filters that the scheduler can use. + +An unordered list of the filter classes the gyan scheduler may apply. Only the +filters specified in the 'scheduler_enabled_filters' option will be used, but +any filter appearing in that option must also be included in this list. + +By default, this is set to all filters that are included with gyan. + +This option is only used by the FilterScheduler and its subclasses; if you use +a different scheduler, this option has no effect. + +Possible values: + +* A list of zero or more strings, where each string corresponds to the name of + a filter that may be used for selecting a host + +Related options: + +* scheduler_enabled_filters +"""), + cfg.ListOpt("enabled_filters", + default=[ + "AvailabilityZoneFilter", + "CPUFilter", + "RamFilter", + "ComputeFilter", + "DiskFilter", + ], + help=""" +Filters that the scheduler will use. + +An ordered list of filter class names that will be used for filtering +hosts. Ignore the word 'default' in the name of this option: these filters will +*always* be applied, and they will be applied in the order they are listed so +place your most restrictive filters first to make the filtering process more +efficient. + +This option is only used by the FilterScheduler and its subclasses; if you use +a different scheduler, this option has no effect. + +Possible values: + +* A list of zero or more strings, where each string corresponds to the name of + a filter to be used for selecting a host + +Related options: + +* All of the filters in this option *must* be present in the + 'scheduler_available_filters' option, or a SchedulerHostFilterNotFound + exception will be raised. +"""), +] + + +def register_opts(conf): + conf.register_group(scheduler_group) + conf.register_opts(scheduler_opts, group=scheduler_group) + + +def list_opts(): + return {scheduler_group: scheduler_opts} diff --git a/gyan/conf/services.py b/gyan/conf/services.py new file mode 100644 index 0000000..9971b22 --- /dev/null +++ b/gyan/conf/services.py @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + + +periodic_opts = [ + cfg.IntOpt('periodic_interval_max', + default=60, + help='Max interval size between periodic tasks execution in ' + 'seconds.'), + cfg.IntOpt('service_down_time', + default=180, + help='Max interval size between periodic tasks execution in ' + 'seconds.') +] + + +ALL_OPTS = (periodic_opts) + + +def register_opts(conf): + conf.register_opts(ALL_OPTS) + + +def list_opts(): + return {"DEFAULT": ALL_OPTS} diff --git a/gyan/conf/ssl.py b/gyan/conf/ssl.py new file mode 100644 index 0000000..6a0af10 --- /dev/null +++ b/gyan/conf/ssl.py @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg +from oslo_service import sslutils + + +def register_opts(conf): + sslutils.register_opts(conf) + + +def list_opts(): + group_name, ssl_opts = sslutils.list_opts()[0] + ssl_group = cfg.OptGroup(name=group_name, + title='Options for the ssl') + return { + ssl_group: ssl_opts + } diff --git a/gyan/conf/utils.py b/gyan/conf/utils.py new file mode 100644 index 0000000..6f07d06 --- /dev/null +++ b/gyan/conf/utils.py @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy +# of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo_config import cfg + + +utils_opts = [ + cfg.StrOpt('rootwrap_config', + default="/etc/gyan/rootwrap.conf", + help='Path to the rootwrap configuration file to use for ' + 'running commands as root.'), +] + + +def register_opts(conf): + conf.register_opts(utils_opts) + + +def list_opts(): + return { + "DEFAULT": utils_opts + } diff --git a/gyan/db/__init__.py b/gyan/db/__init__.py new file mode 100644 index 0000000..3644dad --- /dev/null +++ b/gyan/db/__init__.py @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_db import options + +from gyan.common import paths +import gyan.conf + +_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('gyan.sqlite') + +options.set_defaults(gyan.conf.CONF) +options.set_defaults(gyan.conf.CONF, _DEFAULT_SQL_CONNECTION) diff --git a/gyan/db/api.py b/gyan/db/api.py new file mode 100644 index 0000000..63f4535 --- /dev/null +++ b/gyan/db/api.py @@ -0,0 +1,197 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Base API for Database +""" + +from oslo_db import api as db_api + +from gyan.common import profiler +import gyan.conf + +"""Add the database backend mapping here""" + +CONF = gyan.conf.CONF +_BACKEND_MAPPING = {'sqlalchemy': 'gyan.db.sqlalchemy.api'} +IMPL = db_api.DBAPI.from_config(CONF, + backend_mapping=_BACKEND_MAPPING, + lazy=True) + + +@profiler.trace("db") +def _get_dbdriver_instance(): + """Return a DB API instance.""" + return IMPL + + +@profiler.trace("db") +def list_ml_models(context, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + """List matching ML Models. + + Return a list of the specified columns for all ml models that match + the specified filters. + + :param context: The security context + :param filters: Filters to apply. Defaults to None. + :param limit: Maximum number of ml_models to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: Direction in which results should be sorted. + (asc, desc) + :returns: A list of tuples of the specified columns. + """ + return _get_dbdriver_instance().list_ml_models( + context, filters, limit, marker, sort_key, sort_dir) + + +@profiler.trace("db") +def create_ml_model(context, values): + """Create a new ML Model. + + :param context: The security context + :param values: A dict containing several items used to identify + and track the ML Model + :returns: A ML Model. + """ + return _get_dbdriver_instance().create_ml_model(context, values) + + +@profiler.trace("db") +def get_ml_model_by_uuid(context, ml_model_uuid): + """Return a ML Model. + + :param context: The security context + :param ml_model_uuid: The uuid of a ml model. + :returns: A ML Model. + """ + return _get_dbdriver_instance().get_ml_model_by_uuid( + context, ml_model_uuid) + + +@profiler.trace("db") +def get_ml_model_by_name(context, ml_model_name): + """Return a ML Model. + + :param context: The security context + :param ml_model_name: The name of a ML Model. + :returns: A ML Model. + """ + return _get_dbdriver_instance().get_ml_model_by_name( + context, ml_model_name) + + +@profiler.trace("db") +def destroy_ml_model(context, ml_model_id): + """Destroy a ml model and all associated interfaces. + + :param context: Request context + :param ml_model_id: The id or uuid of a ml model. + """ + return _get_dbdriver_instance().destroy_ml_model(context, ml_model_id) + + +@profiler.trace("db") +def update_ml_model(context, ml_model_id, values): + """Update properties of a ml model. + + :param context: Request context + :param ml_model_id: The id or uuid of a ml model. + :param values: The properties to be updated + :returns: A ML Model. + :raises: MLModelNotFound + """ + return _get_dbdriver_instance().update_ml_model( + context, ml_model_id, values) + + +@profiler.trace("db") +def list_compute_hosts(context, filters=None, limit=None, marker=None, + sort_key=None, sort_dir=None): + """List matching compute hosts. + + Return a list of the specified columns for all compute hosts that match + the specified filters. + + :param context: The security context + :param filters: Filters to apply. Defaults to None. + :param limit: Maximum number of compute nodes to return. + :param marker: the last item of the previous page; we return the next + result set. + :param sort_key: Attribute by which results should be sorted. + :param sort_dir: Direction in which results should be sorted. + (asc, desc) + :returns: A list of tuples of the specified columns. + """ + return _get_dbdriver_instance().list_compute_hosts( + context, filters, limit, marker, sort_key, sort_dir) + + +@profiler.trace("db") +def create_compute_host(context, values): + """Create a new compute host. + + :param context: The security context + :param values: A dict containing several items used to identify + and track the compute node, and several dicts which are + passed into the Drivers when managing this compute host. + :returns: A compute host. + """ + return _get_dbdriver_instance().create_compute_host(context, values) + + +@profiler.trace("db") +def get_compute_host(context, host_uuid): + """Return a compute host. + + :param context: The security context + :param node_uuid: The uuid of a compute node. + :returns: A compute node. + """ + return _get_dbdriver_instance().get_compute_host(context, host_uuid) + + +@profiler.trace("db") +def get_compute_host_by_hostname(context, hostname): + """Return a compute node. + + :param context: The security context + :param hostname: The hostname of a compute node. + :returns: A compute node. + """ + return _get_dbdriver_instance().get_compute_host_by_hostname( + context, hostname) + + +@profiler.trace("db") +def destroy_compute_host(context, host_uuid): + """Destroy a compute node and all associated interfaces. + + :param context: Request context + :param node_uuid: The uuid of a compute node. + """ + return _get_dbdriver_instance().destroy_compute_host(context, host_uuid) + + +@profiler.trace("db") +def update_compute_host(context, host_uuid, values): + """Update properties of a compute node. + + :param context: Request context + :param node_uuid: The uuid of a compute node. + :param values: The properties to be updated + :returns: A compute node. + :raises: ComputeNodeNotFound + """ + return _get_dbdriver_instance().update_compute_host( + context, host_uuid, values) diff --git a/gyan/db/migration.py b/gyan/db/migration.py new file mode 100644 index 0000000..7e87710 --- /dev/null +++ b/gyan/db/migration.py @@ -0,0 +1,47 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Database setup and migration commands.""" + +from stevedore import driver + +import gyan.conf + +_IMPL = None + + +def get_backend(): + global _IMPL + if not _IMPL: + gyan.conf.CONF.import_opt('backend', + 'oslo_db.options', group='database') + _IMPL = driver.DriverManager("gyan.database.migration_backend", + gyan.conf.CONF.database.backend).driver + return _IMPL + + +def upgrade(version=None): + """Migrate the database to `version` or the most recent version.""" + return get_backend().upgrade(version) + + +def version(): + return get_backend().version() + + +def stamp(version): + return get_backend().stamp(version) + + +def revision(message, autogenerate): + return get_backend().revision(message, autogenerate) diff --git a/gyan/db/sqlalchemy/__init__.py b/gyan/db/sqlalchemy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/db/sqlalchemy/alembic.ini b/gyan/db/sqlalchemy/alembic.ini new file mode 100644 index 0000000..4d0b584 --- /dev/null +++ b/gyan/db/sqlalchemy/alembic.ini @@ -0,0 +1,68 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; this defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path +# version_locations = %(here)s/bar %(here)s/bat alembic/versions + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +#sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/gyan/db/sqlalchemy/alembic/README b/gyan/db/sqlalchemy/alembic/README new file mode 100644 index 0000000..94004de --- /dev/null +++ b/gyan/db/sqlalchemy/alembic/README @@ -0,0 +1,11 @@ +Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation + +To create alembic migrations use: +$ gyan-db-manage revision --message "description of revision" --autogenerate + +Stamp db with most recent migration version, without actually running migrations +$ gyan-db-manage stamp head + +Upgrade can be performed by: +$ gyan-db-manage upgrade +$ gyan-db-manage upgrade head diff --git a/gyan/db/sqlalchemy/alembic/env.py b/gyan/db/sqlalchemy/alembic/env.py new file mode 100644 index 0000000..e589101 --- /dev/null +++ b/gyan/db/sqlalchemy/alembic/env.py @@ -0,0 +1,58 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from __future__ import with_statement +from alembic import context +from logging.config import fileConfig + +from gyan.db.sqlalchemy import api as sqla_api +from gyan.db.sqlalchemy import models +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = models.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = sqla_api.get_engine() + with engine.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + render_as_batch=True + ) + + with context.begin_transaction(): + context.run_migrations() + +run_migrations_online() diff --git a/gyan/db/sqlalchemy/alembic/script.py.mako b/gyan/db/sqlalchemy/alembic/script.py.mako new file mode 100644 index 0000000..8323caa --- /dev/null +++ b/gyan/db/sqlalchemy/alembic/script.py.mako @@ -0,0 +1,20 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} diff --git a/gyan/db/sqlalchemy/alembic/versions/cebd81b206ca_add_model_and_host_tables.py b/gyan/db/sqlalchemy/alembic/versions/cebd81b206ca_add_model_and_host_tables.py new file mode 100644 index 0000000..844d5f5 --- /dev/null +++ b/gyan/db/sqlalchemy/alembic/versions/cebd81b206ca_add_model_and_host_tables.py @@ -0,0 +1,48 @@ +"""Add Model and Host tables + +Revision ID: cebd81b206ca +Create Date: 2018-10-09 09:57:20.823110 + +""" + +# revision identifiers, used by Alembic. +revision = 'cebd81b206ca' +down_revision = None +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + 'compute_host', + sa.Column('id', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('hostname', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('type', sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_table( + 'ml_model', + sa.Column('id', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(), nullable=True), + sa.Column('updated_at', sa.DateTime(), nullable=True), + sa.Column('host_id', sa.String(length=255), nullable=True), + sa.Column('name', sa.String(length=255), nullable=True), + sa.Column('project_id', sa.String(length=255), nullable=True), + sa.Column('user_id', sa.String(length=255), nullable=True), + sa.Column('status', sa.String(length=255), nullable=True), + sa.Column('status_reason', sa.Text, nullable=True), + sa.Column('url', sa.Text, nullable=True), + sa.Column('hints', sa.Text, nullable=True), + sa.Column('deployed', sa.Boolean(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.ForeignKeyConstraint(['host_id'], ['compute_host.id']) + ) + + # ### end Alembic commands ### diff --git a/gyan/db/sqlalchemy/api.py b/gyan/db/sqlalchemy/api.py new file mode 100644 index 0000000..125d727 --- /dev/null +++ b/gyan/db/sqlalchemy/api.py @@ -0,0 +1,306 @@ +# Copyright 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""SQLAlchemy storage backend.""" + +from oslo_db import exception as db_exc +from oslo_db.sqlalchemy import session as db_session +from oslo_db.sqlalchemy import utils as db_utils +from oslo_utils import importutils +from oslo_utils import strutils +from oslo_utils import timeutils +from oslo_utils import uuidutils +import sqlalchemy as sa +from sqlalchemy.orm import contains_eager +from sqlalchemy.orm.exc import MultipleResultsFound +from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.sql.expression import desc +from sqlalchemy.sql import func + +from gyan.common import consts +from gyan.common import exception +from gyan.common.i18n import _ +import gyan.conf +from gyan.db.sqlalchemy import models + +profiler_sqlalchemy = importutils.try_import('osprofiler.sqlalchemy') + +CONF = gyan.conf.CONF + +_FACADE = None + + +def _create_facade_lazily(): + global _FACADE + if _FACADE is None: + _FACADE = db_session.enginefacade.get_legacy_facade() + if profiler_sqlalchemy: + if CONF.profiler.enabled and CONF.profiler.trace_sqlalchemy: + profiler_sqlalchemy.add_tracing(sa, _FACADE.get_engine(), "db") + return _FACADE + + +def get_engine(): + facade = _create_facade_lazily() + return facade.get_engine() + + +def get_session(**kwargs): + facade = _create_facade_lazily() + return facade.get_session(**kwargs) + + +def get_backend(): + """The backend is this module itself.""" + return Connection() + + +def model_query(model, *args, **kwargs): + """Query helper for simpler session usage. + + :param session: if present, the session to use + """ + + session = kwargs.get('session') or get_session() + query = session.query(model, *args) + return query + + +def add_identity_filter(query, value): + """Adds an identity filter to a query. + + Filters results by ID, if supplied value is a valid integer. + Otherwise attempts to filter results by UUID. + + :param query: Initial query to add filter to. + :param value: Value for filtering results by. + :return: Modified query. + """ + if strutils.is_int_like(value): + return query.filter_by(id=value) + elif uuidutils.is_uuid_like(value): + return query.filter_by(uuid=value) + else: + raise exception.InvalidIdentity(identity=value) + + +def _paginate_query(model, limit=None, marker=None, sort_key=None, + sort_dir=None, query=None, default_sort_key='id'): + if not query: + query = model_query(model) + sort_keys = [default_sort_key] + if sort_key and sort_key not in sort_keys: + sort_keys.insert(0, sort_key) + try: + query = db_utils.paginate_query(query, model, limit, sort_keys, + marker=marker, sort_dir=sort_dir) + except db_exc.InvalidSortKey: + raise exception.InvalidParameterValue( + _('The sort_key value "%(key)s" is an invalid field for sorting') + % {'key': sort_key}) + return query.all() + + +class Connection(object): + """SqlAlchemy connection.""" + + def __init__(self): + pass + + def _add_project_filters(self, context, query): + if context.is_admin and context.all_projects: + return query + + if context.project_id: + query = query.filter_by(project_id=context.project_id) + else: + query = query.filter_by(user_id=context.user_id) + + return query + + def _add_filters(self, query, model, filters=None, filter_names=None): + """Generic way to add filters to a Gyan model""" + if not filters: + return query + + if not filter_names: + filter_names = [] + + for name in filter_names: + if name in filters: + value = filters[name] + if isinstance(value, list): + column = getattr(model, name) + query = query.filter(column.in_(value)) + else: + column = getattr(model, name) + query = query.filter(column == value) + + return query + + def _add_compute_hosts_filters(self, query, filters): + filter_names = None + return self._add_filters(query, models.ComputeHost, filters=filters, + filter_names=filter_names) + + def list_compute_hosts(self, context, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + query = model_query(models.ComputeHost) + query = self._add_compute_hosts_filters(query, filters) + return _paginate_query(models.ComputeHost, limit, marker, + sort_key, sort_dir, query, + default_sort_key='id') + + def create_compute_host(self, context, values): + # ensure defaults are present for new compute hosts + if not values.get('id'): + values['id'] = uuidutils.generate_uuid() + + compute_host = models.ComputeHost() + compute_host.update(values) + try: + compute_host.save() + except db_exc.DBDuplicateEntry: + raise exception.ComputeHostAlreadyExists( + field='UUID', value=values['uuid']) + return compute_host + + def get_compute_host(self, context, host_uuid): + query = model_query(models.ComputeHost) + query = query.filter_by(id=host_uuid) + try: + return query.one() + except NoResultFound: + raise exception.ComputeHostNotFound( + compute_host=host_uuid) + + def get_compute_host_by_hostname(self, context, hostname): + query = model_query(models.ComputeHost) + query = query.filter_by(hostname=hostname) + try: + return query.one() + except NoResultFound: + raise exception.ComputeHostNotFound( + compute_host=hostname) + except MultipleResultsFound: + raise exception.Conflict('Multiple compute hosts exist with same ' + 'hostname. Please use the uuid instead.') + + def destroy_compute_host(self, context, host_uuid): + session = get_session() + with session.begin(): + query = model_query(models.ComputeHost, session=session) + query = query.filter_by(uuid=host_uuid) + count = query.delete() + if count != 1: + raise exception.ComputeHostNotFound( + compute_host=host_uuid) + + def update_compute_host(self, context, host_uuid, values): + if 'uuid' in values: + msg = _("Cannot overwrite UUID for an existing ComputeHost.") + raise exception.InvalidParameterValue(err=msg) + + return self._do_update_compute_host(host_uuid, values) + + def _do_update_compute_host(self, host_uuid, values): + session = get_session() + with session.begin(): + query = model_query(models.ComputeHost, session=session) + query = query.filter_by(uuid=host_uuid) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.ComputeHostNotFound( + compute_host=host_uuid) + + ref.update(values) + return ref + + def list_ml_models(self, context, filters=None, limit=None, + marker=None, sort_key=None, sort_dir=None): + query = model_query(models.Capsule) + query = self._add_project_filters(context, query) + query = self._add_ml_models_filters(query, filters) + return _paginate_query(models.Capsule, limit, marker, + sort_key, sort_dir, query) + + def create_ml_model(self, context, values): + # ensure defaults are present for new ml_models + if not values.get('uuid'): + values['uuid'] = uuidutils.generate_uuid() + ml_model = models.ML_Model() + ml_model.update(values) + try: + ml_model.save() + except db_exc.DBDuplicateEntry: + raise exception.MLModelAlreadyExists(field='UUID', + value=values['uuid']) + return ml_model + + def get_ml_model_by_uuid(self, context, ml_model_uuid): + query = model_query(models.ML_Model) + query = self._add_project_filters(context, query) + query = query.filter_by(uuid=ml_model_uuid) + try: + return query.one() + except NoResultFound: + raise exception.MLModelNotFound(ml_model=ml_model_uuid) + + def get_ml_model_by_name(self, context, ml_model_name): + query = model_query(models.ML_Model) + query = self._add_project_filters(context, query) + query = query.filter_by(meta_name=ml_model_name) + try: + return query.one() + except NoResultFound: + raise exception.MLModelNotFound(ml_model=ml_model_name) + except MultipleResultsFound: + raise exception.Conflict('Multiple ml_models exist with same ' + 'name. Please use the ml_model uuid ' + 'instead.') + + def destroy_ml_model(self, context, ml_model_id): + session = get_session() + with session.begin(): + query = model_query(models.ML_Model, session=session) + query = add_identity_filter(query, ml_model_id) + count = query.delete() + if count != 1: + raise exception.MLModelNotFound(ml_model_id) + + def update_ml_model(self, context, ml_model_id, values): + if 'uuid' in values: + msg = _("Cannot overwrite UUID for an existing ML Model.") + raise exception.InvalidParameterValue(err=msg) + + return self._do_update_ml_model_id(ml_model_id, values) + + def _do_update_ml_model_id(self, ml_model_id, values): + session = get_session() + with session.begin(): + query = model_query(models.ML_Model, session=session) + query = add_identity_filter(query, ml_model_id) + try: + ref = query.with_lockmode('update').one() + except NoResultFound: + raise exception.MLModelNotFound(ml_model=ml_model_id) + + ref.update(values) + return ref + + def _add_ml_models_filters(self, query, filters): + filter_names = ['uuid', 'project_id', 'user_id'] + return self._add_filters(query, models.ML_Model, filters=filters, + filter_names=filter_names) diff --git a/gyan/db/sqlalchemy/migration.py b/gyan/db/sqlalchemy/migration.py new file mode 100644 index 0000000..e2ac01a --- /dev/null +++ b/gyan/db/sqlalchemy/migration.py @@ -0,0 +1,111 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import alembic +from alembic import config as alembic_config +import alembic.migration as alembic_migration +from oslo_db import exception as db_exc +from oslo_db.sqlalchemy import enginefacade +from oslo_db.sqlalchemy.migration_cli import manager + +from gyan.db.sqlalchemy import models + +import gyan.conf + +_MANAGER = None + + +def _alembic_config(): + path = os.path.join(os.path.dirname(__file__), 'alembic.ini') + config = alembic_config.Config(path) + return config + + +def get_manager(): + global _MANAGER + if not _MANAGER: + alembic_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'alembic.ini')) + migrate_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'alembic')) + migration_config = {'alembic_ini_path': alembic_path, + 'alembic_repo_path': migrate_path, + 'db_url': gyan.conf.CONF.database.connection} + _MANAGER = manager.MigrationManager(migration_config) + + return _MANAGER + + +def version(config=None, engine=None): + """Current database version. + + :returns: Database version + :rtype: string + """ + if engine is None: + engine = enginefacade.get_legacy_facade().get_engine() + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_revision() + + +def upgrade(version): + """Used for upgrading database. + + :param version: Desired database version + :type version: string + """ + version = version or 'head' + + get_manager().upgrade(version) + + +def stamp(revision, config=None): + """Stamps database with provided revision. + + Don't run any migrations. + + :param revision: Should match one from repository or head - to stamp + database with most recent revision + :type revision: string + """ + config = config or _alembic_config() + return alembic.command.stamp(config, revision=revision) + + +def create_schema(config=None, engine=None): + """Create database schema from models description. + + Can be used for initial installation instead of upgrade('head'). + """ + if engine is None: + engine = enginefacade.get_legacy_facade().get_engine() + + if version(engine=engine) is not None: + raise db_exc.DBMigrationError("DB schema is already under version" + " control. Use upgrade() instead") + models.Base.metadata.create_all(engine) + stamp('head', config=config) + + +def revision(message=None, autogenerate=False): + """Creates template for migration. + + :param message: Text that will be used for migration title + :type message: string + :param autogenerate: If True - generates diff based on current database + state + :type autogenerate: bool + """ + return get_manager().revision(message=message, autogenerate=autogenerate) diff --git a/gyan/db/sqlalchemy/models.py b/gyan/db/sqlalchemy/models.py new file mode 100644 index 0000000..b148439 --- /dev/null +++ b/gyan/db/sqlalchemy/models.py @@ -0,0 +1,141 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +SQLAlchemy models for ML INFRA service +""" + +from oslo_db.sqlalchemy import models +from oslo_serialization import jsonutils as json +from oslo_utils import timeutils +import six.moves.urllib.parse as urlparse +from sqlalchemy import Boolean +from sqlalchemy import Column +from sqlalchemy import DateTime +from sqlalchemy.dialects.mysql import MEDIUMTEXT +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import Float +from sqlalchemy import ForeignKey +from sqlalchemy import Index +from sqlalchemy import Integer +from sqlalchemy import orm +from sqlalchemy import schema +from sqlalchemy import sql +from sqlalchemy import String +from sqlalchemy import Text +from sqlalchemy.types import TypeDecorator, TEXT + +import gyan.conf + + +def MediumText(): + return Text().with_variant(MEDIUMTEXT(), 'mysql') + + +def table_args(): + engine_name = urlparse.urlparse(gyan.conf.CONF.database.connection).scheme + if engine_name == 'mysql': + return {'mysql_engine': gyan.conf.CONF.database.mysql_engine, + 'mysql_charset': "utf8"} + return None + + +class JsonEncodedType(TypeDecorator): + """Abstract base type serialized as json-encoded string in db.""" + type = None + impl = TEXT + + def process_bind_param(self, value, dialect): + if value is None: + # Save default value according to current type to keep the + # interface the consistent. + value = self.type() + elif not isinstance(value, self.type): + raise TypeError("%s supposes to store %s objects, but %s given" + % (self.__class__.__name__, + self.type.__name__, + type(value).__name__)) + serialized_value = json.dump_as_bytes(value) + return serialized_value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value + + +class JSONEncodedDict(JsonEncodedType): + """Represents dict serialized as json-encoded string in db.""" + type = dict + + +class JSONEncodedList(JsonEncodedType): + """Represents list serialized as json-encoded string in db.""" + type = list + + +class GyanBase(models.TimestampMixin, + models.ModelBase): + + metadata = None + + def as_dict(self): + d = {} + for c in self.__table__.columns: + d[c.name] = self[c.name] + return d + + def save(self, session=None): + import gyan.db.sqlalchemy.api as db_api + + if session is None: + session = db_api.get_session() + + super(GyanBase, self).save(session) + + +Base = declarative_base(cls=GyanBase) + + +class ML_Model(Base): + """Represents a ML Model.""" + + __tablename__ = 'ml_model' + __table_args__ = ( + schema.UniqueConstraint('id', name='uniq_mlmodel0uuid'), + table_args() + ) + id = Column(Integer, primary_key=True) + project_id = Column(String(255)) + user_id = Column(String(255)) + name = Column(String(255)) + status = Column(String(20)) + status_reason = Column(Text, nullable=True) + task_state = Column(String(20)) + host_id = Column(String(255)) + status_detail = Column(String(50)) + deployed = Column(String(50)) + deployed = Column(Text, nullable=True) + started_at = Column(DateTime) + + +class ComputeHost(Base): + """Represents a compute host. """ + + __tablename__ = 'compute_host' + __table_args__ = ( + table_args() + ) + id = Column(String(36), primary_key=True, nullable=False) + hostname = Column(String(255), nullable=False) + status = Column(String(255), nullable=False) + type = Column(String(255), nullable=False) \ No newline at end of file diff --git a/gyan/ml_model/__init__.py b/gyan/ml_model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/ml_model/driver.py b/gyan/ml_model/driver.py new file mode 100644 index 0000000..86414c4 --- /dev/null +++ b/gyan/ml_model/driver.py @@ -0,0 +1,100 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +from oslo_log import log as logging +from oslo_utils import importutils +from oslo_utils import units + +from gyan.common.i18n import _ +import gyan.conf +from gyan import objects + +LOG = logging.getLogger(__name__) +CONF = gyan.conf.CONF + + +def load_ml_model_driver(ml_model_driver=None): + """Load a ml_model driver module. + + Load the ml_model driver module specified by the ml_model_driver + configuration option or, if supplied, the driver name supplied as an + argument. + :param ml_model_driver: a ml_model driver name to override the config opt + :returns: a MLModelDriver instance + """ + if not ml_model_driver: + ml_model_driver = CONF.ml_model_driver + if not ml_model_driver: + LOG.error("ML Model driver option required, " + "but not specified") + sys.exit(1) + + LOG.info("Loading ML Model driver '%s'", ml_model_driver) + try: + if not ml_model_driver.startswith('gyan.'): + ml_model_driver = 'gyan.ml_model.%s' % ml_model_driver + driver = importutils.import_object(ml_model_driver) + if not isinstance(driver, MLModelDriver): + raise Exception(_('Expected driver of type: %s') % + str(MLModelDriver)) + + return driver + except ImportError: + LOG.exception("Unable to load the ml model driver") + sys.exit(1) + + +class MLModelDriver(object): + """Base class for ml model drivers.""" + + def create(self, context, ml_model, **kwargs): + """Create a ML Model.""" + raise NotImplementedError() + + def delete(self, context, ml_model, force): + """Delete a ML Model.""" + raise NotImplementedError() + + def list(self, context): + """List all ML Models.""" + raise NotImplementedError() + + def show(self, context, ml_model): + """Show the details of a ML Models.""" + raise NotImplementedError() + + def train(self, context, ml_model): + """Train a ML Model.""" + raise NotImplementedError() + + def deploy(self, context, ml_model): + """Deploy a ML Model.""" + raise NotImplementedError() + + def undeploy(self, context, ml_model): + """Undeploy a ML Model.""" + raise NotImplementedError() + + def get_available_hosts(self): + pass + + def get_available_resources(self, host): + pass + + def node_is_available(self, hostname): + """Return whether this compute service manages a particular host.""" + if hostname in self.get_available_hosts(): + return True + return False \ No newline at end of file diff --git a/gyan/ml_model/tensorflow/__init__.py b/gyan/ml_model/tensorflow/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/ml_model/tensorflow/driver.py b/gyan/ml_model/tensorflow/driver.py new file mode 100644 index 0000000..7b41a5a --- /dev/null +++ b/gyan/ml_model/tensorflow/driver.py @@ -0,0 +1,67 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import datetime +import eventlet +import functools +import types + +from docker import errors +from oslo_log import log as logging +from oslo_utils import timeutils +from oslo_utils import uuidutils +import six + +from gyan.common import consts +from gyan.common import exception +from gyan.common.i18n import _ +from gyan.common import utils +from gyan.compute import api as gyan_compute +import gyan.conf +from gyan.ml_model import driver +from gyan import objects + + +CONF = gyan.conf.CONF +LOG = logging.getLogger(__name__) + + +class TensorflowDriver(driver.MLModelDriver): + """Implementation of ml model drivers for Tensorflow.""" + + def __init__(self): + super(driver.MLModelDriver, self).__init__() + self._host = None + + def create(self, context, ml_model): + return ml_model + pass + + + def delete(self, context, ml_model, force): + pass + + def list(self, context): + pass + + def show(self, context, ml_model): + pass + + def train(self, context, ml_model): + pass + + def deploy(self, context, ml_model): + pass + + def undeploy(self, context, ml_model): + pass \ No newline at end of file diff --git a/gyan/objects/__init__.py b/gyan/objects/__init__.py new file mode 100644 index 0000000..9586a6a --- /dev/null +++ b/gyan/objects/__init__.py @@ -0,0 +1,23 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gyan.objects import compute_host +from gyan.objects import ml_model + + +ComputeHost = compute_host.ComputeHost +ML_Model = ml_model.ML_Model + +__all__ = ( + 'ComputeHost', + 'ML_Model' +) diff --git a/gyan/objects/base.py b/gyan/objects/base.py new file mode 100644 index 0000000..d359aaf --- /dev/null +++ b/gyan/objects/base.py @@ -0,0 +1,131 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Gyan common internal object model""" + +from oslo_versionedobjects import base as ovoo_base +from oslo_versionedobjects import fields as ovoo_fields + +from gyan.objects import fields as obj_fields + +remotable_classmethod = ovoo_base.remotable_classmethod +remotable = ovoo_base.remotable + + +class GyanObjectRegistry(ovoo_base.VersionedObjectRegistry): + pass + + +class GyanObject(ovoo_base.VersionedObject): + """Base class and object factory. + + This forms the base of all objects that can be remoted or instantiated + via RPC. Simply defining a class that inherits from this base class + will make it remotely instantiatable. Objects should implement the + necessary "get" classmethod routines as well as "save" object methods + as appropriate. + """ + OBJ_SERIAL_NAMESPACE = 'gyan_object' + OBJ_PROJECT_NAMESPACE = 'gyan' + + def as_dict(self): + return {k: getattr(self, k) + for k in self.fields + if self.obj_attr_is_set(k)} + + +class GyanPersistentObject(object): + """Mixin class for Persistent objects. + + This adds the fields that we use in common for all persistent objects. + """ + fields = { + 'created_at': ovoo_fields.DateTimeField(nullable=True, + tzinfo_aware=False), + 'updated_at': ovoo_fields.DateTimeField(nullable=True, + tzinfo_aware=False), + } + + +class GyanObjectSerializer(ovoo_base.VersionedObjectSerializer): + # Base class to use for object hydration + OBJ_BASE_CLASS = GyanObject + + +class ObjectListBase(ovoo_base.ObjectListBase): + # NOTE: These are for transition to using the oslo + # base object and can be removed when we move to it. + @classmethod + def _obj_primitive_key(cls, field): + return 'gyan_object.%s' % field + + @classmethod + def _obj_primitive_field(cls, primitive, field, + default=obj_fields.UnspecifiedDefault): + key = cls._obj_primitive_key(field) + if default == obj_fields.UnspecifiedDefault: + return primitive[key] + else: + return primitive.get(key, default) + + +def obj_to_primitive(obj): + """Recursively turn an object into a python primitive. + + A GyanObject becomes a dict, and anything that implements ObjectListBase + becomes a list. + """ + if isinstance(obj, ObjectListBase): + return [obj_to_primitive(x) for x in obj] + elif isinstance(obj, GyanObject): + result = {} + for key in obj.obj_fields: + if obj.obj_attr_is_set(key) or key in obj.obj_extra_fields: + result[key] = obj_to_primitive(getattr(obj, key)) + return result + else: + return obj + + +def obj_equal_prims(obj_1, obj_2, ignore=None): + """Compare two primitives for equivalence ignoring some keys. + + This operation tests the primitives of two objects for equivalence. + Object primitives may contain a list identifying fields that have been + changed - this is ignored in the comparison. The ignore parameter lists + any other keys to be ignored. + + :param:obj1: The first object in the comparison + :param:obj2: The second object in the comparison + :param:ignore: A list of fields to ignore + :returns: True if the primitives are equal ignoring changes + and specified fields, otherwise False. + """ + + def _strip(prim, keys): + if isinstance(prim, dict): + for k in keys: + prim.pop(k, None) + for v in prim.values(): + _strip(v, keys) + if isinstance(prim, list): + for v in prim: + _strip(v, keys) + return prim + + if ignore is not None: + keys = ['gyan_object.changes'] + ignore + else: + keys = ['gyan_object.changes'] + prim_1 = _strip(obj_1.obj_to_primitive(), keys) + prim_2 = _strip(obj_2.obj_to_primitive(), keys) + return prim_1 == prim_2 diff --git a/gyan/objects/compute_host.py b/gyan/objects/compute_host.py new file mode 100644 index 0000000..dbf81d5 --- /dev/null +++ b/gyan/objects/compute_host.py @@ -0,0 +1,119 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_serialization import jsonutils +from oslo_versionedobjects import fields + +from gyan.db import api as dbapi +from gyan.objects import base + + +@base.GyanObjectRegistry.register +class ComputeHost(base.GyanPersistentObject, base.GyanObject): + # Version 1: Initial Version + VERSION = '1' + + fields = { + 'id': fields.UUIDField(read_only=True, nullable=False), + 'hostname': fields.StringField(nullable=False), + 'status': fields.StringField(nullable=False), + 'type': fields.StringField(nullable=False) + } + + @staticmethod + def _from_db_object(context, compute_node, db_compute_node): + """Converts a database entity to a formal object.""" + fields = set(compute_node.fields) + for field in fields: + setattr(compute_node, field, db_compute_node[field]) + + compute_node.obj_reset_changes(recursive=True) + return compute_node + + @staticmethod + def _from_db_object_list(db_objects, cls, context): + """Converts a list of database entities to a list of formal objects.""" + return [ComputeHost._from_db_object(context, cls(context), obj) + for obj in db_objects] + + @base.remotable + def create(self, context): + """Create a compute node record in the DB. + + :param context: Security context. + + """ + values = self.obj_get_changes() + + db_compute_host = dbapi.create_compute_host(context, values) + self._from_db_object(context, self, db_compute_host) + + @base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + """Find a compute node based on uuid. + + :param uuid: the uuid of a compute node. + :param context: Security context + :returns: a :class:`ComputeHost` object. + """ + db_compute_node = dbapi.get_compute_host(context, uuid) + compute_node = ComputeHost._from_db_object( + context, cls(context), db_compute_node) + return compute_node + + @base.remotable_classmethod + def get_by_name(cls, context, hostname): + db_compute_node = dbapi.get_compute_host_by_hostname( + context, hostname) + return cls._from_db_object(context, cls(), db_compute_node) + + @base.remotable_classmethod + def list(cls, context, limit=None, marker=None, + sort_key=None, sort_dir=None, filters=None): + """Return a list of ComputeHost objects. + + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: filters when list resource providers. + :returns: a list of :class:`ComputeHost` object. + + """ + db_compute_nodes = dbapi.list_compute_hosts( + context, limit=limit, marker=marker, sort_key=sort_key, + sort_dir=sort_dir, filters=filters) + return ComputeHost._from_db_object_list( + db_compute_nodes, cls, context) + + @base.remotable + def destroy(self, context=None): + """Delete the ComputeHost from the DB. + + :param context: Security context. + """ + dbapi.destroy_compute_host(context, self.uuid) + self.obj_reset_changes(recursive=True) + + @base.remotable + def save(self, context=None): + """Save updates to this ComputeHost. + + Updates will be made column by column based on the result + of self.what_changed(). + + :param context: Security context. + """ + updates = self.obj_get_changes() + dbapi.update_compute_host(context, self.uuid, updates) + self.obj_reset_changes(recursive=True) \ No newline at end of file diff --git a/gyan/objects/fields.py b/gyan/objects/fields.py new file mode 100644 index 0000000..07dc6da --- /dev/null +++ b/gyan/objects/fields.py @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import six + +from oslo_serialization import jsonutils as json +from oslo_versionedobjects import fields + +UnspecifiedDefault = fields.UnspecifiedDefault + + +class BaseGyanEnum(fields.Enum): + def __init__(self, **kwargs): + super(BaseGyanEnum, self).__init__(valid_values=self.__class__.ALL) + + +class ListOfIntegersField(fields.AutoTypedField): + AUTO_TYPE = fields.List(fields.Integer()) + + +class Json(fields.FieldType): + def coerce(self, obj, attr, value): + if isinstance(value, six.string_types): + loaded = json.loads(value) + return loaded + return value + + def from_primitive(self, obj, attr, value): + return self.coerce(obj, attr, value) + + def to_primitive(self, obj, attr, value): + return json.dump_as_bytes(value) + + +class JsonField(fields.AutoTypedField): + AUTO_TYPE = Json() diff --git a/gyan/objects/ml_model.py b/gyan/objects/ml_model.py new file mode 100644 index 0000000..3420ba8 --- /dev/null +++ b/gyan/objects/ml_model.py @@ -0,0 +1,155 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging +from oslo_versionedobjects import fields + +from gyan.common import exception +from gyan.common.i18n import _ +from gyan.db import api as dbapi +from gyan.objects import base +from gyan.objects import fields as z_fields + + +LOG = logging.getLogger(__name__) + +@base.GyanObjectRegistry.register +class ML_Model(base.GyanPersistentObject, base.GyanObject): + VERSION = '1' + + fields = { + 'id': fields.UUIDField(nullable=True), + 'name': fields.StringField(nullable=True), + 'project_id': fields.StringField(nullable=True), + 'user_id': fields.StringField(nullable=True), + 'status': fields.StringField(nullable=True), + 'status_reason': fields.StringField(nullable=True), + 'url': fields.StringField(nullable=True), + 'deployed': fields.BooleanField(nullable=True), + 'node': fields.UUIDField(nullable=True), + 'hints': fields.StringField(nullable=True), + 'created_at': fields.DateTimeField(tzinfo_aware=False, nullable=True), + 'updated_at': fields.DateTimeField(tzinfo_aware=False, nullable=True) + } + + @staticmethod + def _from_db_object(ml_model, db_ml_model): + """Converts a database entity to a formal object.""" + for field in ml_model.fields: + setattr(ml_model, field, db_ml_model[field]) + + ml_model.obj_reset_changes() + return ml_model + + @staticmethod + def _from_db_object_list(db_objects, cls, context): + """Converts a list of database entities to a list of formal objects.""" + return [ML_Model._from_db_object(cls(context), obj) + for obj in db_objects] + + @base.remotable_classmethod + def get_by_uuid(cls, context, uuid): + """Find a ml model based on uuid and return a :class:`ML_Model` object. + + :param uuid: the uuid of a ml model. + :param context: Security context + :returns: a :class:`ML_Model` object. + """ + db_ml_model = dbapi.get_ml_model_by_uuid(context, uuid) + ml_model = ML_Model._from_db_object(cls(context), db_ml_model) + return ml_model + + @base.remotable_classmethod + def get_by_name(cls, context, name): + """Find a ml model based on name and return a Ml model object. + + :param name: the logical name of a ml model. + :param context: Security context + :returns: a :class:`ML_Model` object. + """ + db_ml_model = dbapi.get_ml_model_by_name(context, name) + ml_model = ML_Model._from_db_object(cls(context), db_ml_model) + return ml_model + + @base.remotable_classmethod + def list(cls, context, limit=None, marker=None, + sort_key=None, sort_dir=None, filters=None): + """Return a list of ML Model objects. + + :param context: Security context. + :param limit: maximum number of resources to return in a single result. + :param marker: pagination marker for large data sets. + :param sort_key: column to sort results by. + :param sort_dir: direction to sort. "asc" or "desc". + :param filters: filters when list ml models, the filter name could be + 'name', 'project_id', 'user_id'. + :returns: a list of :class:`ML_Model` object. + + """ + db_ml_models = dbapi.list_ml_models( + context, limit=limit, marker=marker, sort_key=sort_key, + sort_dir=sort_dir, filters=filters) + return ML_Model._from_db_object_list(db_ml_models, cls, context) + + @base.remotable_classmethod + def list_by_host(cls, context, host): + """Return a list of ML Model objects by host. + + :param context: Security context. + :param host: A compute host. + :returns: a list of :class:`ML_Model` object. + + """ + db_ml_models = dbapi.list_ml_models(context, filters={'host': host}) + return ML_Model._from_db_object_list(db_ml_models, cls, context) + + def create(self, context): + """Create a ML_Model record in the DB. + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: ML_Model(context) + + """ + values = self.obj_get_changes() + db_ml_model = dbapi.create_ml_model(context, values) + self._from_db_object(self, db_ml_model) + + @base.remotable + def destroy(self, context=None): + """Delete the ML_Model from the DB. + + :param context: Security context. NOTE: This should only + be used internally by the indirection_api. + Unfortunately, RPC requires context as the first + argument, even though we don't use it. + A context should be set when instantiating the + object, e.g.: ML Model(context) + """ + dbapi.destroy_ml_model(context, self.uuid) + self.obj_reset_changes() + + def obj_load_attr(self, attrname): + if not self._context: + raise exception.OrphanedObjectError(method='obj_load_attr', + objtype=self.obj_name()) + + LOG.debug("Lazy-loading '%(attr)s' on %(name)s uuid %(uuid)s", + {'attr': attrname, + 'name': self.obj_name(), + 'uuid': self.uuid, + }) + + self.obj_reset_changes([attrname]) diff --git a/gyan/servicegroup/__init__.py b/gyan/servicegroup/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gyan/servicegroup/gyan_service_periodic.py b/gyan/servicegroup/gyan_service_periodic.py new file mode 100644 index 0000000..779f269 --- /dev/null +++ b/gyan/servicegroup/gyan_service_periodic.py @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Gyan Service Layer""" + +from oslo_log import log +from oslo_service import periodic_task + +from gyan.common import context +from gyan import objects + + +LOG = log.getLogger(__name__) + + +class GyanServicePeriodicTasks(periodic_task.PeriodicTasks): + """Gyan periodic Task class + + Any periodic task job need to be added into this class + """ + + def __init__(self, conf, binary): + self.gyan_service_ref = None + self.host = conf.compute.host + self.binary = binary + super(GyanServicePeriodicTasks, self).__init__(conf) + + +def setup(conf, binary, tg): + pt = GyanServicePeriodicTasks(conf, binary) + tg.add_dynamic_timer( + pt.run_periodic_tasks, + periodic_interval_max=conf.periodic_interval_max, + context=None) diff --git a/gyan/version.py b/gyan/version.py new file mode 100644 index 0000000..43bb374 --- /dev/null +++ b/gyan/version.py @@ -0,0 +1,18 @@ +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import absolute_import +import pbr.version + +version_info = pbr.version.VersionInfo('gyan') +version_string = version_info.version_string \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 8f21586..1220cb1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,3 +35,15 @@ input_file = gyan/locale/gyan.pot keywords = _ gettext ngettext l_ lazy_gettext mapping_file = babel.cfg output_file = gyan/locale/gyan.pot + +[entry_points] +console_scripts = + gyan-api = gyan.cmd.api:main + gyan-compute = gyan.cmd.compute:main + gyan-db-manage = gyan.cmd.db_manage:main + +wsgi_scripts = + gyan-api-wsgi = gyan.api.wsgi:init_application + +gyan.database.migration_backend = + sqlalchemy = gyan.db.sqlalchemy.migration diff --git a/tox.ini b/tox.ini index 71cf575..0a65bbd 100644 --- a/tox.ini +++ b/tox.ini @@ -35,6 +35,13 @@ commands = deps = -r{toxinidir}/doc/requirements.txt commands = sphinx-build -W -b html doc/source doc/build/html + +[testenv:genconfig] +basepython = python3 +envdir = {toxworkdir}/venv +commands = + oslo-config-generator --config-file etc/gyan/gyan-config-generator.conf + [testenv:releasenotes] deps = {[testenv:docs]deps} commands =