openstack
/
trove
Code Issues Proposed changes

Merge "Server side of module maintenance commands"

This commit is contained in:
Jenkins 2016-03-03 10:39:29 +00:00 committed by Gerrit Code Review
parent 44dec7ddf1 bf3fb085cc
commit dffbf77135
25 changed files with 2016 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
test-requirements.txt
View File

@ -23,3 +23,4 @@ pymongo>=3.0.2 # Apache-2.0
redis>=2.10.0 # MIT
psycopg2>=2.5 # LGPL/ZPL
cassandra-driver>=2.1.4 # Apache-2.0
pycrypto>=2.6 # Public Domain

28
trove/common/api.py
View File

@ -23,6 +23,7 @@ from trove.datastore.service import DatastoreController
from trove.flavor.service import FlavorController
from trove.instance.service import InstanceController
from trove.limits.service import LimitsController
from trove.module.service import ModuleController
from trove.versions import VersionsController
@ -39,6 +40,7 @@ class API(wsgi.Router):
self._limits_router(mapper)
self._backups_router(mapper)
self._configurations_router(mapper)
self._modules_router(mapper)
def _versions_router(self, mapper):
versions_resource = VersionsController().create_resource()
@ -184,6 +186,32 @@ class API(wsgi.Router):
action="delete",
conditions={'method': ['DELETE']})
def _modules_router(self, mapper):
modules_resource = ModuleController().create_resource()
mapper.resource("modules", "/{tenant_id}/modules",
controller=modules_resource)
mapper.connect("/{tenant_id}/modules",
controller=modules_resource,
action="index",
conditions={'method': ['GET']})
mapper.connect("/{tenant_id}/modules",
controller=modules_resource,
action="create",
conditions={'method': ['POST']})
mapper.connect("/{tenant_id}/modules/{id}",
controller=modules_resource,
action="show",
conditions={'method': ['GET']})
mapper.connect("/{tenant_id}/modules/{id}",
controller=modules_resource,
action="update",
conditions={'method': ['PUT']})
mapper.connect("/{tenant_id}/modules/{id}",
controller=modules_resource,
action="delete",
conditions={'method': ['DELETE']})
def _configurations_router(self, mapper):
parameters_resource = ParametersController().create_resource()
path = '/{tenant_id}/datastores/versions/{version}/parameters'

69
trove/common/apischema.py
View File

@ -528,6 +528,75 @@ guest_log = {
}
}
module_non_empty_string = {
"type": "string",
"minLength": 1,
"maxLength": 65535,
"pattern": "^.*.+.*$"
}
module = {
"create": {
"name": "module:create",
"type": "object",
"required": ["module"],
"properties": {
"module": {
"type": "object",
"required": ["name", "module_type", "contents"],
"additionalProperties": True,
"properties": {
"name": non_empty_string,
"module_type": non_empty_string,
"contents": module_non_empty_string,
"description": non_empty_string,
"datastore": {
"type": "object",
"properties": {
"type": non_empty_string,
"version": non_empty_string
}
},
"auto_apply": boolean_string,
"all_tenants": boolean_string,
"visible": boolean_string,
"live_update": boolean_string,
}
}
}
},
"update": {
"name": "module:update",
"type": "object",
"required": ["module"],
"properties": {
"module": {
"type": "object",
"required": [],
"additionalProperties": True,
"properties": {
"name": non_empty_string,
"type": non_empty_string,
"contents": module_non_empty_string,
"description": non_empty_string,
"datastore": {
"type": "object",
"additionalProperties": True,
"properties": {
"type": non_empty_string,
"version": non_empty_string
}
},
"auto_apply": boolean_string,
"all_tenants": boolean_string,
"visible": boolean_string,
"live_update": boolean_string,
}
}
}
},
}
configuration = {
"create": {
"name": "configuration:create",

6
trove/common/cfg.py
View File

@ -131,6 +131,8 @@ common_opts = [
help='Page size for listing backups.'),
cfg.IntOpt('configurations_page_size', default=20,
help='Page size for listing configurations.'),
cfg.IntOpt('modules_page_size', default=20,
help='Page size for listing modules.'),
cfg.IntOpt('agent_call_low_timeout', default=5,
help="Maximum time (in seconds) to wait for Guest Agent 'quick'"
"requests (such as retrieving a list of users or "
@ -399,6 +401,10 @@ common_opts = [
cfg.IntOpt('timeout_wait_for_service', default=120,
help='Maximum time (in seconds) to wait for a service to '
'become alive.'),
cfg.StrOpt('module_aes_cbc_key', default='module_aes_cbc_key',
help='OpenSSL aes_cbc key for module encryption.'),
cfg.StrOpt('module_types', default='test, hidden_test',
help='A list of module types supported.'),
cfg.StrOpt('guest_log_container_name',
default='database_logs',
help='Name of container that stores guest log components.'),

26
trove/common/exception.py
View File

@ -496,6 +496,32 @@ class ReplicaSourceDeleteForbidden(Forbidden):
"replicas.")
class ModuleTypeNotFound(NotFound):
message = _("Module type '%(module_type)s' was not found.")
class ModuleAppliedToInstance(BadRequest):
message = _("A module cannot be deleted or its contents modified if it "
"has been applied to a non-terminated instance, unless the "
"module has been marked as 'live_update.' "
"Please remove the module from all non-terminated "
"instances and try again.")
class ModuleAlreadyExists(BadRequest):
message = _("A module with the name '%(name)s' already exists for "
"datastore '%(datastore)s' and datastore version "
"'%(ds_version)s'")
class ModuleAccessForbidden(Forbidden):
message = _("You must be admin to %(action)s a module with these "
"options. %(options)s")
class ClusterNotFound(NotFound):
message = _("Cluster '%(cluster)s' cannot be found.")

45
trove/common/utils.py
View File

@ -14,8 +14,12 @@
# under the License.
"""I totally stole most of this from melange, thx guys!!!"""
import base64
import collections
from Crypto.Cipher import AES
from Crypto import Random
import datetime
import hashlib
import inspect
import os
import shutil
@ -327,3 +331,44 @@ def is_collection(item):
"""
return (isinstance(item, collections.Iterable) and
not isinstance(item, types.StringTypes))
# Encryption/decryption handling methods
IV_BIT_COUNT = 16
def encode_string(data_str):
byte_array = bytearray(data_str)
return base64.b64encode(byte_array)
def decode_string(data_str):
return base64.b64decode(data_str)
# Pad the data string to an multiple of pad_size
def pad_for_encryption(data_str, pad_size=IV_BIT_COUNT):
pad_count = pad_size - (len(data_str) % pad_size)
return data_str + chr(pad_count) * pad_count
# Unpad the data string by stripping off excess characters
def unpad_after_decryption(data_str):
return data_str[:len(data_str) - ord(data_str[-1])]
def encrypt_string(data_str, key, iv_bit_count=IV_BIT_COUNT):
md5_key = hashlib.md5(key).hexdigest()
iv = encode_string(Random.new().read(iv_bit_count))[:iv_bit_count]
aes = AES.new(md5_key, AES.MODE_CBC, iv)
data_str = pad_for_encryption(data_str, iv_bit_count)
encrypted_str = aes.encrypt(data_str)
return iv + encrypted_str
def decrypt_string(data_str, key, iv_bit_count=IV_BIT_COUNT):
md5_key = hashlib.md5(key).hexdigest()
iv = data_str[:iv_bit_count]
aes = AES.new(md5_key, AES.MODE_CBC, iv)
decrypted_str = aes.decrypt(data_str[iv_bit_count:])
return unpad_after_decryption(decrypted_str)

4
trove/common/wsgi.py
View File

@ -319,6 +319,7 @@ class Controller(object):
webob.exc.HTTPForbidden: [
exception.ReplicaSourceDeleteForbidden,
exception.BackupTooLarge,
exception.ModuleAccessForbidden,
],
webob.exc.HTTPBadRequest: [
exception.InvalidModelError,
@ -328,6 +329,8 @@ class Controller(object):
exception.DatabaseAlreadyExists,
exception.UserAlreadyExists,
exception.LocalStorageNotSpecified,
exception.ModuleAlreadyExists,
exception.ModuleAppliedToInstance,
],
webob.exc.HTTPNotFound: [
exception.NotFound,
@ -340,6 +343,7 @@ class Controller(object):
exception.ClusterNotFound,
exception.DatastoreNotFound,
exception.SwiftNotFound,
exception.ModuleTypeNotFound,
],
webob.exc.HTTPConflict: [
exception.BackupNotCompleteError,

4
trove/db/sqlalchemy/mappers.py
View File

@ -70,6 +70,10 @@ def map(engine, models):
orm.mapper(models['datastore_configuration_parameters'],
Table('datastore_configuration_parameters', meta,
autoload=True))
orm.mapper(models['modules'],
Table('modules', meta, autoload=True))
orm.mapper(models['instance_modules'],
Table('instance_modules', meta, autoload=True))
def mapping_exists(model):

84
trove/db/sqlalchemy/migrate_repo/versions/037_modules.py Normal file
View File

@ -0,0 +1,84 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from sqlalchemy import ForeignKey
from sqlalchemy.schema import Column
from sqlalchemy.schema import MetaData
from sqlalchemy.schema import UniqueConstraint
from trove.db.sqlalchemy.migrate_repo.schema import Boolean
from trove.db.sqlalchemy.migrate_repo.schema import create_tables
from trove.db.sqlalchemy.migrate_repo.schema import DateTime
from trove.db.sqlalchemy.migrate_repo.schema import drop_tables
from trove.db.sqlalchemy.migrate_repo.schema import String
from trove.db.sqlalchemy.migrate_repo.schema import Table
from trove.db.sqlalchemy.migrate_repo.schema import Text
meta = MetaData()
modules = Table(
'modules',
meta,
Column('id', String(length=64), primary_key=True, nullable=False),
Column('name', String(length=255), nullable=False),
Column('type', String(length=255), nullable=False),
Column('contents', Text(), nullable=False),
Column('description', String(length=255)),
Column('tenant_id', String(length=64), nullable=True),
Column('datastore_id', String(length=64), nullable=True),
Column('datastore_version_id', String(length=64), nullable=True),
Column('auto_apply', Boolean(), default=0, nullable=False),
Column('visible', Boolean(), default=1, nullable=False),
Column('live_update', Boolean(), default=0, nullable=False),
Column('md5', String(length=32), nullable=False),
Column('created', DateTime(), nullable=False),
Column('updated', DateTime(), nullable=False),
Column('deleted', Boolean(), default=0, nullable=False),
Column('deleted_at', DateTime()),
UniqueConstraint(
'type', 'tenant_id', 'datastore_id', 'datastore_version_id',
'name', 'deleted_at',
name='UQ_type_tenant_datastore_datastore_version_name'),
)
instance_modules = Table(
'instance_modules',
meta,
Column('id', String(length=64), primary_key=True, nullable=False),
Column('instance_id', String(length=64),
ForeignKey('instances.id', ondelete="CASCADE",
onupdate="CASCADE"), nullable=False),
Column('module_id', String(length=64),
ForeignKey('modules.id', ondelete="CASCADE",
onupdate="CASCADE"), nullable=False),
Column('md5', String(length=32), nullable=False),
Column('created', DateTime(), nullable=False),
Column('updated', DateTime(), nullable=False),
Column('deleted', Boolean(), default=0, nullable=False),
Column('deleted_at', DateTime()),
)
def upgrade(migrate_engine):
meta.bind = migrate_engine
Table('instances', meta, autoload=True)
create_tables([modules, instance_modules])
def downgrade(migrate_engine):
meta.bind = migrate_engine
drop_tables([instance_modules, modules])

2
trove/db/sqlalchemy/session.py
View File

@ -48,6 +48,7 @@ def configure_db(options, models_mapper=None):
from trove.extensions.security_group import models as secgrp_models
from trove.guestagent import models as agent_models
from trove.instance import models as base_models
from trove.module import models as module_models
from trove.quota import models as quota_models
model_modules = [
@ -62,6 +63,7 @@ def configure_db(options, models_mapper=None):
configurations_models,
conductor_models,
cluster_models,
module_models
]
models = {}

0
trove/module/__init__.py Normal file
View File

273
trove/module/models.py Normal file
View File

@ -0,0 +1,273 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
"""Model classes that form the core of Module functionality."""
from datetime import datetime
import hashlib
from trove.common import cfg
from trove.common import exception
from trove.common.i18n import _
from trove.common import utils
from trove.datastore import models as datastore_models
from trove.db import models
from trove.instance import models as instances_models
from oslo_log import log as logging
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class Modules(object):
DEFAULT_LIMIT = CONF.modules_page_size
ENCRYPT_KEY = CONF.module_aes_cbc_key
VALID_MODULE_TYPES = CONF.module_types
MATCH_ALL_NAME = 'all'
@staticmethod
def load(context):
if context is None:
raise TypeError("Argument context not defined.")
elif id is None:
raise TypeError("Argument is not defined.")
if context.is_admin:
db_info = DBModule.find_all(deleted=False)
if db_info.count() == 0:
LOG.debug("No modules found for admin user")
else:
db_info = DBModule.find_all(
tenant_id=context.tenant, visible=True, deleted=False)
if db_info.count() == 0:
LOG.debug("No modules found for tenant %s" % context.tenant)
limit = utils.pagination_limit(
context.limit, Modules.DEFAULT_LIMIT)
data_view = DBModule.find_by_pagination(
'modules', db_info, 'foo', limit=limit, marker=context.marker)
next_marker = data_view.next_page_marker
return data_view.collection, next_marker
class Module(object):
def __init__(self, context, module_id):
self.context = context
self.module_id = module_id
@staticmethod
def create(context, name, module_type, contents,
description, tenant_id, datastore,
datastore_version, auto_apply, visible, live_update):
if module_type not in Modules.VALID_MODULE_TYPES:
raise exception.ModuleTypeNotFound(module_type=module_type)
Module.validate_action(
context, 'create', tenant_id, auto_apply, visible)
datastore_id, datastore_version_id = Module.validate_datastore(
datastore, datastore_version)
if Module.key_exists(
name, module_type, tenant_id,
datastore_id, datastore_version_id):
datastore_str = datastore_id or Modules.MATCH_ALL_NAME
ds_version_str = datastore_version_id or Modules.MATCH_ALL_NAME
raise exception.ModuleAlreadyExists(
name=name, datastore=datastore_str, ds_version=ds_version_str)
md5, processed_contents = Module.process_contents(contents)
module = DBModule.create(
name=name,
type=module_type,
contents=processed_contents,
description=description,
tenant_id=tenant_id,
datastore_id=datastore_id,
datastore_version_id=datastore_version_id,
auto_apply=auto_apply,
visible=visible,
live_update=live_update,
md5=md5)
return module
# Certain fields require admin access to create/change/delete
@staticmethod
def validate_action(context, action_str, tenant_id, auto_apply, visible):
error_str = None
if not context.is_admin:
option_strs = []
if tenant_id is None:
option_strs.append(_("Tenant: %s") % Modules.MATCH_ALL_NAME)
if auto_apply:
option_strs.append(_("Auto: %s") % auto_apply)
if not visible:
option_strs.append(_("Visible: %s") % visible)
if option_strs:
error_str = "(" + " ".join(option_strs) + ")"
if error_str:
raise exception.ModuleAccessForbidden(
action=action_str, options=error_str)
@staticmethod
def validate_datastore(datastore, datastore_version):
datastore_id = None
datastore_version_id = None
if datastore:
ds, ds_ver = datastore_models.get_datastore_version(
type=datastore, version=datastore_version)
datastore_id = ds.id
if datastore_version:
datastore_version_id = ds_ver.id
elif datastore_version:
msg = _("Cannot specify version without datastore")
raise exception.BadRequest(message=msg)
return datastore_id, datastore_version_id
@staticmethod
def key_exists(name, module_type, tenant_id, datastore_id,
datastore_version_id):
try:
DBModule.find_by(
name=name, type=module_type, tenant_id=tenant_id,
datastore_id=datastore_id,
datastore_version_id=datastore_version_id,
deleted=False)
return True
except exception.ModelNotFoundError:
return False
# We encrypt the contents (which should be encoded already, since it
# might be in binary format) and then encode them again so they can
# be stored in a text field in the Trove database.
@staticmethod
def process_contents(contents):
md5 = hashlib.md5(contents).hexdigest()
encrypted_contents = utils.encrypt_string(
contents, Modules.ENCRYPT_KEY)
return md5, utils.encode_string(encrypted_contents)
@staticmethod
def delete(context, module):
Module.validate_action(
context, 'delete',
module.tenant_id, module.auto_apply, module.visible)
Module.enforce_live_update(module.id, module.live_update, module.md5)
module.deleted = True
module.deleted_at = datetime.utcnow()
module.save()
@staticmethod
def enforce_live_update(module_id, live_update, md5):
if not live_update:
instances = DBInstanceModules.find_all(
id=module_id, md5=md5, deleted=False).all()
if instances:
raise exception.ModuleAppliedToInstance()
@staticmethod
def load(context, module_id):
try:
if context.is_admin:
return DBModule.find_by(id=module_id, deleted=False)
else:
return DBModule.find_by(
id=module_id, tenant_id=context.tenant, visible=True,
deleted=False)
except exception.ModelNotFoundError:
# See if we have the module in the 'all' tenant section
if not context.is_admin:
try:
return DBModule.find_by(
id=module_id, tenant_id=None, visible=True,
deleted=False)
except exception.ModelNotFoundError:
pass # fall through to the raise below
msg = _("Module with ID %s could not be found.") % module_id
raise exception.ModelNotFoundError(msg)
@staticmethod
def update(context, module, original_module):
Module.enforce_live_update(
original_module.id, original_module.live_update,
original_module.md5)
do_update = False
if module.contents != original_module.contents:
md5, processed_contents = Module.process_contents(module.contents)
do_update = (original_module.live_update and
md5 != original_module.md5)
module.md5 = md5
module.contents = processed_contents
else:
module.contents = original_module.contents
# we don't allow any changes to 'admin'-type modules, even if
# the values changed aren't the admin ones.
access_tenant_id = (None if (original_module.tenant_id is None or
module.tenant_id is None)
else module.tenant_id)
access_auto_apply = original_module.auto_apply or module.auto_apply
access_visible = original_module.visible and module.visible
Module.validate_action(
context, 'update',
access_tenant_id, access_auto_apply, access_visible)
ds_id, ds_ver_id = Module.validate_datastore(
module.datastore_id, module.datastore_version_id)
if module.datastore_id:
module.datastore_id = ds_id
if module.datastore_version_id:
module.datastore_version_id = ds_ver_id
module.updated = datetime.utcnow()
DBModule.save(module)
if do_update:
Module.reapply_on_all_instances(context, module)
@staticmethod
def reapply_on_all_instances(context, module):
"""Reapply a module on all its instances, if required."""
if module.live_update:
instance_modules = DBInstanceModules.find_all(
id=module.id, deleted=False).all()
LOG.debug(
"All instances with module '%s' applied: %s"
% (module.id, instance_modules))
for instance_module in instance_modules:
if instance_module.md5 != module.md5:
LOG.debug("Applying module '%s' to instance: %s"
% (module.id, instance_module.instance_id))
instance = instances_models.Instance.load(
context, instance_module.instance_id)
instance.apply_module(module)
class DBModule(models.DatabaseModelBase):
_data_fields = [
'id', 'name', 'type', 'contents', 'description',
'tenant_id', 'datastore_id', 'datastore_version_id',
'auto_apply', 'visible', 'live_update',
'md5', 'created', 'updated', 'deleted', 'deleted_at']
class DBInstanceModules(models.DatabaseModelBase):
_data_fields = [
'id', 'instance_id', 'module_id', 'md5', 'created',
'updated', 'deleted', 'deleted_at']
def persisted_models():
return {'modules': DBModule, 'instance_modules': DBInstanceModules}

123
trove/module/service.py Normal file
View File

@ -0,0 +1,123 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import copy
from oslo_log import log as logging
import trove.common.apischema as apischema
from trove.common import cfg
from trove.common.i18n import _
from trove.common import pagination
from trove.common import wsgi
from trove.module import models
from trove.module import views
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
class ModuleController(wsgi.Controller):
schemas = apischema.module
def index(self, req, tenant_id):
context = req.environ[wsgi.CONTEXT_KEY]
modules, marker = models.Modules.load(context)
view = views.ModulesView(modules)
paged = pagination.SimplePaginatedDataView(req.url, 'modules',
view, marker)
return wsgi.Result(paged.data(), 200)
def show(self, req, tenant_id, id):
LOG.info(_("Showing module %s") % id)
context = req.environ[wsgi.CONTEXT_KEY]
module = models.Module.load(context, id)
module.instance_count = models.DBInstanceModules.find_all(
id=module.id, md5=module.md5,
deleted=False).count()
return wsgi.Result(
views.DetailedModuleView(module).data(), 200)
def create(self, req, body, tenant_id):
name = body['module']['name']
LOG.info(_("Creating module '%s'") % name)
context = req.environ[wsgi.CONTEXT_KEY]
module_type = body['module']['module_type']
contents = body['module']['contents']
description = body['module'].get('description')
all_tenants = body['module'].get('all_tenants', 0)
module_tenant_id = None if all_tenants else tenant_id
datastore = body['module'].get('datastore', {}).get('type', None)
ds_version = body['module'].get('datastore', {}).get('version', None)
auto_apply = body['module'].get('auto_apply', 0)
visible = body['module'].get('visible', 1)
live_update = body['module'].get('live_update', 0)
module = models.Module.create(
context, name, module_type, contents,
description, module_tenant_id, datastore, ds_version,
auto_apply, visible, live_update)
view_data = views.DetailedModuleView(module)
return wsgi.Result(view_data.data(), 200)
def delete(self, req, tenant_id, id):
LOG.info(_("Deleting module %s") % id)
context = req.environ[wsgi.CONTEXT_KEY]
module = models.Module.load(context, id)
models.Module.delete(context, module)
return wsgi.Result(None, 200)
def update(self, req, body, tenant_id, id):
LOG.info(_("Updating module %s") % id)
context = req.environ[wsgi.CONTEXT_KEY]
module = models.Module.load(context, id)
original_module = copy.deepcopy(module)
if 'name' in body['module']:
module.name = body['module']['name']
if 'module_type' in body['module']:
module.type = body['module']['module_type']
if 'contents' in body['module']:
module.contents = body['module']['contents']
if 'description' in body['module']:
module.description = body['module']['description']
if 'all_tenants' in body['module']:
module.tenant_id = (None if body['module']['all_tenants']
else tenant_id)
if 'datastore' in body['module']:
if 'type' in body['module']['datastore']:
module.datastore_id = body['module']['datastore']['type']
if 'version' in body['module']['datastore']:
module.datastore_version_id = (
body['module']['datastore']['version'])
if 'auto_apply' in body['module']:
module.auto_apply = body['module']['auto_apply']
if 'visible' in body['module']:
module.visible = body['module']['visible']
if 'live_update' in body['module']:
module.live_update = body['module']['live_update']
models.Module.update(context, module, original_module)
view_data = views.DetailedModuleView(module)
return wsgi.Result(view_data.data(), 200)

101
trove/module/views.py Normal file
View File

@ -0,0 +1,101 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from oslo_log import log as logging
from trove.datastore import models as datastore_models
from trove.module import models
LOG = logging.getLogger(__name__)
class ModuleView(object):
def __init__(self, module):
self.module = module
def data(self):
module_dict = dict(
id=self.module.id,
name=self.module.name,
type=self.module.type,
description=self.module.description,
tenant_id=self.module.tenant_id,
datastore_id=self.module.datastore_id,
datastore_version_id=self.module.datastore_version_id,
auto_apply=self.module.auto_apply,
md5=self.module.md5,
created=self.module.created,
updated=self.module.updated)
# add extra data to make results more legible
if self.module.tenant_id:
# This should be the tenant name, but until we figure out where
# to get it from, use the tenant_id
tenant = self.module.tenant_id
else:
tenant = models.Modules.MATCH_ALL_NAME
module_dict["tenant"] = tenant
datastore = self.module.datastore_id
datastore_version = self.module.datastore_version_id
if datastore:
ds, ds_ver = (
datastore_models.get_datastore_version(
type=datastore, version=datastore_version))
datastore = ds.name
if datastore_version:
datastore_version = ds_ver.name
else:
datastore_version = models.Modules.MATCH_ALL_NAME
else:
datastore = models.Modules.MATCH_ALL_NAME
datastore_version = models.Modules.MATCH_ALL_NAME
module_dict["datastore"] = datastore
module_dict["datastore_version"] = datastore_version
return {"module": module_dict}
class ModulesView(object):
def __init__(self, modules):
self.modules = modules
def data(self):
data = []
for module in self.modules:
data.append(self.data_for_module(module))
return {"modules": data}
def data_for_module(self, module):
view = ModuleView(module)
return view.data()['module']
class DetailedModuleView(ModuleView):
def __init__(self, module):
super(DetailedModuleView, self).__init__(module)
def data(self):
return_value = super(DetailedModuleView, self).data()
module_dict = return_value["module"]
module_dict["visible"] = self.module.visible
module_dict["live_update"] = self.module.live_update
if hasattr(self.module, 'instance_count'):
module_dict["instance_count"] = self.module.instance_count
return {"module": module_dict}

18
trove/tests/int_tests.py
View File

@ -40,6 +40,7 @@ from trove.tests.scenario.groups import guest_log_group
from trove.tests.scenario.groups import instance_actions_group
from trove.tests.scenario.groups import instance_create_group
from trove.tests.scenario.groups import instance_delete_group
from trove.tests.scenario.groups import module_group
from trove.tests.scenario.groups import negative_cluster_actions_group
from trove.tests.scenario.groups import replication_group
from trove.tests.scenario.groups import root_actions_group
@ -155,6 +156,16 @@ guest_log_groups.extend([guest_log_group.GROUP])
instance_actions_groups = list(instance_create_groups)
instance_actions_groups.extend([instance_actions_group.GROUP])
instance_module_groups = list(instance_create_groups)
instance_module_groups.extend([module_group.GROUP_INSTANCE_MODULE])
module_groups = list(instance_create_groups)
module_groups.extend([module_group.GROUP])
module_create_groups = list(base_groups)
module_create_groups.extend([module_group.GROUP_MODULE,
module_group.GROUP_MODULE_DELETE])
replication_groups = list(instance_create_groups)
replication_groups.extend([replication_group.GROUP])
@ -166,9 +177,9 @@ user_actions_groups.extend([user_actions_group.GROUP])
# groups common to all datastores
common_groups = list(instance_actions_groups)
common_groups.extend([guest_log_groups])
common_groups.extend([guest_log_groups, module_groups])
# Register: Module based groups
# Register: Component based groups
register(["backup"], backup_groups)
register(["cluster"], cluster_actions_groups)
register(["configuration"], configuration_groups)
@ -176,6 +187,9 @@ register(["database"], database_actions_groups)
register(["guest_log"], guest_log_groups)
register(["instance", "instance_actions"], instance_actions_groups)
register(["instance_create"], instance_create_groups)
register(["instance_module"], instance_module_groups)
register(["module"], module_groups)
register(["module_create"], module_create_groups)
register(["replication"], replication_groups)
register(["root"], root_actions_groups)
register(["user"], user_actions_groups)

2
trove/tests/scenario/groups/instance_delete_group.py
View File

@ -20,6 +20,7 @@ from trove.tests.scenario.groups import configuration_group
from trove.tests.scenario.groups import database_actions_group
from trove.tests.scenario.groups import instance_actions_group
from trove.tests.scenario.groups import instance_create_group
from trove.tests.scenario.groups import module_group
from trove.tests.scenario.groups import replication_group
from trove.tests.scenario.groups import root_actions_group
from trove.tests.scenario.groups.test_group import TestGroup
@ -35,6 +36,7 @@ GROUP = "scenario.instance_delete_group"
configuration_group.GROUP,
database_actions_group.GROUP,
instance_actions_group.GROUP,
module_group.GROUP,
replication_group.GROUP,
root_actions_group.GROUP,
user_actions_group.GROUP])

344
trove/tests/scenario/groups/module_group.py Normal file
View File

@ -0,0 +1,344 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from proboscis import test
from trove.tests.scenario.groups import instance_create_group
from trove.tests.scenario.groups.test_group import TestGroup
GROUP = "scenario.module_all_group"
GROUP_MODULE = "scenario.module_group"
GROUP_MODULE_DELETE = "scenario.module_delete_group"
GROUP_INSTANCE_MODULE = "scenario.instance_module_group"
@test(groups=[GROUP, GROUP_MODULE])
class ModuleGroup(TestGroup):
"""Test Module functionality."""
def __init__(self):
super(ModuleGroup, self).__init__(
'module_runners', 'ModuleRunner')
@test(groups=[GROUP, GROUP_MODULE])
def module_delete_existing(self):
"""Delete all previous test modules."""
self.test_runner.run_module_delete_existing()
@test(groups=[GROUP, GROUP_MODULE])
def module_create_bad_type(self):
"""Ensure create module fails with invalid type."""
self.test_runner.run_module_create_bad_type()
@test(groups=[GROUP, GROUP_MODULE])
def module_create_non_admin_auto(self):
"""Ensure create auto_apply module fails for non-admin."""
self.test_runner.run_module_create_non_admin_auto()
@test(groups=[GROUP, GROUP_MODULE])
def module_create_non_admin_all_tenant(self):
"""Ensure create all tenant module fails for non-admin."""
self.test_runner.run_module_create_non_admin_all_tenant()
@test(groups=[GROUP, GROUP_MODULE])
def module_create_non_admin_hidden(self):
"""Ensure create hidden module fails for non-admin."""
self.test_runner.run_module_create_non_admin_hidden()
@test(groups=[GROUP, GROUP_MODULE])
def module_create_bad_datastore(self):
"""Ensure create module fails with invalid datastore."""
self.test_runner.run_module_create_bad_datastore()
@test(groups=[GROUP, GROUP_MODULE])
def module_create_bad_datastore_version(self):
"""Ensure create module fails with invalid datastore_version."""
self.test_runner.run_module_create_bad_datastore_version()
@test(groups=[GROUP, GROUP_MODULE])
def module_create_missing_datastore(self):
"""Ensure create module fails with missing datastore."""
self.test_runner.run_module_create_missing_datastore()
@test(groups=[GROUP, GROUP_MODULE],
runs_after=[module_delete_existing])
def module_create(self):
"""Check that create module works."""
self.test_runner.run_module_create()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create])
def module_create_dupe(self):
"""Ensure create with duplicate info fails."""
self.test_runner.run_module_create_dupe()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create])
def module_show(self):
"""Check that show module works."""
self.test_runner.run_module_show()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create])
def module_show_unauth_user(self):
"""Ensure that show module for unauth user fails."""
self.test_runner.run_module_show_unauth_user()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create])
def module_list(self):
"""Check that list modules works."""
self.test_runner.run_module_list()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create])
def module_list_unauth_user(self):
"""Ensure that list module for unauth user fails."""
self.test_runner.run_module_list_unauth_user()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create],
runs_after=[module_list])
def module_create_admin_all(self):
"""Check that create module works with all admin options."""
self.test_runner.run_module_create_admin_all()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create],
runs_after=[module_create_admin_all])
def module_create_admin_hidden(self):
"""Check that create module works with hidden option."""
self.test_runner.run_module_create_admin_hidden()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create],
runs_after=[module_create_admin_hidden])
def module_create_admin_auto(self):
"""Check that create module works with auto option."""
self.test_runner.run_module_create_admin_auto()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create],
runs_after=[module_create_admin_auto])
def module_create_admin_live_update(self):
"""Check that create module works with live-update option."""
self.test_runner.run_module_create_admin_live_update()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create],
runs_after=[module_create_admin_live_update])
def module_create_all_tenant(self):
"""Check that create 'all' tenants with datastore module works."""
self.test_runner.run_module_create_all_tenant()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create],
runs_after=[module_create_all_tenant, module_list_unauth_user])
def module_create_different_tenant(self):
"""Check that create with same name on different tenant works."""
self.test_runner.run_module_create_different_tenant()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create_all_tenant],
runs_after=[module_create_different_tenant])
def module_list_again(self):
"""Check that list modules skips invisible modules."""
self.test_runner.run_module_list_again()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create_admin_hidden])
def module_show_invisible(self):
"""Ensure that show invisible module for non-admin fails."""
self.test_runner.run_module_show_invisible()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create_all_tenant],
runs_after=[module_create_different_tenant])
def module_list_admin(self):
"""Check that list modules for admin works."""
self.test_runner.run_module_list_admin()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_create],
runs_after=[module_show])
def module_update(self):
"""Check that update module works."""
self.test_runner.run_module_update()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update])
def module_update_same_contents(self):
"""Check that update module with same contents works."""
self.test_runner.run_module_update_same_contents()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_same_contents])
def module_update_auto_toggle(self):
"""Check that update module works for auto apply toggle."""
self.test_runner.run_module_update_auto_toggle()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_auto_toggle])
def module_update_all_tenant_toggle(self):
"""Check that update module works for all tenant toggle."""
self.test_runner.run_module_update_all_tenant_toggle()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_all_tenant_toggle])
def module_update_invisible_toggle(self):
"""Check that update module works for invisible toggle."""
self.test_runner.run_module_update_invisible_toggle()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_unauth(self):
"""Ensure update module fails for unauth user."""
self.test_runner.run_module_update_unauth()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_auto(self):
"""Ensure update module to auto_apply fails for non-admin."""
self.test_runner.run_module_update_non_admin_auto()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_auto_off(self):
"""Ensure update module to auto_apply off fails for non-admin."""
self.test_runner.run_module_update_non_admin_auto_off()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_auto_any(self):
"""Ensure any update module to auto_apply fails for non-admin."""
self.test_runner.run_module_update_non_admin_auto_any()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_all_tenant(self):
"""Ensure update module to all tenant fails for non-admin."""
self.test_runner.run_module_update_non_admin_all_tenant()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_all_tenant_off(self):
"""Ensure update module to all tenant off fails for non-admin."""
self.test_runner.run_module_update_non_admin_all_tenant_off()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_all_tenant_any(self):
"""Ensure any update module to all tenant fails for non-admin."""
self.test_runner.run_module_update_non_admin_all_tenant_any()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_invisible(self):
"""Ensure update module to invisible fails for non-admin."""
self.test_runner.run_module_update_non_admin_invisible()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_invisible_off(self):
"""Ensure update module to invisible off fails for non-admin."""
self.test_runner.run_module_update_non_admin_invisible_off()
@test(groups=[GROUP, GROUP_MODULE],
depends_on=[module_update],
runs_after=[module_update_invisible_toggle])
def module_update_non_admin_invisible_any(self):
"""Ensure any update module to invisible fails for non-admin."""
self.test_runner.run_module_update_non_admin_invisible_any()
@test(depends_on_groups=[instance_create_group.GROUP,
GROUP_MODULE],
groups=[GROUP, GROUP_INSTANCE_MODULE])
class ModuleInstanceGroup(TestGroup):
"""Test Instance Module functionality."""
def __init__(self):
super(ModuleInstanceGroup, self).__init__(
'module_runners', 'ModuleRunner')
@test(depends_on_groups=[GROUP_MODULE],
groups=[GROUP, GROUP_MODULE_DELETE])
class ModuleDeleteGroup(TestGroup):
"""Test Module Delete functionality."""
def __init__(self):
super(ModuleDeleteGroup, self).__init__(
'module_runners', 'ModuleRunner')
@test(groups=[GROUP, GROUP_MODULE_DELETE])
def module_delete_non_existent(self):
"""Ensure delete non-existent module fails."""
self.test_runner.run_module_delete_non_existent()
@test(groups=[GROUP, GROUP_MODULE_DELETE])
def module_delete_unauth_user(self):
"""Ensure delete module by unauth user fails."""
self.test_runner.run_module_delete_unauth_user()
@test(groups=[GROUP, GROUP_MODULE_DELETE],
runs_after=[module_delete_unauth_user])
def module_delete_hidden_by_non_admin(self):
"""Ensure delete hidden module by non-admin user fails."""
self.test_runner.run_module_delete_hidden_by_non_admin()
@test(groups=[GROUP, GROUP_MODULE_DELETE],
runs_after=[module_delete_hidden_by_non_admin])
def module_delete_all_tenant_by_non_admin(self):
"""Ensure delete all tenant module by non-admin user fails."""
self.test_runner.run_module_delete_all_tenant_by_non_admin()
@test(groups=[GROUP, GROUP_MODULE_DELETE],
runs_after=[module_delete_all_tenant_by_non_admin])
def module_delete_auto_by_non_admin(self):
"""Ensure delete auto-apply module by non-admin user fails."""
self.test_runner.run_module_delete_auto_by_non_admin()
@test(groups=[GROUP, GROUP_MODULE_DELETE],
runs_after=[module_delete_auto_by_non_admin])
def module_delete(self):
"""Check that delete module works."""
self.test_runner.run_module_delete_auto_by_non_admin()
@test(groups=[GROUP, GROUP_MODULE_DELETE],
runs_after=[module_delete])
def module_delete_all(self):
"""Check that delete module works for admin."""
self.test_runner.run_module_delete()
@test(groups=[GROUP, GROUP_MODULE_DELETE],
runs_after=[module_delete_all])
def module_delete_existing(self):
"""Delete all remaining test modules."""
self.test_runner.run_module_delete_existing()

7
trove/tests/scenario/helpers/test_helper.py
View File

@ -433,3 +433,10 @@ class TestHelper(object):
"""Return a valid password that can be used by a 'root' user.
"""
return "RootTestPass"
##############
# Module related
##############
def get_valid_module_type(self):
"""Return a valid module type."""
return "test"

634
trove/tests/scenario/runners/module_runners.py Normal file
View File

@ -0,0 +1,634 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from proboscis import SkipTest
from troveclient.compat import exceptions
from trove.common import utils
from trove.module import models
from trove.tests.scenario.runners.test_runners import TestRunner
# Variables here are set up to be used across multiple groups,
# since each group will instantiate a new runner
test_modules = []
module_count_prior_to_create = 0
module_admin_count_prior_to_create = 0
module_other_count_prior_to_create = 0
module_create_count = 0
module_admin_create_count = 0
module_other_create_count = 0
class ModuleRunner(TestRunner):
def __init__(self):
self.TIMEOUT_MODULE_APPLY = 60 * 10
super(ModuleRunner, self).__init__(
sleep_time=10, timeout=self.TIMEOUT_MODULE_APPLY)
self.MODULE_NAME = 'test_module_1'
self.MODULE_DESC = 'test description'
self.MODULE_CONTENTS = utils.encode_string(
'mode=echo\nkey=mysecretkey\n')
self.temp_module = None
self._module_type = None
@property
def module_type(self):
if not self._module_type:
self._module_type = self.test_helper.get_valid_module_type()
return self._module_type
@property
def main_test_module(self):
if not test_modules or not test_modules[0]:
SkipTest("No main module created")
return test_modules[0]
def run_module_delete_existing(self):
modules = self.admin_client.modules.list()
for module in modules:
if module.name.startswith(self.MODULE_NAME):
self.admin_client.modules.delete(module.id)
def run_module_create_bad_type(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.create,
self.MODULE_NAME, 'invalid-type', self.MODULE_CONTENTS)
def run_module_create_non_admin_auto(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.create,
self.MODULE_NAME, self.module_type, self.MODULE_CONTENTS,
auto_apply=True)
def run_module_create_non_admin_all_tenant(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.create,
self.MODULE_NAME, self.module_type, self.MODULE_CONTENTS,
all_tenants=True)
def run_module_create_non_admin_hidden(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.create,
self.MODULE_NAME, self.module_type, self.MODULE_CONTENTS,
visible=False)
def run_module_create_bad_datastore(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.create,
self.MODULE_NAME, self.module_type, self.MODULE_CONTENTS,
datastore='bad-datastore')
def run_module_create_bad_datastore_version(
self, expected_exception=exceptions.BadRequest,
expected_http_code=400):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.create,
self.MODULE_NAME, self.module_type, self.MODULE_CONTENTS,
datastore=self.instance_info.dbaas_datastore,
datastore_version='bad-datastore-version')
def run_module_create_missing_datastore(
self, expected_exception=exceptions.BadRequest,
expected_http_code=400):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.create,
self.MODULE_NAME, self.module_type, self.MODULE_CONTENTS,
datastore_version=self.instance_info.dbaas_datastore_version)
def run_module_create(self):
# Necessary to test that the count increases.
global module_count_prior_to_create
global module_admin_count_prior_to_create
global module_other_count_prior_to_create
module_count_prior_to_create = len(
self.auth_client.modules.list())
module_admin_count_prior_to_create = len(
self.admin_client.modules.list())
module_other_count_prior_to_create = len(
self.unauth_client.modules.list())
self.assert_module_create(
self.auth_client,
name=self.MODULE_NAME,
module_type=self.module_type,
contents=self.MODULE_CONTENTS,
description=self.MODULE_DESC)
def assert_module_create(self, client, name=None, module_type=None,
contents=None, description=None,
all_tenants=False,
datastore=None, datastore_version=None,
auto_apply=False,
live_update=False, visible=True):
result = client.modules.create(
name, module_type, contents,
description=description,
all_tenants=all_tenants,
datastore=datastore, datastore_version=datastore_version,
auto_apply=auto_apply,
live_update=live_update, visible=visible)
global module_create_count
global module_admin_create_count
global module_other_create_count
if (client == self.auth_client or
(client == self.admin_client and visible)):
module_create_count += 1
elif not visible:
module_admin_create_count += 1
else:
module_other_create_count += 1
global test_modules
test_modules.append(result)
tenant_id = None
tenant = models.Modules.MATCH_ALL_NAME
if not all_tenants:
tenant, tenant_id = self.get_client_tenant(client)
# TODO(peterstac) we don't support tenant name yet ...
tenant = tenant_id
datastore = datastore or models.Modules.MATCH_ALL_NAME
datastore_version = datastore_version or models.Modules.MATCH_ALL_NAME
self.validate_module(
result, validate_all=False,
expected_name=name,
expected_module_type=module_type,
expected_description=description,
expected_tenant=tenant,
expected_tenant_id=tenant_id,
expected_datastore=datastore,
expected_ds_version=datastore_version,
expected_auto_apply=auto_apply)
def validate_module(self, module, validate_all=False,
expected_name=None,
expected_module_type=None,
expected_description=None,
expected_tenant=None,
expected_tenant_id=None,
expected_datastore=None,
expected_datastore_id=None,
expected_ds_version=None,
expected_ds_version_id=None,
expected_all_tenants=None,
expected_auto_apply=None,
expected_live_update=None,
expected_visible=None,
expected_contents=None):
if expected_all_tenants:
expected_tenant = expected_tenant or models.Modules.MATCH_ALL_NAME
if expected_name:
self.assert_equal(expected_name, module.name,
'Unexpected module name')
if expected_module_type:
self.assert_equal(expected_module_type, module.type,
'Unexpected module type')
if expected_description:
self.assert_equal(expected_description, module.description,
'Unexpected module description')
if expected_tenant_id:
self.assert_equal(expected_tenant_id, module.tenant_id,
'Unexpected tenant id')
if expected_tenant:
self.assert_equal(expected_tenant, module.tenant,
'Unexpected tenant name')
if expected_datastore:
self.assert_equal(expected_datastore, module.datastore,
'Unexpected datastore')
if expected_ds_version:
self.assert_equal(expected_ds_version,
module.datastore_version,
'Unexpected datastore version')
if expected_auto_apply is not None:
self.assert_equal(expected_auto_apply, module.auto_apply,
'Unexpected auto_apply')
if validate_all:
if expected_datastore_id:
self.assert_equal(expected_datastore_id, module.datastore_id,
'Unexpected datastore id')
if expected_ds_version_id:
self.assert_equal(expected_ds_version_id,
module.datastore_version_id,
'Unexpected datastore version id')
if expected_live_update is not None:
self.assert_equal(expected_live_update, module.live_update,
'Unexpected live_update')
if expected_visible is not None:
self.assert_equal(expected_visible, module.visible,
'Unexpected visible')
def run_module_create_dupe(
self, expected_exception=exceptions.BadRequest,
expected_http_code=400):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.create,
self.MODULE_NAME, self.module_type, self.MODULE_CONTENTS)
def run_module_show(self):
test_module = self.main_test_module
result = self.auth_client.modules.get(test_module.id)
self.validate_module(
result, validate_all=True,
expected_name=test_module.name,
expected_module_type=test_module.type,
expected_description=test_module.description,
expected_tenant=test_module.tenant,
expected_datastore=test_module.datastore,
expected_ds_version=test_module.datastore_version,
expected_auto_apply=test_module.auto_apply,
expected_live_update=False,
expected_visible=True)
def run_module_show_unauth_user(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
self.assert_raises(
expected_exception, None,
self.unauth_client.modules.get, self.main_test_module.id)
# we're using a different client, so we'll check the return code
# on it explicitly, instead of depending on 'assert_raises'
self.assert_client_code(expected_http_code=expected_http_code,
client=self.unauth_client)
def run_module_list(self):
self.assert_module_list(
self.auth_client,
module_count_prior_to_create + module_create_count)
def assert_module_list(self, client, expected_count,
skip_validation=False):
module_list = client.modules.list()
self.assert_equal(expected_count, len(module_list),
"Wrong number of modules for list")
if not skip_validation:
for module in module_list:
if module.name != self.MODULE_NAME:
continue
test_module = self.main_test_module
self.validate_module(
module, validate_all=False,
expected_name=test_module.name,
expected_module_type=test_module.type,
expected_description=test_module.description,
expected_tenant=test_module.tenant,
expected_datastore=test_module.datastore,
expected_ds_version=test_module.datastore_version,
expected_auto_apply=test_module.auto_apply)
def run_module_list_unauth_user(self):
self.assert_module_list(self.unauth_client, 0)
def run_module_create_admin_all(self):
self.assert_module_create(
self.admin_client,
name=self.MODULE_NAME + '_admin_apply',
module_type=self.module_type,
contents=self.MODULE_CONTENTS,
description=(self.MODULE_DESC + ' admin apply'),
all_tenants=True,
visible=False,
auto_apply=True)
def run_module_create_admin_hidden(self):
self.assert_module_create(
self.admin_client,
name=self.MODULE_NAME + '_hidden',
module_type=self.module_type,
contents=self.MODULE_CONTENTS,
description=self.MODULE_DESC + ' hidden',
visible=False)
def run_module_create_admin_auto(self):
self.assert_module_create(
self.admin_client,
name=self.MODULE_NAME + '_auto',
module_type=self.module_type,
contents=self.MODULE_CONTENTS,
description=self.MODULE_DESC + ' hidden',
auto_apply=True)
def run_module_create_admin_live_update(self):
self.assert_module_create(
self.admin_client,
name=self.MODULE_NAME + '_live',
module_type=self.module_type,
contents=self.MODULE_CONTENTS,
description=(self.MODULE_DESC + ' live update'),
live_update=True)
def run_module_create_all_tenant(self):
self.assert_module_create(
self.admin_client,
name=self.MODULE_NAME + '_all_tenant',
module_type=self.module_type,
contents=self.MODULE_CONTENTS,
description=self.MODULE_DESC + ' all tenant',
all_tenants=True,
datastore=self.instance_info.dbaas_datastore,
datastore_version=self.instance_info.dbaas_datastore_version)
def run_module_create_different_tenant(self):
self.assert_module_create(
self.unauth_client,
name=self.MODULE_NAME,
module_type=self.module_type,
contents=self.MODULE_CONTENTS,
description=self.MODULE_DESC)
def run_module_list_again(self):
self.assert_module_list(
self.auth_client,
# TODO(peterstac) remove the '-1' once the list is fixed to
# include 'all' tenant modules
module_count_prior_to_create + module_create_count - 1,
skip_validation=True)
def run_module_show_invisible(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
module = self._find_invisible_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.get, module.id)
def _find_invisible_module(self):
def _match(mod):
return not mod.visible and mod.tenant_id and not mod.auto_apply
return self._find_module(_match, "Could not find invisible module")
def _find_module(self, match_fn, not_found_message):
module = None
for test_module in test_modules:
if match_fn(test_module):
module = test_module
break
if not module:
self.fail(not_found_message)
return module
def run_module_list_admin(self):
self.assert_module_list(
self.admin_client,
(module_admin_count_prior_to_create +
module_create_count +
module_admin_create_count +
module_other_create_count),
skip_validation=True)
def run_module_update(self):
self.assert_module_update(
self.auth_client,
self.main_test_module.id,
description=self.MODULE_DESC + " modified")
def run_module_update_same_contents(self):
old_md5 = self.main_test_module.md5
self.assert_module_update(
self.auth_client,
self.main_test_module.id,
contents=self.MODULE_CONTENTS)
self.assert_equal(old_md5, self.main_test_module.md5,
"MD5 changed with same contents")
def run_module_update_auto_toggle(self):
module = self._find_auto_apply_module()
toggle_off_args = {'auto_apply': False}
toggle_on_args = {'auto_apply': True}
self.assert_module_toggle(module, toggle_off_args, toggle_on_args)
def assert_module_toggle(self, module, toggle_off_args, toggle_on_args):
# First try to update the module based on the change
# (this should toggle the state and allow non-admin access)
self.assert_module_update(
self.admin_client, module.id, **toggle_off_args)
# Now we can update using the non-admin client
self.assert_module_update(
self.auth_client, module.id, description='Updated by auth')
# Now set it back
self.assert_module_update(
self.admin_client, module.id, description=module.description,
**toggle_on_args)
def run_module_update_all_tenant_toggle(self):
module = self._find_all_tenant_module()
toggle_off_args = {'all_tenants': False}
toggle_on_args = {'all_tenants': True}
self.assert_module_toggle(module, toggle_off_args, toggle_on_args)
def run_module_update_invisible_toggle(self):
module = self._find_invisible_module()
toggle_off_args = {'visible': True}
toggle_on_args = {'visible': False}
self.assert_module_toggle(module, toggle_off_args, toggle_on_args)
def assert_module_update(self, client, module_id, **kwargs):
result = client.modules.update(module_id, **kwargs)
global test_modules
found = False
index = -1
for test_module in test_modules:
index += 1
if test_module.id == module_id:
found = True
break
if not found:
self.fail("Could not find updated module in module list")
test_modules[index] = result
expected_args = {}
for key, value in kwargs.items():
new_key = 'expected_' + key
expected_args[new_key] = value
self.validate_module(result, **expected_args)
def run_module_update_unauth(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
self.assert_raises(
expected_exception, expected_http_code,
self.unauth_client.modules.update,
self.main_test_module.id, description='Upd')
def run_module_update_non_admin_auto(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update,
self.main_test_module.id, visible=False)
def run_module_update_non_admin_auto_off(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
module = self._find_auto_apply_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update, module.id, auto_apply=False)
def _find_auto_apply_module(self):
def _match(mod):
return mod.auto_apply and mod.tenant_id and mod.visible
return self._find_module(_match, "Could not find auto-apply module")
def run_module_update_non_admin_auto_any(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
module = self._find_auto_apply_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update, module.id, description='Upd')
def run_module_update_non_admin_all_tenant(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update,
self.main_test_module.id, all_tenants=True)
def run_module_update_non_admin_all_tenant_off(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
module = self._find_all_tenant_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update, module.id, all_tenants=False)
def _find_all_tenant_module(self):
def _match(mod):
return mod.tenant_id is None and mod.visible
return self._find_module(_match, "Could not find all tenant module")
def run_module_update_non_admin_all_tenant_any(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
module = self._find_all_tenant_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update, module.id, description='Upd')
def run_module_update_non_admin_invisible(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update,
self.main_test_module.id, visible=False)
def run_module_update_non_admin_invisible_off(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
module = self._find_invisible_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update, module.id, visible=True)
def run_module_update_non_admin_invisible_any(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
module = self._find_invisible_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.update, module.id, description='Upd')
# ModuleDeleteGroup methods
def run_module_delete_non_existent(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.delete, 'bad_id')
def run_module_delete_unauth_user(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
self.assert_raises(
expected_exception, expected_http_code,
self.unauth_client.modules.delete, self.main_test_module.id)
def run_module_delete_hidden_by_non_admin(
self, expected_exception=exceptions.NotFound,
expected_http_code=404):
module = self._find_invisible_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.delete, module.id)
def run_module_delete_all_tenant_by_non_admin(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
module = self._find_all_tenant_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.delete, module.id)
def run_module_delete_auto_by_non_admin(
self, expected_exception=exceptions.Forbidden,
expected_http_code=403):
module = self._find_auto_apply_module()
self.assert_raises(
expected_exception, expected_http_code,
self.auth_client.modules.delete, module.id)
def run_module_delete(self):
expected_count = len(self.auth_client.modules.list()) - 1
test_module = test_modules.pop(0)
self.assert_module_delete(self.auth_client, test_module.id,
expected_count)
def run_module_delete_admin(self):
start_count = count = len(self.admin_client.modules.list())
for test_module in test_modules:
count -= 1
self.report.log("Deleting module '%s' (tenant: %s)" % (
test_module.name, test_module.tenant_id))
self.assert_module_delete(self.admin_client, test_module.id, count)
self.assert_not_equal(start_count, count, "Nothing was deleted")
count = len(self.admin_client.modules.list())
self.assert_equal(module_admin_count_prior_to_create, count,
"Wrong number of admin modules after deleting all")
count = len(self.auth_client.modules.list())
self.assert_equal(module_count_prior_to_create, count,
"Wrong number of modules after deleting all")
def assert_module_delete(self, client, module_id, expected_count):
client.modules.delete(module_id)
count = len(client.modules.list())
self.assert_equal(expected_count, count,
"Wrong number of modules after delete")

7
trove/tests/scenario/runners/test_runners.py
View File

@ -197,6 +197,13 @@ class TestRunner(object):
auth_version='2.0',
os_options=os_options)
def get_client_tenant(self, client):
tenant_name = client.real_client.client.tenant
service_url = client.real_client.client.service_url
su_parts = service_url.split('/')
tenant_id = su_parts[-1]
return tenant_name, tenant_id
def assert_raises(self, expected_exception, expected_http_code,
client_cmd, *cmd_args, **cmd_kwargs):
asserts.assert_raises(expected_exception, client_cmd,

39
trove/tests/unittests/common/test_utils.py
View File

@ -13,7 +13,10 @@
# License for the specific language governing permissions and limitations
# under the License.
#
from Crypto import Random
from mock import Mock
from testtools import ExpectedException
from trove.common import exception
from trove.common import utils
@ -79,3 +82,39 @@ class TestTroveExecuteWithTimeout(trove_testtools.TestCase):
def test_pagination_limit(self):
self.assertEqual(5, utils.pagination_limit(5, 9))
self.assertEqual(5, utils.pagination_limit(9, 5))
def test_encode_decode_string(self):
random_data = bytearray(Random.new().read(12))
data = ['abc', 'numbers01234', '\x00\xFF\x00\xFF\xFF\x00', random_data]
for datum in data:
encoded_data = utils.encode_string(datum)
decoded_data = utils.decode_string(encoded_data)
self. assertEqual(datum, decoded_data,
"Encode/decode failed")
def test_pad_unpad(self):
for size in range(1, 100):
data_str = 'a' * size
padded_str = utils.pad_for_encryption(data_str, utils.IV_BIT_COUNT)
self.assertEqual(0, len(padded_str) % utils.IV_BIT_COUNT,
"Padding not successful")
unpadded_str = utils.unpad_after_decryption(padded_str)
self.assertEqual(data_str, unpadded_str,
"String mangled after pad/unpad")
def test_encryp_decrypt(self):
key = 'my_secure_key'
for size in range(1, 100):
orig_str = ''
for index in range(1, size):
orig_str += Random.new().read(1)
orig_encoded = utils.encode_string(orig_str)
encrypted = utils.encrypt_string(orig_encoded, key)
encoded = utils.encode_string(encrypted)
decoded = utils.decode_string(encoded)
decrypted = utils.decrypt_string(decoded, key)
final_decoded = utils.decode_string(decrypted)
self.assertEqual(orig_str, final_decoded,
"String did not match original")

0
trove/tests/unittests/module/__init__.py Normal file
View File

80
trove/tests/unittests/module/test_module_controller.py Normal file
View File

@ -0,0 +1,80 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import jsonschema
from testtools.matchers import Is, Equals
from trove.module.service import ModuleController
from trove.tests.unittests import trove_testtools
class TestModuleController(trove_testtools.TestCase):
def setUp(self):
super(TestModuleController, self).setUp()
self.controller = ModuleController()
self.module = {
"module": {
"name": 'test_module',
"module_type": 'test',
"contents": 'my_contents\n',
}
}
def verify_errors(self, errors, msg=None, properties=None, path=None):
msg = msg or []
properties = properties or []
self.assertThat(len(errors), Is(len(msg)))
i = 0
while i < len(msg):
self.assertIn(errors[i].message, msg)
if path:
self.assertThat(path, Equals(properties[i]))
else:
self.assertThat(errors[i].path.pop(), Equals(properties[i]))
i += 1
def test_get_schema_create(self):
schema = self.controller.get_schema('create', {'module': {}})
self.assertIsNotNone(schema)
self.assertTrue('module' in schema['properties'])
def test_validate_create_complete(self):
body = self.module
schema = self.controller.get_schema('create', body)
validator = jsonschema.Draft4Validator(schema)
self.assertTrue(validator.is_valid(body))
def test_validate_create_blankname(self):
body = self.module
body['module']['name'] = " "
schema = self.controller.get_schema('create', body)
validator = jsonschema.Draft4Validator(schema)
self.assertFalse(validator.is_valid(body))
errors = sorted(validator.iter_errors(body), key=lambda e: e.path)
self.assertThat(len(errors), Is(1))
self.assertThat(errors[0].message,
Equals("' ' does not match '^.*[0-9a-zA-Z]+.*$'"))
def test_validate_create_invalid_name(self):
body = self.module
body['module']['name'] = "$#$%^^"
schema = self.controller.get_schema('create', body)
validator = jsonschema.Draft4Validator(schema)
self.assertFalse(validator.is_valid(body))
errors = sorted(validator.iter_errors(body), key=lambda e: e.path)
self.assertEqual(1, len(errors))
self.assertIn("'$#$%^^' does not match '^.*[0-9a-zA-Z]+.*$'",
errors[0].message)

50
trove/tests/unittests/module/test_module_models.py Normal file
View File

@ -0,0 +1,50 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from mock import Mock, patch
from trove.common import cfg
from trove.module import models
from trove.taskmanager import api as task_api
from trove.tests.unittests import trove_testtools
from trove.tests.unittests.util import util
CONF = cfg.CONF
class CreateModuleTest(trove_testtools.TestCase):
@patch.object(task_api.API, 'get_client', Mock(return_value=Mock()))
def setUp(self):
util.init_db()
self.context = Mock()
self.name = "name"
self.module_type = 'test'
self.contents = 'my_contents\n'
super(CreateModuleTest, self).setUp()
@patch.object(task_api.API, 'get_client', Mock(return_value=Mock()))
def tearDown(self):
super(CreateModuleTest, self).tearDown()
def test_can_create_module(self):
module = models.Module.create(
self.context,
self.name, self.module_type, self.contents,
'my desc', 'my_tenant', None, None, False, True, False)
self.assertIsNotNone(module)
module.delete()

71
trove/tests/unittests/module/test_module_views.py Normal file
View File

@ -0,0 +1,71 @@
# Copyright 2016 Tesora, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from mock import Mock, patch
from trove.datastore import models
from trove.module.views import DetailedModuleView
from trove.tests.unittests import trove_testtools
class ModuleViewsTest(trove_testtools.TestCase):
def setUp(self):
super(ModuleViewsTest, self).setUp()
def tearDown(self):
super(ModuleViewsTest, self).tearDown()
class DetailedModuleViewTest(trove_testtools.TestCase):
def setUp(self):
super(DetailedModuleViewTest, self).setUp()
self.module = Mock()
self.module.name = 'test_module'
self.module.type = 'test'
self.module.md5 = 'md5-hash'
self.module.created = 'Yesterday'
self.module.updated = 'Now'
self.module.datastore = 'mysql'
self.module.datastore_version = '5.6'
self.module.auto_apply = False
self.module.tenant_id = 'my_tenant'
def tearDown(self):
super(DetailedModuleViewTest, self).tearDown()
def test_data(self):
datastore = Mock()
datastore.name = self.module.datastore
ds_version = Mock()
ds_version.name = self.module.datastore_version
with patch.object(models, 'get_datastore_version',
Mock(return_value=(datastore, ds_version))):
view = DetailedModuleView(self.module)
result = view.data()
self.assertEqual(self.module.name, result['module']['name'])
self.assertEqual(self.module.type, result['module']['type'])
self.assertEqual(self.module.md5, result['module']['md5'])
self.assertEqual(self.module.created, result['module']['created'])
self.assertEqual(self.module.updated, result['module']['updated'])
self.assertEqual(self.module.datastore_version,
result['module']['datastore_version'])
self.assertEqual(self.module.datastore,
result['module']['datastore'])
self.assertEqual(self.module.auto_apply,
result['module']['auto_apply'])
self.assertEqual(self.module.tenant_id,
result['module']['tenant_id'])