x
/
ranger
Code Issues Proposed changes

Merge "Removed RDS service call (cont'd)"

This commit is contained in:
Zuul 2020-02-24 21:45:51 +00:00 committed by Gerrit Code Review
parent 93c873bc96 2cba5155c7
commit 308b9af12d
22 changed files with 245 additions and 378 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
orm/base_config.py
View File

@ -44,6 +44,9 @@ OrmOpts = [
cfg.StrOpt('ranger_agent_client_cert_path',
default='',
help='Ranger Agent certificate path.'),
cfg.IntOpt('resource_status_max_interval',
default=2,
help='Interval in seconds.'),
cfg.StrOpt('log_location',
default='/var/log/ranger',
help='Orm log directory.'),

35
orm/common/orm_common/model/models.py
View File

@ -1,17 +1,42 @@
class ResourceMetaData(object):
def __init__(self, checksum, virtual_size, size):
self.size = size
self.virtual_size = virtual_size
self.checksum = checksum
def as_dict(self):
return self.__dict__
class ResourceStatusModel(object):
def __init__(self,
timestamp,
region,
status,
transaction_id,
resource_id,
ord_notifier,
err_msg,
err_code,
operation,
err_msg):
resource_extra_metadata=None):
self.timestamp = timestamp
self.region = region
self.status = status
self.ord_transaction_id = transaction_id
self.resource_id = resource_id
self.operation = operation
self.ord_notifier_id = ord_notifier
self.error_msg = err_msg
self.error_code = err_code
self.operation = operation
if resource_extra_metadata:
self.resource_extra_metadata = ResourceMetaData(
checksum=resource_extra_metadata[0].checksum,
virtual_size=resource_extra_metadata[0].virtual_size,
size=resource_extra_metadata[0].size)
else:
self.resource_extra_metadata = None
def as_dict(self):
return self.__dict__
@ -19,12 +44,12 @@ class ResourceStatusModel(object):
class StatusModel(object):
def __init__(self, status):
self.regions_status = status
self.regions = status
self.status = self._get_aggregated_status()
def _get_aggregated_status(self):
is_pending = False
for region in self.regions_status:
for region in self.regions:
if (region.status == 'Error' and
region.operation.strip() != 'delete'):
# If a region had an error, the aggregated status is 'Error'
@ -37,6 +62,6 @@ class StatusModel(object):
if is_pending:
return 'Pending'
else:
# If self.regions_status is empty, the result will still be
# If self.regions is empty, the result will still be
# 'Success' but the server returns 404 Not Found
return 'Success'

20
orm/common/orm_common/sql_alchemy/db_models.py
View File

@ -1,5 +1,5 @@
from oslo_db.sqlalchemy import models
from sqlalchemy import BigInteger, Column, ForeignKey, Integer, Text
from sqlalchemy import BigInteger, Column, Integer, Text
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
@ -9,24 +9,6 @@ class CommonBaseModel(models.ModelBase):
__table_args__ = {'mysql_engine': 'InnoDB'}
class ImageMetadData(Base, CommonBaseModel):
__tablename__ = 'image_metadata'
image_meta_data_id = Column(ForeignKey('resource_status.id'),
primary_key=True)
checksum = Column(Text, primary_key=False)
virtual_size = Column(Text, primary_key=False)
size = Column(Text, primary_key=False)
def __json__(self):
return dict(
image_meta_data_id=self.image_meta_data_id,
checksum=self.checksum,
virtual_size=self.virtual_size,
size=self.size
)
class ResourceStatus(Base, CommonBaseModel):
__tablename__ = 'resource_status'

39
orm/common/orm_common/sql_alchemy/resource_status_record.py
View File

@ -3,6 +3,8 @@ from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel
from orm.common.orm_common.sql_alchemy.db_models import ResourceStatus
import time
from oslo_config import cfg
logger = logging.getLogger(__name__)
@ -32,26 +34,26 @@ class ResourceStatusRecord:
timestamp = int(time.time()) * 1000
# assume same time period for all resource types
# max_interval_time_in_seconds = \
# conf.region_resource_id_status.max_interval_time.default * 60
max_interval_time_in_seconds = 2 * 60
ref_timestamp = (int(time.time()) -
max_interval_time_in_seconds) * 1000
max_interval_in_seconds = cfg.CONF.resource_status_max_interval * 60
ref_timestamp = (int(time.time()) - max_interval_in_seconds) * 1000
try:
records = self.session.query(
ResourceStatus.id,
ResourceStatus.resource_id,
ResourceStatus.timestamp,
ResourceStatus.region,
ResourceStatus.status,
ResourceStatus.timestamp,
ResourceStatus.operation,
ResourceStatus.err_msg).filter(
ResourceStatus.transaction_id,
ResourceStatus.resource_id,
ResourceStatus.ord_notifier,
ResourceStatus.err_msg,
ResourceStatus.err_code,
ResourceStatus.operation).filter(
ResourceStatus.resource_id.in_(resource_uuids)).all()
if records:
for id, resource_id, region, status, timestamp, operation, \
err_msg in records:
for id, timestamp, region, status, transaction, resource, \
notifier, err_msg, err_code, operation in records:
if (status == "Submitted" and
timestamp < ref_timestamp):
timestamp = timestamp
@ -63,14 +65,17 @@ class ResourceStatusRecord:
timestamp,
region,
status,
resource_id,
operation,
err_msg)
transaction,
resource,
notifier,
err_msg,
err_code,
operation)
if resource_id in records_model:
records_model[resource_id].append(status_model)
if resource in records_model:
records_model[resource].append(status_model)
else:
records_model[resource_id] = [status_model]
records_model[resource] = [status_model]
for id, model in records_model.items():
statuses_model[id] = StatusModel(model)

16
orm/services/customer_manager/cms_rest/data/sql_alchemy/customer_record.py
View File

@ -88,22 +88,6 @@ class CustomerRecord:
else:
return None
def get_customers_status_by_uuids(self, uuid_str):
results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec
" FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str))
cust_region_dict = {}
if results:
resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results)
# using resource_status_dict, create cust_region_dict with resource_id as key and (region, status) as value
for v in list(resource_status_dict.values()):
if v[0] in cust_region_dict:
cust_region_dict[v[0]].append(v[1:])
else:
cust_region_dict[v[0]] = [v[1:]]
results.close()
return cust_region_dict
def delete_customer_by_uuid(self, uuid):
try:
result = self.session.query(Customer).filter(

2
orm/services/customer_manager/cms_rest/data/sql_alchemy/group_record.py
View File

@ -79,7 +79,7 @@ class GroupRecord:
def get_groups_status_by_uuids(self, uuid_str):
cmd = "SELECT id, resource_id, region, status FROM " \
"rds_resource_status_view WHERE resource_id IN (%s)"
"resource_status WHERE resource_id IN (%s)"
results = self.session.connection().execute(cmd % uuid_str)
group_region = {}

4
orm/services/customer_manager/cms_rest/logic/customer_logic.py
View File

@ -630,9 +630,9 @@ class CustomerLogic(object):
wsme_customer = sql_customer.to_wsme()
wsme_customer.status = 'no regions'
if status_model and status_model.regions_status:
if status_model and status_model.regions:
for region in wsme_customer.regions:
for status in status_model.regions_status:
for status in status_model.regions:
if status.region == region.name:
region.status = status.status
if status.error_msg:

5
orm/services/customer_manager/scripts/db_scripts/ranger_cms_create_db.sql
View File

@ -90,11 +90,6 @@ create table if not exists user_role
index region_id (region_id),
index user_id (user_id));
create or replace view rds_resource_status_view AS
(
SELECT id, resource_id, region, status,
err_code, operation from resource_status);
create table if not exists cms_domain
(
id integer auto_increment not null,

18
orm/services/flavor_manager/fms_rest/data/sql_alchemy/flavor/flavor_record.py
View File

@ -147,24 +147,6 @@ class FlavorRecord:
LOG.log_exception(message, exception)
raise
def get_flavors_status_by_uuids(self, uuid_str):
results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec
" FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str))
flvr_region_dict = {}
if results:
resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results)
# using resource_status_dict, create flvr_region_dict with resource_id as key and (region, status) as value
for v in list(resource_status_dict.values()):
if v[0] in flvr_region_dict:
flvr_region_dict[v[0]].append(v[1:])
else:
flvr_region_dict[v[0]] = [v[1:]]
results.close()
return flvr_region_dict
def get_flavors_by_criteria(self, **criteria):
try:

4
orm/services/flavor_manager/fms_rest/logic/flavor_logic.py
View File

@ -873,10 +873,10 @@ def update_region_statuses(flavor, region_names, status_model):
flavor.status = 'no regions'
error_status = submitted_status = success_status = 0
if status_model and status_model.regions_status:
if status_model and status_model.regions:
# get region status if region in flavor_regions_list
if region_names:
for status in status_model.regions_status:
for status in status_model.regions:
# check that the region read from RDS is in the
# flavor_regions_list
if status.region in region_names:

11
orm/services/flavor_manager/scripts/db_scripts/ranger_fms_create_db.sql
View File

@ -99,14 +99,3 @@ create table if not exists flavor_option
foreign key (flavor_internal_id) references flavor(internal_id) ON DELETE CASCADE ON UPDATE NO ACTION,
primary key (flavor_internal_id,key_name)
);
#
#*****
#* MySql script for Creating View rds_resource_status_view
#*****
create or replace view rds_resource_status_view AS
(
SELECT ID, RESOURCE_ID, REGION,STATUS,
ERR_CODE,OPERATION from resource_status);

204
orm/services/image_manager/ims/logic/image_logic.py
View File

@ -1,11 +1,14 @@
import time
from orm.common.orm_common.injector import injector
from orm.common.orm_common.utils import utils
from orm.services.image_manager.ims.logger import get_logger
from orm.services.image_manager.ims.logic.error_base import ErrorStatus, NotFoundError
from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import ImageCustomer, ImageRegion
from orm.services.image_manager.ims.persistency.wsme.models import (ImageSummary, ImageSummaryResponse,
ImageWrapper, RegionWrapper)
from orm.services.image_manager.ims.logic.error_base import \
ErrorStatus, NotFoundError
from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import \
ImageCustomer, ImageRegion
from orm.services.image_manager.ims.persistency.wsme.models import (
ImageSummary, ImageSummaryResponse, ImageWrapper, RegionWrapper)
from orm.services.image_manager.ims.utils import utils as ImsUtils
import oslo_db
@ -63,13 +66,13 @@ def send_to_rds_if_needed(sql_image, existing_region_names, http_action,
image_dict = sql_image.get_proxy_dict()
update_region_actions(image_dict, existing_region_names, http_action)
if image_dict['regions'] or len(existing_region_names) > 0:
LOG.debug("Image is valid to send to RDS - sending to RDS Proxy ")
LOG.debug("Image is valid, sending to RDS Proxy ")
rds_proxy.send_image(image_dict, transaction_id, http_action)
else:
LOG.debug("Group with no regions - wasn't send to RDS Proxy " + str(
LOG.debug("Group with no regions, not sending to RDS Proxy " + str(
sql_image))
else:
LOG.debug("Image with no regions - wasn't send to RDS Proxy " + str(
LOG.debug("Image with no regions, not sending to RDS Proxy " + str(
sql_image))
@ -87,9 +90,10 @@ def update_image(image_wrapper, image_uuid, transaction_id, http_action="put"):
sql_image = image_rec.get_image_by_id(image_uuid)
if sql_image is None:
raise NotFoundError(status_code=404,
message="Image {0} does not exist for update".format(
image_uuid))
raise NotFoundError(
status_code=404,
message="Image {0} does not exist for update".format(
image_uuid))
image_wrapper.validate_update(sql_image, new_image)
@ -120,51 +124,48 @@ def update_image(image_wrapper, image_uuid, transaction_id, http_action="put"):
raise
@di.dependsOn('rds_proxy')
@di.dependsOn('data_manager')
def delete_image_by_uuid(image_uuid, transaction_id):
rds_proxy, DataManager = di.resolver.unpack(delete_image_by_uuid)
DataManager = di.resolver.unpack(delete_image_by_uuid)
datamanager = DataManager()
try:
datamanager.begin_transaction()
image_rec = datamanager.get_record('image')
sql_image = image_rec.get_image_by_id(image_uuid)
if sql_image is None:
raise NotFoundError(message="Image '{}' not found".format(image_uuid))
raise NotFoundError(
message="Image '{}' not found".format(image_uuid))
image_existing_region_names = sql_image.get_existing_region_names()
if len(image_existing_region_names) > 0:
# Do not delete a flavor that still has some regions
# Do not delete an image that still has region(s)
raise ErrorStatus(405,
"Cannot delete a image with regions. "
"Please delete the regions first and then "
"delete the image. ")
# Get status from RDS
image_status = rds_proxy.get_status(sql_image.id, False)
status_resp = None
if image_status.status_code == 200:
status_resp = image_status.json()['status']
LOG.debug('RDS returned status: {}'.format(status_resp))
elif image_status.status_code == 404:
status_resp = 'Success'
# Get status from resource status table
uuid = [sql_image.id]
resource_status_dict = utils.get_resource_status_from_db(
datamanager.get_session(), uuid)
status_model = resource_status_dict.get(sql_image.id)
if status_model:
status = status_model.status
LOG.debug('Status from resource_status table: {}'.format(status))
else:
# fail to get status from rds
raise ErrorStatus(500, "fail to get status for this resource "
"deleting image not allowed ")
# Image not found in table, that means it never had any
# regions. So it is OK to delete it
status = 'Success'
LOG.debug('Resource not found in table, so it is OK to delete')
if status_resp != 'Success':
if status != 'Success':
raise ErrorStatus(405, "not allowed as aggregate status "
"have to be Success (either the deletion"
" failed on one of the regions or it is "
"still in progress)")
"have to be Success (either the deletion "
"failed on one of the regions or it is "
"still in progress)")
image_rec.delete_image_by_id(image_uuid)
datamanager.flush() # i want to get any exception created by this
@ -193,7 +194,8 @@ def add_regions(image_uuid, regions, transaction_id):
existing_region_names = sql_image.get_existing_region_names()
for region in regions.regions:
db_region = ImageRegion(region_name=region.name, region_type=region.type)
db_region = ImageRegion(region_name=region.name,
region_type=region.type)
sql_image.add_region(db_region)
datamanager.flush() # i want to get any exception created by
@ -236,7 +238,8 @@ def replace_regions(image_uuid, regions, transaction_id):
datamanager.flush()
for region in regions.regions:
db_region = ImageRegion(region_name=region.name, region_type=region.type)
db_region = ImageRegion(region_name=region.name,
region_type=region.type)
sql_image.add_region(db_region)
datamanager.flush() # i want to get any exception created by
# previous actions against the database
@ -335,7 +338,8 @@ def add_customers(image_uuid, customers, transaction_id):
return ret_image
except Exception as exp:
if 'conflicts with persistent instance' in str(exp) or 'Duplicate entry' in str(exp):
if 'conflicts with persistent instance' in str(exp) or \
'Duplicate entry' in str(exp):
raise ErrorStatus(409, "Duplicate Customer for Image")
LOG.log_exception("ImageLogic - Failed to add Customers", exp)
datamanager.rollback()
@ -373,7 +377,8 @@ def replace_customers(image_uuid, customers, transaction_id):
return ret_image
except Exception as exp:
if 'conflicts with persistent instance' in str(exp) or 'Duplicate entry' in str(exp):
if 'conflicts with persistent instance' in str(exp) or \
'Duplicate entry' in str(exp):
raise ErrorStatus(409, "Duplicate Customer for Image")
LOG.log_exception("ImageLogic - Failed to add Customers", exp)
datamanager.rollback()
@ -392,7 +397,8 @@ def delete_customer(image_uuid, customer_id, transaction_id):
raise ErrorStatus(404, 'image {0} not found'.format(image_uuid))
# if trying to delete the only one Customer then return value error
if sql_image.visibility == "public":
raise ValueError("Image {} is public, no customers".format(image_uuid))
raise ValueError(
"Image {} is public, no customers".format(image_uuid))
if len(sql_image.customers) == 1 and \
sql_image.customers[0].customer_id == customer_id:
@ -414,15 +420,32 @@ def delete_customer(image_uuid, customer_id, transaction_id):
raise
def set_resource_status(sql_image, status_model):
image_wrapper = ImageWrapper.from_db_model(sql_image)
image_wrapper.image.status = 'no regions'
if status_model and status_model.regions:
for region in image_wrapper.image.regions:
for status in status_model.regions:
if status.region == region.name:
region.status = status.status
if status.error_msg:
region.set_error_message(status.error_msg)
image_wrapper.image.status = status_model.status
return image_wrapper
@di.dependsOn('data_manager')
@di.dependsOn('rds_proxy')
def get_image_by_uuid(image_uuid, query_by_id_or_name=False):
"""This function includes an optional boolean parameter "query_by_id_or_name".
If query_by_id_or_name evaluates to true, IMS logic will fetch the image
record whose "image_uuid" parameter value matches either the image id or name value.
Otherwise it defaults to query by image ID value only.
"""This function includes an optional boolean parameter
query_by_id_or_name. If query_by_id_or_name evaluates to true, IMS logic
will fetch the image record whose "image_uuid" parameter value matches
either the image id or name value. Otherwise, it defaults to query by
image ID value only.
"""
DataManager, rds_proxy = di.resolver.unpack(get_image_by_uuid)
DataManager = di.resolver.unpack(get_image_by_uuid)
datamanager = DataManager()
LOG.debug("Get image by uuid : {}".format(image_uuid))
@ -431,51 +454,36 @@ def get_image_by_uuid(image_uuid, query_by_id_or_name=False):
datamanager.begin_transaction()
image_rec = datamanager.get_record('image')
# Only the get_image API will pass the optional parameter and set it to true
# Only the get_image API will pass the optional parameter and
# set it to true
if query_by_id_or_name:
sql_image = image_rec.get_image(image_uuid)
else:
# all other image APIs will NOT pass the optional parameter
sql_image = image_rec.get_image_by_id(image_uuid)
if not sql_image:
raise NotFoundError(status_code=404,
message="Image {0} not found ".format(
image_uuid))
image_wrapper = ImageWrapper.from_db_model(sql_image)
if not sql_image:
raise NotFoundError(
status_code=404,
message="Image {0} not found ".format(image_uuid))
# Get the status from resource table
uuid = [sql_image.id]
resource_status_dict = utils.get_resource_status_from_db(
datamanager.get_session(), uuid)
status_model = resource_status_dict.get(sql_image.id)
image_wrapper = set_resource_status(sql_image, status_model)
# get image own link
image_wrapper.image.links, image_wrapper.image.self_link = ImsUtils.get_server_links(image_uuid)
image_wrapper.image.links, image_wrapper.image.self_link = \
ImsUtils.get_server_links(image_uuid)
# convert time stamp format to human readable time
image_wrapper.image.created_at = ImsUtils.convert_time_human(
image_wrapper.image.created_at)
image_wrapper.image.updated_at = ImsUtils.convert_time_human(
image_wrapper.image.updated_at)
# Get the status from RDS
image_status = rds_proxy.get_status(image_wrapper.image.id, False)
if image_status.status_code == 404:
# image not on rds resource table - applicable to images created with no region assigned
image_wrapper.image.status = 'no regions'
elif image_status.status_code == 200:
image_status = image_status.json()
if image_wrapper.image.regions:
image_wrapper.image.status = image_status['status']
else:
image_wrapper.image.status = 'no regions'
# update status for all regions
for result_regions in image_wrapper.image.regions:
for status_region in image_status['regions']:
if result_regions.name == status_region['region']:
result_regions.status = status_region['status']
if status_region['error_msg']:
result_regions.set_error_message(status_region['error_msg'])
# status codes not falling under 404 (not found) or 200 (success)
else:
raise ErrorStatus(500, "unsuccessful GET - failed to get status for this resource")
except NotFoundError as exp:
datamanager.rollback()
@ -491,11 +499,10 @@ def get_image_by_uuid(image_uuid, query_by_id_or_name=False):
@di.dependsOn('data_manager')
@di.dependsOn('rds_proxy')
def get_image_list_by_params(visibility, region, Customer):
DataManager, rds_proxy = di.resolver.unpack(get_image_list_by_params)
DataManager = di.resolver.unpack(get_image_list_by_params)
datamanager = DataManager()
try:
image_record = datamanager.get_record('image')
sql_images = image_record.get_images_by_criteria(visibility=visibility,
@ -504,35 +511,16 @@ def get_image_list_by_params(visibility, region, Customer):
response = ImageSummaryResponse()
if sql_images:
uuids = ','.join(str("\'" + sql_image.id + "\'")
for sql_image in sql_images if sql_image and sql_image.id)
resource_status_dict = image_record.get_images_status_by_uuids(uuids)
uuids = [sql_image.id for sql_image in sql_images]
resource_status_dict = utils.get_resource_status_from_db(
datamanager.get_session(), uuids)
for sql_image in sql_images:
image = ImageSummary.from_db_model(sql_image)
if sql_image.id:
# rds_region_list contains tuples - each containing the regions associated with the image
# along with the region status
rds_region_list = resource_status_dict.get(sql_image.id)
status_model = resource_status_dict.get(sql_image.id)
wsme_image = set_resource_status(sql_image, status_model)
image_summary = ImageSummary.from_wsme(wsme_image)
response.images.append(image_summary)
if rds_region_list and image.regions:
# set image.status to 'error' if any of the regions has an 'Error' status'
# else, if any region status shows 'Submitted' then set image status to 'Pending'
# otherwise image status is 'Success'
error_status = [item for item in rds_region_list if item[1] == 'Error']
submitted_status = [item for item in rds_region_list if item[1] == 'Submitted']
success_status = [item for item in rds_region_list if item[1] == 'Success']
if len(error_status) > 0:
image.status = 'Error'
elif len(submitted_status) > 0:
image.status = 'Pending'
elif len(success_status) > 0:
image.status = 'Success'
else:
image.status = 'no regions'
response.images.append(image)
return response
except ErrorStatus as exp:
@ -605,10 +593,12 @@ def enable_image(image_uuid, int_enabled, transaction_id):
return ret_image
except ErrorStatus as exp:
LOG.log_exception("ImageLogic - Failed to change image activation value", exp)
LOG.log_exception(
"ImageLogic - Failed to change image activation value", exp)
datamanager.rollback()
raise exp
except Exception as exp:
LOG.log_exception("ImageLogic - Failed to change image activation value", exp)
LOG.log_exception(
"ImageLogic - Failed to change image activation value", exp)
datamanager.rollback()
raise exp

15
orm/services/image_manager/ims/persistency/sql_alchemy/image/image_record.py
View File

@ -94,21 +94,6 @@ class ImageRecord(Record):
LOG.log_exception(message, exception)
raise
def get_images_status_by_uuids(self, uuid_str):
results = self.session.connection().execute("SELECT id, resource_id, region, status" # nosec
" FROM rds_resource_status_view WHERE resource_id IN ({})".format(uuid_str))
img_region_dict = {}
if results:
resource_status_dict = dict((id, (resource_id, region, status)) for id, resource_id, region, status in results)
# using resource_status_dict, create img_region_dict with resource_id as key and (region, status) as value
for v in list(resource_status_dict.values()):
if v[0] in img_region_dict:
img_region_dict[v[0]].append(v[1:])
else:
img_region_dict[v[0]] = [v[1:]]
results.close()
return img_region_dict
def create_images_by_visibility_query(self, visibility):
try:
query = self.session.query(Image).filter(Image.visibility == visibility)

13
orm/services/image_manager/ims/persistency/wsme/models.py
View File

@ -494,6 +494,19 @@ class ImageSummary(Model):
return image
@staticmethod
def from_wsme(wsme_image):
image = ImageSummary()
image.status = wsme_image.image.status
image.id = wsme_image.image.id
image.name = wsme_image.image.name
image.visibility = wsme_image.image.visibility
image.regions = []
for wsme_region in wsme_image.image.regions:
image.regions.append(wsme_region.name)
return image
class ImageSummaryResponse(Model):
images = wsme.wsattr([ImageSummary], mandatory=True)

10
orm/services/image_manager/scripts/db_scripts/create_db.sql
View File

@ -84,13 +84,3 @@ create table if not exists image_customer
primary key (image_id,customer_id),
foreign key (image_id) references image(id) ON DELETE CASCADE ON UPDATE NO ACTION
);
#
#*****
#* MySql script for Creating View rds_resource_status_view
#*****
create or replace view rds_resource_status_view AS
(
SELECT ID, RESOURCE_ID, REGION,STATUS,
ERR_CODE,OPERATION from resource_status);

69
orm/services/resource_distributor/rds/services/model/region_resource_id_status.py
View File

@ -1,50 +1,5 @@
class ResourceMetaData(object):
def __init__(self, checksum, virtual_size, size):
self.size = size
self.virtual_size = virtual_size
self.checksum = checksum
def as_dict(self):
return self.__dict__
class ResourceStatusModel(object):
def __init__(self,
timestamp,
region,
status,
transaction_id,
resource_id,
ord_notifier,
err_msg,
err_code,
operation,
resource_extra_metadata=None):
self.timestamp = timestamp
self.region = region
self.status = status
self.ord_transaction_id = transaction_id
self.resource_id = resource_id
self.ord_notifier_id = ord_notifier
self.error_msg = err_msg
self.error_code = err_code
self.operation = operation
if resource_extra_metadata:
self.resource_extra_metadata = ResourceMetaData(
checksum=resource_extra_metadata[0].checksum,
virtual_size=resource_extra_metadata[0].virtual_size,
size=resource_extra_metadata[0].size
)
else:
self.resource_extra_metadata = None
def as_dict(self):
return self.__dict__
class ResourceTemplateModel(object):
def __init__(self,
resource_id,
@ -62,30 +17,6 @@ class ResourceTemplateModel(object):
return self.__dict__
class StatusModel(object):
def __init__(self, status):
self.regions = status
self.status = self._get_aggregated_status()
def _get_aggregated_status(self):
is_pending = False
for region in self.regions:
if region.status == 'Error' and region.operation.strip() != 'delete':
# If a region had an error, the aggregated status is 'Error'
return 'Error'
elif region.status == 'Submitted':
# Just set the flag but don't return, because there might be
# an error in any of the next iterations
is_pending = True
if is_pending:
return 'Pending'
else:
# If self.regions is empty, the result will still be 'Success' but the
# server returns 404 Not Found
return 'Success'
class RegionEndPointData(object):
"""class method endpoints data"""

3
orm/services/resource_distributor/rds/storage/mysql/region_resource_id_status.py
View File

@ -5,8 +5,9 @@ import time
from oslo_db.sqlalchemy.enginefacade import LegacyEngineFacade
from pecan import conf
from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel
from orm.services.resource_distributor.rds.services.model.region_resource_id_status \
import ResourceStatusModel, StatusModel, RegionEndPointData
import RegionEndPointData
from orm.services.resource_distributor.rds.storage import region_resource_id_status
from sqlalchemy import BigInteger, BLOB, Column, ForeignKey, Integer, String, Text
from sqlalchemy.ext.declarative.api import declarative_base

2
orm/tests/unit/cms/test_customer_logic.py
View File

@ -42,7 +42,7 @@ class ResourceStatusModel(object):
class StatusModel(object):
def __init__(self, status='Success'):
self.regions_status = [ResourceStatusModel()]
self.regions = [ResourceStatusModel()]
self.status = status

118
orm/tests/unit/ims/logic/test_image_logic.py
View File

@ -1,3 +1,4 @@
from orm.common.orm_common.utils import utils
from orm.services.image_manager.ims.logic import image_logic
from orm.services.image_manager.ims.persistency.sql_alchemy.db_models import Image
from orm.services.image_manager.ims.persistency.wsme import models
@ -81,51 +82,46 @@ image_status_dict = {'regions': [{
class TestImageLogic(FunctionalTest):
@mock.patch.object(image_logic, 'di')
def test_get_image_by_uuid_image_not_found(self, mock_di):
mock_rds_proxy = mock.MagicMock()
my_get_image = mock.MagicMock()
my_get_image.get_image_by_id.return_value = None
my_get_record = mock.MagicMock()
my_get_record.get_record.return_value = my_get_image
my_dm = mock.MagicMock(return_value=my_get_record)
_, mock_data_manager = \
get_data_manager_mock(mock_sql_image=None)
mock_di.resolver.unpack.return_value = (mock_data_manager)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
try:
image_logic.get_image_by_uuid('te')
except image_logic.ErrorStatus as e:
self.assertEqual(e.status_code, 404)
@mock.patch.object(utils, 'get_resource_status_from_db')
@mock.patch.object(image_logic.ImsUtils, 'get_server_links',
return_value=["ip", "path"])
@mock.patch.object(image_logic, 'di')
@mock.patch.object(image_logic, 'ImageWrapper')
def test_get_image_by_uuid_image_no_status(self, mock_image,
mock_di, mock_links):
mock_rds_proxy = mock.MagicMock()
mock_rds_proxy.get_status.return_value = RDSGetStatus(status_code=404)
def test_get_image_by_uuid_image_no_status(
self, mock_image, mock_di, mock_links, mock_get_resource):
_, mock_data_manager = get_data_manager_mock()
mock_di.resolver.unpack.return_value = (mock_data_manager)
mock_image.from_db_model.return_value = ImageWrapperTest()
my_get_image = mock.MagicMock()
my_get_record = mock.MagicMock()
my_get_record.get_record.return_value = my_get_image
my_dm = mock.MagicMock(return_value=my_get_record)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
mock_get_resource.return_value = {}
result = image_logic.get_image_by_uuid('test')
self.assertEqual(result.image.status, 'no regions')
@mock.patch.object(utils, 'get_resource_status_from_db')
@mock.patch.object(image_logic.ImsUtils, 'get_server_links',
return_value=["ip", "path"])
@mock.patch.object(image_logic, 'di')
@mock.patch.object(image_logic, 'ImageWrapper')
def test_get_image_by_uuid_image_sanity(self, mock_image,
mock_di, mock_links):
mock_rds_proxy = mock.MagicMock()
mock_rds_proxy.get_status.return_value = RDSGetStatus()
my_get_image = mock.MagicMock()
my_get_record = mock.MagicMock()
my_get_record.get_record.return_value = my_get_image
my_dm = mock.MagicMock(return_value=my_get_record)
def test_get_image_by_uuid_image_sanity(
self, mock_image, mock_di, mock_links, mock_get_resource):
_, mock_data_manager = get_data_manager_mock()
mock_di.resolver.unpack.return_value = (mock_data_manager)
status_model = mock.MagicMock()
status_model.status = 'Success'
resource_status_dict = {'some_id': status_model}
mock_get_resource.return_value = resource_status_dict
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
result = image_logic.get_image_by_uuid('test')
self.assertEqual(result.image.status, 'Success')
@ -133,50 +129,50 @@ class TestImageLogic(FunctionalTest):
class TestDeleteImageLogic(FunctionalTest):
"""test delete image."""
@mock.patch.object(image_logic, 'update_region_actions', return_value=True)
@mock.patch.object(utils, 'get_resource_status_from_db')
@mock.patch.object(image_logic, 'di')
def test_delete_image_success(self, mock_di, mock_update_region):
mock_rds_proxy, mock_data_manager = get_data_manager_mock(
def test_delete_image_success(self, mock_di, mock_get_resource):
_, mock_data_manager = get_data_manager_mock(
get_existing_region_names=[])
mock_rds_proxy.get_status.return_value = RdsResponse()
mock_di.resolver.unpack.return_value = (mock_rds_proxy,
mock_data_manager)
global regions
regions = []
mock_di.resolver.unpack.return_value = (mock_data_manager)
status_model = mock.MagicMock()
status_model.status = 'Success'
resource_status_dict = {'some_id': status_model}
mock_get_resource.return_value = resource_status_dict
image_logic.delete_image_by_uuid("image_uuid", "transaction_id")
@mock.patch.object(image_logic, 'update_region_actions', return_value=True)
@mock.patch.object(utils, 'get_resource_status_from_db')
@mock.patch.object(image_logic, 'di')
def test_delete_image_success_nords(self, mock_di, mock_update_region):
mock_rds_proxy, mock_data_manager = \
get_data_manager_mock(imagejson={"regions": {}},
get_existing_region_names=[])
mock_rds_proxy.get_status.return_value = RdsResponse()
mock_di.resolver.unpack.return_value = (mock_rds_proxy,
mock_data_manager)
def test_delete_image_success_nords(self, mock_di, mock_get_resource):
_, mock_data_manager = get_data_manager_mock(
get_existing_region_names=[])
mock_di.resolver.unpack.return_value = (mock_data_manager)
mock_get_resource.return_value = {}
image_logic.delete_image_by_uuid("image_uuid", "transaction_id")
@mock.patch.object(image_logic, 'update_region_actions', return_value=True)
@mock.patch.object(image_logic, 'di')
def test_delete_image_notfound_error(self, mock_di, mock_update_region):
mock_rds_proxy, mock_data_manager = \
def test_delete_image_notfound_error(self, mock_di):
_, mock_data_manager = \
get_data_manager_mock(mock_sql_image=None)
mock_di.resolver.unpack.return_value = (mock_rds_proxy,
mock_data_manager)
mock_di.resolver.unpack.return_value = (mock_data_manager)
try:
image_logic.delete_image_by_uuid("image_uuid", "transaction_id")
except Exception as e:
self.assertEqual(404, e.status_code)
@mock.patch.object(image_logic, 'update_region_actions',
side_effect=ValueError('test'))
@mock.patch.object(image_logic, 'di')
def test_delete_image_other_error(self, mock_di, mock_update_region):
mock_rds_proxy, mock_data_manager = get_data_manager_mock()
mock_di.resolver.unpack.return_value = (mock_rds_proxy,
mock_data_manager)
self.assertRaises(image_logic.ErrorStatus, image_logic.delete_image_by_uuid,
"image_uuid", "transaction_id")
def test_delete_image_other_error(self, mock_di):
_, mock_data_manager = get_data_manager_mock()
mock_di.resolver.unpack.return_value = (mock_data_manager)
self.assertRaises(image_logic.ErrorStatus,
image_logic.delete_image_by_uuid,
"image_uuid",
"transaction_id")
class TestUpdateImage(FunctionalTest):
@ -310,14 +306,13 @@ class TestActivateImageLogic(FunctionalTest):
class TestListImageLogic(FunctionalTest):
@mock.patch.object(image_logic, 'di')
def test_list_image_not_found(self, mock_di):
mock_rds_proxy = mock.MagicMock()
my_get_image = mock.MagicMock()
my_get_image.get_image.return_value = None
my_get_record = mock.MagicMock()
my_get_record.get_record.side_effect = image_logic.ErrorStatus(404, 'a')
my_dm = mock.MagicMock(return_value=my_get_record)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
mock_di.resolver.unpack.return_value = my_dm
try:
image_logic.get_image_list_by_params('a', 'b', 'c')
except image_logic.ErrorStatus as e:
@ -326,13 +321,12 @@ class TestListImageLogic(FunctionalTest):
@mock.patch.object(image_logic, 'di')
@mock.patch.object(image_logic, 'ImageWrapper')
def test_list_image_error(self, mock_image, mock_di):
mock_rds_proxy = mock.MagicMock()
my_get_image = mock.MagicMock()
my_get_record = mock.MagicMock()
my_get_record.get_record.side_effect = SystemError()
my_dm = mock.MagicMock(return_value=my_get_record)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
mock_di.resolver.unpack.return_value = my_dm
try:
image_logic.get_image_list_by_params('a', 'b', 'c')
except Exception as e:
@ -341,17 +335,12 @@ class TestListImageLogic(FunctionalTest):
@mock.patch.object(image_logic, 'di')
@mock.patch.object(image_logic, 'ImageWrapper')
def test_list_image_sanity(self, mock_image, mock_di):
imagejson = [{"regions": {"name": "mdt1"}}]
mock_rds_proxy = mock.MagicMock()
mock_rds_proxy.get_status.return_value = RDSGetStatus()
mock_data_manager = mock.MagicMock()
mock_image_rec = mock.MagicMock()
image_json = mock.MagicMock()
image_json.return_value = imagejson
mock_image_rec.get_images_by_criteria.return_value = Image()
my_dm = mock.MagicMock(mock_data_manager)
mock_di.resolver.unpack.return_value = my_dm, mock_rds_proxy
mock_di.resolver.unpack.return_value = my_dm
result = image_logic.get_image_list_by_params('a', 'b', 'c')
self.assertEqual(len(result.images), 0)
@ -688,6 +677,7 @@ def get_data_manager_mock(get_existing_region_names={"name": "mdt1"},
if mock_sql_image:
mock_sql_image = mock.MagicMock()
mock_sql_image.__json__ = image_json
mock_sql_image.id = 'some_id'
mock_sql_image.visibility = visibility
mock_sql_image.protected = protected
mock_sql_image.get_proxy_dict = mock.MagicMock(return_value={'regions': regions})

3
orm/tests/unit/rds/controllers/v1/status/test_get_resource_status.py
View File

@ -1,9 +1,10 @@
"""unittest get resource status."""
from mock import MagicMock
import orm.services.resource_distributor.rds.controllers.v1.status.get_resource as resource
from orm.services.resource_distributor.rds.services.model.region_resource_id_status import ResourceStatusModel, StatusModel
from orm.tests.unit.rds.controllers.v1.functional_test import FunctionalTest
from orm.common.orm_common.model.models import ResourceStatusModel, StatusModel
class EmptyModel(object):
"""mock class."""

23
orm/tests/unit/rds/services/model/test_region_resource_id_status.py
View File

@ -1,12 +1,13 @@
import unittest
from orm.common.orm_common.model import models
from orm.services.resource_distributor.rds.services.model import region_resource_id_status
class TestModel(unittest.TestCase):
def test_model_as_dict(self):
model = region_resource_id_status.ResourceStatusModel(1, 2, 3, 4, 5, 6, 7, 8,
'create')
model = models.ResourceStatusModel(
1, 2, 3, 4, 5, 6, 7, 8, 'create')
expected_dict = {
'timestamp': 1,
'region': 2,
@ -42,19 +43,19 @@ class TestResourceTemplateModel(unittest.TestCase):
class TestStatusModel(unittest.TestCase):
def test_get_aggregated_status_error(self):
model = region_resource_id_status.ResourceStatusModel(1, 2, 'Error', 4, 5, 6, 7, 8,
'create')
status_model = region_resource_id_status.StatusModel([model])
model = models.ResourceStatusModel(
1, 2, 'Error', 4, 5, 6, 7, 8, 'create')
status_model = models.StatusModel([model])
self.assertEqual(status_model.status, 'Error')
def test_get_aggregated_status_pending(self):
model = region_resource_id_status.ResourceStatusModel(1, 2, 'Submitted', 4, 5, 6, 7,
8, 'create')
status_model = region_resource_id_status.StatusModel([model])
model = models.ResourceStatusModel(
1, 2, 'Submitted', 4, 5, 6, 7, 8, 'create')
status_model = models.StatusModel([model])
self.assertEqual(status_model.status, 'Pending')
def test_get_aggregated_status_success(self):
model = region_resource_id_status.ResourceStatusModel(1, 2, 'Success', 4, 5, 6, 7, 8,
'create')
status_model = region_resource_id_status.StatusModel([model])
model = models.ResourceStatusModel(
1, 2, 'Success', 4, 5, 6, 7, 8, 'create')
status_model = models.StatusModel([model])
self.assertEqual(status_model.status, 'Success')

6
orm/tests/unit/rds/services/test_create_resource.py
View File

@ -2,10 +2,10 @@
import unittest
from unittest.mock import patch
from orm.common.orm_common.model.models import (ResourceStatusModel,
ResourceMetaData,
StatusModel)
from orm.services.resource_distributor.rds.services import resource as ResourceService
from orm.services.resource_distributor.rds.services.model.region_resource_id_status import (ResourceStatusModel,
ResourceMetaData,
StatusModel)
result = ResourceStatusModel(
status="success", timestamp="123456789", region="name",