openstack
/
masakari
Code Issues Proposed changes

Merge "vm moves for host notification"

This commit is contained in:
Zuul 2023-01-28 13:19:42 +00:00 committed by Gerrit Code Review
parent 73e35991ff 23c6c31409
commit e8db24c637
22 changed files with 1390 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

100
masakari/api/openstack/ha/vmoves.py Normal file
View File

@ -0,0 +1,100 @@
# Copyright(c) 2022 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The VM Move API extension."""
from http import HTTPStatus
from webob import exc
from masakari.api.openstack import common
from masakari.api.openstack import extensions
from masakari.api.openstack import wsgi
from masakari import exception
from masakari.ha import api as vmove_api
from masakari.policies import vmoves as vmove_policies
ALIAS = "vmoves"
class VMovesController(wsgi.Controller):
"""The VM move API controller for the Instance HA."""
def __init__(self):
self.api = vmove_api.VMoveAPI()
@extensions.expected_errors((HTTPStatus.BAD_REQUEST, HTTPStatus.FORBIDDEN,
HTTPStatus.NOT_FOUND))
def index(self, req, notification_id):
"""Returns a list of vmoves."""
context = req.environ['masakari.context']
context.can(vmove_policies.VMOVES % 'index')
try:
filters = {}
limit, marker = common.get_limit_and_marker(req)
sort_keys, sort_dirs = common.get_sort_params(req.params)
if 'status' in req.params:
filters['status'] = req.params['status']
if 'type' in req.params:
filters['type'] = req.params['type']
vmoves = self.api.get_all(context,
notification_id,
filters=filters,
sort_keys=sort_keys,
sort_dirs=sort_dirs,
limit=limit,
marker=marker)
except exception.MarkerNotFound as ex:
raise exc.HTTPBadRequest(explanation=ex.format_message())
except exception.Invalid as e:
raise exc.HTTPBadRequest(explanation=e.format_message())
except exception.NotificationNotFound as ex:
raise exc.HTTPNotFound(explanation=ex.format_message())
return {'vmoves': vmoves}
@extensions.expected_errors((HTTPStatus.FORBIDDEN, HTTPStatus.NOT_FOUND))
def show(self, req, notification_id, id):
"""Shows the details of one vmove."""
context = req.environ['masakari.context']
context.can(vmove_policies.VMOVES % 'detail')
try:
vmove = self.api.get_vmove(context, notification_id, id)
except exception.VMoveNotFound as e:
raise exc.HTTPNotFound(explanation=e.format_message())
return {'vmove': vmove}
class VMoves(extensions.V1APIExtensionBase):
"""vmoves controller"""
name = "vmoves"
alias = ALIAS
version = 1
def get_resources(self):
parent = {'member_name': 'notification',
'collection_name': 'notifications'}
resources = [
extensions.ResourceExtension(
'vmoves', VMovesController(), parent=parent,
member_name='vmove')]
return resources
def get_controller_extensions(self):
return []

78
masakari/db/api.py
View File

@ -363,6 +363,84 @@ def notification_delete(context, notification_uuid):
return IMPL.notification_delete(context, notification_uuid)
# vmoves related db apis
def vmoves_get_all_by_filters(
context, filters=None, sort_keys=None, sort_dirs=None,
limit=None, marker=None):
"""Get all vm moves that match the filters.
:param context: context to query under
:param filters: filters for the query in the form of key/value
:param sort_keys: list of attributes by which results should be sorted,
paired with corresponding item in sort_dirs
:param sort_dirs: list of directions in which results should be sorted,
paired with corresponding item in sort_keys
:param limit: maximum number of items to return
:param marker: the last item of the previous page, used to determine the
next page of results to return
:returns: list of dictionary-like objects containing all vm moves
"""
return IMPL.vmoves_get_all_by_filters(context, filters=filters,
sort_keys=sort_keys,
sort_dirs=sort_dirs,
limit=limit,
marker=marker)
def vmove_get_by_uuid(context, vmove_uuid):
"""Get one vm move information by uuid.
:param context: context to query under
:param uuid: uuid of the vm move
:returns: dictionary-like object containing one vm move
:raises: exception.VMoveNotFound if the vm move with given
'uuid' doesn't exist
"""
return IMPL.vmove_get_by_uuid(context, vmove_uuid)
def vmove_create(context, values):
"""Create one vm move.
:param context: context to query under
:param values: dictionary of the vm move attributes to create
:returns: dictionary-like object containing created one vm move
"""
return IMPL.vmove_create(context, values)
def vmove_update(context, uuid, values):
"""Update one vm move information in the database.
:param context: context to query under
:param uuid: uuid of the vm move to be updated
:param values: dictionary of the vm move attributes to be updated
:returns: dictionary-like object containing updated one vm move
:raises: exception.VMoveNotFound if the vm move with given
'uuid' doesn't exist
"""
return IMPL.vmove_update(context, uuid, values)
def vmove_delete(context, uuid):
"""Delete one vm move.
:param context: context to query under
:param uuid: uuid of the vm move to be delete
:raises exception.VMoveNotFound if the vm move not exist.
"""
return IMPL.vmove_delete(context, uuid)
def purge_deleted_rows(context, age_in_days, max_rows):
"""Purge the soft deleted rows.

103
masakari/db/sqlalchemy/api.py
View File

@ -618,6 +618,109 @@ def notification_delete(context, notification_uuid):
raise exception.NotificationNotFound(id=notification_uuid)
# db apis for vm moves
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@main_context_manager.reader
def vmoves_get_all_by_filters(
context, filters=None, sort_keys=None,
sort_dirs=None, limit=None, marker=None):
if limit == 0:
return []
sort_keys, sort_dirs = _process_sort_params(sort_keys,
sort_dirs)
filters = filters or {}
query = model_query(context, models.VMove)
if 'notification_uuid' in filters:
query = query.filter(models.VMove.notification_uuid == filters[
'notification_uuid'])
if 'type' in filters:
query = query.filter(models.VMove.type == filters[
'type'])
if 'status' in filters:
status = filters['status']
if isinstance(status, (list, tuple, set, frozenset)):
column_attr = getattr(models.VMove, 'status')
query = query.filter(column_attr.in_(status))
else:
query = query.filter(models.VMove.status == status)
marker_row = None
if marker is not None:
marker_row = model_query(context,
models.VMove
).filter_by(id=marker).first()
if not marker_row:
raise exception.MarkerNotFound(marker=marker)
try:
query = sqlalchemyutils.paginate_query(query, models.VMove,
limit,
sort_keys,
marker=marker_row,
sort_dirs=sort_dirs)
except db_exc.InvalidSortKey as err:
raise exception.InvalidSortKey(err)
return query.all()
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@main_context_manager.reader
def vmove_get_by_uuid(context, uuid):
return _vmove_get_by_uuid(context, uuid)
def _vmove_get_by_uuid(context, uuid):
query = model_query(context, models.VMove).filter_by(uuid=uuid)
result = query.first()
if not result:
raise exception.VMoveNotFound(id=uuid)
return result
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@main_context_manager.writer
def vmove_create(context, values):
vm_move = models.VMove()
vm_move.update(values)
vm_move.save(session=context.session)
return _vmove_get_by_uuid(context, vm_move.uuid)
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@main_context_manager.writer
def vmove_update(context, uuid, values):
vm_move = _vmove_get_by_uuid(context, uuid)
vm_move.update(values)
vm_move.save(session=context.session)
return _vmove_get_by_uuid(context, vm_move.uuid)
@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
@main_context_manager.writer
def vmove_delete(context, vmove_uuid):
count = model_query(context, models.VMove
).filter_by(uuid=vmove_uuid
).soft_delete(synchronize_session=False)
if count == 0:
raise exception.VMoveNotFound(id=vmove_uuid)
class DeleteFromSelect(sa_sql.expression.UpdateBase):
def __init__(self, table, select, column):
self.table = table

52
masakari/db/sqlalchemy/migrate_repo/versions/008_add_vm_moves_table.py Normal file
View File

@ -0,0 +1,52 @@
# Copyright(c) 2022 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from migrate.changeset import UniqueConstraint
from sqlalchemy import Column, MetaData, Table
from sqlalchemy import Integer, DateTime, String, Text
def define_vm_moves_table(meta):
vm_moves = Table('vmoves',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('deleted_at', DateTime),
Column('deleted', Integer),
Column('id', Integer, primary_key=True, nullable=False),
Column('uuid', String(36), nullable=False),
Column('notification_uuid', String(36), nullable=False),
Column('instance_uuid', String(36), nullable=False),
Column('instance_name', String(255), nullable=False),
Column('source_host', String(255), nullable=True),
Column('dest_host', String(255), nullable=True),
Column('start_time', DateTime, nullable=True),
Column('end_time', DateTime, nullable=True),
Column('type', String(36), nullable=True),
Column('status', String(36), nullable=True),
Column('message', Text, nullable=True),
UniqueConstraint('uuid', name='uniq_vmove0uuid'),
mysql_engine='InnoDB',
mysql_charset='utf8',
extend_existing=True)
return vm_moves
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
table = define_vm_moves_table(meta)
table.create()

22
masakari/db/sqlalchemy/models.py
View File

@ -141,3 +141,25 @@ class Notification(BASE, MasakariAPIBase, models.SoftDeleteMixin):
'ignored', 'finished', name='notification_status'),
nullable=False)
source_host_uuid = Column(String(36), nullable=False)
class VMove(BASE, MasakariAPIBase, models.SoftDeleteMixin):
"""Represents one vm move."""
__tablename__ = 'vmoves'
__table_args__ = (
schema.UniqueConstraint('uuid',
name='uniq_vmove0uuid'),
)
id = Column(Integer, primary_key=True, autoincrement=True)
uuid = Column(String(36), nullable=False)
notification_uuid = Column(String(36), nullable=False)
instance_uuid = Column(String(36), nullable=False)
instance_name = Column(String(255), nullable=False)
source_host = Column(String(255), nullable=True)
dest_host = Column(String(255), nullable=True)
start_time = Column(DateTime, nullable=True)
end_time = Column(DateTime, nullable=True)
type = Column(String(36), nullable=True)
status = Column(String(255), nullable=True)
message = Column(Text)

134
masakari/engine/drivers/taskflow/host_failure.py
View File

@ -22,12 +22,15 @@ from oslo_log import log as logging
from oslo_service import loopingcall
from oslo_utils import excutils
from oslo_utils import strutils
from oslo_utils import timeutils
from taskflow.patterns import linear_flow
from taskflow import retry
import masakari.conf
from masakari.engine.drivers.taskflow import base
from masakari import exception
from masakari import objects
from masakari.objects import fields
from masakari import utils
@ -61,15 +64,14 @@ class DisableComputeServiceTask(base.MasakariTask):
class PrepareHAEnabledInstancesTask(base.MasakariTask):
"""Get all HA_Enabled instances."""
default_provides = set(["instance_list"])
def __init__(self, context, novaclient, **kwargs):
kwargs['requires'] = ["host_name"]
kwargs['requires'] = ["host_name", "notification_uuid"]
super(PrepareHAEnabledInstancesTask, self).__init__(context,
novaclient,
**kwargs)
def execute(self, host_name):
def execute(self, host_name, notification_uuid):
def _filter_instances(instance_list):
ha_enabled_instances = []
non_ha_enabled_instances = []
@ -131,21 +133,28 @@ class PrepareHAEnabledInstancesTask(base.MasakariTask):
LOG.info(msg)
raise exception.SkipHostRecoveryException(message=msg)
# persist vm moves
for instance in instance_list:
vmove = objects.VMove(context=self.context)
vmove.instance_uuid = instance.id
vmove.instance_name = instance.name
vmove.notification_uuid = notification_uuid
vmove.source_host = host_name
vmove.status = fields.VMoveStatus.PENDING
vmove.type = fields.VMoveType.EVACUATION
vmove.create()
# List of instance UUID
instance_list = [instance.id for instance in instance_list]
msg = "Instances to be evacuated are: '%s'" % ','.join(instance_list)
self.update_details(msg, 1.0)
return {
"instance_list": instance_list,
}
class EvacuateInstancesTask(base.MasakariTask):
def __init__(self, context, novaclient, **kwargs):
kwargs['requires'] = ["host_name", "instance_list"]
kwargs['requires'] = ["host_name", "notification_uuid"]
self.update_host_method = kwargs['update_host_method']
super(EvacuateInstancesTask, self).__init__(context, novaclient,
**kwargs)
@ -185,9 +194,27 @@ class EvacuateInstancesTask(base.MasakariTask):
finally:
periodic_call_stopped.stop()
def _evacuate_and_confirm(self, context, instance, host_name,
failed_evacuation_instances,
def _evacuate_and_confirm(self, context, vmove,
reserved_host=None):
def _update_vmove(vmove, status=None, start_time=None,
end_time=None, dest_host=None,
message=None):
if status:
vmove.status = status
if start_time:
vmove.start_time = start_time
if end_time:
vmove.end_time = end_time
if dest_host:
vmove.dest_host = dest_host
if message:
vmove.message = message
vmove.save()
instance_uuid = vmove.instance_uuid
instance = self.novaclient.get_server(context, instance_uuid)
# Before locking the instance check whether it is already locked
# by user, if yes don't lock the instance
instance_already_locked = self.novaclient.get_server(
@ -208,7 +235,7 @@ class EvacuateInstancesTask(base.MasakariTask):
raise exception.InstanceEvacuateFailed(
instance_uuid=instance.id)
if instance_host != host_name:
if instance_host != vmove.source_host:
if ((old_vm_state == 'error' and
new_vm_state == 'active') or
old_vm_state == new_vm_state):
@ -265,7 +292,11 @@ class EvacuateInstancesTask(base.MasakariTask):
if vm_state == 'active':
stop_instance = False
# evacuate the instance
# start to evacuate the instance
_update_vmove(
vmove,
status=fields.VMoveStatus.ONGOING,
start_time=timeutils.utcnow())
self.novaclient.evacuate_instance(context, instance.id,
target=reserved_host)
@ -279,29 +310,46 @@ class EvacuateInstancesTask(base.MasakariTask):
if vm_state == 'error':
self.novaclient.reset_instance_state(
context, instance.id)
instance = self.novaclient.get_server(context, instance_uuid)
dest_host = getattr(
instance, "OS-EXT-SRV-ATTR:hypervisor_hostname")
_update_vmove(
vmove,
status=fields.VMoveStatus.SUCCEEDED,
dest_host=dest_host)
except etimeout.Timeout:
# Instance is not stop in the expected time_limit.
failed_evacuation_instances.append(instance.id)
msg = "Failed reason: timeout."
_update_vmove(
vmove,
status=fields.VMoveStatus.FAILED,
message=msg)
except Exception as e:
# Exception is raised while resetting instance state or
# evacuating the instance itself.
LOG.warning(str(e))
failed_evacuation_instances.append(instance.id)
_update_vmove(
vmove,
status=fields.VMoveStatus.FAILED,
message=str(e))
finally:
_update_vmove(vmove, end_time=timeutils.utcnow())
if not instance_already_locked:
# Unlock the server after evacuation and confirmation
self.novaclient.unlock_server(context, instance.id)
def execute(self, host_name, instance_list, reserved_host=None):
def execute(self, host_name, notification_uuid, reserved_host=None):
all_vmoves = objects.VMoveList.get_all_vmoves(
self.context, notification_uuid, status=fields.VMoveStatus.PENDING)
instance_list = [i.instance_uuid for i in all_vmoves]
msg = ("Start evacuation of instances from failed host '%(host_name)s'"
", instance uuids are: '%(instance_list)s'") % {
'host_name': host_name, 'instance_list': ','.join(instance_list)}
self.update_details(msg)
def _do_evacuate(context, host_name, instance_list,
def _do_evacuate(context, host_name,
reserved_host=None):
failed_evacuation_instances = []
if reserved_host:
msg = "Enabling reserved host: '%s'" % reserved_host
self.update_details(msg, 0.1)
@ -338,40 +386,41 @@ class EvacuateInstancesTask(base.MasakariTask):
context, reserved_host, enable=True)
# Set reserved property of reserved_host to False
self.update_host_method(self.context, reserved_host)
self.update_host_method(context, reserved_host)
thread_pool = greenpool.GreenPool(
CONF.host_failure_recovery_threads)
for instance_id in instance_list:
msg = "Evacuation of instance started: '%s'" % instance_id
self.update_details(msg, 0.5)
instance = self.novaclient.get_server(self.context,
instance_id)
thread_pool.spawn_n(self._evacuate_and_confirm, context,
instance, host_name,
failed_evacuation_instances,
reserved_host)
nonlocal all_vmoves
for vmove in all_vmoves:
msg = ("Evacuation of instance started: '%s'"
% vmove.instance_uuid)
self.update_details(msg, 0.5)
thread_pool.spawn_n(self._evacuate_and_confirm, self.context,
vmove, reserved_host)
thread_pool.waitall()
evacuated_instances = list(set(instance_list).difference(set(
failed_evacuation_instances)))
all_vmoves = objects.VMoveList.get_all_vmoves(
self.context, notification_uuid)
if evacuated_instances:
evacuated_instances.sort()
succeeded_vmoves = [i.instance_uuid for i in all_vmoves
if i.status == fields.VMoveStatus.SUCCEEDED]
if succeeded_vmoves:
succeeded_vmoves.sort()
msg = ("Successfully evacuate instances '%(instance_list)s' "
"from host '%(host_name)s'") % {
'instance_list': ','.join(evacuated_instances),
'instance_list': ','.join(succeeded_vmoves),
'host_name': host_name}
self.update_details(msg, 0.7)
if failed_evacuation_instances:
failed_vmoves = [i.instance_uuid for i in
all_vmoves if i.status == fields.VMoveStatus.FAILED]
if failed_vmoves:
msg = ("Failed to evacuate instances "
"'%(failed_evacuation_instances)s' from host "
"'%(instance_list)s' from host "
"'%(host_name)s'") % {
'failed_evacuation_instances':
','.join(failed_evacuation_instances),
'instance_list': ','.join(failed_vmoves),
'host_name': host_name}
self.update_details(msg, 0.7)
raise exception.HostRecoveryFailureException(
@ -383,18 +432,19 @@ class EvacuateInstancesTask(base.MasakariTask):
lock_name = reserved_host if reserved_host else None
@utils.synchronized(lock_name)
def do_evacuate_with_reserved_host(context, host_name, instance_list,
reserved_host):
_do_evacuate(self.context, host_name, instance_list,
def do_evacuate_with_reserved_host(context, host_name,
notification_uuid, reserved_host):
_do_evacuate(context, host_name,
reserved_host=reserved_host)
if lock_name:
do_evacuate_with_reserved_host(self.context, host_name,
instance_list, reserved_host)
notification_uuid,
reserved_host)
else:
# No need to acquire lock on reserved_host when recovery_method is
# 'auto' as the selection of compute host will be decided by nova.
_do_evacuate(self.context, host_name, instance_list)
_do_evacuate(self.context, host_name)
def get_auto_flow(context, novaclient, process_what):

8
masakari/exception.py
View File

@ -281,6 +281,14 @@ class ComputeNotFoundByName(NotFound):
"be found.")
class VMoveNotFound(NotFound):
msg_fmt = _("No vm move with id %(id)s.")
class NotificationWithoutVMoves(Invalid):
msg_fmt = _("This notification %(id)s without vm moves.")
class FailoverSegmentExists(MasakariException):
msg_fmt = _("Failover segment with name %(name)s already exists.")

33
masakari/ha/api.py
View File

@ -393,3 +393,36 @@ class NotificationAPI(object):
get_notification_recovery_workflow_details(
context, notification))
return notification
class VMoveAPI(object):
"""The vmoves API to manage vmoves"""
def _is_valid_notification(self, context, notification_uuid):
notification = objects.Notification.get_by_uuid(
context, notification_uuid)
if notification.type != fields.NotificationType.COMPUTE_HOST:
raise exception.NotificationWithoutVMoves(id=notification_uuid)
def get_all(self, context, notification_uuid, filters=None,
sort_keys=None, sort_dirs=None, limit=None, marker=None):
"""Get all vmoves by filters"""
self._is_valid_notification(context, notification_uuid)
filters['notification_uuid'] = notification_uuid
vmoves = objects.VMoveList.get_all(
context, filters, sort_keys, sort_dirs, limit, marker)
return vmoves
def get_vmove(self, context, notification_uuid, vmove_uuid):
"""Get one vmove."""
self._is_valid_notification(context, notification_uuid)
if uuidutils.is_uuid_like(vmove_uuid):
LOG.debug("Fetching vmove by uuid %s", vmove_uuid)
vmove = objects.VMove.get_by_uuid(context, vmove_uuid)
else:
LOG.debug("Failed to fetch vmove by uuid %s", vmove_uuid)
raise exception.VMoveNotFound(id=vmove_uuid)
return vmove

1
masakari/objects/__init__.py
View File

@ -21,3 +21,4 @@ def register_all():
__import__('masakari.objects.host')
__import__('masakari.objects.notification')
__import__('masakari.objects.segment')
__import__('masakari.objects.vmove')

58
masakari/objects/fields.py
View File

@ -278,3 +278,61 @@ class EventNotificationPriorityField(BaseEnumField):
class EventNotificationPhaseField(BaseEnumField):
AUTO_TYPE = EventNotificationPhase()
class VMoveType(Enum):
"""Represents possible types for VMoves."""
EVACUATION = "evacuation"
MIGRATION = "migration"
LIVE_MIGRATION = "live_migration"
ALL = (EVACUATION, MIGRATION, LIVE_MIGRATION)
def __init__(self):
super(VMoveType,
self).__init__(valid_values=VMoveType.ALL)
@classmethod
def index(cls, value):
"""Return an index into the Enum given a value."""
return cls.ALL.index(value)
@classmethod
def from_index(cls, index):
"""Return the Enum value at a given index."""
return cls.ALL[index]
class VMoveTypeField(BaseEnumField):
AUTO_TYPE = VMoveType()
class VMoveStatus(Enum):
"""Represents possible statuses for VMoves."""
PENDING = "pending"
ONGOING = "ongoing"
IGNORED = "ignored"
FAILED = "failed"
SUCCEEDED = "succeeded"
ALL = (PENDING, ONGOING, IGNORED, FAILED, SUCCEEDED)
def __init__(self):
super(VMoveStatus,
self).__init__(valid_values=VMoveStatus.ALL)
@classmethod
def index(cls, value):
"""Return an index into the Enum given a value."""
return cls.ALL.index(value)
@classmethod
def from_index(cls, index):
"""Return the Enum value at a given index."""
return cls.ALL[index]
class VMoveStatusField(BaseEnumField):
AUTO_TYPE = VMoveStatus()

115
masakari/objects/vmove.py Normal file
View File

@ -0,0 +1,115 @@
# Copyright(c) 2022 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_log import log as logging
from oslo_utils import uuidutils
from masakari import db
from masakari import exception
from masakari import objects
from masakari.objects import base
from masakari.objects import fields
LOG = logging.getLogger(__name__)
@base.MasakariObjectRegistry.register
class VMove(base.MasakariPersistentObject, base.MasakariObject,
base.MasakariObjectDictCompat):
VERSION = '1.0'
fields = {
'id': fields.IntegerField(),
'uuid': fields.UUIDField(),
'notification_uuid': fields.UUIDField(),
'instance_uuid': fields.UUIDField(),
'instance_name': fields.StringField(),
'source_host': fields.StringField(nullable=True),
'dest_host': fields.StringField(nullable=True),
'start_time': fields.DateTimeField(nullable=True),
'end_time': fields.DateTimeField(nullable=True),
'type': fields.VMoveTypeField(nullable=True),
'status': fields.VMoveStatusField(nullable=True),
'message': fields.StringField(nullable=True),
}
@staticmethod
def _from_db_object(context, vmove, db_vmove):
for key in vmove.fields:
setattr(vmove, key, db_vmove[key])
vmove._context = context
vmove.obj_reset_changes()
return vmove
@base.remotable_classmethod
def get_by_uuid(cls, context, uuid):
db_inst = db.vmove_get_by_uuid(context, uuid)
return cls._from_db_object(context, cls(), db_inst)
@base.remotable
def create(self):
if self.obj_attr_is_set('id'):
raise exception.ObjectActionError(action='create',
reason='already created')
updates = self.masakari_obj_get_changes()
if 'uuid' not in updates:
updates['uuid'] = uuidutils.generate_uuid()
vmove = db.vmove_create(self._context, updates)
self._from_db_object(self._context, self, vmove)
@base.remotable
def save(self):
updates = self.masakari_obj_get_changes()
updates.pop('id', None)
vmove = db.vmove_update(self._context, self.uuid, updates)
self._from_db_object(self._context, self, vmove)
@base.MasakariObjectRegistry.register
class VMoveList(base.ObjectListBase, base.MasakariObject):
VERSION = '1.0'
fields = {
'objects': fields.ListOfObjectsField('VMove'),
}
@base.remotable_classmethod
def get_all(cls, ctxt, filters=None, sort_keys=None,
sort_dirs=None, limit=None, marker=None):
groups = db.vmoves_get_all_by_filters(ctxt, filters=filters,
sort_keys=sort_keys,
sort_dirs=sort_dirs,
limit=limit,
marker=marker)
return base.obj_make_list(ctxt, cls(ctxt), objects.VMove,
groups)
@base.remotable_classmethod
def get_all_vmoves(cls, ctxt, notification_uuid, status=None):
filters = {
'notification_uuid': notification_uuid
}
if status:
filters['status'] = status
groups = db.vmoves_get_all_by_filters(ctxt, filters=filters)
return base.obj_make_list(ctxt, cls(ctxt), objects.VMove,
groups)

4
masakari/policies/__init__.py
View File

@ -22,6 +22,7 @@ from masakari.policies import hosts
from masakari.policies import notifications
from masakari.policies import segments
from masakari.policies import versions
from masakari.policies import vmoves
def list_rules():
@ -31,5 +32,6 @@ def list_rules():
hosts.list_rules(),
notifications.list_rules(),
segments.list_rules(),
versions.list_rules()
versions.list_rules(),
vmoves.list_rules()
)

54
masakari/policies/vmoves.py Normal file
View File

@ -0,0 +1,54 @@
# Copyright(c) 2022 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_policy import policy
from masakari.policies import base
VMOVES = 'os_masakari_api:vmoves:%s'
rules = [
policy.DocumentedRuleDefault(
name=VMOVES % 'index',
check_str=base.RULE_ADMIN_API,
description="Lists IDs, notification_id, instance_id, source_host, "
"dest_host, status and type for all VM moves.",
operations=[
{
'method': 'GET',
'path': '/notifications/{notification_id}/vmoves'
}
]),
policy.DocumentedRuleDefault(
name=VMOVES % 'detail',
check_str=base.RULE_ADMIN_API,
description="Shows details for one VM move.",
operations=[
{
'method': 'GET',
'path': '/notifications/{notification_id}/vmoves/'
'{vmove_id}'
}
]),
policy.RuleDefault(
name=VMOVES % 'discoverable',
check_str=base.RULE_ADMIN_API,
description="VM moves API extensions to change the API.",
),
]
def list_rules():
return rules

205
masakari/tests/unit/api/openstack/ha/test_vmoves.py Normal file
View File

@ -0,0 +1,205 @@
# Copyright(c) 2022 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the vmoves api."""
from unittest import mock
import ddt
from webob import exc
from masakari.api.openstack.ha import vmoves
from masakari import exception
from masakari.ha import api as ha_api
from masakari.objects import base as obj_base
from masakari.objects import notification as notification_obj
from masakari.objects import vmove as vmove_obj
from masakari import test
from masakari.tests.unit.api.openstack import fakes
from masakari.tests.unit import fakes as fakes_data
from masakari.tests import uuidsentinel
def _make_vmove_obj(vmove_dict):
return vmove_obj.VMove(**vmove_dict)
def _make_vmoves_list(vmove_list):
return vmove_obj.VMove(objects=[
_make_vmove_obj(a) for a in vmove_list])
@ddt.ddt
class VMoveTestCase(test.TestCase):
"""Test Case for vmove api."""
bad_request = exception.ValidationError
def _set_up(self):
self.controller = vmoves.VMovesController()
self.req = fakes.HTTPRequest.blank(
'/v1/notifications/%s/vmoves' % (
uuidsentinel.fake_notification1),
use_admin_context=True)
self.context = self.req.environ['masakari.context']
def setUp(self):
super(VMoveTestCase, self).setUp()
self._set_up()
self.host_type_notification = fakes_data.create_fake_notification(
id=1,
type="COMPUTE_HOST",
source_host_uuid=uuidsentinel.fake_host_1,
status="running",
notification_uuid=uuidsentinel.fake_host_type_notification,
payload={'event': 'STOPPED',
'host_status': 'NORMAL',
'cluster_status': 'ONLINE'}
)
self.vm_type_notification = fakes_data.create_fake_notification(
id=1,
type="VM",
source_host_uuid=uuidsentinel.fake_host_2,
status="running",
notification_uuid=uuidsentinel.fake_vm_type_notification,
payload={'event': 'STOPPED',
'host_status': 'NORMAL',
'cluster_status': 'ONLINE'}
)
self.vmove_1 = fakes_data.create_fake_vmove(
id=1,
uuid=uuidsentinel.fake_vmove_1,
notification_uuid=self.host_type_notification.notification_uuid,
instance_uuid=uuidsentinel.fake_instance_1,
instance_name='vm-1',
source_host='node01',
dest_host='node02',
start_time='2022-11-22 14:50:22',
end_time="2022-11-22 14:50:35",
type="evacuation",
status='succeeded',
message=None
)
self.vmove_2 = fakes_data.create_fake_vmove(
id=1,
uuid=uuidsentinel.fake_vmove_1,
notification_uuid=self.host_type_notification.notification_uuid,
instance_uuid=uuidsentinel.fake_instance_1,
instance_name='vm-1',
source_host='node01',
dest_host='node02',
start_time="2022-11-22 14:50:23",
end_time="2022-11-22 14:50:38",
type="evacuation",
status='succeeded',
message=None
)
self.vmove_list = [self.vmove_1, self.vmove_2]
self.vmove_list_obj = _make_vmoves_list(self.vmove_list)
@property
def app(self):
return fakes.wsgi_app_v1(init_only='vmoves')
def _assert_vmove_data(self, expected, actual):
self.assertTrue(obj_base.obj_equal_prims(expected, actual),
"The vmove objects were not equal")
@mock.patch.object(notification_obj.Notification, 'get_by_uuid')
@mock.patch.object(ha_api.VMoveAPI, 'get_all')
def test_index(self, mock_get_all, mock_notification):
mock_notification.return_value = mock.Mock()
mock_get_all.return_value = self.vmove_list
result = self.controller.index(
self.req, uuidsentinel.fake_host_type_notification)
result = result['vmoves']
self._assert_vmove_data(self.vmove_list_obj,
_make_vmoves_list(result))
@ddt.data('sort_key', 'sort_dir')
@mock.patch.object(notification_obj.Notification, 'get_by_uuid',
return_value=mock.Mock())
def test_index_invalid(self, sort_by, mock_notification):
req = fakes.HTTPRequest.blank(
'/v1/notifications/%s/vmoves?%s=abcd' % (
uuidsentinel.fake_notification, sort_by),
use_admin_context=True)
self.assertRaises(exc.HTTPBadRequest, self.controller.index, req,
uuidsentinel.fake_notification1)
@mock.patch.object(notification_obj.Notification, 'get_by_uuid')
@mock.patch.object(ha_api.VMoveAPI, 'get_all')
def test_index_with_valid_notification(self, mock_get_all,
mock_notification):
mock_notification.return_value = mock.Mock()
mock_get_all.side_effect = exception.NotificationWithoutVMoves
req = fakes.HTTPRequest.blank('/v1/notifications/%s/vmoves' % (
uuidsentinel.fake_vm_type_notification), use_admin_context=True)
self.assertRaises(exc.HTTPBadRequest, self.controller.index, req,
uuidsentinel.fake_notification1)
@mock.patch.object(ha_api.VMoveAPI, 'get_vmove')
def test_show(self, mock_get_vmove):
mock_get_vmove.return_value = self.vmove_1
result = self.controller.show(self.req,
uuidsentinel.fake_notification1,
uuidsentinel.fake_vmove_1)
vmove = result['vmove']
self._assert_vmove_data(self.vmove_1,
_make_vmove_obj(vmove))
@mock.patch.object(ha_api.VMoveAPI, 'get_vmove')
def test_show_with_non_existing_id(self, mock_get_vmove):
mock_get_vmove.side_effect = exception.VMoveNotFound(id="2")
self.assertRaises(exc.HTTPNotFound,
self.controller.show, self.req,
uuidsentinel.fake_notification1, "2")
class VMoveTestCasePolicyNotAuthorized(test.NoDBTestCase):
"""Test Case for vmove non admin."""
def _set_up(self):
self.controller = vmoves.VMovesController()
self.req = fakes.HTTPRequest.blank(
'/v1/notifications/%s/vmoves' % (
uuidsentinel.fake_notification1))
self.context = self.req.environ['masakari.context']
def setUp(self):
super(VMoveTestCasePolicyNotAuthorized, self).setUp()
self._set_up()
def _check_rule(self, exc, rule_name):
self.assertEqual(
"Policy doesn't allow %s to be performed." % rule_name,
exc.format_message())
def test_index_no_admin(self):
rule_name = "os_masakari_api:vmoves:index"
self.policy.set_rules({rule_name: "project:non_fake"})
exc = self.assertRaises(exception.PolicyNotAuthorized,
self.controller.index,
self.req, uuidsentinel.fake_notification1)
self._check_rule(exc, rule_name)
def test_show_no_admin(self):
rule_name = "os_masakari_api:vmoves:detail"
self.policy.set_rules({rule_name: "project:non_fake"})
exc = self.assertRaises(exception.PolicyNotAuthorized,
self.controller.show,
self.req, uuidsentinel.fake_notification1,
uuidsentinel.fake_vmove_1)
self._check_rule(exc, rule_name)

118
masakari/tests/unit/db/test_db_api.py
View File

@ -501,3 +501,121 @@ class NotificationsTestCase(test.TestCase, ModelsObjectComparatorMixin):
self.assertRaises(exception.InvalidSortKey,
db.notifications_get_all_by_filters,
context=self.ctxt, sort_keys=['invalid_sort_key'])
class VMoveTestCase(test.TestCase, ModelsObjectComparatorMixin):
def setUp(self):
super(VMoveTestCase, self).setUp()
self.ctxt = context.get_admin_context()
def _get_fake_values(self):
return {
'uuid': uuidsentinel.vmove,
'notification_uuid': uuidsentinel.notification,
'instance_uuid': uuidsentinel.instance_uuid,
'instance_name': 'fake_instance',
'source_host': 'fake_host1',
'dest_host': 'fake_host2',
'start_time': None,
'end_time': None,
'type': 'evacuation',
'status': 'pending',
'message': None
}
def _get_fake_values_list(self):
return [
{'uuid': uuidsentinel.vmove_1,
'notification_uuid': uuidsentinel.notification,
'instance_uuid': uuidsentinel.instance_uuid_1,
'instance_name': 'fake_instance1',
'source_host': 'fake_host1',
'dest_host': None,
'start_time': None,
'end_time': None,
'type': 'evacuation',
'status': 'pending',
'message': None},
{'uuid': uuidsentinel.vmove_2,
'notification_uuid': uuidsentinel.notification,
'instance_uuid': uuidsentinel.instance_uuid_2,
'instance_name': 'fake_instance2',
'source_host': 'fake_host1',
'dest_host': None,
'start_time': None,
'end_time': None,
'type': 'evacuation',
'status': 'pending',
'message': None},
{'uuid': uuidsentinel.vmove_3,
'notification_uuid': uuidsentinel.notification,
'instance_uuid': uuidsentinel.instance_uuid_3,
'instance_name': 'fake_instance3',
'source_host': 'fake_host1',
'dest_host': None,
'start_time': None,
'end_time': None,
'type': 'evacuation',
'status': 'pending',
'message': None}]
def _create_vmove(self, values):
return db.vmove_create(self.ctxt, values)
def _test_get_vmove(self, method, filter):
vmoves = [self._create_vmove(p)
for p in self._get_fake_values_list()]
for vmove in vmoves:
real_vmove = method(self.ctxt, vmove[filter])
self._assertEqualObjects(vmove, real_vmove)
def test_vmove_create(self):
vmove = self._create_vmove(self._get_fake_values())
self.assertIsNotNone(vmove['uuid'])
ignored_keys = ['deleted', 'created_at', 'updated_at', 'deleted_at',
'id']
self._assertEqualObjects(vmove, self._get_fake_values(),
ignored_keys)
def test_vmove_get_by_uuid(self):
self._test_get_vmove(db.vmove_get_by_uuid, 'uuid')
def test_vmove_update(self):
update = {'dest_host': 'fake_host2', 'status': 'succeeded'}
updated = {'uuid': uuidsentinel.vmove,
'notification_uuid': uuidsentinel.notification,
'instance_uuid': uuidsentinel.instance_uuid,
'instance_name': 'fake_instance',
'source_host': 'fake_host1',
'dest_host': 'fake_host2',
'start_time': None,
'end_time': None,
'type': 'evacuation',
'status': 'succeeded',
'message': None}
ignored_keys = ['deleted', 'created_at', 'updated_at', 'deleted_at',
'id']
self._create_vmove(self._get_fake_values())
db.vmove_update(self.ctxt, uuidsentinel.vmove, update)
vmove_updated = db.vmove_get_by_uuid(
self.ctxt, uuidsentinel.vmove)
self._assertEqualObjects(updated, vmove_updated, ignored_keys)
def test_vmove_not_found(self):
self._create_vmove(self._get_fake_values())
self.assertRaises(exception.VMoveNotFound,
db.vmove_get_by_uuid, self.ctxt,
500)
def test_invalid_marker(self):
[self._create_vmove(p) for p in self._get_fake_values_list()]
self.assertRaises(exception.MarkerNotFound,
db.vmoves_get_all_by_filters,
context=self.ctxt, marker=6)
def test_invalid_sort_key(self):
[self._create_vmove(p) for p in self._get_fake_values_list()]
self.assertRaises(exception.InvalidSortKey,
db.vmoves_get_all_by_filters,
context=self.ctxt, sort_keys=['invalid_sort_key'])

13
masakari/tests/unit/db/test_migrations.py
View File

@ -217,6 +217,19 @@ class MasakariMigrationsCheckers(test_migrations.WalkVersionsMixin):
def _check_007(self, engine, data):
self.assertColumnExists(engine, 'failover_segments', 'enabled')
def _check_008(self, engine, data):
self.assertColumnExists(engine, 'vmoves', 'uuid')
self.assertColumnExists(engine, 'vmoves', 'notification_uuid')
self.assertColumnExists(engine, 'vmoves', 'instance_uuid')
self.assertColumnExists(engine, 'vmoves', 'instance_name')
self.assertColumnExists(engine, 'vmoves', 'source_host')
self.assertColumnExists(engine, 'vmoves', 'dest_host')
self.assertColumnExists(engine, 'vmoves', 'start_time')
self.assertColumnExists(engine, 'vmoves', 'end_time')
self.assertColumnExists(engine, 'vmoves', 'type')
self.assertColumnExists(engine, 'vmoves', 'status')
self.assertColumnExists(engine, 'vmoves', 'message')
class TestMasakariMigrationsSQLite(
MasakariMigrationsCheckers,

178
masakari/tests/unit/engine/drivers/taskflow/test_host_failure_flow.py
View File

@ -27,8 +27,12 @@ from masakari import context
from masakari.engine.drivers.taskflow import host_failure
from masakari.engine import manager
from masakari import exception
from masakari import objects
from masakari.objects import fields
from masakari.objects import vmove as vmove_obj
from masakari import test
from masakari.tests.unit import fakes
from masakari.tests import uuidsentinel
CONF = conf.CONF
@ -50,13 +54,19 @@ class HostFailureTestCase(test.TestCase):
self.override_config("evacuate_all_instances",
False, "host_failure")
self.instance_host = "fake-host"
self.notification_uuid = uuidsentinel.fake_notification
self.novaclient = nova.API()
self.fake_client = fakes.FakeNovaClient()
self.disabled_reason = CONF.host_failure.service_disable_reason
def _verify_instance_evacuated(self, old_instance_list):
for server in old_instance_list:
instance = self.novaclient.get_server(self.ctxt, server)
def _verify_instance_evacuated(self):
all_vmoves = objects.VMoveList.get_all_vmoves(
self.ctxt, self.notification_uuid)
for vmove in all_vmoves:
instance = self.novaclient.get_server(self.ctxt,
vmove.instance_uuid)
if getattr(instance, 'OS-EXT-STS:vm_state') in \
['active', 'stopped', 'error']:
@ -92,10 +102,15 @@ class HostFailureTestCase(test.TestCase):
def _test_instance_list(self, instances_evacuation_count):
task = host_failure.PrepareHAEnabledInstancesTask(self.ctxt,
self.novaclient)
instances = task.execute(self.instance_host)
instance_uuid_list = []
for instance_id in instances['instance_list']:
instance = self.novaclient.get_server(self.ctxt, instance_id)
task.execute(self.instance_host, self.notification_uuid)
all_vmoves = objects.VMoveList.get_all_vmoves(
self.ctxt,
self.notification_uuid)
for vmove in all_vmoves:
instance = self.novaclient.get_server(self.ctxt,
vmove.instance_uuid)
if CONF.host_failure.ignore_instances_in_error_state:
self.assertNotEqual("error",
getattr(instance, "OS-EXT-STS:vm_state"))
@ -104,34 +119,25 @@ class HostFailureTestCase(test.TestCase):
.ha_enabled_instance_metadata_key)
self.assertTrue(instance.metadata.get(ha_enabled_key, False))
instance_uuid_list.append(instance.id)
self.assertEqual(instances_evacuation_count, len(all_vmoves))
self.assertEqual(instances_evacuation_count,
len(instances['instance_list']))
return {
"instance_list": instance_uuid_list,
}
def _evacuate_instances(self, instance_list, mock_enable_disable,
reserved_host=None):
def _evacuate_instances(self, mock_enable_disable, reserved_host=None):
task = host_failure.EvacuateInstancesTask(
self.ctxt, self.novaclient,
update_host_method=manager.update_host_method)
old_instance_list = copy.deepcopy(instance_list['instance_list'])
if reserved_host:
task.execute(self.instance_host,
instance_list['instance_list'],
self.notification_uuid,
reserved_host=reserved_host)
self.assertTrue(mock_enable_disable.called)
else:
task.execute(
self.instance_host, instance_list['instance_list'])
self.instance_host, self.notification_uuid)
# make sure instance is active and has different host
self._verify_instance_evacuated(old_instance_list)
self._verify_instance_evacuated()
@mock.patch.object(nova.API, "get_server")
@mock.patch.object(nova.API, "evacuate_instance")
@ -150,20 +156,24 @@ class HostFailureTestCase(test.TestCase):
return fake_server
failed_evacuation_instances = []
fake_instance = self.fake_client.servers.create(
id="1", host=self.instance_host, ha_enabled=True)
vmove = vmove_obj.VMove(context=self.ctxt)
vmove.instance_uuid = fake_instance.id
vmove.instance_name = fake_instance.name
vmove.notification_uuid = self.notification_uuid
vmove.source_host = self.instance_host
vmove.status = fields.VMoveStatus.PENDING
vmove.type = fields.VMoveType.EVACUATION
vmove.create()
_mock_get.side_effect = [
get_fake_server(fake_instance, 'active'),
get_fake_server(fake_instance, 'error'),
]
task._evacuate_and_confirm(self.ctxt, fake_instance,
self.instance_host,
failed_evacuation_instances)
self.assertIn(fake_instance.id, failed_evacuation_instances)
expected_log = 'Failed to evacuate instance %s' % fake_instance.id
_mock_log.warning.assert_called_once_with(expected_log)
task._evacuate_and_confirm(self.ctxt, vmove)
self.assertEqual(fields.VMoveStatus.FAILED, vmove.status)
@mock.patch('masakari.compute.nova.novaclient')
@mock.patch('masakari.engine.drivers.taskflow.base.MasakariTask.'
@ -184,10 +194,10 @@ class HostFailureTestCase(test.TestCase):
self._test_disable_compute_service(mock_enable_disable)
# execute PrepareHAEnabledInstancesTask
instance_list = self._test_instance_list(2)
self._test_instance_list(2)
# execute EvacuateInstancesTask
self._evacuate_instances(instance_list, mock_enable_disable)
self._evacuate_instances(mock_enable_disable)
# verify progress details
_mock_notify.assert_has_calls([
@ -202,9 +212,9 @@ class HostFailureTestCase(test.TestCase):
"considered for evacuation. Total count is: '2'", 0.8),
mock.call("Instances to be evacuated are: '1,2'", 1.0),
mock.call("Start evacuation of instances from failed host "
"'fake-host', instance uuids are: '1,2'"),
mock.call("Evacuation of instance started: '1'", 0.5),
"'fake-host', instance uuids are: '2,1'"),
mock.call("Evacuation of instance started: '2'", 0.5),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Successfully evacuate instances '1,2' from host "
"'fake-host'", 0.7),
mock.call('Evacuation process completed!', 1.0)
@ -236,10 +246,10 @@ class HostFailureTestCase(test.TestCase):
self._test_disable_compute_service(mock_enable_disable)
# execute PrepareHAEnabledInstancesTask
instance_list = self._test_instance_list(1)
self._test_instance_list(1)
# execute EvacuateInstancesTask
self._evacuate_instances(instance_list, mock_enable_disable)
self._evacuate_instances(mock_enable_disable)
# verify progress details
_mock_notify.assert_has_calls([
@ -283,12 +293,12 @@ class HostFailureTestCase(test.TestCase):
self._test_disable_compute_service(mock_enable_disable)
# execute PrepareHAEnabledInstancesTask
instance_list = self._test_instance_list(2)
self._test_instance_list(2)
# execute EvacuateInstancesTask
with mock.patch.object(manager, "update_host_method") as mock_save:
self._evacuate_instances(
instance_list, mock_enable_disable,
mock_enable_disable,
reserved_host=reserved_host.name)
self.assertEqual(1, mock_save.call_count)
self.assertIn(reserved_host.name,
@ -299,22 +309,22 @@ class HostFailureTestCase(test.TestCase):
mock.call("Disabling compute service on host: 'fake-host'"),
mock.call("Disabled compute service on host: 'fake-host'", 1.0),
mock.call('Preparing instances for evacuation'),
mock.call("Total instances running on failed host 'fake-host' is 2"
"", 0.3),
mock.call("Total instances running on failed host 'fake-host' is "
"2", 0.3),
mock.call("Total HA Enabled instances count: '1'", 0.6),
mock.call("Total Non-HA Enabled instances count: '1'", 0.7),
mock.call("All instances (HA Enabled/Non-HA Enabled) should be "
"considered for evacuation. Total count is: '2'", 0.8),
mock.call("Instances to be evacuated are: '1,2'", 1.0),
mock.call("Start evacuation of instances from failed host "
"'fake-host', instance uuids are: '1,2'"),
"'fake-host', instance uuids are: '2,1'"),
mock.call("Enabling reserved host: 'fake-reserved-host'", 0.1),
mock.call('Add host fake-reserved-host to aggregate fake_agg',
0.2),
mock.call('Added host fake-reserved-host to aggregate fake_agg',
0.3),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Evacuation of instance started: '2'", 0.5),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Successfully evacuate instances '1,2' from host "
"'fake-host'", 0.7),
mock.call('Evacuation process completed!', 1.0)
@ -348,12 +358,12 @@ class HostFailureTestCase(test.TestCase):
self._test_disable_compute_service(mock_enable_disable)
# execute PrepareHAEnabledInstancesTask
instance_list = self._test_instance_list(2)
self._test_instance_list(2)
# execute EvacuateInstancesTask
with mock.patch.object(manager, "update_host_method") as mock_save:
self._evacuate_instances(
instance_list, mock_enable_disable,
mock_enable_disable,
reserved_host=reserved_host.name)
self.assertEqual(1, mock_save.call_count)
self.assertIn(reserved_host.name,
@ -374,7 +384,7 @@ class HostFailureTestCase(test.TestCase):
"considered for evacuation. Total count is: '2'", 0.8),
mock.call("Instances to be evacuated are: '1,2'", 1.0),
mock.call("Start evacuation of instances from failed host "
"'fake-host', instance uuids are: '1,2'"),
"'fake-host', instance uuids are: '2,1'"),
mock.call("Enabling reserved host: 'fake-reserved-host'", 0.1),
mock.call('Add host fake-reserved-host to aggregate fake_agg_1',
0.2),
@ -384,8 +394,8 @@ class HostFailureTestCase(test.TestCase):
0.2),
mock.call('Added host fake-reserved-host to aggregate fake_agg_2',
0.3),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Evacuation of instance started: '2'", 0.5),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Successfully evacuate instances '1,2' from host "
"'fake-host'", 0.7),
mock.call('Evacuation process completed!', 1.0)
@ -421,12 +431,12 @@ class HostFailureTestCase(test.TestCase):
self._test_disable_compute_service(mock_enable_disable)
# execute PrepareHAEnabledInstancesTask
instance_list = self._test_instance_list(1)
self._test_instance_list(1)
# execute EvacuateInstancesTask
with mock.patch.object(manager, "update_host_method") as mock_save:
self._evacuate_instances(
instance_list, mock_enable_disable,
mock_enable_disable,
reserved_host=reserved_host.name)
self.assertEqual(1, mock_save.call_count)
self.assertIn(reserved_host.name,
@ -476,23 +486,17 @@ class HostFailureTestCase(test.TestCase):
power_state=power_state,
ha_enabled=True)
instance_uuid_list = []
for instance in self.fake_client.servers.list():
instance_uuid_list.append(instance.id)
instance_list = {
"instance_list": instance_uuid_list,
}
self._test_instance_list(2)
# execute EvacuateInstancesTask
self._evacuate_instances(instance_list, mock_enable_disable)
self._evacuate_instances(mock_enable_disable)
# verify progress details
_mock_notify.assert_has_calls([
mock.call("Start evacuation of instances from failed host "
"'fake-host', instance uuids are: '1,2'"),
mock.call("Evacuation of instance started: '1'", 0.5),
"'fake-host', instance uuids are: '2,1'"),
mock.call("Evacuation of instance started: '2'", 0.5),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Successfully evacuate instances '1,2' from host "
"'fake-host'", 0.7),
mock.call('Evacuation process completed!', 1.0)
@ -519,10 +523,10 @@ class HostFailureTestCase(test.TestCase):
ha_enabled=True)
# execute PrepareHAEnabledInstancesTask
instance_list = self._test_instance_list(1)
self._test_instance_list(1)
# execute EvacuateInstancesTask
self._evacuate_instances(instance_list, mock_enable_disable)
self._evacuate_instances(mock_enable_disable)
# verify progress details
_mock_notify.assert_has_calls([
@ -565,7 +569,7 @@ class HostFailureTestCase(test.TestCase):
task = host_failure.PrepareHAEnabledInstancesTask(self.ctxt,
self.novaclient)
self.assertRaises(exception.SkipHostRecoveryException, task.execute,
self.instance_host)
self.instance_host, self.notification_uuid)
# verify progress details
_mock_notify.assert_has_calls([
@ -598,7 +602,7 @@ class HostFailureTestCase(test.TestCase):
task = host_failure.PrepareHAEnabledInstancesTask(self.ctxt,
self.novaclient)
self.assertRaises(exception.SkipHostRecoveryException, task.execute,
self.instance_host)
self.instance_host, self.notification_uuid)
# verify progress details
_mock_notify.assert_has_calls([
@ -628,13 +632,7 @@ class HostFailureTestCase(test.TestCase):
host=self.instance_host,
ha_enabled=True)
instance_uuid_list = []
for instance in self.fake_client.servers.list():
instance_uuid_list.append(instance.id)
instance_list = {
"instance_list": instance_uuid_list,
}
self._test_instance_list(1)
def fake_get_server(context, host):
# assume that while evacuating instance goes into error state
@ -646,7 +644,7 @@ class HostFailureTestCase(test.TestCase):
# execute EvacuateInstancesTask
self.assertRaises(
exception.HostRecoveryFailureException,
self._evacuate_instances, instance_list, mock_enable_disable)
self._evacuate_instances, mock_enable_disable)
# verify progress details
_mock_notify.assert_has_calls([
@ -674,7 +672,7 @@ class HostFailureTestCase(test.TestCase):
task = host_failure.PrepareHAEnabledInstancesTask(self.ctxt,
self.novaclient)
self.assertRaises(exception.SkipHostRecoveryException, task.execute,
self.instance_host)
self.instance_host, self.notification_uuid)
# verify progress details
_mock_notify.assert_has_calls([
@ -715,34 +713,19 @@ class HostFailureTestCase(test.TestCase):
task_state='fake_task_state',
power_state=None,
ha_enabled=True)
instance_uuid_list = []
for instance in self.fake_client.servers.list():
instance_uuid_list.append(instance.id)
instance_list = {
"instance_list": instance_uuid_list,
}
self._test_instance_list(3)
# execute EvacuateInstancesTask
self._evacuate_instances(instance_list, mock_enable_disable)
reset_calls = [('1', 'active'),
('2', 'stopped'),
('3', 'error'),
('3', 'stopped')]
stop_calls = ['2', '3']
self.assertEqual(reset_calls,
self.fake_client.servers.reset_state_calls)
self.assertEqual(stop_calls,
self.fake_client.servers.stop_calls)
self._evacuate_instances(mock_enable_disable)
# verify progress details
_mock_notify.assert_has_calls([
mock.call("Start evacuation of instances from failed host "
"'fake-host', instance uuids are: '1,2,3'"),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Evacuation of instance started: '2'", 0.5),
"'fake-host', instance uuids are: '3,2,1'"),
mock.call("Evacuation of instance started: '3'", 0.5),
mock.call("Evacuation of instance started: '2'", 0.5),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Successfully evacuate instances '1,2,3' from host "
"'fake-host'", 0.7),
mock.call('Evacuation process completed!', 1.0)
@ -773,12 +756,12 @@ class HostFailureTestCase(test.TestCase):
self._test_disable_compute_service(mock_enable_disable)
# execute PrepareHAEnabledInstancesTask
instance_list = self._test_instance_list(2)
self._test_instance_list(2)
# execute EvacuateInstancesTask
with mock.patch.object(manager, "update_host_method") as mock_save:
self._evacuate_instances(
instance_list, mock_enable_disable,
mock_enable_disable,
reserved_host=reserved_host.name)
self.assertEqual(1, mock_save.call_count)
self.assertIn(reserved_host.name,
@ -797,14 +780,14 @@ class HostFailureTestCase(test.TestCase):
"considered for evacuation. Total count is: '2'", 0.8),
mock.call("Instances to be evacuated are: '1,2'", 1.0),
mock.call("Start evacuation of instances from failed host "
"'fake-host', instance uuids are: '1,2'"),
"'fake-host', instance uuids are: '2,1'"),
mock.call("Enabling reserved host: 'fake-reserved-host'", 0.1),
mock.call('Add host fake-reserved-host to aggregate fake_agg',
0.2),
mock.call('Added host fake-reserved-host to aggregate fake_agg',
0.3),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Evacuation of instance started: '2'", 0.5),
mock.call("Evacuation of instance started: '1'", 0.5),
mock.call("Successfully evacuate instances '1,2' from host "
"'fake-host'", 0.7),
mock.call('Evacuation process completed!', 1.0)
@ -822,16 +805,11 @@ class HostFailureTestCase(test.TestCase):
task_state=None,
power_state=None,
ha_enabled=True)
instance_uuid_list = []
for instance in self.fake_client.servers.list():
instance_uuid_list.append(instance.id)
instance_list = {
"instance_list": instance_uuid_list,
}
self._test_instance_list(1)
# execute EvacuateInstancesTask
self._evacuate_instances(instance_list, mock_enable_disable)
self._evacuate_instances(mock_enable_disable)
# If vm_state=stopped and task_state=None, reset_state and stop API
# will not be called.

29
masakari/tests/unit/fakes.py
View File

@ -45,6 +45,7 @@ class FakeNovaClient(object):
self.id = id
self.uuid = uuid or uuidutils.generate_uuid()
self.host = host
self.name = 'fake_instance'
setattr(self, 'OS-EXT-SRV-ATTR:hypervisor_hostname', host)
setattr(self, 'OS-EXT-STS:vm_state', vm_state)
setattr(self, 'OS-EXT-STS:task_state', task_state)
@ -226,3 +227,31 @@ def create_fake_notification_progress_details(
return objects.NotificationProgressDetails(
name=name, uuid=uuid, progress=progress, state=state,
progress_details=progress_details)
def create_fake_vmove(
id=1,
uuid=uuidsentinel.fake_vmove,
notification_uuid=uuidsentinel.fake_notification,
instance_uuid=uuidsentinel.fake_instance,
instance_name='fake_instance',
source_host='fake_host_1',
dest_host='fake_host_2',
start_time=None,
end_time=None,
type='evacuation',
status='pending',
message=None):
return objects.VMove(
id=id,
uuid=uuid,
notification_uuid=notification_uuid,
instance_uuid=instance_uuid,
instance_name=instance_name,
source_host=source_host,
dest_host=dest_host,
start_time=start_time,
end_time=end_time,
type=type,
status=status,
message=message)

104
masakari/tests/unit/ha/test_api.py
View File

@ -32,6 +32,7 @@ from masakari.objects import fields
from masakari.objects import host as host_obj
from masakari.objects import notification as notification_obj
from masakari.objects import segment as segment_obj
from masakari.objects import vmove as vmove_obj
from masakari import test
from masakari.tests.unit.api.openstack import fakes
from masakari.tests.unit import fakes as fakes_data
@ -40,6 +41,10 @@ from masakari.tests import uuidsentinel
NOW = timeutils.utcnow().replace(microsecond=0)
def _make_vmove_obj(vmove_dict):
return vmove_obj.VMove(**vmove_dict)
def _make_segment_obj(segment_dict):
return segment_obj.FailoverSegment(**segment_dict)
@ -938,3 +943,102 @@ class NotificationAPITestCase(test.NoDBTestCase):
self.assertRaises(exception.InvalidInput,
self.notification_api.get_all,
self.context, self.req)
class VMoveAPITestCase(test.NoDBTestCase):
"""Test Case for vmove api."""
def setUp(self):
super(VMoveAPITestCase, self).setUp()
self.vmove_api = ha_api.VMoveAPI()
self.req = fakes.HTTPRequest.blank(
'/v1/notifications/%s/vmoves' % (
uuidsentinel.fake_notification),
use_admin_context=True)
self.context = self.req.environ['masakari.context']
self.notification = fakes_data.create_fake_notification(
id=1,
type="COMPUTE_HOST",
source_host_uuid=uuidsentinel.fake_host_1,
status="running",
notification_uuid=uuidsentinel.fake_notification,
payload={'event': 'STOPPED',
'host_status': 'NORMAL',
'cluster_status': 'ONLINE'}
)
self.vmove = fakes_data.create_fake_vmove(
id=1,
uuid=uuidsentinel.fake_vmove,
notification_uuid=self.notification.notification_uuid,
instance_uuid=uuidsentinel.fake_instance_1,
instance_name='vm1',
source_host='host_1',
dest_host='host_2',
start_time='2022-11-22 14:50:22',
end_time='2022-11-22 14:50:22',
type='evacuation',
status='succeeded',
message=None
)
def _assert_vmove_data(self, expected, actual):
self.assertTrue(obj_base.obj_equal_prims(expected, actual),
"The vmove objects were not equal")
@mock.patch.object(vmove_obj.VMoveList, 'get_all')
@mock.patch.object(notification_obj.Notification, 'get_by_uuid')
def test_get_all(self, mock_get, mock_get_all):
mock_get.return_value = self.notification
fake_vmove_list = [self.vmove]
mock_get_all.return_value = fake_vmove_list
result = self.vmove_api.get_all(self.context,
uuidsentinel.fake_notification,
filters={},
sort_keys=['created_at'],
sort_dirs=['desc'],
limit=None,
marker=None)
for i in range(len(result)):
self._assert_vmove_data(
fake_vmove_list[i], _make_vmove_obj(result[i]))
@mock.patch.object(vmove_obj.VMoveList, 'get_all')
@mock.patch.object(notification_obj.Notification, 'get_by_uuid')
def test_get_all_with_invalid_notification(self, mock_get, mock_get_all):
vm_type_notification = fakes_data.create_fake_notification(
id=1,
type="VM",
source_host_uuid=uuidsentinel.fake_host_2,
status="running",
notification_uuid=uuidsentinel.fake_vm_type_notification,
payload={'event': 'STOPPED',
'host_status': 'NORMAL',
'cluster_status': 'ONLINE'}
)
mock_get.return_value = vm_type_notification
self.assertRaises(exception.NotificationWithoutVMoves,
self.vmove_api.get_all, self.context,
uuidsentinel.fake_vm_type_notification)
@mock.patch.object(vmove_obj.VMove, 'get_by_uuid')
@mock.patch.object(notification_obj.Notification, 'get_by_uuid')
def test_get_vmove(self, mock_get, mock_get_vmove):
mock_get.return_value = self.notification
mock_get_vmove.return_value = self.vmove
result = self.vmove_api.get_vmove(
self.context,
uuidsentinel.fake_notification,
uuidsentinel.fake_vmove)
self._assert_vmove_data(self.vmove, result)
@mock.patch.object(vmove_obj.VMove, 'get_by_uuid')
@mock.patch.object(notification_obj.Notification, 'get_by_uuid')
def test_get_vmove_not_found(self, mock_get_notification, mock_get_vmove):
mock_get_notification.return_value = self.notification
self.assertRaises(exception.VMoveNotFound,
self.vmove_api.get_vmove, self.context,
uuidsentinel.fake_notification,
"123")

4
masakari/tests/unit/objects/test_objects.py
View File

@ -674,7 +674,9 @@ object_data = {
'MyOwnedObject': '1.0-fec853730bd02d54cc32771dd67f08a0',
'SegmentApiNotification': '1.0-1187e93f564c5cca692db76a66cda2a6',
'SegmentApiPayload': '1.1-e34e1c772e16e9ad492067ee98607b1d',
'SegmentApiPayloadBase': '1.1-6a1db76f3e825f92196fc1a11508d886'
'SegmentApiPayloadBase': '1.1-6a1db76f3e825f92196fc1a11508d886',
'VMove': '1.0-5c4d8667b5612b8a49adc065f8961aa2',
'VMoveList': '1.0-63fff36dee683c7a1555798cb233ad3f'
}

120
masakari/tests/unit/objects/test_vmoves.py Normal file
View File

@ -0,0 +1,120 @@
# Copyright(c) 2022 Inspur
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from unittest import mock
from oslo_utils import timeutils
from masakari import exception
from masakari.objects import vmove
from masakari.tests.unit.objects import test_objects
from masakari.tests import uuidsentinel
NOW = timeutils.utcnow().replace(microsecond=0)
fake_vmove = {
'created_at': NOW,
'updated_at': None,
'deleted_at': None,
'deleted': False,
'id': 1,
'uuid': uuidsentinel.fake_vmove,
'notification_uuid': uuidsentinel.fake_notification,
'instance_uuid': uuidsentinel.fake_instance,
'instance_name': 'fake_vm',
'source_host': 'fake_host1',
'dest_host': None,
'start_time': None,
'end_time': None,
'status': 'pending',
'type': 'evacuation',
'message': None
}
class TestVMoveObject(test_objects._LocalTest):
@mock.patch('masakari.db.vmove_get_by_uuid')
def test_get_by_uuid(self, mock_api_get):
mock_api_get.return_value = fake_vmove
vmove_obj = vmove.VMove.get_by_uuid(
self.context, uuidsentinel.fake_vmove)
self.compare_obj(vmove_obj, fake_vmove)
mock_api_get.assert_called_once_with(self.context,
uuidsentinel.fake_vmove)
def _vmove_create_attributes(self):
vmove_obj = vmove.VMove(context=self.context)
vmove_obj.uuid = uuidsentinel.fake_vmove
vmove_obj.notification_uuid = uuidsentinel.fake_notification
vmove_obj.instance_uuid = uuidsentinel.fake_instance
vmove_obj.instance_name = 'fake_vm1'
vmove_obj.source_host = 'fake_host1'
vmove_obj.status = 'pending'
vmove_obj.type = 'evacuation'
return vmove_obj
@mock.patch('masakari.db.vmove_create')
def test_create(self, mock_vmove_create):
mock_vmove_create.return_value = fake_vmove
vmove_obj = self._vmove_create_attributes()
vmove_obj.create()
self.compare_obj(vmove_obj, fake_vmove)
mock_vmove_create.assert_called_once_with(self.context, {
'uuid': uuidsentinel.fake_vmove,
'notification_uuid': uuidsentinel.fake_notification,
'instance_uuid': uuidsentinel.fake_instance,
'instance_name': 'fake_vm1',
'source_host': 'fake_host1',
'status': 'pending',
'type': 'evacuation'
})
@mock.patch('masakari.db.vmoves_get_all_by_filters')
def test_get_limit_and_marker_invalid_marker(self, mock_api_get):
vmove_uuid = uuidsentinel.fake_vmove
mock_api_get.side_effect = (exception.
MarkerNotFound(marker=vmove_uuid))
self.assertRaises(exception.MarkerNotFound,
vmove.VMoveList.get_all,
self.context, limit=5, marker=vmove_uuid)
@mock.patch('masakari.db.vmove_update')
def test_save(self, mock_vmove_update):
mock_vmove_update.return_value = fake_vmove
vmove_obj = self._vmove_create_attributes()
vmove_obj.uuid = uuidsentinel.fake_vmove
vmove_obj.save()
self.compare_obj(vmove_obj, fake_vmove)
(mock_vmove_update.assert_called_once_with(
self.context, uuidsentinel.fake_vmove,
{'uuid': uuidsentinel.fake_vmove,
'notification_uuid': uuidsentinel.fake_notification,
'instance_uuid': uuidsentinel.fake_instance,
'instance_name': 'fake_vm1',
'source_host': 'fake_host1',
'status': 'pending',
'type': 'evacuation'}))

2
setup.cfg
View File

@ -61,7 +61,7 @@ masakari.api.v1.extensions =
segments = masakari.api.openstack.ha.segments:Segments
hosts = masakari.api.openstack.ha.hosts:Hosts
notifications = masakari.api.openstack.ha.notifications:Notifications
vmoves = masakari.api.openstack.ha.vmoves:VMoves
masakari.driver =
taskflow_driver = masakari.engine.drivers.taskflow:TaskFlowDriver