add enabled to segment

Sometimes, operators want to temporarily close instance-ha function.
This patch add 'enabled' to segment. If the segment 'enabled' value
is set False, all notifications of this segment will be ignored
and no recovery methods will execuate.

Change-Id: I561a2519626fa1beae1e3033a6de510cea8f3fac
Implements: BP enable-to-segment
This commit is contained in:
suzhengwei 2020-01-02 15:29:37 +08:00 committed by sue
parent 7f76081ccf
commit fe88eae9cb
19 changed files with 183 additions and 62 deletions

View File

@ -39,7 +39,8 @@ from masakari.i18n import _
REST_API_VERSION_HISTORY = """REST API Version History:
* 1.0 - Initial version.
* 1.1 - Add support for getting notification progress details
* 1.1 - Add support for getting notification progress details.
* 1.2 - Add enabled option to segment.
"""
# The minimum and maximum versions of the API supported
@ -48,7 +49,7 @@ REST_API_VERSION_HISTORY = """REST API Version History:
# Note: This only applies for the v1 API once microversions
# support is fully merged.
_MIN_API_VERSION = "1.0"
_MAX_API_VERSION = "1.1"
_MAX_API_VERSION = "1.2"
DEFAULT_API_VERSION = _MIN_API_VERSION

View File

@ -45,6 +45,9 @@ create = copy.deepcopy(_base)
create['properties']['segment']['required'] = ['name', 'recovery_method',
'service_type']
create_v12 = copy.deepcopy(create)
create_v12['properties']['segment']['properties']['enabled'] = \
parameter_types.boolean
update = copy.deepcopy(_base)
update['properties']['segment']['anyOf'] = [{'required': ['name']},
@ -52,3 +55,6 @@ update['properties']['segment']['anyOf'] = [{'required': ['name']},
{'required': ['recovery_method']},
{'required': ['service_type']},
]
update_v12 = copy.deepcopy(update)
update_v12['properties']['segment']['anyOf'].append({'required': ['enabled']})

View File

@ -47,10 +47,9 @@ class SegmentsController(wsgi.Controller):
sort_keys, sort_dirs = common.get_sort_params(req.params)
filters = {}
if 'recovery_method' in req.params:
filters['recovery_method'] = req.params['recovery_method']
if 'service_type' in req.params:
filters['service_type'] = req.params['service_type']
for field in ['recovery_method', 'service_type', 'enabled']:
if field in req.params:
filters[field] = req.params[field]
segments = self.api.get_all(context, filters=filters,
sort_keys=sort_keys,
@ -77,7 +76,8 @@ class SegmentsController(wsgi.Controller):
@wsgi.response(http.CREATED)
@extensions.expected_errors((http.FORBIDDEN, http.CONFLICT))
@validation.schema(schema.create)
@validation.schema(schema.create, '1.0', '1.1')
@validation.schema(schema.create_v12, '1.2')
def create(self, req, body):
"""Creates a new failover segment."""
context = req.environ['masakari.context']
@ -92,7 +92,8 @@ class SegmentsController(wsgi.Controller):
@extensions.expected_errors((http.FORBIDDEN, http.NOT_FOUND,
http.CONFLICT))
@validation.schema(schema.update)
@validation.schema(schema.update, '1.0', '1.1')
@validation.schema(schema.update_v12, '1.2')
def update(self, req, id, body):
"""Updates the existing segment."""
context = req.environ['masakari.context']

View File

@ -208,6 +208,9 @@ def failover_segment_get_all_by_filters(
if 'service_type' in filters:
query = query.filter(models.FailoverSegment.service_type == filters[
'service_type'])
if 'enabled' in filters:
query = query.filter(models.FailoverSegment.enabled == filters[
'enabled'])
marker_row = None
if marker is not None:

View File

@ -0,0 +1,26 @@
# Copyright 2020 Inspur.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import Column, MetaData, Table
from sqlalchemy import Boolean
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
segments_table = Table('failover_segments', meta, autoload=True)
enable_column = Column('enabled', Boolean, default=True)
segments_table.create_column(enable_column)

View File

@ -87,6 +87,7 @@ class FailoverSegment(BASE, MasakariAPIBase, models.SoftDeleteMixin):
uuid = Column(String(36), nullable=False)
name = Column(String(255), nullable=False)
service_type = Column(String(255), nullable=False)
enabled = Column(Boolean, default=True)
description = Column(Text)
recovery_method = Column(Enum('auto', 'reserved_host', 'auto_priority',
'rh_priority',

View File

@ -328,6 +328,20 @@ class MasakariManager(manager.Manager):
def process_notification(self, context, notification=None):
"""Processes the notification"""
host = objects.Host.get_by_uuid(
context, notification.source_host_uuid)
if not host.failover_segment.enabled:
update_data = {
'status': fields.NotificationStatus.IGNORED,
}
notification.update(update_data)
notification.save()
msg = ('Notification %(notification_uuid)s of type: %(type)s '
'is ignored, because the failover segment is disabled.',
{'notification_uuid': notification.notification_uuid,
'type': notification.type})
raise exception.FailoverSegmentDisabled(msg)
self._process_notification(context, notification)
@periodic_task.periodic_task(

View File

@ -370,3 +370,7 @@ class HostNotFoundUnderFailoverSegment(HostNotFound):
class InstanceEvacuateFailed(MasakariException):
msg_fmt = _("Failed to evacuate instance %(instance_uuid)s")
class FailoverSegmentDisabled(MasakariException):
msg_fmt = _('Failover segment is disabled.')

View File

@ -93,6 +93,8 @@ class FailoverSegmentAPI(object):
segment.description = segment_data.get('description')
segment.recovery_method = segment_data.get('recovery_method')
segment.service_type = segment_data.get('service_type')
segment.enabled = strutils.bool_from_string(
segment_data.get('enabled', True), strict=True)
try:
segment.create()

View File

@ -27,9 +27,11 @@ class SegmentApiPayloadBase(base.NotificationPayloadBase):
'service_type': ('segment', 'service_type'),
'description': ('segment', 'description'),
'recovery_method': ('segment', 'recovery_method'),
'enabled': ('segment', 'enabled'),
}
# Version 1.0: Initial version
VERSION = '1.0'
# Version 1.1: Add 'enabled' field
VERSION = '1.1'
fields = {
'id': fields.IntegerField(),
'uuid': fields.UUIDField(),
@ -37,6 +39,7 @@ class SegmentApiPayloadBase(base.NotificationPayloadBase):
'service_type': fields.StringField(),
'description': fields.StringField(nullable=True),
'recovery_method': fields.FailoverSegmentRecoveryMethodField(),
'enabled': fields.BooleanField(),
}
def __init__(self, segment, **kwargs):
@ -48,7 +51,7 @@ class SegmentApiPayloadBase(base.NotificationPayloadBase):
class SegmentApiPayload(SegmentApiPayloadBase):
# No SCHEMA as all the additional fields are calculated
VERSION = '1.0'
VERSION = '1.1'
fields = {
'fault': fields.ObjectField('ExceptionPayload', nullable=True),
}

View File

@ -15,6 +15,7 @@
from oslo_log import log as logging
from oslo_utils import uuidutils
from oslo_utils import versionutils
from masakari.api import utils as api_utils
from masakari import db
@ -29,17 +30,27 @@ LOG = logging.getLogger(__name__)
@base.MasakariObjectRegistry.register
class FailoverSegment(base.MasakariPersistentObject, base.MasakariObject,
base.MasakariObjectDictCompat):
VERSION = '1.0'
# 1.0, init
# 1.1, add enabled field
VERSION = '1.1'
fields = {
'id': fields.IntegerField(),
'uuid': fields.UUIDField(),
'name': fields.StringField(),
'service_type': fields.StringField(),
'enabled': fields.BooleanField(default=True),
'description': fields.StringField(nullable=True),
'recovery_method': fields.FailoverSegmentRecoveryMethodField(),
}
def obj_make_compatible(self, primitive, target_version):
super(FailoverSegment, self).obj_make_compatible(primitive,
target_version)
target_version = versionutils.convert_version_to_tuple(target_version)
if target_version < (1, 1) and 'enabled' in primitive:
del primitive['enabled']
@staticmethod
def _from_db_object(context, segment, db_segment):
for key in segment.fields:

View File

@ -62,7 +62,8 @@ class FailoverSegmentsTestCase(test.TestCase, ModelsObjectComparatorMixin):
'name': 'fake_name',
'service_type': 'fake_service_type',
'description': 'fake_description',
'recovery_method': 'auto'
'recovery_method': 'auto',
'enabled': True
}
def _get_fake_values_list(self):
@ -105,12 +106,15 @@ class FailoverSegmentsTestCase(test.TestCase, ModelsObjectComparatorMixin):
db.failover_segment_get_by_name, 'name')
def test_failover_segment_update(self):
update = {'name': 'updated_name', 'description': 'updated_desc'}
update = {'name': 'updated_name',
'description': 'updated_desc',
'enabled': False}
updated = {'uuid': uuidsentinel.fake_uuid,
'name': 'updated_name',
'service_type': 'fake_service_type',
'description': 'updated_desc',
'recovery_method': 'auto'}
'recovery_method': 'auto',
'enabled': False}
ignored_keys = ['deleted', 'created_at', 'updated_at', 'deleted_at',
'id']
self._create_failover_segment(self._get_fake_values())

View File

@ -210,6 +210,9 @@ class MasakariMigrationsCheckers(test_migrations.WalkVersionsMixin):
self.assertColumnExists(engine, 'atomdetails', 'revert_results')
self.assertColumnExists(engine, 'atomdetails', 'revert_failure')
def _check_007(self, engine, data):
self.assertColumnExists(engine, 'failover_segments', 'enabled')
class TestMasakariMigrationsSQLite(MasakariMigrationsCheckers,
test_base.DbTestCase):

View File

@ -64,6 +64,12 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
return exc
# else the workflow executed successfully
def _get_fake_host(self, segment_enabled):
segment = fakes.create_fake_failover_segment(enabled=segment_enabled)
host = fakes.create_fake_host()
host.failover_segment = segment
return host
def _get_process_type_notification(self):
return fakes.create_fake_notification(
type="PROCESS", id=1, payload={
@ -84,6 +90,20 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
status="new",
notification_uuid=uuidsentinel.fake_notification)
@mock.patch.object(host_obj.Host, "get_by_uuid")
@mock.patch.object(notification_obj.Notification, "save")
@mock.patch.object(engine_utils, 'notify_about_notification_update')
def test_process_notification_with_segment_disabled(
self, mock_notify_about_notification_update,
mock_notification_save, mock_host_get, mock_notification_get):
notification = _get_vm_type_notification()
mock_notification_get.return_value = notification
mock_host_get.return_value = self._get_fake_host(
segment_enabled=False)
self.assertRaises(exception.FailoverSegmentDisabled,
self.engine.process_notification,
self.context, notification)
@mock.patch("masakari.engine.drivers.taskflow."
"TaskFlowDriver.execute_instance_failure")
@mock.patch.object(notification_obj.Notification, "save")
@ -94,8 +114,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_instance_failure.side_effect = self._fake_notification_workflow()
notification = _get_vm_type_notification()
mock_notification_get.return_value = notification
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("finished", notification.status)
mock_instance_failure.assert_called_once_with(
self.context, notification.payload.get('instance_uuid'),
@ -123,8 +143,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
exc=exception.InstanceRecoveryFailureException)
notification = _get_vm_type_notification()
mock_notification_get.return_value = notification
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("error", notification.status)
mock_instance_failure.assert_called_once_with(
self.context, notification.payload.get('instance_uuid'),
@ -156,8 +176,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
notification_uuid=uuidsentinel.fake_notification)
mock_notification_get.return_value = notification
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("ignored", notification.status)
@mock.patch("masakari.engine.drivers.taskflow."
@ -173,8 +193,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_notification_get.return_value = notification
mock_instance_failure.side_effect = self._fake_notification_workflow(
exc=exception.SkipInstanceRecoveryException)
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("finished", notification.status)
mock_instance_failure.assert_called_once_with(
self.context, notification.payload.get('instance_uuid'),
@ -204,8 +224,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_process_failure.side_effect = self._fake_notification_workflow()
fake_host = fakes.create_fake_host()
mock_host_obj.return_value = fake_host
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("finished", notification.status)
mock_host_save.assert_called_once()
mock_process_failure.assert_called_once_with(
@ -240,8 +260,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_host_obj.return_value = fake_host
mock_process_failure.side_effect = self._fake_notification_workflow(
exc=exception.SkipProcessRecoveryException)
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("finished", notification.status)
mock_host_save.assert_called_once()
action = fields.EventNotificationAction.NOTIFICATION_PROCESS
@ -272,8 +292,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_host_obj.return_value = fake_host
mock_process_failure.side_effect = self._fake_notification_workflow(
exc=exception.ProcessRecoveryFailureException)
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("error", notification.status)
mock_host_save.assert_called_once()
e = exception.ProcessRecoveryFailureException('Failed to execute '
@ -304,8 +324,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
notification.payload['event'] = 'started'
fake_host = fakes.create_fake_host()
mock_host_obj.return_value = fake_host
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("finished", notification.status)
self.assertFalse(mock_process_failure.called)
action = fields.EventNotificationAction.NOTIFICATION_PROCESS
@ -328,8 +348,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
notification = self._get_process_type_notification()
mock_notification_get.return_value = notification
notification.payload['event'] = 'other'
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("ignored", notification.status)
self.assertFalse(mock_process_failure.called)
action = fields.EventNotificationAction.NOTIFICATION_PROCESS
@ -362,8 +382,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_get_all.return_value = None
fake_host.failover_segment = fakes.create_fake_failover_segment()
mock_host_obj.return_value = fake_host
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
update_data_by_host_failure = {
'on_maintenance': True,
@ -407,8 +427,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
notification = self._get_compute_host_type_notification()
mock_notification_get.return_value = notification
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
update_data_by_host_failure = {
'on_maintenance': True,
@ -456,8 +476,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_notification_get.return_value = notification
mock_host_failure.side_effect = self._fake_notification_workflow()
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
update_data_by_host_failure = {
'on_maintenance': True,
@ -510,8 +530,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
reserved_host_list = [host.name for host in
reserved_host_object_list]
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
update_data_by_host_failure = {
'on_maintenance': True,
@ -559,8 +579,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_host_obj.return_value = fake_host
mock_host_failure.side_effect = self._fake_notification_workflow(
exc=exception.HostRecoveryFailureException)
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
update_data_by_host_failure = {
'on_maintenance': True,
@ -606,8 +626,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
# mock_host_failure.side_effect = str(e)
mock_host_failure.side_effect = self._fake_notification_workflow(
exc=exception.SkipHostRecoveryException)
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
update_data_by_host_failure = {
'on_maintenance': True,
@ -635,8 +655,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
notification = self._get_compute_host_type_notification()
mock_notification_get.return_value = notification
notification.payload['event'] = 'started'
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("finished", notification.status)
self.assertFalse(mock_host_failure.called)
action = fields.EventNotificationAction.NOTIFICATION_PROCESS
@ -659,8 +679,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
notification = self._get_compute_host_type_notification()
mock_notification_get.return_value = notification
notification.payload['event'] = 'other'
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("ignored", notification.status)
self.assertFalse(mock_host_failure.called)
action = fields.EventNotificationAction.NOTIFICATION_PROCESS
@ -689,8 +709,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
id=1, uuid=uuidsentinel.fake_ins, host='fake_host',
vm_state='paused', ha_enabled=True)
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("ignored", notification.status)
self.assertFalse(mock_stop_server.called)
msg = ("Recovery of instance '%(instance_uuid)s' is ignored as it is "
@ -725,8 +745,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
id=1, uuid=uuidsentinel.fake_ins, host='fake_host',
vm_state='rescued', ha_enabled=True)
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
self.assertEqual("ignored", notification.status)
self.assertFalse(mock_stop_server.called)
msg = ("Recovery of instance '%(instance_uuid)s' is ignored as it is "
@ -752,8 +772,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
status="failed")
with mock.patch("masakari.engine.manager.LOG.warning") as mock_log:
self.engine.process_notification(self.context,
notification=noti_new)
self.engine._process_notification(self.context,
notification=noti_new)
mock_log.assert_called_once()
args = mock_log.call_args[0]
expected_log = ("Processing of notification is skipped to avoid "
@ -817,8 +837,8 @@ class EngineManagerUnitTestCase(test.NoDBTestCase):
mock_notification_save.side_effect = [notification, notification_new]
with mock.patch("masakari.engine.manager.LOG.warning") as mock_log:
self.engine.process_notification(self.context,
notification=notification)
self.engine._process_notification(self.context,
notification=notification)
mock_log.assert_called_once()
args = mock_log.call_args[0]
expected_log = ("Notification '%(uuid)s' ignored as host_status"

View File

@ -206,10 +206,11 @@ def create_fake_host(**updates):
def create_fake_failover_segment(name='fake_segment', id=1, description=None,
service_type='COMPUTE',
recovery_method="auto",
uuid=uuidsentinel.fake_segment):
uuid=uuidsentinel.fake_segment,
enabled=True):
return objects.FailoverSegment(
name=name, id=id, description=description, service_type=service_type,
recovery_method=recovery_method, uuid=uuid)
recovery_method=recovery_method, uuid=uuid, enabled=enabled)
def create_fake_notification_progress_details(

View File

@ -34,6 +34,7 @@ fake_segment_dict = {
'recovery_method': 'auto',
'description': 'fake',
'service_type': 'CINDER',
'enabled': True,
'id': 123,
'uuid': uuidsentinel.fake_segment,
'created_at': NOW,

View File

@ -653,7 +653,7 @@ class TestRegistry(test.NoDBTestCase):
# they come with a corresponding version bump in the affected
# objects
object_data = {
'FailoverSegment': '1.0-5e8b8bc8840b35439b5f2b621482d15d',
'FailoverSegment': '1.1-9cecc07c111f647b32d560f19f1f5db9',
'FailoverSegmentList': '1.0-dfc5c6f5704d24dcaa37b0bbb03cbe60',
'Host': '1.2-f05735b156b687bc916d46b551bc45e3',
'HostList': '1.0-25ebe1b17fbd9f114fae8b6a10d198c0',
@ -673,8 +673,8 @@ object_data = {
'MyObj': '1.6-ee7b607402fbfb3390a92ab7199e0d88',
'MyOwnedObject': '1.0-fec853730bd02d54cc32771dd67f08a0',
'SegmentApiNotification': '1.0-1187e93f564c5cca692db76a66cda2a6',
'SegmentApiPayload': '1.0-4c85836a1c2e4069b9dc84fa029a4657',
'SegmentApiPayloadBase': '1.0-93a7c8b78d0e9ea3f6811d4ed75fa799'
'SegmentApiPayload': '1.1-e34e1c772e16e9ad492067ee98607b1d',
'SegmentApiPayloadBase': '1.1-6a1db76f3e825f92196fc1a11508d886'
}

View File

@ -38,7 +38,8 @@ fake_segment = {
'name': 'foo-segment',
'service_type': 'COMPUTE',
'description': 'fake-description',
'recovery_method': 'auto'
'recovery_method': 'auto',
'enabled': True
}
@ -282,3 +283,14 @@ class TestFailoverSegmentObject(test_objects._LocalTest):
mock.call(self.context, segment_object, action=action,
phase=phase_start)]
mock_notify_about_segment_api.assert_has_calls(notify_calls)
def test_obj_make_compatible(self):
segment_obj = segment.FailoverSegment(context=self.context)
segment_obj.name = "foo-segment"
segment_obj.id = 123
segment_obj.uuid = uuidsentinel.fake_segment
segment_obj.enabled = True
primitive = segment_obj.obj_to_primitive('1.1')
self.assertIn('enabled', primitive['masakari_object.data'])
primitive = segment_obj.obj_to_primitive('1.0')
self.assertNotIn('enabled', primitive['masakari_object.data'])

View File

@ -0,0 +1,8 @@
---
features:
- |
Sometimes, operators want to temporarily disable instance-ha function.
This version adds 'enabled' to segment. If the segment 'enabled' value
is set False, all notifications of this segment will be ignored
and no recovery methods will execute.