Merge "Add Loki service plugin for optional DB havoc"
This commit is contained in:
commit
6ace0d42d3
|
@ -0,0 +1,38 @@
|
|||
..
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
not use this file except in compliance with the License. You may obtain
|
||||
a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
License for the specific language governing permissions and limitations
|
||||
under the License.
|
||||
|
||||
|
||||
Convention for heading levels in Neutron devref:
|
||||
======= Heading 0 (reserved for the title in a document)
|
||||
------- Heading 1
|
||||
~~~~~~~ Heading 2
|
||||
+++++++ Heading 3
|
||||
''''''' Heading 4
|
||||
(Avoid deeper levels because they do not render well.)
|
||||
|
||||
|
||||
Transient DB Failure Injection
|
||||
==============================
|
||||
|
||||
Neutron has a service plugin to inject random delays and Deadlock exceptions
|
||||
into normal Neutron operations. The service plugin is called 'Loki' and is
|
||||
located under neutron.services.loki.loki_plugin.
|
||||
|
||||
To enable the plugin, just add 'loki' to the list of service_plugins in your
|
||||
neutron-server neutron.conf file.
|
||||
|
||||
The plugin will inject a Deadlock exception on database flushes with a 1/50
|
||||
probability and a delay of 1 second with a 1/200 probability when SQLAlchemy
|
||||
objects are loaded into the persistent state from the DB. The goal is to ensure
|
||||
the code is tolerant of these transient delays/failures that will be experienced
|
||||
in busy production (and Galera) systems.
|
|
@ -91,6 +91,7 @@ Testing
|
|||
fullstack_testing
|
||||
testing_coverage
|
||||
template_model_sync_test
|
||||
db_transient_failure_injection
|
||||
|
||||
Module Reference
|
||||
----------------
|
||||
|
|
|
@ -0,0 +1,49 @@
|
|||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import random
|
||||
import time
|
||||
|
||||
from neutron.db import api as db_api
|
||||
from neutron.services import service_base
|
||||
|
||||
from oslo_db import exception as db_exc
|
||||
from oslo_log import log as logging
|
||||
from sqlalchemy.orm import session as se
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LokiPlugin(service_base.ServicePluginBase):
|
||||
"""Loki brings us the gift of sporadic database failures and delays."""
|
||||
|
||||
def __init__(self):
|
||||
super(LokiPlugin, self).__init__()
|
||||
db_api.sqla_listen(se.Session, 'before_flush', self.random_deadlock)
|
||||
db_api.sqla_listen(se.Session, 'loaded_as_persistent',
|
||||
self.random_delay)
|
||||
|
||||
def random_deadlock(self, session, flush_context, instances):
|
||||
if random.randrange(0, 51) > 49: # 1/50 probability
|
||||
raise db_exc.DBDeadlock()
|
||||
|
||||
def random_delay(self, session, instance):
|
||||
if random.randrange(0, 201) > 199: # 1/200 probability
|
||||
LOG.debug("Loki has delayed loading of instance %s", instance)
|
||||
time.sleep(1)
|
||||
|
||||
def get_plugin_type(self):
|
||||
return "loki"
|
||||
|
||||
def get_plugin_description(self):
|
||||
return "Injects deadlocks and delays into database operations."
|
|
@ -81,6 +81,7 @@ neutron.service_plugins =
|
|||
revisions = neutron.services.revisions.revision_plugin:RevisionPlugin
|
||||
timestamp = neutron.services.timestamp.timestamp_plugin:TimeStampPlugin
|
||||
trunk = neutron.services.trunk.plugin:TrunkPlugin
|
||||
loki = neutron.services.loki.loki_plugin:LokiPlugin
|
||||
neutron.qos.notification_drivers =
|
||||
message_queue = neutron.services.qos.notification_drivers.message_queue:RpcQosServiceNotificationDriver
|
||||
neutron.ml2.type_drivers =
|
||||
|
|
Loading…
Reference in New Issue