Adding retry decorator for plugin operations

This follows the same design pattern and uses the same decorator
used by Neutron. The plugin operations will be retried when
the following exceptions occur:
sqlalchemy.orm.exc.StaleDataError
oslo_db.exception.DBDeadlock
oslo_db.exception.DBConnectionError
oslo_db.exception.DBDuplicateEntry
oslo_db.exception.RetryRequest
oslo_db.exception.DBError (with 1035 code for SAVEPOINT errors)

Each operation will be tried 10 times before the actual exception
is raised. At that point the exception will be replaced by a
GroupPolicyDriverException and returned to the client. Retry logic
and decorator is implemented in Neutron and used as is.

Change-Id: I42e5a62e81f423bbc3a131953245b789cdb1c1c8
(cherry picked from commit ead7821d06)
This commit is contained in:
Sumit Naiksatam 2017-03-30 20:28:58 -07:00
parent eec74ade30
commit ff30eab869
2 changed files with 90 additions and 13 deletions

View File

@ -16,6 +16,7 @@ from neutron._i18n import _LE
from neutron._i18n import _LW
from neutron.api.v2 import attributes as nattr
from neutron import context as n_ctx
from neutron.db import api as db_api
from neutron.extensions import portbindings
from neutron import manager as n_manager
from neutron.plugins.common import constants as pconst
@ -458,6 +459,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
{'port_attributes': port_attributes})
@log.log_method_call
@db_api.retry_db_errors
def create_policy_target(self, context, policy_target):
self._ensure_tenant(context, policy_target['policy_target'])
self._add_fixed_ips_to_port_attributes(policy_target)
@ -487,6 +489,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_target(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_policy_target(self, context, policy_target_id, policy_target):
self._add_fixed_ips_to_port_attributes(policy_target)
session = context.session
@ -512,6 +515,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_target(context, policy_target_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_policy_target(self, context, policy_target_id):
session = context.session
with session.begin(subtransactions=True):
@ -532,11 +536,13 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
policy_target_id)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_target(self, context, policy_target_id, fields=None):
return self._get_resource(context, 'policy_target', policy_target_id,
'PolicyTargetContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_targets(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -546,6 +552,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_policy_target_group(self, context, policy_target_group):
self._ensure_tenant(context,
policy_target_group['policy_target_group'])
@ -576,6 +583,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_target_group(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_policy_target_group(self, context, policy_target_group_id,
policy_target_group):
session = context.session
@ -615,6 +623,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_target_group(context, policy_target_group_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_policy_target_group(self, context, policy_target_group_id):
session = context.session
with session.begin(subtransactions=True):
@ -670,6 +679,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
policy_target_group_id)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_target_group(self, context, policy_target_group_id,
fields=None):
return self._get_resource(context, 'policy_target_group',
@ -677,6 +687,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
'PolicyTargetGroupContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_target_groups(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -686,6 +697,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_application_policy_group(self, context,
application_policy_group):
self._ensure_tenant(
@ -716,6 +728,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_application_policy_group(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_application_policy_group(self, context,
application_policy_group_id,
application_policy_group):
@ -748,6 +761,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
application_policy_group_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_application_policy_group(self, context,
application_policy_group_id):
session = context.session
@ -770,6 +784,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
application_policy_group_id)
@log.log_method_call
@db_api.retry_db_errors
def get_application_policy_group(self, context,
application_policy_group_id, fields=None):
return self._get_resource(context, 'application_policy_group',
@ -778,6 +793,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_application_policy_groups(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -787,6 +803,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
sorts=sorts, limit=limit, marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_l2_policy(self, context, l2_policy):
self._ensure_tenant(context, l2_policy['l2_policy'])
session = context.session
@ -813,6 +830,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_l2_policy(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_l2_policy(self, context, l2_policy_id, l2_policy):
session = context.session
with session.begin(subtransactions=True):
@ -836,6 +854,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_l2_policy(context, l2_policy_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_l2_policy(self, context, l2_policy_id):
session = context.session
with session.begin(subtransactions=True):
@ -855,12 +874,14 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
"for l2_policy %s"), l2_policy_id)
@log.log_method_call
@db_api.retry_db_errors
def get_l2_policy(self, context, l2_policy_id, fields=None):
return self._get_resource(context, 'l2_policy',
l2_policy_id,
'L2PolicyContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_l2_policies(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -870,6 +891,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_network_service_policy(self, context, network_service_policy):
self._ensure_tenant(
context, network_service_policy['network_service_policy'])
@ -902,6 +924,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_network_service_policy(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_network_service_policy(self, context, network_service_policy_id,
network_service_policy):
session = context.session
@ -931,6 +954,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
network_service_policy_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_network_service_policy(
self, context, network_service_policy_id):
session = context.session
@ -953,6 +977,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
"for network_service_policy %s"), network_service_policy_id)
@log.log_method_call
@db_api.retry_db_errors
def get_network_service_policy(self, context, network_service_policy_id,
fields=None):
return self._get_resource(context, 'network_service_policy',
@ -960,6 +985,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
'NetworkServicePolicyContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_network_service_policies(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -969,6 +995,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_l3_policy(self, context, l3_policy):
self._ensure_tenant(context, l3_policy['l3_policy'])
session = context.session
@ -997,6 +1024,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_l3_policy(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_l3_policy(self, context, l3_policy_id, l3_policy):
session = context.session
with session.begin(subtransactions=True):
@ -1021,6 +1049,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_l3_policy(context, l3_policy_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_l3_policy(self, context, l3_policy_id, check_unused=False):
session = context.session
with session.begin(subtransactions=True):
@ -1045,12 +1074,14 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return True
@log.log_method_call
@db_api.retry_db_errors
def get_l3_policy(self, context, l3_policy_id, fields=None):
return self._get_resource(context, 'l3_policy',
l3_policy_id,
'L3PolicyContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_l3_policies(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -1060,6 +1091,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_policy_classifier(self, context, policy_classifier):
self._ensure_tenant(context,
policy_classifier['policy_classifier'])
@ -1090,6 +1122,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_classifier(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_policy_classifier(self, context, id, policy_classifier):
session = context.session
with session.begin(subtransactions=True):
@ -1114,6 +1147,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_classifier(context, id)
@log.log_method_call
@db_api.retry_db_errors
def delete_policy_classifier(self, context, id):
session = context.session
with session.begin(subtransactions=True):
@ -1133,6 +1167,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
"for policy_classifier %s"), id)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_classifier(self, context, policy_classifier_id,
fields=None):
return self._get_resource(context, 'policy_classifier',
@ -1140,6 +1175,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
'PolicyClassifierContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_classifiers(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -1149,6 +1185,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_policy_action(self, context, policy_action):
self._ensure_tenant(context, policy_action['policy_action'])
session = context.session
@ -1178,6 +1215,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_action(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_policy_action(self, context, id, policy_action):
session = context.session
with session.begin(subtransactions=True):
@ -1203,6 +1241,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_action(context, id)
@log.log_method_call
@db_api.retry_db_errors
def delete_policy_action(self, context, id):
session = context.session
with session.begin(subtransactions=True):
@ -1221,12 +1260,14 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
"for policy_action %s"), id)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_action(self, context, policy_action_id, fields=None):
return self._get_resource(context, 'policy_action',
policy_action_id,
'PolicyActionContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_actions(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -1236,6 +1277,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_policy_rule(self, context, policy_rule):
self._ensure_tenant(context, policy_rule['policy_rule'])
session = context.session
@ -1264,6 +1306,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_rule(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_policy_rule(self, context, id, policy_rule):
session = context.session
with session.begin(subtransactions=True):
@ -1287,6 +1330,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_rule(context, id)
@log.log_method_call
@db_api.retry_db_errors
def delete_policy_rule(self, context, id):
session = context.session
with session.begin(subtransactions=True):
@ -1306,12 +1350,14 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
"for policy_rule %s"), id)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_rule(self, context, policy_rule_id, fields=None):
return self._get_resource(context, 'policy_rule',
policy_rule_id,
'PolicyRuleContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_rules(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -1321,6 +1367,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_policy_rule_set(self, context, policy_rule_set):
self._ensure_tenant(context, policy_rule_set['policy_rule_set'])
session = context.session
@ -1350,6 +1397,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_rule_set(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_policy_rule_set(self, context, id, policy_rule_set):
session = context.session
with session.begin(subtransactions=True):
@ -1374,6 +1422,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_policy_rule_set(context, id)
@log.log_method_call
@db_api.retry_db_errors
def delete_policy_rule_set(self, context, id):
session = context.session
with session.begin(subtransactions=True):
@ -1392,12 +1441,14 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
"for policy_rule_set %s"), id)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_rule_set(self, context, policy_rule_set_id, fields=None):
return self._get_resource(context, 'policy_rule_set',
policy_rule_set_id,
'PolicyRuleSetContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_policy_rule_sets(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -1407,6 +1458,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_external_segment(self, context, external_segment):
self._ensure_tenant(context, external_segment['external_segment'])
session = context.session
@ -1439,6 +1491,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_external_segment(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_external_segment(self, context, external_segment_id,
external_segment):
session = context.session
@ -1469,6 +1522,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_external_segment(context, external_segment_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_external_segment(self, context, external_segment_id):
session = context.session
with session.begin(subtransactions=True):
@ -1492,12 +1546,14 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return True
@log.log_method_call
@db_api.retry_db_errors
def get_external_segment(self, context, external_segment_id, fields=None):
return self._get_resource(context, 'external_segment',
external_segment_id,
'ExternalSegmentContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_external_segments(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -1507,6 +1563,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_external_policy(self, context, external_policy):
self._ensure_tenant(context, external_policy['external_policy'])
session = context.session
@ -1536,6 +1593,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_external_policy(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_external_policy(self, context, external_policy_id,
external_policy):
session = context.session
@ -1563,6 +1621,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_external_policy(context, external_policy_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_external_policy(self, context, external_policy_id,
check_unused=False):
session = context.session
@ -1583,12 +1642,14 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
"for external_policy %s"), external_policy_id)
@log.log_method_call
@db_api.retry_db_errors
def get_external_policy(self, context, external_policy_id, fields=None):
return self._get_resource(context, 'external_policy',
external_policy_id,
'ExternalPolicyContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_external_policies(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):
@ -1598,6 +1659,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
marker=marker, page_reverse=page_reverse)
@log.log_method_call
@db_api.retry_db_errors
def create_nat_pool(self, context, nat_pool):
self._ensure_tenant(context, nat_pool['nat_pool'])
session = context.session
@ -1624,6 +1686,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_nat_pool(context, result['id'])
@log.log_method_call
@db_api.retry_db_errors
def update_nat_pool(self, context, nat_pool_id, nat_pool):
session = context.session
with session.begin(subtransactions=True):
@ -1645,6 +1708,7 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
return self.get_nat_pool(context, nat_pool_id)
@log.log_method_call
@db_api.retry_db_errors
def delete_nat_pool(self, context, nat_pool_id, check_unused=False):
session = context.session
with session.begin(subtransactions=True):
@ -1664,12 +1728,14 @@ class GroupPolicyPlugin(group_policy_mapping_db.GroupPolicyMappingDbPlugin):
nat_pool_id)
@log.log_method_call
@db_api.retry_db_errors
def get_nat_pool(self, context, nat_pool_id, fields=None):
return self._get_resource(context, 'nat_pool',
nat_pool_id,
'NatPoolContext', fields=fields)
@log.log_method_call
@db_api.retry_db_errors
def get_nat_pools(self, context, filters=None, fields=None,
sorts=None, limit=None, marker=None,
page_reverse=False):

View File

@ -13,6 +13,7 @@
from neutron._i18n import _LE
from neutron._i18n import _LI
from neutron.common import exceptions as n_exc
from neutron.db import api as db_api
from oslo_config import cfg
from oslo_log import log
from oslo_policy import policy as oslo_policy
@ -122,21 +123,31 @@ class PolicyDriverManager(stevedore.named.NamedExtensionManager):
for driver in drivers:
try:
getattr(driver.obj, method_name)(context)
except (gp_exc.GroupPolicyException, n_exc.NeutronException,
oslo_policy.PolicyNotAuthorized):
with excutils.save_and_reraise_exception():
except Exception as e:
if db_api.is_retriable(e):
with excutils.save_and_reraise_exception():
LOG.debug(
"Policy driver '%(name)s' failed in"
" %(method)s, operation will be retried",
{'name': driver.name, 'method': method_name}
)
elif isinstance(e, gp_exc.GroupPolicyException) or isinstance(
e, n_exc.NeutronException) or isinstance(
e, oslo_policy.PolicyNotAuthorized):
with excutils.save_and_reraise_exception():
LOG.exception(
_LE("Policy driver '%(name)s' failed in"
" %(method)s"),
{'name': driver.name, 'method': method_name}
)
else:
error = True
# We are eating a non-GBP/non-Neutron exception here
LOG.exception(
_LE("Policy driver '%(name)s' failed in %(method)s"),
{'name': driver.name, 'method': method_name}
)
except Exception:
# We are eating a non-GBP/non-Neutron exception here
LOG.exception(
_LE("Policy driver '%(name)s' failed in %(method)s"),
{'name': driver.name, 'method': method_name})
error = True
if not continue_on_failure:
break
{'name': driver.name, 'method': method_name})
if not continue_on_failure:
break
if error:
raise gp_exc.GroupPolicyDriverError(method=method_name)