From 67e3b7ce3d3707ec9d8e51ae7f7f7a6e2ec493ce Mon Sep 17 00:00:00 2001 From: Robert Kukura Date: Thu, 29 Nov 2018 11:52:59 -0500 Subject: [PATCH] [AIM] Improve validation output Validation now catches any exception that leaks from driver validation methods, reports the error, and exits indicating failure. Binding of ports during validation is more verbose, indicating which ports its attempting to bind and how many failed. Failure to bind any ports now results in a "failed binding ports" rather than "failed unrepairable" result, along with additonal information on how to proceed. Change-Id: I85a180d2a06d7f4442c47424a1c02bc307d1fc0e --- .../ml2plus/drivers/apic_aim/mechanism_driver.py | 16 ++++++++++++++-- .../drivers/cisco/apic/aim_validation.py | 7 +++++++ .../grouppolicy/group_policy_driver_api.py | 1 + .../services/grouppolicy/test_aim_validation.py | 8 +++++++- 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/mechanism_driver.py b/gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/mechanism_driver.py index 1a6fe7857..bd890f6da 100644 --- a/gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/mechanism_driver.py +++ b/gbpservice/neutron/plugins/ml2plus/drivers/apic_aim/mechanism_driver.py @@ -4874,16 +4874,28 @@ class ApicMechanismDriver(api_plus.MechanismDriver, # REVISIT: Deal with distributed port bindings? Also, consider # moving this to the ML2Plus plugin or to a base validation # manager, as it is not specific to this mechanism driver. + failure_count = 0 + failure_hosts = set() for port_id, in (mgr.actual_session.query(models.PortBinding.port_id). filter(models.PortBinding.host != '', models.PortBinding.vif_type == portbindings.VIF_TYPE_UNBOUND)): + mgr.output("Attempting to bind port %s" % port_id) # REVISIT: Use the more efficient get_bound_port_contexts, # which is not available in stable/newton? pc = self.plugin.get_bound_port_context( mgr.actual_context, port_id) if (pc.vif_type == portbindings.VIF_TYPE_BINDING_FAILED or pc.vif_type == portbindings.VIF_TYPE_UNBOUND): - mgr.validation_failed( - "unable to bind port %(port)s on host %(host)s" % + mgr.bind_ports_failed( + "Unable to bind port %(port)s on host %(host)s" % {'port': port_id, 'host': pc.host}) + failure_count += 1 + failure_hosts.add(pc.host) + if failure_count: + mgr.output( + "Failed to bind %s ports on hosts %s. See log for details. " + "Make sure L2 agents are alive, and re-run validation to try " + "binding them again." % (failure_count, list(failure_hosts))) + else: + mgr.output("All ports are bound") diff --git a/gbpservice/neutron/services/grouppolicy/drivers/cisco/apic/aim_validation.py b/gbpservice/neutron/services/grouppolicy/drivers/cisco/apic/aim_validation.py index 8ff49f250..e48e0fae7 100644 --- a/gbpservice/neutron/services/grouppolicy/drivers/cisco/apic/aim_validation.py +++ b/gbpservice/neutron/services/grouppolicy/drivers/cisco/apic/aim_validation.py @@ -122,6 +122,9 @@ class ValidationManager(object): raise RollbackTransaction() except RollbackTransaction: pass + except Exception as exc: + self.output("Validation failed with exception: %s" % exc) + return api.VALIDATION_FAILED_UNREPAIRABLE # Bind unbound ports outside transaction. if (self.repair and @@ -225,6 +228,10 @@ class ValidationManager(object): self.output("Failed UNREPAIRABLE due to %s" % reason) self.result = api.VALIDATION_FAILED_UNREPAIRABLE + def bind_ports_failed(self, message): + self.output(message) + self.result = api.VALIDATION_FAILED_BINDING_PORTS + def _validate_aim_resources(self): for resource_class in self._expected_aim_resources.keys(): self._validate_aim_resource_class(resource_class) diff --git a/gbpservice/neutron/services/grouppolicy/group_policy_driver_api.py b/gbpservice/neutron/services/grouppolicy/group_policy_driver_api.py index ee4574737..e33f037bd 100644 --- a/gbpservice/neutron/services/grouppolicy/group_policy_driver_api.py +++ b/gbpservice/neutron/services/grouppolicy/group_policy_driver_api.py @@ -22,6 +22,7 @@ VALIDATION_PASSED = "passed" VALIDATION_REPAIRED = "repaired" VALIDATION_FAILED_REPAIRABLE = "failed repairable" VALIDATION_FAILED_UNREPAIRABLE = "failed unrepairable" +VALIDATION_FAILED_BINDING_PORTS = "failed binding ports" @six.add_metaclass(abc.ABCMeta) diff --git a/gbpservice/neutron/tests/unit/services/grouppolicy/test_aim_validation.py b/gbpservice/neutron/tests/unit/services/grouppolicy/test_aim_validation.py index 32d03f92b..9c7126cd1 100644 --- a/gbpservice/neutron/tests/unit/services/grouppolicy/test_aim_validation.py +++ b/gbpservice/neutron/tests/unit/services/grouppolicy/test_aim_validation.py @@ -64,6 +64,12 @@ class AimValidationTestMixin(object): api.VALIDATION_FAILED_UNREPAIRABLE, self.av_mgr.validate(repair=True)) + def _validate_fails_binding_ports(self): + # Repair should fail. + self.assertEqual( + api.VALIDATION_FAILED_BINDING_PORTS, + self.av_mgr.validate(repair=True)) + def _test_aim_resource(self, resource, unexpected_attr_name='name', unexpected_attr_value='unexpected', test_unexpected_monitored=True): @@ -862,7 +868,7 @@ class TestNeutronMapping(AimValidationTestCase): 'host': 'yyy'}) self.db_session.query(ml2_models.PortBinding).filter_by( port_id=port['id']).update({'host': 'yyy'}) - self._validate_unrepairable() + self._validate_fails_binding_ports() def test_legacy_cleanup(self): # Create external network.