Merge "Enrich integration with ironic networking features"
This commit is contained in:
commit
3158fdca20
|
@ -0,0 +1,38 @@
|
||||||
|
---
|
||||||
|
features:
|
||||||
|
- |
|
||||||
|
Adds support to specify additional parameters for Bare Metal ports when
|
||||||
|
registering nodes.
|
||||||
|
|
||||||
|
The ``mac`` key in nodes_json (instackenv.json) is replaced by the new
|
||||||
|
``ports`` key. Each port-entry supports the following keys: ``address``,
|
||||||
|
``physical_network`` and ``local_link_connection``. (The keys in ``ports``
|
||||||
|
mirror a subset off the `Bare Metal service API <https://developer.openstack.org/api-ref/baremetal/#ports-ports>`_
|
||||||
|
.)
|
||||||
|
|
||||||
|
Example specifying port mac address only::
|
||||||
|
|
||||||
|
"ports": [
|
||||||
|
{
|
||||||
|
"address": "52:54:00:87:c8:2e"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
|
||||||
|
Example specifying additional parameters::
|
||||||
|
|
||||||
|
"ports": [
|
||||||
|
{
|
||||||
|
"address": "52:54:00:87:c8:2f",
|
||||||
|
"physical_network": "network",
|
||||||
|
"local_link_connection": {
|
||||||
|
"switch_info": "switch",
|
||||||
|
"port_id": "gi1/0/11",
|
||||||
|
"switch_id": "a6:18:66:33:cb:49"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
deprecations:
|
||||||
|
- |
|
||||||
|
The ``mac`` key in nodes_json is replaced by ``ports``. The ``ports`` key
|
||||||
|
expect a list of dictionaries specifying ``address`` (mac address), and
|
||||||
|
optional keys ``physical_network`` and ``local_link_connection``.
|
|
@ -46,7 +46,7 @@ class RegisterOrUpdateNodes(base.TripleOAction):
|
||||||
def __init__(self, nodes_json, remove=False, kernel_name=None,
|
def __init__(self, nodes_json, remove=False, kernel_name=None,
|
||||||
ramdisk_name=None, instance_boot_option='local'):
|
ramdisk_name=None, instance_boot_option='local'):
|
||||||
super(RegisterOrUpdateNodes, self).__init__()
|
super(RegisterOrUpdateNodes, self).__init__()
|
||||||
self.nodes_json = nodes_json
|
self.nodes_json = nodes.convert_nodes_json_mac_to_ports(nodes_json)
|
||||||
self.remove = remove
|
self.remove = remove
|
||||||
self.instance_boot_option = instance_boot_option
|
self.instance_boot_option = instance_boot_option
|
||||||
self.kernel_name = kernel_name
|
self.kernel_name = kernel_name
|
||||||
|
@ -83,7 +83,7 @@ class ValidateNodes(base.TripleOAction):
|
||||||
|
|
||||||
def __init__(self, nodes_json):
|
def __init__(self, nodes_json):
|
||||||
super(ValidateNodes, self).__init__()
|
super(ValidateNodes, self).__init__()
|
||||||
self.nodes_json = nodes_json
|
self.nodes_json = nodes.convert_nodes_json_mac_to_ports(nodes_json)
|
||||||
|
|
||||||
def run(self, context):
|
def run(self, context):
|
||||||
try:
|
try:
|
||||||
|
|
|
@ -352,7 +352,7 @@ class GenerateFencingParametersAction(base.TripleOAction):
|
||||||
def __init__(self, nodes_json, os_auth, delay,
|
def __init__(self, nodes_json, os_auth, delay,
|
||||||
ipmi_level, ipmi_cipher, ipmi_lanplus):
|
ipmi_level, ipmi_cipher, ipmi_lanplus):
|
||||||
super(GenerateFencingParametersAction, self).__init__()
|
super(GenerateFencingParametersAction, self).__init__()
|
||||||
self.nodes_json = nodes_json
|
self.nodes_json = nodes.convert_nodes_json_mac_to_ports(nodes_json)
|
||||||
self.os_auth = os_auth
|
self.os_auth = os_auth
|
||||||
self.delay = delay
|
self.delay = delay
|
||||||
self.ipmi_level = ipmi_level
|
self.ipmi_level = ipmi_level
|
||||||
|
@ -369,10 +369,10 @@ class GenerateFencingParametersAction(base.TripleOAction):
|
||||||
for node in self.nodes_json:
|
for node in self.nodes_json:
|
||||||
node_data = {}
|
node_data = {}
|
||||||
params = {}
|
params = {}
|
||||||
if "mac" in node:
|
if "ports" in node:
|
||||||
# Not all Ironic drivers present a MAC address, so we only
|
# Not all Ironic drivers present a MAC address, so we only
|
||||||
# capture it if it's present
|
# capture it if it's present
|
||||||
mac_addr = node["mac"][0]
|
mac_addr = node['ports'][0]['address']
|
||||||
node_data["host_mac"] = mac_addr
|
node_data["host_mac"] = mac_addr
|
||||||
|
|
||||||
# If the MAC isn't in the hostmap, this node hasn't been
|
# If the MAC isn't in the hostmap, this node hasn't been
|
||||||
|
|
|
@ -258,9 +258,9 @@ class NodesTest(base.TestCase):
|
||||||
|
|
||||||
def _get_node(self):
|
def _get_node(self):
|
||||||
return {'cpu': '1', 'memory': '2048', 'disk': '30', 'arch': 'amd64',
|
return {'cpu': '1', 'memory': '2048', 'disk': '30', 'arch': 'amd64',
|
||||||
'mac': ['aaa'], 'pm_addr': 'foo.bar', 'pm_user': 'test',
|
'ports': [{'address': 'aaa'}], 'pm_addr': 'foo.bar',
|
||||||
'pm_password': 'random', 'pm_type': 'ipmi', 'name': 'node1',
|
'pm_user': 'test', 'pm_password': 'random', 'pm_type': 'ipmi',
|
||||||
'capabilities': 'num_nics:6'}
|
'name': 'node1', 'capabilities': 'num_nics:6'}
|
||||||
|
|
||||||
def test_register_all_nodes_ironic_no_hw_stats(self):
|
def test_register_all_nodes_ironic_no_hw_stats(self):
|
||||||
node_list = [self._get_node()]
|
node_list = [self._get_node()]
|
||||||
|
@ -287,7 +287,8 @@ class NodesTest(base.TestCase):
|
||||||
resource_class='baremetal',
|
resource_class='baremetal',
|
||||||
properties=node_properties)
|
properties=node_properties)
|
||||||
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
||||||
address='aaa', physical_network='ctlplane')
|
address='aaa', physical_network='ctlplane',
|
||||||
|
local_link_connection=None)
|
||||||
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
||||||
ironic.port.create.assert_has_calls([port_call])
|
ironic.port.create.assert_has_calls([port_call])
|
||||||
|
|
||||||
|
@ -311,7 +312,8 @@ class NodesTest(base.TestCase):
|
||||||
resource_class='baremetal',
|
resource_class='baremetal',
|
||||||
properties=node_properties)
|
properties=node_properties)
|
||||||
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
||||||
address='aaa', physical_network='ctlplane')
|
address='aaa', physical_network='ctlplane',
|
||||||
|
local_link_connection=None)
|
||||||
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
||||||
ironic.port.create.assert_has_calls([port_call])
|
ironic.port.create.assert_has_calls([port_call])
|
||||||
|
|
||||||
|
@ -341,7 +343,8 @@ class NodesTest(base.TestCase):
|
||||||
resource_class='baremetal',
|
resource_class='baremetal',
|
||||||
properties=node_properties)
|
properties=node_properties)
|
||||||
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
||||||
address='aaa', physical_network='ctlplane')
|
address='aaa', physical_network='ctlplane',
|
||||||
|
local_link_connection=None)
|
||||||
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
||||||
ironic.port.create.assert_has_calls([port_call])
|
ironic.port.create.assert_has_calls([port_call])
|
||||||
|
|
||||||
|
@ -365,7 +368,8 @@ class NodesTest(base.TestCase):
|
||||||
resource_class='baremetal',
|
resource_class='baremetal',
|
||||||
uuid="abcdef")
|
uuid="abcdef")
|
||||||
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
||||||
address='aaa', physical_network='ctlplane')
|
address='aaa', physical_network='ctlplane',
|
||||||
|
local_link_connection=None)
|
||||||
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
||||||
ironic.port.create.assert_has_calls([port_call])
|
ironic.port.create.assert_has_calls([port_call])
|
||||||
|
|
||||||
|
@ -390,7 +394,8 @@ class NodesTest(base.TestCase):
|
||||||
resource_class='baremetal',
|
resource_class='baremetal',
|
||||||
properties=node_properties)
|
properties=node_properties)
|
||||||
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
||||||
address='aaa', physical_network='ctlplane')
|
address='aaa', physical_network='ctlplane',
|
||||||
|
local_link_connection=None)
|
||||||
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
||||||
ironic.port.create.assert_has_calls([port_call])
|
ironic.port.create.assert_has_calls([port_call])
|
||||||
|
|
||||||
|
@ -425,7 +430,8 @@ class NodesTest(base.TestCase):
|
||||||
resource_class='baremetal',
|
resource_class='baremetal',
|
||||||
**interfaces)
|
**interfaces)
|
||||||
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
||||||
address='aaa', physical_network='ctlplane')
|
address='aaa', local_link_connection=None,
|
||||||
|
physical_network='ctlplane')
|
||||||
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
ironic.node.create.assert_has_calls([pxe_node, mock.ANY])
|
||||||
ironic.port.create.assert_has_calls([port_call])
|
ironic.port.create.assert_has_calls([port_call])
|
||||||
|
|
||||||
|
@ -589,7 +595,7 @@ class NodesTest(base.TestCase):
|
||||||
|
|
||||||
def test_register_node_update(self):
|
def test_register_node_update(self):
|
||||||
node = self._get_node()
|
node = self._get_node()
|
||||||
node['mac'][0] = node['mac'][0].upper()
|
node['ports'][0]['address'] = node['ports'][0]['address'].upper()
|
||||||
ironic = mock.MagicMock()
|
ironic = mock.MagicMock()
|
||||||
node_map = {'mac': {'aaa': 1}}
|
node_map = {'mac': {'aaa': 1}}
|
||||||
|
|
||||||
|
@ -781,6 +787,38 @@ class NodesTest(base.TestCase):
|
||||||
'redfish_username': 'test',
|
'redfish_username': 'test',
|
||||||
'redfish_system_id': '/redfish/v1/Systems/1'})
|
'redfish_system_id': '/redfish/v1/Systems/1'})
|
||||||
|
|
||||||
|
def test_register_ironic_node_with_physical_network(self):
|
||||||
|
node = self._get_node()
|
||||||
|
node['ports'] = [{'physical_network': 'subnet1', 'address': 'aaa'}]
|
||||||
|
ironic = mock.MagicMock()
|
||||||
|
nodes.register_ironic_node(node, client=ironic)
|
||||||
|
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
||||||
|
address='aaa', physical_network='subnet1',
|
||||||
|
local_link_connection=None)
|
||||||
|
ironic.port.create.assert_has_calls([port_call])
|
||||||
|
|
||||||
|
def test_register_ironic_node_with_local_link_connection(self):
|
||||||
|
node = self._get_node()
|
||||||
|
node['ports'] = [
|
||||||
|
{
|
||||||
|
'local_link_connection': {
|
||||||
|
"switch_info": "switch",
|
||||||
|
"port_id": "port1",
|
||||||
|
"switch_id": "bbb"
|
||||||
|
},
|
||||||
|
'physical_network': 'subnet1',
|
||||||
|
'address': 'aaa'
|
||||||
|
}
|
||||||
|
]
|
||||||
|
ironic = mock.MagicMock()
|
||||||
|
nodes.register_ironic_node(node, client=ironic)
|
||||||
|
port_call = mock.call(node_uuid=ironic.node.create.return_value.uuid,
|
||||||
|
address='aaa', physical_network='subnet1',
|
||||||
|
local_link_connection={"switch_info": "switch",
|
||||||
|
"port_id": "port1",
|
||||||
|
"switch_id": "bbb"})
|
||||||
|
ironic.port.create.assert_has_calls([port_call])
|
||||||
|
|
||||||
def test_clean_up_extra_nodes_ironic(self):
|
def test_clean_up_extra_nodes_ironic(self):
|
||||||
node = collections.namedtuple('node', ['uuid'])
|
node = collections.namedtuple('node', ['uuid'])
|
||||||
client = mock.MagicMock()
|
client = mock.MagicMock()
|
||||||
|
@ -859,8 +897,10 @@ VALID_NODE_JSON = [
|
||||||
'pm_password': 'p@$$w0rd',
|
'pm_password': 'p@$$w0rd',
|
||||||
'pm_port': 1234,
|
'pm_port': 1234,
|
||||||
'ipmi_priv_level': 'USER',
|
'ipmi_priv_level': 'USER',
|
||||||
'mac': ['aa:bb:cc:dd:ee:ff',
|
'ports': [
|
||||||
'11:22:33:44:55:66'],
|
{'address': 'aa:bb:cc:dd:ee:ff'},
|
||||||
|
{'address': '11:22:33:44:55:66'}
|
||||||
|
],
|
||||||
'name': 'foobar1',
|
'name': 'foobar1',
|
||||||
'capabilities': {'foo': 'bar'},
|
'capabilities': {'foo': 'bar'},
|
||||||
'kernel_id': 'kernel1',
|
'kernel_id': 'kernel1',
|
||||||
|
@ -871,8 +911,10 @@ VALID_NODE_JSON = [
|
||||||
'pm_password': 'p@$$w0rd',
|
'pm_password': 'p@$$w0rd',
|
||||||
'pm_port': 1234,
|
'pm_port': 1234,
|
||||||
'ipmi_priv_level': 'USER',
|
'ipmi_priv_level': 'USER',
|
||||||
'mac': ['dd:ee:ff:aa:bb:cc',
|
'ports': [
|
||||||
'44:55:66:11:22:33'],
|
{'address': 'dd:ee:ff:aa:bb:cc'},
|
||||||
|
{'address': '44:55:66:11:22:33'}
|
||||||
|
],
|
||||||
'name': 'foobar2',
|
'name': 'foobar2',
|
||||||
'capabilities': {'foo': 'bar'},
|
'capabilities': {'foo': 'bar'},
|
||||||
'kernel_id': 'kernel1',
|
'kernel_id': 'kernel1',
|
||||||
|
@ -881,7 +923,9 @@ VALID_NODE_JSON = [
|
||||||
'pm_addr': '1.2.3.4',
|
'pm_addr': '1.2.3.4',
|
||||||
'pm_user': 'root',
|
'pm_user': 'root',
|
||||||
'pm_password': 'p@$$w0rd',
|
'pm_password': 'p@$$w0rd',
|
||||||
'mac': ['22:22:22:22:22:22'],
|
'ports': [
|
||||||
|
{'address': '22:22:22:22:22:22'}
|
||||||
|
],
|
||||||
'capabilities': 'foo:bar,foo1:bar1',
|
'capabilities': 'foo:bar,foo1:bar1',
|
||||||
'cpu': 2,
|
'cpu': 2,
|
||||||
'memory': 1024,
|
'memory': 1024,
|
||||||
|
@ -932,7 +976,9 @@ class TestValidateNodes(base.TestCase):
|
||||||
'pm_addr': '1.1.1.1',
|
'pm_addr': '1.1.1.1',
|
||||||
'pm_user': 'root',
|
'pm_user': 'root',
|
||||||
'pm_password': 'p@$$w0rd',
|
'pm_password': 'p@$$w0rd',
|
||||||
'mac': ['42']},
|
'ports': [
|
||||||
|
{'address': '42'}]
|
||||||
|
},
|
||||||
]
|
]
|
||||||
self.assertRaisesRegex(exception.InvalidNode,
|
self.assertRaisesRegex(exception.InvalidNode,
|
||||||
'MAC address 42 is invalid',
|
'MAC address 42 is invalid',
|
||||||
|
@ -944,12 +990,16 @@ class TestValidateNodes(base.TestCase):
|
||||||
'pm_addr': '1.1.1.1',
|
'pm_addr': '1.1.1.1',
|
||||||
'pm_user': 'root',
|
'pm_user': 'root',
|
||||||
'pm_password': 'p@$$w0rd',
|
'pm_password': 'p@$$w0rd',
|
||||||
'mac': ['11:22:33:44:55:66']},
|
'ports': [
|
||||||
|
{'address': '11:22:33:44:55:66'}
|
||||||
|
]},
|
||||||
{'pm_type': 'ipmi',
|
{'pm_type': 'ipmi',
|
||||||
'pm_addr': '1.2.1.1',
|
'pm_addr': '1.2.1.1',
|
||||||
'pm_user': 'user',
|
'pm_user': 'user',
|
||||||
'pm_password': 'p@$$w0rd',
|
'pm_password': 'p@$$w0rd',
|
||||||
'mac': ['11:22:33:44:55:66']},
|
'ports': [
|
||||||
|
{'address': '11:22:33:44:55:66'}
|
||||||
|
]},
|
||||||
]
|
]
|
||||||
self.assertRaisesRegex(exception.InvalidNode,
|
self.assertRaisesRegex(exception.InvalidNode,
|
||||||
'MAC 11:22:33:44:55:66 is not unique',
|
'MAC 11:22:33:44:55:66 is not unique',
|
||||||
|
|
|
@ -34,6 +34,20 @@ _KNOWN_INTERFACE_FIELDS = [
|
||||||
CTLPLANE_NETWORK = 'ctlplane'
|
CTLPLANE_NETWORK = 'ctlplane'
|
||||||
|
|
||||||
|
|
||||||
|
def convert_nodes_json_mac_to_ports(nodes_json):
|
||||||
|
for node in nodes_json:
|
||||||
|
if node.get('mac'):
|
||||||
|
LOG.warning('Key mac is deprecated, please use ports.')
|
||||||
|
for address in node['mac']:
|
||||||
|
try:
|
||||||
|
node['ports'].append({'address': address})
|
||||||
|
except KeyError:
|
||||||
|
node['ports'] = [{'address': address}]
|
||||||
|
del node['mac']
|
||||||
|
|
||||||
|
return nodes_json
|
||||||
|
|
||||||
|
|
||||||
class DriverInfo(object):
|
class DriverInfo(object):
|
||||||
"""Class encapsulating field conversion logic."""
|
"""Class encapsulating field conversion logic."""
|
||||||
DEFAULTS = {}
|
DEFAULTS = {}
|
||||||
|
@ -232,9 +246,9 @@ class SshDriverInfo(DriverInfo):
|
||||||
|
|
||||||
def validate(self, node):
|
def validate(self, node):
|
||||||
super(SshDriverInfo, self).validate(node)
|
super(SshDriverInfo, self).validate(node)
|
||||||
if not node.get('mac'):
|
if not node.get('ports')[0]['address']:
|
||||||
raise exception.InvalidNode(
|
raise exception.InvalidNode(
|
||||||
'Nodes with SSH drivers require at least one MAC')
|
'Nodes with SSH drivers require at least one PORT')
|
||||||
|
|
||||||
|
|
||||||
class iBootDriverInfo(PrefixedDriverInfo):
|
class iBootDriverInfo(PrefixedDriverInfo):
|
||||||
|
@ -356,9 +370,14 @@ def register_ironic_node(node, client):
|
||||||
LOG.debug('Registering node %s with ironic.', node_id)
|
LOG.debug('Registering node %s with ironic.', node_id)
|
||||||
ironic_node = client.node.create(**create_map)
|
ironic_node = client.node.create(**create_map)
|
||||||
|
|
||||||
for mac in node.get("mac", []):
|
for port in node.get('ports', []):
|
||||||
client.port.create(address=mac, physical_network=CTLPLANE_NETWORK,
|
LOG.debug('Creating Bare Metal port for node: %s, with properties: %s.'
|
||||||
node_uuid=ironic_node.uuid)
|
% (ironic_node.uuid, port))
|
||||||
|
client.port.create(
|
||||||
|
address=port.get('address'),
|
||||||
|
physical_network=port.get('physical_network', 'ctlplane'),
|
||||||
|
local_link_connection=port.get('local_link_connection'),
|
||||||
|
node_uuid=ironic_node.uuid)
|
||||||
|
|
||||||
validation = client.node.validate(ironic_node.uuid)
|
validation = client.node.validate(ironic_node.uuid)
|
||||||
if not validation.power['result']:
|
if not validation.power['result']:
|
||||||
|
@ -388,9 +407,9 @@ def _populate_node_mapping(client):
|
||||||
|
|
||||||
def _get_node_id(node, handler, node_map):
|
def _get_node_id(node, handler, node_map):
|
||||||
candidates = set()
|
candidates = set()
|
||||||
for mac in node.get('mac', []):
|
for port in node.get('ports', []):
|
||||||
try:
|
try:
|
||||||
candidates.add(node_map['mac'][mac.lower()])
|
candidates.add(node_map['mac'][port['address'].lower()])
|
||||||
except KeyError:
|
except KeyError:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
@ -527,15 +546,16 @@ def validate_nodes(nodes_list):
|
||||||
except exception.InvalidNode as exc:
|
except exception.InvalidNode as exc:
|
||||||
failures.append((index, exc))
|
failures.append((index, exc))
|
||||||
|
|
||||||
for mac in node.get('mac', ()):
|
for port in node.get('ports', ()):
|
||||||
if not netutils.is_valid_mac(mac):
|
if not netutils.is_valid_mac(port['address']):
|
||||||
failures.append((index, 'MAC address %s is invalid' % mac))
|
failures.append((index, 'MAC address %s is invalid' %
|
||||||
|
port['address']))
|
||||||
|
|
||||||
if mac in macs:
|
if port['address'] in macs:
|
||||||
failures.append(
|
failures.append(
|
||||||
(index, 'MAC %s is not unique' % mac))
|
(index, 'MAC %s is not unique' % port['address']))
|
||||||
else:
|
else:
|
||||||
macs.add(mac)
|
macs.add(port['address'])
|
||||||
|
|
||||||
unique_id = handler.unique_id_from_fields(node)
|
unique_id = handler.unique_id_from_fields(node)
|
||||||
if unique_id:
|
if unique_id:
|
||||||
|
@ -569,7 +589,7 @@ def validate_nodes(nodes_list):
|
||||||
for field in node:
|
for field in node:
|
||||||
converted = handler.convert_key(field)
|
converted = handler.convert_key(field)
|
||||||
if (converted is None and field not in _NON_DRIVER_FIELDS and
|
if (converted is None and field not in _NON_DRIVER_FIELDS and
|
||||||
field not in ('mac', 'pm_type')):
|
field not in ('ports', 'pm_type')):
|
||||||
failures.append((index, 'Unknown field %s' % field))
|
failures.append((index, 'Unknown field %s' % field))
|
||||||
|
|
||||||
if failures:
|
if failures:
|
||||||
|
|
Loading…
Reference in New Issue