Merge "SR-IOV: Add Agent QoS driver to support bandwidth limit" into feature/qos
This commit is contained in:
commit
43166cc4c5
|
@ -253,8 +253,13 @@ with them.
|
|||
Agent backends
|
||||
--------------
|
||||
|
||||
At the moment, QoS is supported by Open vSwitch backend only, so
|
||||
QosOVSAgentDriver is the only driver that implements QosAgentDriver interface.
|
||||
At the moment, QoS is supported by Open vSwitch and SR-IOV ml2 drivers.
|
||||
|
||||
Each agent backend defines a QoS driver that implements the QosAgentDriver
|
||||
interface:
|
||||
|
||||
* Open vSwitch (QosOVSAgentDriver);
|
||||
* SR-IOV (QosSRIOVAgentDriver).
|
||||
|
||||
|
||||
Open vSwitch
|
||||
|
@ -274,6 +279,24 @@ That approach is less flexible than linux-htb, Queues and OvS QoS profiles,
|
|||
which we may explore in the future, but which will need to be used in
|
||||
combination with openflow rules.
|
||||
|
||||
SR-IOV
|
||||
~~~~~~
|
||||
|
||||
SR-IOV bandwidth limit implementation relies on the new pci_lib function:
|
||||
|
||||
* set_vf_max_rate
|
||||
|
||||
As the name of the function suggests, the limit is applied on a Virtual
|
||||
Function (VF).
|
||||
|
||||
ip link interface has the following limitation for bandwidth limit: it uses
|
||||
Mbps as units of bandwidth measurement, not kbps, and does not support float
|
||||
numbers. So in case the limit is set to something less than 1000 kbps, it's set
|
||||
to 1 Mbps only. If the limit is set to something that does not divide to 1000
|
||||
kbps chunks, then the effective limit is rounded to the nearest integer Mbps
|
||||
value.
|
||||
|
||||
|
||||
Configuration
|
||||
=============
|
||||
|
||||
|
|
|
@ -354,3 +354,17 @@ class ESwitchManager(object):
|
|||
{"device_mac": device_mac, "pci_slot": pci_slot})
|
||||
embedded_switch = None
|
||||
return embedded_switch
|
||||
|
||||
def get_pci_slot_by_mac(self, device_mac):
|
||||
"""Get pci slot by mac.
|
||||
|
||||
Get pci slot by device mac
|
||||
@param device_mac: device mac
|
||||
"""
|
||||
result = None
|
||||
for pci_slot, embedded_switch in self.pci_slot_map.items():
|
||||
used_device_mac = embedded_switch.get_pci_device(pci_slot)
|
||||
if used_device_mac == device_mac:
|
||||
result = pci_slot
|
||||
break
|
||||
return result
|
||||
|
|
|
@ -0,0 +1,84 @@
|
|||
# Copyright 2015 Mellanox Technologies, Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from oslo_log import log as logging
|
||||
|
||||
from neutron.agent.l2.extensions import qos
|
||||
from neutron.i18n import _LE, _LI, _LW
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.agent.common import (
|
||||
exceptions as exc)
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.agent import eswitch_manager as esm
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.mech_driver import (
|
||||
mech_driver)
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class QosSRIOVAgentDriver(qos.QosAgentDriver):
|
||||
|
||||
_SUPPORTED_RULES = (
|
||||
mech_driver.SriovNicSwitchMechanismDriver.supported_qos_rule_types)
|
||||
|
||||
def __init__(self):
|
||||
super(QosSRIOVAgentDriver, self).__init__()
|
||||
self.eswitch_mgr = None
|
||||
|
||||
def initialize(self):
|
||||
self.eswitch_mgr = esm.ESwitchManager()
|
||||
|
||||
def create(self, port, qos_policy):
|
||||
self._handle_rules('create', port, qos_policy)
|
||||
|
||||
def update(self, port, qos_policy):
|
||||
self._handle_rules('update', port, qos_policy)
|
||||
|
||||
def delete(self, port, qos_policy):
|
||||
# TODO(QoS): consider optimizing flushing of all QoS rules from the
|
||||
# port by inspecting qos_policy.rules contents
|
||||
self._delete_bandwidth_limit(port)
|
||||
|
||||
def _handle_rules(self, action, port, qos_policy):
|
||||
for rule in qos_policy.rules:
|
||||
if rule.rule_type in self._SUPPORTED_RULES:
|
||||
handler_name = ("".join(("_", action, "_", rule.rule_type)))
|
||||
handler = getattr(self, handler_name)
|
||||
handler(port, rule)
|
||||
else:
|
||||
LOG.warning(_LW('Unsupported QoS rule type for %(rule_id)s: '
|
||||
'%(rule_type)s; skipping'),
|
||||
{'rule_id': rule.id, 'rule_type': rule.rule_type})
|
||||
|
||||
def _create_bandwidth_limit(self, port, rule):
|
||||
self._update_bandwidth_limit(port, rule)
|
||||
|
||||
def _update_bandwidth_limit(self, port, rule):
|
||||
pci_slot = port['profile'].get('pci_slot')
|
||||
device = port['device']
|
||||
self._set_vf_max_rate(device, pci_slot, rule.max_kbps)
|
||||
|
||||
def _delete_bandwidth_limit(self, port):
|
||||
pci_slot = port['profile'].get('pci_slot')
|
||||
device = port['device']
|
||||
self._set_vf_max_rate(device, pci_slot)
|
||||
|
||||
def _set_vf_max_rate(self, device, pci_slot, max_kbps=0):
|
||||
if self.eswitch_mgr.device_exists(device, pci_slot):
|
||||
try:
|
||||
self.eswitch_mgr.set_device_max_rate(
|
||||
device, pci_slot, max_kbps)
|
||||
except exc.SriovNicError:
|
||||
LOG.exception(
|
||||
_LE("Failed to set device %s max rate"), device)
|
||||
else:
|
||||
LOG.info(_LI("No device with MAC %s defined on agent."), device)
|
|
@ -26,6 +26,7 @@ from oslo_log import log as logging
|
|||
import oslo_messaging
|
||||
from oslo_service import loopingcall
|
||||
|
||||
from neutron.agent.l2.extensions import manager as ext_manager
|
||||
from neutron.agent import rpc as agent_rpc
|
||||
from neutron.agent import securitygroups_rpc as sg_rpc
|
||||
from neutron.common import config as common_config
|
||||
|
@ -34,7 +35,7 @@ from neutron.common import topics
|
|||
from neutron.common import utils as n_utils
|
||||
from neutron import context
|
||||
from neutron.i18n import _LE, _LI, _LW
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.agent.common import config # noqa
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.agent.common import config
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.agent.common \
|
||||
import exceptions as exc
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.agent import eswitch_manager as esm
|
||||
|
@ -72,12 +73,13 @@ class SriovNicSwitchAgent(object):
|
|||
polling_interval):
|
||||
|
||||
self.polling_interval = polling_interval
|
||||
self.conf = cfg.CONF
|
||||
self.setup_eswitch_mgr(physical_devices_mappings,
|
||||
exclude_devices)
|
||||
configurations = {'device_mappings': physical_devices_mappings}
|
||||
self.agent_state = {
|
||||
'binary': 'neutron-sriov-nic-agent',
|
||||
'host': cfg.CONF.host,
|
||||
'host': self.conf.host,
|
||||
'topic': n_constants.L2_AGENT_TOPIC,
|
||||
'configurations': configurations,
|
||||
'agent_type': n_constants.AGENT_TYPE_NIC_SWITCH,
|
||||
|
@ -92,6 +94,10 @@ class SriovNicSwitchAgent(object):
|
|||
self.sg_agent = sg_rpc.SecurityGroupAgentRpc(self.context,
|
||||
self.sg_plugin_rpc)
|
||||
self._setup_rpc()
|
||||
self.ext_manager = self._create_agent_extension_manager(
|
||||
self.connection)
|
||||
# The initialization is complete; we can start receiving messages
|
||||
self.connection.consume_in_threads()
|
||||
# Initialize iteration counter
|
||||
self.iter_num = 0
|
||||
|
||||
|
@ -111,7 +117,8 @@ class SriovNicSwitchAgent(object):
|
|||
[topics.SECURITY_GROUP, topics.UPDATE]]
|
||||
self.connection = agent_rpc.create_consumers(self.endpoints,
|
||||
self.topic,
|
||||
consumers)
|
||||
consumers,
|
||||
start_listening=False)
|
||||
|
||||
report_interval = cfg.CONF.AGENT.report_interval
|
||||
if report_interval:
|
||||
|
@ -129,6 +136,12 @@ class SriovNicSwitchAgent(object):
|
|||
except Exception:
|
||||
LOG.exception(_LE("Failed reporting state!"))
|
||||
|
||||
def _create_agent_extension_manager(self, connection):
|
||||
ext_manager.register_opts(self.conf)
|
||||
mgr = ext_manager.AgentExtensionsManager(self.conf)
|
||||
mgr.initialize(connection, 'sriov')
|
||||
return mgr
|
||||
|
||||
def setup_eswitch_mgr(self, device_mappings, exclude_devices={}):
|
||||
self.eswitch_mgr = esm.ESwitchManager()
|
||||
self.eswitch_mgr.discover_devices(device_mappings, exclude_devices)
|
||||
|
@ -225,6 +238,7 @@ class SriovNicSwitchAgent(object):
|
|||
profile.get('pci_slot'),
|
||||
device_details['admin_state_up'],
|
||||
spoofcheck)
|
||||
self.ext_manager.handle_port(self.context, device_details)
|
||||
else:
|
||||
LOG.info(_LI("Device with MAC %s not defined on plugin"),
|
||||
device)
|
||||
|
@ -235,6 +249,16 @@ class SriovNicSwitchAgent(object):
|
|||
for device in devices:
|
||||
LOG.info(_LI("Removing device with mac_address %s"), device)
|
||||
try:
|
||||
pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(device)
|
||||
if pci_slot:
|
||||
profile = {'pci_slot': pci_slot}
|
||||
port = {'device': device, 'profile': profile}
|
||||
self.ext_manager.delete_port(self.context, port)
|
||||
else:
|
||||
LOG.warning(_LW("Failed to find pci slot for device "
|
||||
"%(device)s; skipping extension port "
|
||||
"cleanup"), device)
|
||||
|
||||
dev_details = self.plugin_rpc.update_device_down(self.context,
|
||||
device,
|
||||
self.agent_id,
|
||||
|
|
|
@ -24,6 +24,7 @@ from neutron.plugins.common import constants as p_const
|
|||
from neutron.plugins.ml2 import driver_api as api
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.mech_driver \
|
||||
import exceptions as exc
|
||||
from neutron.services.qos import qos_consts
|
||||
|
||||
|
||||
LOG = log.getLogger(__name__)
|
||||
|
@ -61,6 +62,8 @@ class SriovNicSwitchMechanismDriver(api.MechanismDriver):
|
|||
|
||||
"""
|
||||
|
||||
supported_qos_rule_types = [qos_consts.RULE_TYPE_BANDWIDTH_LIMIT]
|
||||
|
||||
def __init__(self,
|
||||
agent_type=constants.AGENT_TYPE_NIC_SWITCH,
|
||||
vif_type=portbindings.VIF_TYPE_HW_VEB,
|
||||
|
|
|
@ -0,0 +1,92 @@
|
|||
# Copyright 2015 Mellanox Technologies, Ltd
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import mock
|
||||
from oslo_utils import uuidutils
|
||||
|
||||
from neutron import context
|
||||
from neutron.objects.qos import policy
|
||||
from neutron.objects.qos import rule
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.agent.common import exceptions
|
||||
from neutron.plugins.ml2.drivers.mech_sriov.agent.extension_drivers import (
|
||||
qos_driver)
|
||||
from neutron.tests import base
|
||||
|
||||
|
||||
class QosSRIOVAgentDriverTestCase(base.BaseTestCase):
|
||||
|
||||
ASSIGNED_MAC = '00:00:00:00:00:66'
|
||||
PCI_SLOT = '0000:06:00.1'
|
||||
|
||||
def setUp(self):
|
||||
super(QosSRIOVAgentDriverTestCase, self).setUp()
|
||||
self.context = context.get_admin_context()
|
||||
self.qos_driver = qos_driver.QosSRIOVAgentDriver()
|
||||
self.qos_driver.initialize()
|
||||
self.qos_driver.eswitch_mgr = mock.Mock()
|
||||
self.qos_driver.eswitch_mgr.set_device_max_rate = mock.Mock()
|
||||
self.max_rate_mock = self.qos_driver.eswitch_mgr.set_device_max_rate
|
||||
self.rule = self._create_bw_limit_rule_obj()
|
||||
self.qos_policy = self._create_qos_policy_obj([self.rule])
|
||||
self.port = self._create_fake_port()
|
||||
|
||||
def _create_bw_limit_rule_obj(self):
|
||||
rule_obj = rule.QosBandwidthLimitRule()
|
||||
rule_obj.id = uuidutils.generate_uuid()
|
||||
rule_obj.max_kbps = 2
|
||||
rule_obj.max_burst_kbps = 200
|
||||
rule_obj.obj_reset_changes()
|
||||
return rule_obj
|
||||
|
||||
def _create_qos_policy_obj(self, rules):
|
||||
policy_dict = {'id': uuidutils.generate_uuid(),
|
||||
'tenant_id': uuidutils.generate_uuid(),
|
||||
'name': 'test',
|
||||
'description': 'test',
|
||||
'shared': False,
|
||||
'rules': rules}
|
||||
policy_obj = policy.QosPolicy(self.context, **policy_dict)
|
||||
policy_obj.obj_reset_changes()
|
||||
return policy_obj
|
||||
|
||||
def _create_fake_port(self):
|
||||
return {'port_id': uuidutils.generate_uuid(),
|
||||
'profile': {'pci_slot': self.PCI_SLOT},
|
||||
'device': self.ASSIGNED_MAC}
|
||||
|
||||
def test_create_rule(self):
|
||||
self.qos_driver.create(self.port, self.qos_policy)
|
||||
self.max_rate_mock.assert_called_once_with(
|
||||
self.ASSIGNED_MAC, self.PCI_SLOT, self.rule.max_kbps)
|
||||
|
||||
def test_update_rule(self):
|
||||
self.qos_driver.update(self.port, self.qos_policy)
|
||||
self.max_rate_mock.assert_called_once_with(
|
||||
self.ASSIGNED_MAC, self.PCI_SLOT, self.rule.max_kbps)
|
||||
|
||||
def test_delete_rules(self):
|
||||
self.qos_driver.delete(self.port, self.qos_policy)
|
||||
self.max_rate_mock.assert_called_once_with(
|
||||
self.ASSIGNED_MAC, self.PCI_SLOT, 0)
|
||||
|
||||
def test__set_vf_max_rate_captures_sriov_failure(self):
|
||||
self.max_rate_mock.side_effect = exceptions.SriovNicError()
|
||||
self.qos_driver._set_vf_max_rate(self.ASSIGNED_MAC, self.PCI_SLOT)
|
||||
|
||||
def test__set_vf_max_rate_unknown_device(self):
|
||||
with mock.patch.object(self.qos_driver.eswitch_mgr, 'device_exists',
|
||||
return_value=False):
|
||||
self.qos_driver._set_vf_max_rate(self.ASSIGNED_MAC, self.PCI_SLOT)
|
||||
self.assertFalse(self.max_rate_mock.called)
|
|
@ -194,6 +194,26 @@ class TestESwitchManagerApi(base.BaseTestCase):
|
|||
'device_mac': self.WRONG_MAC})
|
||||
self.assertFalse(result)
|
||||
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
|
||||
"PciDeviceIPWrapper.get_assigned_macs",
|
||||
return_value=[ASSIGNED_MAC])
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
|
||||
"eswitch_manager.PciOsWrapper.is_assigned_vf",
|
||||
return_value=True)
|
||||
def test_get_pci_slot_by_existing_mac(self, *args):
|
||||
pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(self.ASSIGNED_MAC)
|
||||
self.assertIsNotNone(pci_slot)
|
||||
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
|
||||
"PciDeviceIPWrapper.get_assigned_macs",
|
||||
return_value=[ASSIGNED_MAC])
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
|
||||
"eswitch_manager.PciOsWrapper.is_assigned_vf",
|
||||
return_value=True)
|
||||
def test_get_pci_slot_by_not_existing_mac(self, *args):
|
||||
pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(self.WRONG_MAC)
|
||||
self.assertIsNone(pci_slot)
|
||||
|
||||
|
||||
class TestEmbSwitch(base.BaseTestCase):
|
||||
DEV_NAME = "eth2"
|
||||
|
|
|
@ -49,7 +49,13 @@ class TestSriovAgent(base.BaseTestCase):
|
|||
|
||||
self.agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0)
|
||||
|
||||
def test_treat_devices_removed_with_existed_device(self):
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
|
||||
"PciDeviceIPWrapper.get_assigned_macs",
|
||||
return_value=[DEVICE_MAC])
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
|
||||
"eswitch_manager.PciOsWrapper.is_assigned_vf",
|
||||
return_value=True)
|
||||
def test_treat_devices_removed_with_existed_device(self, *args):
|
||||
agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0)
|
||||
devices = [DEVICE_MAC]
|
||||
with mock.patch.object(agent.plugin_rpc,
|
||||
|
@ -63,7 +69,13 @@ class TestSriovAgent(base.BaseTestCase):
|
|||
self.assertFalse(resync)
|
||||
self.assertTrue(fn_udd.called)
|
||||
|
||||
def test_treat_devices_removed_with_not_existed_device(self):
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
|
||||
"PciDeviceIPWrapper.get_assigned_macs",
|
||||
return_value=[DEVICE_MAC])
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
|
||||
"eswitch_manager.PciOsWrapper.is_assigned_vf",
|
||||
return_value=True)
|
||||
def test_treat_devices_removed_with_not_existed_device(self, *args):
|
||||
agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0)
|
||||
devices = [DEVICE_MAC]
|
||||
with mock.patch.object(agent.plugin_rpc,
|
||||
|
@ -77,7 +89,13 @@ class TestSriovAgent(base.BaseTestCase):
|
|||
self.assertFalse(resync)
|
||||
self.assertTrue(fn_udd.called)
|
||||
|
||||
def test_treat_devices_removed_failed(self):
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib."
|
||||
"PciDeviceIPWrapper.get_assigned_macs",
|
||||
return_value=[DEVICE_MAC])
|
||||
@mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent."
|
||||
"eswitch_manager.PciOsWrapper.is_assigned_vf",
|
||||
return_value=True)
|
||||
def test_treat_devices_removed_failed(self, *args):
|
||||
agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0)
|
||||
devices = [DEVICE_MAC]
|
||||
with mock.patch.object(agent.plugin_rpc,
|
||||
|
|
|
@ -200,6 +200,7 @@ neutron.agent.l2.extensions =
|
|||
qos = neutron.agent.l2.extensions.qos:QosAgentExtension
|
||||
neutron.qos.agent_drivers =
|
||||
ovs = neutron.plugins.ml2.drivers.openvswitch.agent.extension_drivers.qos_driver:QosOVSAgentDriver
|
||||
sriov = neutron.plugins.ml2.drivers.mech_sriov.agent.extension_drivers.qos_driver:QosSRIOVAgentDriver
|
||||
# These are for backwards compat with Icehouse notification_driver configuration values
|
||||
oslo.messaging.notify.drivers =
|
||||
neutron.openstack.common.notifier.log_notifier = oslo_messaging.notify._impl_log:LogDriver
|
||||
|
|
Loading…
Reference in New Issue