From 3108d28eb3b50999d662765d5d8b90df760ff580 Mon Sep 17 00:00:00 2001 From: songwenping Date: Thu, 21 May 2020 15:04:57 +0800 Subject: [PATCH] Add Inspur FPGA driver Please see the test report in https://wiki.openstack.org/wiki/Cyborg/TestReport/InspurFPGA This patch implemented Inspur FPGA driver in Cyborg. Currently we don't support program with this card by Cyborg, the operator can bind the card to guest and use it in guest, so BSP data reported is ignored now. The spec is already merged. Please see: https://specs.openstack.org/openstack/cyborg-specs/specs/victoria/approved/inspur-fpga-driver-proposal.html Story: 2007772 Task: 39997 Change-Id: I7cc3482a4a6ac6fb0ca6965b7ce18bc37b122f63 --- cyborg/accelerator/drivers/fpga/base.py | 3 +- .../drivers/fpga/inspur/__init__.py | 0 .../accelerator/drivers/fpga/inspur/driver.py | 40 +++++ .../drivers/fpga/inspur/sysinfo.py | 154 ++++++++++++++++++ cyborg/accelerator/drivers/fpga/utils.py | 2 +- cyborg/conf/agent.py | 3 +- .../drivers/fpga/inspur/__init__.py | 0 .../drivers/fpga/inspur/test_driver.py | 112 +++++++++++++ .../accelerator/drivers/fpga/test_base.py | 2 + setup.cfg | 1 + 10 files changed, 314 insertions(+), 3 deletions(-) create mode 100644 cyborg/accelerator/drivers/fpga/inspur/__init__.py create mode 100644 cyborg/accelerator/drivers/fpga/inspur/driver.py create mode 100644 cyborg/accelerator/drivers/fpga/inspur/sysinfo.py create mode 100644 cyborg/tests/unit/accelerator/drivers/fpga/inspur/__init__.py create mode 100644 cyborg/tests/unit/accelerator/drivers/fpga/inspur/test_driver.py diff --git a/cyborg/accelerator/drivers/fpga/base.py b/cyborg/accelerator/drivers/fpga/base.py index d0ba512f..71b037c3 100644 --- a/cyborg/accelerator/drivers/fpga/base.py +++ b/cyborg/accelerator/drivers/fpga/base.py @@ -20,7 +20,8 @@ Cyborg FPGA driver implementation. from cyborg.accelerator.drivers.fpga import utils -VENDOR_MAPS = {"0x8086": "intel"} +VENDOR_MAPS = {"0x8086": "intel", + "1bd4": 'inspur'} class FPGADriver(object): diff --git a/cyborg/accelerator/drivers/fpga/inspur/__init__.py b/cyborg/accelerator/drivers/fpga/inspur/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/accelerator/drivers/fpga/inspur/driver.py b/cyborg/accelerator/drivers/fpga/inspur/driver.py new file mode 100644 index 00000000..ed448215 --- /dev/null +++ b/cyborg/accelerator/drivers/fpga/inspur/driver.py @@ -0,0 +1,40 @@ +# Copyright 2020 Inspur, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Cyborg Inspur FPGA driver implementation. +""" + +from oslo_log import log as logging + +from cyborg.accelerator.drivers.fpga.base import FPGADriver +from cyborg.accelerator.drivers.fpga.inspur import sysinfo + +LOG = logging.getLogger(__name__) + + +class InspurFPGADriver(FPGADriver): + """Base class for FPGA drivers. + + This is just a virtual FPGA drivers interface. + Vendor should implement their specific drivers. + """ + VENDOR = "inspur" + + def __init__(self, *args, **kwargs): + pass + + def discover(self): + return sysinfo.fpga_tree() diff --git a/cyborg/accelerator/drivers/fpga/inspur/sysinfo.py b/cyborg/accelerator/drivers/fpga/inspur/sysinfo.py new file mode 100644 index 00000000..3949e36f --- /dev/null +++ b/cyborg/accelerator/drivers/fpga/inspur/sysinfo.py @@ -0,0 +1,154 @@ +# Copyright 2020 Inspur, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +""" +Cyborg Inspur FPGA driver implementation. +""" + +import re + +from oslo_concurrency import processutils +from oslo_serialization import jsonutils + +from cyborg.accelerator.common import utils +from cyborg.common import constants +from cyborg.objects.driver_objects import driver_attach_handle +from cyborg.objects.driver_objects import driver_attribute +from cyborg.objects.driver_objects import driver_controlpath_id +from cyborg.objects.driver_objects import driver_deployable +from cyborg.objects.driver_objects import driver_device +import cyborg.privsep + +INSPUR_FPGA_FLAGS = ["Inspur Electronic Information Industry Co., Ltd.", + "Processing accelerators"] +INSPUR_FPGA_INFO_PATTERN = re.compile( + r"(?P[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:" + r"[0-9a-fA-F]{2}\.[0-9a-fA-F]) " + r"(?P.*) [\[].*]: (?P.*) .*" + r"[\[](?P[0-9a-fA-F]" + r"{4}):(?P[0-9a-fA-F]{4})].*") + +VENDOR_MAPS = {"1bd4": "inspur"} + + +@cyborg.privsep.sys_admin_pctxt.entrypoint +def lspci_privileged(): + cmd = ['lspci', '-nnn', '-D'] + return processutils.execute(*cmd) + + +def get_pci_devices(pci_flags, vendor_id=None): + device_for_vendor_out = [] + all_device_out = [] + lspci_out = lspci_privileged()[0].split('\n') + for i in range(len(lspci_out)): + if any(x in lspci_out[i] for x in pci_flags): + all_device_out.append(lspci_out[i]) + if vendor_id and vendor_id in lspci_out[i]: + device_for_vendor_out.append(lspci_out[i]) + return device_for_vendor_out if vendor_id else all_device_out + + +def get_traits(vendor_id, product_id): + """Generate traits for FPGAs. + : param vendor_id: vendor_id of FPGA, eg."1bd4" + : param product_id: product_id of FPGA, eg."a115". + Example FPGA traits: + {traits:["CUSTOM_FPGA_INSPUR", "CUSTOM_FPGA_PRODUCT_ID_A115"]} + """ + traits = [] + traits.append("CUSTOM_FPGA_" + VENDOR_MAPS.get(vendor_id, "").upper()) + traits.append("CUSTOM_FPGA_PRODUCT_ID_" + product_id.upper()) + # TODO(wenping) Currently we don't support program by Cyborg. The operator + # can bind Inspur FPGA card to guest and then use it in guest. So the BSP + # data reported is ignored here now. + return {"traits": traits} + + +def fpga_tree(): + fpga_list = [] + fpgas = get_pci_devices(INSPUR_FPGA_FLAGS) + for fpga in fpgas: + m = INSPUR_FPGA_INFO_PATTERN.match(fpga) + if m: + fpga_dict = m.groupdict() + # generate traits info + traits = get_traits( + fpga_dict["vendor_id"], fpga_dict["product_id"]) + fpga_dict["rc"] = constants.RESOURCES["FPGA"] + fpga_dict.update(traits) + fpga_list.append(_generate_driver_device(fpga_dict)) + return fpga_list + + +def _generate_driver_device(fpga): + driver_device_obj = driver_device.DriverDevice() + driver_device_obj.vendor = fpga["vendor_id"] + driver_device_obj.model = fpga.get('model', 'miss model info') + std_board_info = {'product_id': fpga.get('product_id', None), + 'controller': fpga.get('controller', None)} + vendor_board_info = { + 'vendor_info': fpga.get('vendor_info', 'fpga_vb_info')} + driver_device_obj.std_board_info = jsonutils.dumps(std_board_info) + driver_device_obj.vendor_board_info = jsonutils.dumps(vendor_board_info) + driver_device_obj.type = constants.DEVICE_FPGA + driver_device_obj.stub = fpga.get('stub', False) + driver_device_obj.controlpath_id = _generate_controlpath_id(fpga) + driver_device_obj.deployable_list = _generate_dep_list(fpga) + return driver_device_obj + + +def _generate_controlpath_id(fpga): + driver_cpid = driver_controlpath_id.DriverControlPathID() + driver_cpid.cpid_type = "PCI" + driver_cpid.cpid_info = utils.pci_str_to_json(fpga["devices"]) + return driver_cpid + + +def _generate_dep_list(fpga): + dep_list = [] + driver_dep = driver_deployable.DriverDeployable() + driver_dep.attribute_list = _generate_attribute_list(fpga) + driver_dep.attach_handle_list = [] + driver_dep.name = fpga.get('name', '') + '_' + fpga["devices"] + driver_dep.driver_name = VENDOR_MAPS.get(fpga["vendor_id"]).upper() + driver_dep.num_accelerators = 1 + driver_dep.attach_handle_list = [_generate_attach_handle(fpga)] + dep_list.append(driver_dep) + return dep_list + + +def _generate_attribute_list(fpga): + attr_list = [] + for k, v in fpga.items(): + if k == "rc": + driver_attr = driver_attribute.DriverAttribute() + driver_attr.key, driver_attr.value = k, v + attr_list.append(driver_attr) + if k == "traits": + values = fpga.get(k, []) + for index, val in enumerate(values): + driver_attr = driver_attribute.DriverAttribute( + key="trait" + str(index), value=val) + attr_list.append(driver_attr) + return attr_list + + +def _generate_attach_handle(fpga): + driver_ah = driver_attach_handle.DriverAttachHandle() + driver_ah.attach_type = constants.AH_TYPE_PCI + driver_ah.attach_info = utils.pci_str_to_json(fpga["devices"]) + driver_ah.in_use = False + return driver_ah diff --git a/cyborg/accelerator/drivers/fpga/utils.py b/cyborg/accelerator/drivers/fpga/utils.py index cef4ef14..75562ba9 100644 --- a/cyborg/accelerator/drivers/fpga/utils.py +++ b/cyborg/accelerator/drivers/fpga/utils.py @@ -21,7 +21,7 @@ import glob import re -VENDORS = ["intel"] # can extend, such as ["intel", "xilinx"] +VENDORS = ["intel", "inspur"] # can extend, such as ["intel", "xilinx"] SYS_FPGA_PATH = "/sys/class/fpga" VENDORS_PATTERN = re.compile("|".join(["(%s)" % v for v in VENDORS])) diff --git a/cyborg/conf/agent.py b/cyborg/conf/agent.py index e8eceb54..26f6179f 100644 --- a/cyborg/conf/agent.py +++ b/cyborg/conf/agent.py @@ -21,7 +21,8 @@ opts = [ cfg.ListOpt('enabled_drivers', default=[], help=_('The accelerator drivers enabled on this agent. Such ' - 'as intel_fpga_driver, nvidia_gpu_driver, etc.')), + 'as intel_fpga_driver, inspur_fpga_driver,' + 'nvidia_gpu_driver, etc.')), ] opt_group = cfg.OptGroup(name='agent', diff --git a/cyborg/tests/unit/accelerator/drivers/fpga/inspur/__init__.py b/cyborg/tests/unit/accelerator/drivers/fpga/inspur/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cyborg/tests/unit/accelerator/drivers/fpga/inspur/test_driver.py b/cyborg/tests/unit/accelerator/drivers/fpga/inspur/test_driver.py new file mode 100644 index 00000000..e11fbf89 --- /dev/null +++ b/cyborg/tests/unit/accelerator/drivers/fpga/inspur/test_driver.py @@ -0,0 +1,112 @@ +# Copyright 2020 Inspur, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_serialization import jsonutils +from unittest import mock + +from cyborg.accelerator.drivers.fpga.inspur.driver import InspurFPGADriver +from cyborg.tests import base + +INSPUR_FPGA_INFO = ("0000:86:00.0 Processing accelerators [1200]: " + "Inspur Electronic Information Industry Co., Ltd. " + "Device [1bd4:a115] (rev 04)") + + +class stdout(object): + def readlines(self): + return [INSPUR_FPGA_INFO] + + +class p(object): + def __init__(self): + self.stdout = stdout() + + def wait(self): + pass + + +class TestInspurFPGADriver(base.TestCase): + + def setUp(self): + super(TestInspurFPGADriver, self).setUp() + self.p = p() + + @mock.patch('cyborg.accelerator.drivers.fpga.' + 'inspur.sysinfo.lspci_privileged') + def test_discover(self, mock_devices_for_vendor): + mock_devices_for_vendor.return_value = self.p.stdout.readlines() + fpga_list = InspurFPGADriver().discover() + self.assertEqual(1, len(fpga_list)) + attach_handle_list = [ + {'attach_type': 'PCI', + 'attach_info': '{"bus": "86", ' + '"device": "00", ' + '"domain": "0000", ' + '"function": "0"}', + 'in_use': False} + ] + attribute_list = [ + {'key': 'rc', 'value': 'FPGA'}, + {'key': 'trait0', 'value': 'CUSTOM_FPGA_INSPUR'}, + {'key': 'trait1', 'value': 'CUSTOM_FPGA_PRODUCT_ID_A115'}, + ] + expected = { + 'vendor': '1bd4', + 'type': 'FPGA', + 'std_board_info': {"controller": "Processing accelerators", + "product_id": "a115"}, + 'vendor_board_info': {"vendor_info": "fpga_vb_info"}, + 'deployable_list': + [ + { + 'num_accelerators': 1, + 'driver_name': 'INSPUR', + 'name': + 'Inspur Electronic Information Industry Co., Ltd.' + ' Device_0000:86:00.0', + 'attach_handle_list': attach_handle_list, + 'attribute_list': attribute_list + }, + ], + 'controlpath_id': {'cpid_info': '{"bus": "86", ' + '"device": "00", ' + '"domain": "0000", ' + '"function": "0"}', + 'cpid_type': 'PCI'} + } + fpga_obj = fpga_list[0] + fpga_dict = fpga_obj.as_dict() + fpga_dep_list = fpga_dict['deployable_list'] + fpga_attach_handle_list = ( + fpga_dep_list[0].as_dict()['attach_handle_list']) + fpga_attribute_list = fpga_dep_list[0].as_dict()['attribute_list'] + attri_obj_data = [] + [attri_obj_data.append(attr.as_dict()) for attr in fpga_attribute_list] + attribute_actual_data = sorted(attri_obj_data, key=lambda i: i['key']) + self.assertEqual(expected['vendor'], fpga_dict['vendor']) + self.assertEqual(expected['controlpath_id'], + fpga_dict['controlpath_id']) + self.assertEqual(expected['std_board_info'], + jsonutils.loads(fpga_dict['std_board_info'])) + self.assertEqual(expected['vendor_board_info'], + jsonutils.loads(fpga_dict['vendor_board_info'])) + self.assertEqual(expected['deployable_list'][0]['num_accelerators'], + fpga_dep_list[0].as_dict()['num_accelerators']) + self.assertEqual(expected['deployable_list'][0]['name'], + fpga_dep_list[0].as_dict()['name']) + self.assertEqual(expected['deployable_list'][0]['driver_name'], + fpga_dep_list[0].as_dict()['driver_name']) + self.assertEqual(attach_handle_list[0], + fpga_attach_handle_list[0].as_dict()) + self.assertEqual(attribute_list, attribute_actual_data) diff --git a/cyborg/tests/unit/accelerator/drivers/fpga/test_base.py b/cyborg/tests/unit/accelerator/drivers/fpga/test_base.py index c70a987d..76868fd9 100644 --- a/cyborg/tests/unit/accelerator/drivers/fpga/test_base.py +++ b/cyborg/tests/unit/accelerator/drivers/fpga/test_base.py @@ -14,12 +14,14 @@ from cyborg.accelerator.drivers.fpga.base import FPGADriver from cyborg.accelerator.drivers.fpga.intel.driver import IntelFPGADriver # noqa +from cyborg.accelerator.drivers.fpga.inspur.driver import InspurFPGADriver # noqa from cyborg.tests import base class TestFPGADriver(base.TestCase): def test_create(self): FPGADriver.create("intel") + FPGADriver.create("inspur") self.assertRaises(LookupError, FPGADriver.create, "xilinx") def test_discover(self): diff --git a/setup.cfg b/setup.cfg index 28a9f9ea..4b8b2719 100644 --- a/setup.cfg +++ b/setup.cfg @@ -47,6 +47,7 @@ cyborg.database.migration_backend = cyborg.accelerator.driver = intel_fpga_driver = cyborg.accelerator.drivers.fpga.intel.driver:IntelFPGADriver + inspur_fpga_driver = cyborg.accelerator.drivers.fpga.inspur.driver:InspurFPGADriver nvmf_spdk_driver = cyborg.accelerator.drivers.spdk.nvmf.nvmf:NVMFDRIVER nvidia_gpu_driver = cyborg.accelerator.drivers.gpu.nvidia.driver:NVIDIAGPUDriver fake_driver = cyborg.accelerator.drivers.fake:FakeDriver