From 6f28ec2946aec12fb49841854b884b784c34dd24 Mon Sep 17 00:00:00 2001 From: Ilya Chukhnakov Date: Tue, 7 Feb 2017 18:20:49 +0300 Subject: [PATCH] Add support for DPDK This patch adds support for running Neutron with DPDK-enabled OpenVSwitch. Change-Id: I75cd577d946c4f47d37cafbb5cb7caeda0d9c16c --- docker/openvswitch-base/Dockerfile.j2 | 88 +++++++++++++++++++---- service/files/defaults.yaml | 9 +++ service/files/ml2_conf.ini.j2 | 7 ++ service/files/ovs-ensure-configured.sh.j2 | 65 +++++++++++++++-- service/files/ovs-set-bridge-ip.sh.j2 | 20 ++++++ service/openvswitch-vswitchd.yaml | 20 ++++++ 6 files changed, 191 insertions(+), 18 deletions(-) create mode 100644 service/files/ovs-set-bridge-ip.sh.j2 diff --git a/docker/openvswitch-base/Dockerfile.j2 b/docker/openvswitch-base/Dockerfile.j2 index 85d07db..7088858 100644 --- a/docker/openvswitch-base/Dockerfile.j2 +++ b/docker/openvswitch-base/Dockerfile.j2 @@ -2,19 +2,83 @@ FROM {{ image_spec("base-tools") }} MAINTAINER {{ maintainer }} {% if ovs_version == "system" %} -RUN apt-get -y install --no-install-recommends \ +RUN apt-get update && apt-get -y install --no-install-recommends \ openvswitch-switch \ && apt-get clean +{% elif dpdk_version != "none" %} +RUN apt-get update && apt-get -y install --no-install-recommends \ + gcc \ + libc6-dev \ + libssl-dev \ + libcap-ng-dev \ + libatomic1 \ + pciutils \ + make \ + module-init-tools \ + xz-utils \ + && echo Installing DPDK \ + && curl -LO {{ url.dpdk }}/dpdk-{{ dpdk_version }}.tar.xz \ + && tar -xf dpdk-{{ dpdk_version }}.tar.xz \ + && rm -f dpdk-{{ dpdk_version }}.tar.xz \ + && rename "s/-stable//" dpdk-* \ + && cd dpdk-{{ dpdk_version }} \ + && sed -i "/UIO\\|KMOD\\|KNI\\|POWER/s/=.*/=n/" config/common_linuxapp \ + && make install T=x86_64-native-linuxapp-gcc DESTDIR=install \ + && cd tools \ + && install dpdk-devbind.py dpdk-pmdinfo.py /usr/bin \ + && cd ../.. \ + \ + && echo Installing OVS \ + && curl -LO {{ url.ovs }}/openvswitch-{{ ovs_version }}.tar.gz \ + && tar -xf openvswitch-{{ ovs_version }}.tar.gz \ + && rm -f openvswitch-{{ ovs_version }}.tar.gz \ + && cd openvswitch-{{ ovs_version }} \ + && ./configure \ + --with-dpdk=../dpdk-{{ dpdk_version }}/x86_64-native-linuxapp-gcc \ + --prefix=/usr \ + --localstatedir=/var \ + --sysconfdir=/etc \ + && make && make install && make clean \ + && cd .. \ + \ + && echo Cleanup \ + && rm -rf dpdk-{{ dpdk_version }} \ + && rm -rf openvswitch-{{ ovs_version }} \ + && apt-get purge -y \ + gcc \ + libc6-dev \ + make \ + xz-utils \ + && apt-get autoremove -y \ + && apt-get clean {% else %} -RUN apt-get update && \ - apt-get -y install --no-install-recommends gcc libc6-dev libssl-dev libcap-ng-dev make module-init-tools && \ - curl -Lo openvswitch-{{ ovs_version }}.tar.gz {{ url.ovs }}/openvswitch-{{ ovs_version }}.tar.gz && \ - tar -xvzf openvswitch-{{ ovs_version }}.tar.gz && \ - rm -rf openvswitch-{{ ovs_version }}.tar.gz && \ - cd openvswitch-{{ ovs_version }} && \ - ./configure --prefix=/usr --localstatedir=/var --sysconfdir=/etc && \ - make && make install && make clean && \ - rm -rf openvswitch-{{ ovs_version }} && \ - apt-get purge -y gcc libc6-dev libssl-dev libcap-ng-dev && \ - apt-get clean +RUN apt-get update && apt-get -y install --no-install-recommends \ + gcc \ + libc6-dev \ + libssl-dev \ + libcap-ng-dev \ + libatomic1 \ + make \ + module-init-tools \ + && echo Installing OVS \ + && curl -LO {{ url.ovs }}/openvswitch-{{ ovs_version }}.tar.gz \ + && tar -xf openvswitch-{{ ovs_version }}.tar.gz \ + && rm -f openvswitch-{{ ovs_version }}.tar.gz \ + && cd openvswitch-{{ ovs_version }} \ + && ./configure \ + --prefix=/usr \ + --localstatedir=/var \ + --sysconfdir=/etc \ + && make && make install && make clean \ + && cd .. \ + \ + && echo Cleanup \ + && rm -rf openvswitch-{{ ovs_version }} \ + && apt-get purge -y \ + gcc \ + libc6-dev \ + make \ + xz-utils \ + && apt-get autoremove -y \ + && apt-get clean {% endif %} diff --git a/service/files/defaults.yaml b/service/files/defaults.yaml index 7aa015b..9821629 100644 --- a/service/files/defaults.yaml +++ b/service/files/defaults.yaml @@ -22,6 +22,13 @@ configs: interface: "eth2" flat: true vlan_range: false + dpdk: false + dpdk: + driver: vfio-pci + pmd_cpu_mask: 4 + core_cpu_mask: 2 + mem_channels: 4 + socket_mem: 2048,2048 debug: false plugin_agent: "openvswitch" l3_ha: false @@ -120,7 +127,9 @@ sources: versions: karaf_version: "0.5.3-SNAPSHOT" ovs_version: "system" + dpdk_version: "none" url: ovs: http://openvswitch.org/releases opendaylight: https://nexus.opendaylight.org/content/repositories/opendaylight.snapshot/org/opendaylight/integration/distribution-karaf + dpdk: http://fast.dpdk.org/rel diff --git a/service/files/ml2_conf.ini.j2 b/service/files/ml2_conf.ini.j2 index 218bd13..a0cd558 100644 --- a/service/files/ml2_conf.ini.j2 +++ b/service/files/ml2_conf.ini.j2 @@ -41,7 +41,11 @@ vxlan_group = 239.1.1.1 [securitygroup] {% if neutron.plugin_agent == "openvswitch" %} +{% if True in neutron.physnets|map(attribute="dpdk") %} +firewall_driver = openvswitch +{% else %} firewall_driver = neutron.agent.linux.iptables_firewall.OVSHybridIptablesFirewallDriver +{% endif %} {% elif neutron.plugin_agent == "linuxbridge" %} firewall_driver = neutron.agent.linux.iptables_firewall.IptablesFirewallDriver {% endif %} @@ -57,6 +61,9 @@ enable_distributed_routing = true {% endif %} [ovs] +{% if True in neutron.physnets|map(attribute="dpdk") %} +datapath_type = netdev +{% endif %} ovsdb_interface = {{ neutron.ovsdb.interface }} ovsdb_connection = {{ neutron.ovsdb.connection }} bridge_mappings = diff --git a/service/files/ovs-ensure-configured.sh.j2 b/service/files/ovs-ensure-configured.sh.j2 index 54aa2f8..85263a3 100644 --- a/service/files/ovs-ensure-configured.sh.j2 +++ b/service/files/ovs-ensure-configured.sh.j2 @@ -1,21 +1,74 @@ #!/bin/bash +set -ex + +{% if True in neutron.physnets|map(attribute="dpdk") %} +modprobe {{ neutron.dpdk.driver }} +dpdk_init=$(ovs-vsctl get Open_vSwitch . other_config:dpdk-init \ + 2>/dev/null || true) +if [ "$dpdk_init" != "True" ]; then + dpdk_ifnames={{ neutron.physnets + | selectattr("dpdk") + | map(attribute="interface") + | join("|") }} + pci_whitelist=$(dpdk-devbind.py -s \ + | awk "/ if=($dpdk_ifnames) /{print \" -w \"\$1}") + ovs-vsctl --no-wait set Open_vSwitch . \ + other_config:dpdk-init=True \ + other_config:dpdk-hugepage-dir=/dev/hugepages \ + other_config:pmd-cpu-mask={{ neutron.dpdk.pmd_cpu_mask }} \ + other_config:dpdk-lcore-mask={{ neutron.dpdk.core_cpu_mask }} \ + other_config:dpdk-mem-channels={{ neutron.dpdk.mem_channels }} \ + other_config:dpdk-socket-mem={{ neutron.dpdk.socket_mem }} \ + other_config:dpdk-extra=" --proc-type primary $pci_whitelist " +fi +dpdk_ofport=0 +{% endif %} {% for net in neutron.physnets %} bridge={{ net.bridge_name }} -port={{ net.interface }} -ip link set $port up +{% if net.dpdk %} +interface={{ net.interface }} +port_name=dpdk$dpdk_ofport +dpdk_ofport=$((dpdk_ofport+1)) +{% else %} +interface={{ net.interface }} +port_name={{ net.interface }} +ip link set $interface up +{% endif %} -ovs-vsctl br-exists $bridge; rc=$? -if [[ $rc == 2 ]]; then +if ! ovs-vsctl br-exists $bridge; then changed=changed + {% if net.dpdk %} + ovs-vsctl --no-wait add-br $bridge \ + -- set Bridge $bridge datapath_type=netdev + {% else %} ovs-vsctl --no-wait add-br $bridge + {% endif %} fi -if [[ ! $(ovs-vsctl list-ports $bridge) =~ $(echo "\<$port\>") ]]; then +if [[ ! $(ovs-vsctl list-ports $bridge) =~ $(echo "\<$port_name\>") ]]; then changed=changed - ovs-vsctl --no-wait add-port $bridge $port + ip_addrs=$(ip addr show $interface | awk "/inet/{print \$2}"|tr '\n' ' ') + {% if net.dpdk %} + pci_addr=$(dpdk-devbind.py -s|awk "/ if=$interface /{print \$1}") + mac_addr=$(cat /sys/class/net/$interface/address) + dpdk_driver={{ neutron.dpdk.driver }} + + dpdk-devbind.py -b $dpdk_driver $pci_addr + + ovs-vsctl --no-wait add-port $bridge $port_name \ + -- set Interface $port_name \ + type=dpdk \ + options:dpdk-devargs=$pci_addr \ + other_config:pci_address=$pci_addr \ + other_config:driver=$dpdk_driver + ovs-vsctl --no-wait set bridge $bridge other_config:hwaddr=$mac_addr + {% else %} + ovs-vsctl --no-wait add-port $bridge $port_name + {% endif %} + ovs-vsctl --no-wait set bridge $bridge other_config:ipaddrs="$ip_addrs" fi echo $changed diff --git a/service/files/ovs-set-bridge-ip.sh.j2 b/service/files/ovs-set-bridge-ip.sh.j2 new file mode 100644 index 0000000..b0edee4 --- /dev/null +++ b/service/files/ovs-set-bridge-ip.sh.j2 @@ -0,0 +1,20 @@ +#!/bin/bash +set -ex + +{% for net in neutron.physnets %} + +bridge={{ net.bridge_name }} +port={{ net.interface }} + +{% if not net.dpdk %} +ip addr flush dev $port +ip link set dev $port up +{% endif %} + +ip_addrs=$(ovs-vsctl get bridge $bridge other_config:ipaddrs|tr -d \") +for ip_addr in $ip_addrs; do + ip addr add $ip_addr dev $bridge +done +ip link set dev $bridge up + +{% endfor %} diff --git a/service/openvswitch-vswitchd.yaml b/service/openvswitch-vswitchd.yaml index 38e04b9..1ad89b1 100644 --- a/service/openvswitch-vswitchd.yaml +++ b/service/openvswitch-vswitchd.yaml @@ -20,6 +20,17 @@ service: - name: modules type: host path: /lib/modules + # {% if True in neutron.physnets|map(attribute="dpdk") %} + - name: hugepages + type: host + path: /dev/hugepages + - name: dev + type: host + path: /dev + - name: sys + type: host + path: /sys + # {% endif %} pre: - name: vswitchd-bootstrap command: modprobe openvswitch @@ -45,9 +56,14 @@ service: # {% endif %} files: - ovs-ensure-configured.sh + - ovs-set-bridge-ip.sh # {% if neutron.plugin_agent == "opendaylight" %} - ovs-set-managed-by-odl.sh + # {% endif %} post: + - name: ovs-set-bridge-ip + command: /usr/local/bin/ovs-set-bridge-ip.sh + # {% if neutron.plugin_agent == "opendaylight" %} - name: ovs-set-managed-by-odl command: /usr/local/bin/ovs-set-managed-by-odl.sh # {% endif %} @@ -60,3 +76,7 @@ files: path: /usr/local/bin/ovs-set-managed-by-odl.sh content: ovs-set-managed-by-odl.sh.j2 perm: "0755" + ovs-set-bridge-ip.sh: + path: /usr/local/bin/ovs-set-bridge-ip.sh + content: ovs-set-bridge-ip.sh.j2 + perm: "0755"