[Devstack] Rework VMs connection logic
Devstack currently plugs the simulated baremetal VMs into OVS using the
libvirt bridge driver, this caused a problem because libvirt unplugs the
VM from the network when it is turned off. To fix this an extra bridge
was added between the VM and OVS to allow the OVS port to persist even
when the VM was turned off. This patch replaces how the devstack
simulated baremetal VMs are plugged into OVS with a manually created tap
interface, this removes the need for an extra bridge, because manually
created tap interfaces aren't unplugged when the VM is turned off.
Allow to connect several interfaces to a node by setting
IRONIC_VM_INTERFACE_COUNT devstack variable.
Cherry-pick notes: Had to make the ironic-grenade job non-voting so we
can get this to merge. Next patch will remove the ironic-grenade job.
Change-Id: I7d0249efc55edb4f3a69aaa5b101dd80df2a563f
Co-Authored-By: Vasyl Saienko <vsaienko@mirantis.com>
(cherry picked from commit 6df49741dd
)
This commit is contained in:
parent
9ee06e4f8e
commit
966dce6170
|
@ -125,6 +125,8 @@ IRONIC_VM_EMULATOR=${IRONIC_VM_EMULATOR:-'/usr/bin/qemu-system-x86_64'}
|
|||
IRONIC_VM_ENGINE=${IRONIC_VM_ENGINE:-qemu}
|
||||
IRONIC_VM_NETWORK_BRIDGE=${IRONIC_VM_NETWORK_BRIDGE:-brbm}
|
||||
IRONIC_VM_NETWORK_RANGE=${IRONIC_VM_NETWORK_RANGE:-192.0.2.0/24}
|
||||
# Number of NICs to create Ironic node with. Take affect only for neutron network interface.
|
||||
IRONIC_VM_INTERFACE_COUNT=${IRONIC_VM_INTERFACE_COUNT:-1}
|
||||
IRONIC_VM_MACS_CSV_FILE=${IRONIC_VM_MACS_CSV_FILE:-$IRONIC_DATA_DIR/ironic_macs.csv}
|
||||
IRONIC_AUTHORIZED_KEYS_FILE=${IRONIC_AUTHORIZED_KEYS_FILE:-$HOME/.ssh/authorized_keys}
|
||||
IRONIC_CLEAN_NET_NAME=${IRONIC_CLEAN_NET_NAME:-${IRONIC_PROVISION_NETWORK_NAME:-${PRIVATE_NETWORK_NAME}}}
|
||||
|
@ -877,6 +879,21 @@ function configure_ironic {
|
|||
# NOTE(vsaienko) Add stack to libvirt group when installing without nova.
|
||||
if ! is_service_enabled nova; then
|
||||
add_user_to_group $STACK_USER $LIBVIRT_GROUP
|
||||
|
||||
# Add /dev/net/tun to cgroup_device_acls, needed for type=ethernet interfaces
|
||||
if ! sudo grep -q '^cgroup_device_acl' /etc/libvirt/qemu.conf; then
|
||||
cat <<EOF | sudo tee -a /etc/libvirt/qemu.conf
|
||||
cgroup_device_acl = [
|
||||
"/dev/null", "/dev/full", "/dev/zero",
|
||||
"/dev/random", "/dev/urandom",
|
||||
"/dev/ptmx", "/dev/kvm", "/dev/kqemu",
|
||||
"/dev/rtc", "/dev/hpet","/dev/net/tun",
|
||||
"/dev/vfio/vfio",
|
||||
]
|
||||
EOF
|
||||
restart_libvirt
|
||||
fi
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
|
@ -1248,11 +1265,16 @@ function create_bridge_and_vms {
|
|||
vm_opts+=" -L $UEFI_LOADER_PATH -N $UEFI_NVRAM_PATH"
|
||||
fi
|
||||
|
||||
local bridge_mac
|
||||
bridge_mac=$(ip link show dev $IRONIC_VM_NETWORK_BRIDGE | egrep -o "ether [A-Za-z0-9:]+"|sed "s/ether\ //")
|
||||
|
||||
for vm_name in $(_ironic_bm_vm_names); do
|
||||
sudo -E su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/create-node.sh -n $vm_name \
|
||||
-c $IRONIC_VM_SPECS_CPU -m $IRONIC_VM_SPECS_RAM -d $IRONIC_VM_SPECS_DISK \
|
||||
-a $IRONIC_VM_SPECS_CPU_ARCH -b $IRONIC_VM_NETWORK_BRIDGE $vm_opts \
|
||||
-p $vbmc_port -o $pdu_outlet -f $IRONIC_VM_SPECS_DISK_FORMAT $log_arg" >> $IRONIC_VM_MACS_CSV_FILE
|
||||
-a $IRONIC_VM_SPECS_CPU_ARCH -b $IRONIC_VM_NETWORK_BRIDGE $vm_opts -p $vbmc_port -o $pdu_outlet \
|
||||
-i $IRONIC_VM_INTERFACE_COUNT -f $IRONIC_VM_SPECS_DISK_FORMAT -M $PUBLIC_BRIDGE_MTU $log_arg" >> $IRONIC_VM_MACS_CSV_FILE
|
||||
echo " ${bridge_mac} $IRONIC_VM_NETWORK_BRIDGE" >> $IRONIC_VM_MACS_CSV_FILE
|
||||
|
||||
vbmc_port=$((vbmc_port+1))
|
||||
pdu_outlet=$((pdu_outlet+1))
|
||||
done
|
||||
|
@ -1315,7 +1337,10 @@ function enroll_nodes {
|
|||
local node_prefix
|
||||
node_prefix=$(get_ironic_node_prefix)
|
||||
|
||||
local interface_info
|
||||
|
||||
if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
|
||||
interface_info=$(echo $hardware_info | awk '{print $1}')
|
||||
local ironic_node_cpu=$IRONIC_VM_SPECS_CPU
|
||||
local ironic_node_ram=$IRONIC_VM_SPECS_RAM
|
||||
local ironic_node_disk=$IRONIC_VM_SPECS_DISK
|
||||
|
@ -1367,8 +1392,7 @@ function enroll_nodes {
|
|||
fi
|
||||
|
||||
if [[ "$IRONIC_IS_HARDWARE" == "False" ]]; then
|
||||
local mac_address
|
||||
mac_address=$(echo $hardware_info | awk '{print $1}')
|
||||
interface_info=$(echo $hardware_info | awk '{print $1}')
|
||||
|
||||
if is_deployed_by_ipmitool; then
|
||||
local vbmc_port
|
||||
|
@ -1380,17 +1404,16 @@ function enroll_nodes {
|
|||
node_options+=" -i snmp_outlet=$pdu_outlet"
|
||||
fi
|
||||
# Local-link-connection options
|
||||
local llc_opts=""
|
||||
if [[ "${IRONIC_USE_LINK_LOCAL}" == "True" ]]; then
|
||||
local llc_opts=""
|
||||
local switch_info
|
||||
local switch_id
|
||||
local port_id
|
||||
|
||||
switch_info=$(echo $hardware_info |awk '{print $4}')
|
||||
switch_id=$(echo $hardware_info |awk '{print $5}')
|
||||
port_id=$(echo $hardware_info |awk '{print $6}')
|
||||
switch_id=$(echo $hardware_info |awk '{print $4}')
|
||||
switch_info=$(echo $hardware_info |awk '{print $5}')
|
||||
|
||||
llc_opts="-l switch_id=${switch_id} -l switch_info=${switch_info} -l port_id=${port_id}"
|
||||
# NOTE(vsaienko) we will add port_id later in the code.
|
||||
llc_opts="-l switch_id=${switch_id} -l switch_info=${switch_info} "
|
||||
|
||||
local ironic_api_version='--ironic-api-version latest'
|
||||
fi
|
||||
|
@ -1462,6 +1485,8 @@ function enroll_nodes {
|
|||
node_options+=" -i drac_host=$bmc_address -i drac_password=$bmc_passwd\
|
||||
-i drac_username=$bmc_username"
|
||||
fi
|
||||
|
||||
interface_info="${mac_address}"
|
||||
fi
|
||||
|
||||
# First node created will be used for testing in ironic w/o glance
|
||||
|
@ -1494,7 +1519,19 @@ function enroll_nodes {
|
|||
|
||||
# In case we using portgroups, we should API version that support them.
|
||||
# Othervise API will return 406 ERROR
|
||||
ironic $ironic_api_version port-create --address $mac_address --node $node_id $llc_opts
|
||||
# NOTE(vsaienko) interface_info is in the following format here:
|
||||
# mac1,tap-node0i1;mac2,tap-node0i2;...;macN,tap-node0iN
|
||||
for info in ${interface_info//;/ }; do
|
||||
local mac_address=""
|
||||
local port_id=""
|
||||
local llc_port_opt=""
|
||||
mac_address=$(echo $info| awk -F ',' '{print $1}')
|
||||
port_id=$(echo $info| awk -F ',' '{print $2}')
|
||||
if [[ "${IRONIC_USE_LINK_LOCAL}" == "True" ]]; then
|
||||
llc_port_opt+=" -l port_id=${port_id} "
|
||||
fi
|
||||
ironic $ironic_api_version port-create --address $mac_address --node $node_id $llc_opts $llc_port_opt
|
||||
done
|
||||
|
||||
# NOTE(vsaienko) use node-update instead of specifying network_interface
|
||||
# during node creation. If node is added with latest version of API it
|
||||
|
@ -1827,12 +1864,11 @@ function cleanup_baremetal_basic_ops {
|
|||
local vm_name
|
||||
for vm_name in $(_ironic_bm_vm_names); do
|
||||
sudo su $STACK_USER -c "$IRONIC_SCRIPTS_DIR/cleanup-node.sh $vm_name"
|
||||
|
||||
# Cleanup node bridge/interfaces
|
||||
sudo ip link set ovs-$vm_name down
|
||||
sudo ip link set br-$vm_name down
|
||||
sudo ovs-vsctl del-port ovs-$vm_name
|
||||
sudo ip link del dev ovs-$vm_name
|
||||
sudo ip link del dev br-$vm_name
|
||||
for i in $(seq 1 $IRONIC_VM_INTERFACE_COUNT); do
|
||||
sudo ip tuntap del dev tap-${vm_name}i${i} mode tap
|
||||
done
|
||||
done
|
||||
|
||||
sudo ovs-vsctl --if-exists del-br $IRONIC_VM_NETWORK_BRIDGE
|
||||
|
|
|
@ -71,9 +71,8 @@ def main():
|
|||
help="What boot device to use (hd/network).")
|
||||
parser.add_argument('--libvirt-nic-driver', default='virtio',
|
||||
help='The libvirt network driver to use')
|
||||
parser.add_argument('--bridge', default="br-seed",
|
||||
help='The linux bridge name to use for seeding \
|
||||
the baremetal pseudo-node\'s OS image')
|
||||
parser.add_argument('--interface-count', default=1, type=int,
|
||||
help='The number of interfaces to add to VM.'),
|
||||
parser.add_argument('--console-log',
|
||||
help='File to log console')
|
||||
parser.add_argument('--emulator', default=None,
|
||||
|
@ -99,7 +98,7 @@ def main():
|
|||
'memory': args.memory,
|
||||
'cpus': args.cpus,
|
||||
'bootdev': args.bootdev,
|
||||
'bridge': args.bridge,
|
||||
'interface_count': args.interface_count,
|
||||
'nicdriver': args.libvirt_nic_driver,
|
||||
'emulator': args.emulator,
|
||||
'disk_format': args.disk_format,
|
||||
|
|
|
@ -12,10 +12,12 @@ export PS4='+ ${BASH_SOURCE:-}:${FUNCNAME[0]:-}:L${LINENO:-}: '
|
|||
# Keep track of the DevStack directory
|
||||
TOP_DIR=$(cd $(dirname "$0")/.. && pwd)
|
||||
|
||||
while getopts "n:c:m:d:a:b:e:E:p:o:f:l:L:N:" arg; do
|
||||
while getopts "n:c:i:m:M:d:a:b:e:E:p:o:f:l:L:N:" arg; do
|
||||
case $arg in
|
||||
n) NAME=$OPTARG;;
|
||||
c) CPU=$OPTARG;;
|
||||
i) INTERFACE_COUNT=$OPTARG;;
|
||||
M) INTERFACE_MTU=$OPTARG;;
|
||||
m) MEM=$(( 1024 * OPTARG ));;
|
||||
# Extra G to allow fuzz for partition table : flavor size and registered
|
||||
# size need to be different to actual size.
|
||||
|
@ -88,12 +90,15 @@ fi
|
|||
# it will be plugged to OVS.
|
||||
# This is needed in order to have interface in OVS even
|
||||
# when VM is in shutdown state
|
||||
INTERFACE_COUNT=${INTERFACE_COUNT:-1}
|
||||
|
||||
sudo brctl addbr br-$NAME
|
||||
sudo ip link set br-$NAME up
|
||||
sudo ovs-vsctl add-port $BRIDGE ovs-$NAME -- set Interface ovs-$NAME type=internal
|
||||
sudo ip link set ovs-$NAME up
|
||||
sudo brctl addif br-$NAME ovs-$NAME
|
||||
for int in $(seq 1 $INTERFACE_COUNT); do
|
||||
tapif=tap-${NAME}i${int}
|
||||
sudo ip tuntap add dev $tapif mode tap
|
||||
sudo ip link set $tapif mtu $INTERFACE_MTU
|
||||
sudo ip link set $tapif up
|
||||
sudo ovs-vsctl add-port $BRIDGE $tapif
|
||||
done
|
||||
|
||||
if ! virsh list --all | grep -q $NAME; then
|
||||
virsh vol-list --pool $LIBVIRT_STORAGE_POOL | grep -q $VOL_NAME &&
|
||||
|
@ -110,7 +115,8 @@ if ! virsh list --all | grep -q $NAME; then
|
|||
$TOP_DIR/scripts/configure-vm.py \
|
||||
--bootdev network --name $NAME --image "$volume_path" \
|
||||
--arch $ARCH --cpus $CPU --memory $MEM --libvirt-nic-driver $LIBVIRT_NIC_DRIVER \
|
||||
--bridge br-$NAME --disk-format $DISK_FORMAT $VM_LOGGING --engine $ENGINE $UEFI_OPTS $vm_opts >&2
|
||||
--disk-format $DISK_FORMAT $VM_LOGGING --engine $ENGINE $UEFI_OPTS $vm_opts \
|
||||
--interface-count $INTERFACE_COUNT >&2
|
||||
|
||||
# Createa Virtual BMC for the node if IPMI is used
|
||||
if [[ $(type -P vbmc) != "" ]]; then
|
||||
|
@ -119,7 +125,6 @@ if ! virsh list --all | grep -q $NAME; then
|
|||
fi
|
||||
fi
|
||||
|
||||
# echo mac
|
||||
VM_MAC=$(virsh dumpxml $NAME | grep "mac address" | head -1 | cut -d\' -f2)
|
||||
switch_id=$(ip link show dev $BRIDGE | egrep -o "ether [A-Za-z0-9:]+"|sed "s/ether\ //")
|
||||
echo $VM_MAC $VBMC_PORT $PDU_OUTLET $BRIDGE $switch_id ovs-$NAME
|
||||
# echo mac in format mac1,ovs-node-0i1;mac2,ovs-node-0i2;...;macN,ovs-node0iN
|
||||
VM_MAC=$(echo -n $(virsh domiflist $NAME |awk '/tap-/{print $5","$1}')|tr ' ' ';')
|
||||
echo -n "$VM_MAC $VBMC_PORT $PDU_OUTLET"
|
||||
|
|
|
@ -44,11 +44,13 @@
|
|||
<controller type='ide' index='0'>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x1'/>
|
||||
</controller>
|
||||
<interface type='bridge'>
|
||||
<source bridge='{{ bridge }}'/>
|
||||
{% for n in range(1, interface_count+1) %}
|
||||
<interface type='ethernet'>
|
||||
<target dev='{{ "tap-" + name + "i" + n|string }}'/>
|
||||
<model type='{{ nicdriver }}'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
|
||||
<address type='pci' domain='0x0000' bus='0x01' slot='{{ "0x0" + n|string }}' function='0x0'/>
|
||||
</interface>
|
||||
{% endfor %}
|
||||
<input type='mouse' bus='ps2'/>
|
||||
<graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'/>
|
||||
<video>
|
||||
|
|
|
@ -2,7 +2,6 @@
|
|||
name: openstack/ironic
|
||||
check:
|
||||
jobs:
|
||||
- ironic-grenade-dsvm
|
||||
- ironic-tempest-dsvm-ipa-partition-bios-agent_ipmitool-tinyipa
|
||||
- ironic-tempest-dsvm-ipa-partition-bios-pxe_ipmitool-tinyipa
|
||||
- ironic-tempest-dsvm-ipa-partition-uefi-pxe_ipmitool-tinyipa
|
||||
|
@ -10,6 +9,8 @@
|
|||
- ironic-tempest-dsvm-ipa-wholedisk-bios-pxe_ipmitool-tinyipa
|
||||
- ironic-tempest-dsvm-multitenant-network
|
||||
# Non-voting jobs
|
||||
- ironic-grenade-dsvm:
|
||||
voting: false
|
||||
- ironic-tempest-dsvm-ipa-wholedisk-agent_ipmitool-tinyipa-multinode:
|
||||
voting: false
|
||||
- ironic-tempest-dsvm-ipa-wholedisk-bios-pxe_snmp-tinyipa:
|
||||
|
@ -21,7 +22,6 @@
|
|||
gate:
|
||||
queue: ironic
|
||||
jobs:
|
||||
- ironic-grenade-dsvm
|
||||
- ironic-tempest-dsvm-ipa-partition-bios-agent_ipmitool-tinyipa
|
||||
- ironic-tempest-dsvm-ipa-partition-bios-pxe_ipmitool-tinyipa
|
||||
- ironic-tempest-dsvm-ipa-partition-uefi-pxe_ipmitool-tinyipa
|
||||
|
|
Loading…
Reference in New Issue