Overhaul quantum networking for non-seed clouds.
The networking setup was inappropriate for non-seed clouds - it depended on two network interfaces, forced NAT that we don't need and could not be turned off. Additionally there wasn't much clarity between idempotent machine config and non-idempotent service config. Key changes in this patch: * No longer write persistent device files, instead update idempotently. * NAT rules are now setup idempotently on each o-r-c run. * New definitions for the meaning of various config keys to suit working with different cloud layers. Change-Id: Ie86be0fc884b4c4f655d73da345bdee45fcde473
This commit is contained in:
Robert Collins
parent
04a3c267a2
commit
d9a4ea36e1
|
|
@ -1,125 +1,40 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Initialise the quantum server configuration.
|
||||
#
|
||||
# This creates initial networks for quantum based on heat metadata and local
|
||||
# network configuration.
|
||||
#
|
||||
# It is not idempotent.
|
||||
|
||||
set -eux
|
||||
set -o pipefail
|
||||
|
||||
PATH=/usr/local/bin:$PATH
|
||||
source /root/stackrc
|
||||
|
||||
PUBLIC_INTERFACE=$(os-config-applier --key quantum.ovs.public_interface --key-default '')
|
||||
OVS_PHYSICAL_BRIDGE=$(os-config-applier --type raw --key quantum.ovs.physical_bridge --key-default '')
|
||||
OVS_RANGE=$(os-config-applier --type raw --key quantum.ovs.ovs_range --key-default '')
|
||||
PHYSICAL_NETWORK=$(os-config-applier --type raw --key quantum.ovs.physical_network --key-default '')
|
||||
FIXED_RANGE=$(os-config-applier --type raw --key quantum.ovs.fixed_range --key-default '')
|
||||
ALLOCATION_START=$(os-config-applier --type netaddress --key quantum.ovs.fixed_range.start --key-default '')
|
||||
ALLOCATION_END=$(os-config-applier --type netaddress --key quantum.ovs.fixed_range.end --key-default '')
|
||||
NETWORK=$(os-config-applier --type netaddress --key quantum.ovs.ovs_range --key-default 'unset')
|
||||
|
||||
NETWORK_GATEWAY=`python -c 'import netaddr, sys; print netaddr.IPNetwork(sys.argv[1])[1]' $FIXED_RANGE`
|
||||
OVS_FIXED_ADDRESS=${NETWORK_GATEWAY}/${FIXED_RANGE##*/}
|
||||
FIXED_RANGE_START=`python -c 'import netaddr, sys; print netaddr.IPNetwork(sys.argv[1])[2]' $FIXED_RANGE`
|
||||
FIXED_RANGE_END=`python -c 'import netaddr, sys; print netaddr.IPNetwork(sys.argv[1])[-2]' $FIXED_RANGE`
|
||||
ALLOCATION_POOL="start=${FIXED_RANGE_START},end=${FIXED_RANGE_END}"
|
||||
if [ "$NETWORK" == "unset" ] ; then
|
||||
# undercloud / seed cloud - lookup the network device
|
||||
BRIDGE=$(os-config-applier --type raw --key quantum.ovs.physical_bridge)
|
||||
# NB: assumes only one network on the device
|
||||
NETWORK=$(ip addr show dev $BRIDGE | grep ' inet ' | awk '{print $2}')
|
||||
# Route traffic to this machine
|
||||
NETWORK_GATEWAY=`python -c 'import netaddr, sys; print netaddr.IPNetwork(sys.argv[1]).ip' $NETWORK`
|
||||
fi
|
||||
NETWORK_CIDR=`python -c 'import netaddr, sys; print netaddr.IPNetwork(sys.argv[1]).cidr' $NETWORK`
|
||||
|
||||
DISTRO=`lsb_release -si`
|
||||
|
||||
add_bootstack_network_scripts() {
|
||||
# Create $OVS_PHYSICAL_BRIDGE ifcfg file
|
||||
IFCFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$OVS_PHYSICAL_BRIDGE
|
||||
if ! grep -q boot-stack $IFCFG_FILE; then
|
||||
cat >> $IFCFG_FILE <<eof
|
||||
# This interface was installed by the diskimage-builder boot-stack element.
|
||||
DEVICE=$OVS_PHYSICAL_BRIDGE
|
||||
BOOTPROTO=static
|
||||
IPADDR=$OVS_ADDRESS
|
||||
NETMASK=$OVS_NETMASK
|
||||
ONBOOT=yes
|
||||
|
||||
eof
|
||||
fi
|
||||
|
||||
# Create $PUBLIC_INTERFACE ifcfg file
|
||||
IFCFG_FILE=/etc/sysconfig/network-scripts/ifcfg-$PUBLIC_INTERFACE
|
||||
if ! grep -q boot-stack $IFCFG_FILE; then
|
||||
cat >> $IFCFG_FILE <<eof
|
||||
# This interface was installed by the diskimage-builder boot-stack element.
|
||||
DEVICE=$PUBLIC_INTERFACE
|
||||
BOOTPROTO=manual
|
||||
ONBOOT=yes
|
||||
|
||||
eof
|
||||
fi
|
||||
|
||||
# IFUP Configuration
|
||||
IFUP_FILE=/sbin/ifup-local
|
||||
if ! grep -q boot-stack $IFUP_FILE; then
|
||||
cat >> $IFUP_FILE <<eof
|
||||
# This script was installed by the diskimage-builder boot-stack element.
|
||||
if [ \${1} = "$OVS_PHYSICAL_BRIDGE" ]; then
|
||||
iptables -t nat -A PREROUTING -d 169.254.169.254 -p tcp -m tcp --dport 80 -j REDIRECT --to-port 8775
|
||||
iptables -t nat -A POSTROUTING -s $OVS_RANGE ! -d 192.168.122.1/32 -o eth0 -j MASQUERADE
|
||||
ip addr add $OVS_FIXED_ADDRESS dev \${1}
|
||||
if [ -n "$ALLOCATION_START" -a -n "$ALLOCATION_END" ] ; then
|
||||
ALLOCATION_POOL="start=${ALLOCATION_START},end=${ALLOCATION_END}"
|
||||
fi
|
||||
|
||||
if [ \${1} = "$PUBLIC_INTERFACE" ]; then
|
||||
ifconfig \${1} 0.0.0.0 up
|
||||
fi
|
||||
|
||||
eof
|
||||
fi
|
||||
|
||||
chmod +x $IFUP_FILE
|
||||
}
|
||||
|
||||
add_bootstack_interfaces() {
|
||||
if ! grep -q boot-stack /etc/network/interfaces && ! grep -q "iface $PUBLIC_INTERFACE" /etc/network/interfaces; then
|
||||
cat >> /etc/network/interfaces <<eof
|
||||
|
||||
# This interface was installed by the diskimage-builder boot-stack element.
|
||||
auto $OVS_PHYSICAL_BRIDGE
|
||||
iface $OVS_PHYSICAL_BRIDGE inet static
|
||||
address $OVS_ADDRESS
|
||||
netmask $OVS_NETMASK
|
||||
pre-up service openvswitch-switch restart
|
||||
up iptables -t nat -A PREROUTING -d 169.254.169.254 -p tcp -m tcp --dport 80 -j REDIRECT --to-port 8775
|
||||
up iptables -t nat -A POSTROUTING -s $OVS_RANGE -o eth0 -j MASQUERADE
|
||||
up ip addr add $OVS_FIXED_ADDRESS dev \$IFACE
|
||||
|
||||
auto $PUBLIC_INTERFACE
|
||||
iface $PUBLIC_INTERFACE inet manual
|
||||
up ifconfig \$IFACE 0.0.0.0 up
|
||||
|
||||
# Public Bridge
|
||||
# auto eth2
|
||||
# iface eth2 inet manual
|
||||
# up ifconfig \$IFACE 0.0.0.0 up
|
||||
# up ip link set \$IFACE promisc on
|
||||
# down ifconfig \$IFACE down
|
||||
eof
|
||||
elif ! grep -q '# may need to add' /etc/network/interfaces ; then
|
||||
echo "# may need to add 192.0.2.1 to $OVS_PHYSICAL_BRIDGE, see $0" >> /etc/network/interfaces
|
||||
fi
|
||||
}
|
||||
|
||||
if [ -n "$PUBLIC_INTERFACE" ] && [ -n "$OVS_PHYSICAL_BRIDGE" ] ; then
|
||||
OVS_ADDRESS=`python -c 'import netaddr, sys; print netaddr.IPNetwork(sys.argv[1])[1]' $OVS_RANGE`
|
||||
OVS_NETMASK=`python -c 'import netaddr, sys; print netaddr.IPNetwork(sys.argv[1]).netmask' $OVS_RANGE`
|
||||
|
||||
if [[ "Fedora RedHatEnterpriseServer" =~ "$DISTRO" ]]; then
|
||||
add_bootstack_network_scripts
|
||||
elif [[ "Debian Ubuntu" =~ "$DISTRO" ]]; then
|
||||
add_bootstack_interfaces
|
||||
fi
|
||||
fi
|
||||
|
||||
init-quantum-ovs
|
||||
|
||||
if [[ "Fedora RedHatEnterpriseServer" =~ "$DISTRO" ]]; then
|
||||
service network restart
|
||||
elif [[ "Debian Ubuntu" =~ "$DISTRO" ]]; then
|
||||
service networking restart
|
||||
fi
|
||||
|
||||
service nova-bm-dnsmasq restart
|
||||
# Hackish: ensure the server is running. XXX: Test without this.
|
||||
service quantum-server restart
|
||||
|
||||
# TODO: configurable
|
||||
TENANT_ID=$(keystone tenant-list | grep ' admin ' | awk '{print $2}')
|
||||
|
||||
NET_EXTRA="--shared"
|
||||
|
|
@ -143,8 +58,4 @@ while ! NET_ID=$(quantum net-create $NET_NAME $NET_EXTRA | grep ' id ' | awk '{p
|
|||
sleep 1
|
||||
done
|
||||
|
||||
SUBNET_ID=$(quantum subnet-create $SUBNET_EXTRA --ip_version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} --gateway $NETWORK_GATEWAY $NET_ID $FIXED_RANGE | grep ' id ' | awk '{print $4}')
|
||||
|
||||
if [ -n "$OVS_PHYSICAL_BRIDGE" ] ; then
|
||||
ifconfig $OVS_PHYSICAL_BRIDGE up
|
||||
fi
|
||||
SUBNET_ID=$(quantum subnet-create $SUBNET_EXTRA --ip_version 4 ${ALLOCATION_POOL:+--allocation-pool $ALLOCATION_POOL} ${NETWORK_GATEWAY:+--gateway $NETWORK_GATEWAY} $NET_ID $NETWORK_CIDR | grep ' id ' | awk '{print $4}')
|
||||
|
|
|
|||
|
|
@ -12,6 +12,6 @@ service glance-reg restart
|
|||
service nova-api restart
|
||||
|
||||
init-nova
|
||||
os-refresh-config
|
||||
init-quantum
|
||||
|
||||
os-refresh-config
|
||||
|
|
|
|||
|
|
@ -13,11 +13,18 @@ configured via Heat Metadata. For example:
|
|||
physical_network: ctlplane
|
||||
network_vlan_ranges: ctlplane
|
||||
bridge_mappings: ctlplane:br-ctlplane
|
||||
fixed_range: 192.0.2.32/29
|
||||
ovs_range: 192.0.2.0/24
|
||||
fixed_range:
|
||||
start: 192.0.2.2
|
||||
end: 192.0.2.3
|
||||
ovs_range: 198.51.100.0/24
|
||||
|
||||
If public\_interface and physical\_bridge are not set, no bridges will be
|
||||
connected directly. This is normal for quantum hosting virtual machines
|
||||
as the l3 agent is responsible for making this connection. Some of the
|
||||
when using an overlay network (e.g. GRE tunnelling). Some of the
|
||||
other fields will be ignored in this case. Most of them map 1:1 with their
|
||||
counterparts in the OVS section of ovs\_quantum\_plugin.ini
|
||||
|
||||
fixed\_range is used to create an allocation pool on the tenants network, and
|
||||
ovs\_range is used to create the tenants subnet, if set. If not set the subnet
|
||||
from the physical\_bridge is used, and the gateway is set to the
|
||||
physical\_bridge address.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,3 @@
|
|||
#!/bin/bash
|
||||
set -eux
|
||||
init-quantum-ovs
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
#!/bin/bash
|
||||
set -eux
|
||||
init-quantum-ovs
|
||||
service quantum-openvswitch-agent restart
|
||||
|
|
|
|||
|
|
@ -1,30 +1,82 @@
|
|||
#!/bin/bash
|
||||
# This script should remain idempotent as it may be called several times
|
||||
|
||||
# Idempotent script to apply heat configuration to the running network
|
||||
# environment.
|
||||
#
|
||||
# If a bootstrap public_interface_ip is defined in metadata and not attached to
|
||||
# any device then it will be added to the public_interface device if one is
|
||||
# defined.
|
||||
#
|
||||
# An integration bridge for quantum-ovs-agent is created.
|
||||
#
|
||||
# If no physical bridge is defined in metadata, the script will have no
|
||||
# further effect.
|
||||
#
|
||||
# If the defined physical_bridge is not present on the system, one will be
|
||||
# defined via ovs-vsctl.
|
||||
#
|
||||
# The physical bridge will be brought up.
|
||||
#
|
||||
# If a public_interface is defined in metadata:
|
||||
# - it will be made a member of the physical_bridge.
|
||||
# - an ip addresses on public_interface are moved onto physical_bridge.
|
||||
#
|
||||
# An iptables rule to redirect incoming metadata server requests on the public
|
||||
# bridge device is inserted if not present.
|
||||
#
|
||||
# Note that no persistent config file is written to the OS : ovs-vsctl modifies
|
||||
# a persistent database so the bridge device will persist across reboots, but
|
||||
# [on Ubuntu at least] early boot does not bring up ovs-vswitch early enough,
|
||||
# and metadata access will fail.
|
||||
|
||||
set -eux
|
||||
|
||||
PATH=/usr/local/bin:$PATH
|
||||
|
||||
OVS_PHYSICAL_BRIDGE=$(os-config-applier --key quantum.ovs.physical_bridge --type raw --key-default '')
|
||||
PUBLIC_INTERFACE=$(os-config-applier --key quantum.ovs.public_interface --type raw --key-default '')
|
||||
EXTERNAL_BRIDGE=$(os-config-applier --key quantum.ovs.physical_bridge --type raw --key-default '')
|
||||
PHYSICAL_INTERFACE=$(os-config-applier --key quantum.ovs.public_interface --type raw --key-default '')
|
||||
PHYSICAL_INTERFACE_IP=$(os-config-applier --key bootstack.public_interface_ip --type netaddress --key-default '')
|
||||
|
||||
if [ -n "$PHYSICAL_INTERFACE_IP" -a -n "$PHYSICAL_INTERFACE" ] ; then
|
||||
if ! (ip addr show | grep -q $PHYSICAL_INTERFACE_IP) ; then
|
||||
ip addr add $PHYSICAL_INTERFACE_IP dev $PHYSICAL_INTERFACE
|
||||
fi
|
||||
fi
|
||||
|
||||
# Hacky: ensure the switch is running : we should instead arrange for this
|
||||
# script to not be run before it's started.
|
||||
service openvswitch-switch restart || service openvswitch restart
|
||||
|
||||
ovs-vsctl --no-wait -- --may-exist add-br br-int
|
||||
ovs-vsctl --no-wait br-set-external-id br-int bridge-id br-int
|
||||
|
||||
if [ -n "$OVS_PHYSICAL_BRIDGE" ] && [ -n "$PUBLIC_INTERFACE" ]; then
|
||||
ovs-vsctl --no-wait -- --may-exist add-br $OVS_PHYSICAL_BRIDGE
|
||||
ovs-vsctl add-port $OVS_PHYSICAL_BRIDGE $PUBLIC_INTERFACE || echo "port already added?"
|
||||
if [ -z "$EXTERNAL_BRIDGE" ] ; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -n "$OVS_PHYSICAL_BRIDGE" ] && [ -n "$PUBLIC_INTERFACE" ] ; then
|
||||
ovs-vsctl -- --may-exist add-br $EXTERNAL_BRIDGE
|
||||
|
||||
ip link set dev $EXTERNAL_BRIDGE up
|
||||
|
||||
if [ -n "$PHYSICAL_INTERFACE" ] ; then
|
||||
if ovs-vsctl port-to-br $PHYSICAL_INTERFACE ; then
|
||||
EXISTING_PORT=$(ovs-vsctl port-to-br $PHYSICAL_INTERFACE)
|
||||
if [ "$EXISTING_PORT" != "$PHYSICAL_INTERFACE" ] ; then
|
||||
ovs-vsctl del-port $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE
|
||||
fi
|
||||
fi
|
||||
ovs-vsctl add-port $EXTERNAL_BRIDGE $PHYSICAL_INTERFACE
|
||||
ip link set dev $PHYSICAL_INTERFACE up
|
||||
fi
|
||||
|
||||
if [ -n "$PHYSICAL_INTERFACE" ] ; then
|
||||
# Right now we probably are disconnected, need to move all IPs from public interface to bridge
|
||||
for IP in $(ip addr show dev $PUBLIC_INTERFACE | grep ' inet ' | awk '{print $2}'); do
|
||||
ip addr del $IP dev $PUBLIC_INTERFACE
|
||||
|
||||
if ! ip addr show $OVS_PHYSICAL_BRIDGE | grep $IP; then
|
||||
ip addr add $IP dev $OVS_PHYSICAL_BRIDGE
|
||||
for IP in $(ip addr show dev $PHYSICAL_INTERFACE | grep ' inet ' | awk '{print $2}'); do
|
||||
if ! ip addr show $EXTERNAL_BRIDGE | grep $IP; then
|
||||
ip addr add $IP dev $EXTERNAL_BRIDGE
|
||||
fi
|
||||
ip addr del $IP dev $PHYSICAL_INTERFACE
|
||||
done
|
||||
ifconfig $OVS_PHYSICAL_BRIDGE up
|
||||
fi
|
||||
|
||||
service openvswitch-switch restart || service openvswitch restart
|
||||
iptables -t nat -C PREROUTING -d 169.254.169.254/32 -i $EXTERNAL_BRIDGE -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 8775 || iptables -t nat -I PREROUTING -d 169.254.169.254/32 -i $EXTERNAL_BRIDGE -p tcp -m tcp --dport 80 -j REDIRECT --to-ports 8775
|
||||
|
|
|
|||
|
|
@ -16,3 +16,15 @@ Edit config.json to customise it for your deployment environment. The default
|
|||
is configured for nova-baremetal operation in a seed VM. The configuration
|
||||
options are documented in the actual elements that use the configuration - e.g.
|
||||
nova, quantum etc.
|
||||
|
||||
Configuration keys
|
||||
------------------
|
||||
|
||||
bootstack:
|
||||
public\_interface\_ip: 192.0.2.1/24
|
||||
- What IP address to place on the ovs public interface. Only intended for
|
||||
use when the interface will not be otherwise configured.
|
||||
masquerade\_networks: [192.0.2.0]
|
||||
- What networks, if any, to masquerade. When set, all traffic being
|
||||
output from each network to other networks is masqueraded. Traffic
|
||||
to 192.168.122.1 is never masqueraded.
|
||||
|
|
|
|||
|
|
@ -40,6 +40,10 @@
|
|||
},
|
||||
"metadata-proxy": "false"
|
||||
},
|
||||
"bootstack": {
|
||||
"public_interface_ip": "192.0.2.1/24",
|
||||
"masquerade_networks": ["192.0.2.0"]
|
||||
},
|
||||
"cinder": {
|
||||
"db": "mysql://cinder:unset@localhost/cinder",
|
||||
"volume_size_mb": "5000"
|
||||
|
|
@ -48,15 +52,18 @@
|
|||
"host": "127.0.0.1",
|
||||
"ovs_db": "mysql://quantum:unset@localhost/ovs_quantum?charset=utf8",
|
||||
"ovs": {
|
||||
"public_interface": "eth1",
|
||||
"bridge_mappings": "ctlplane:br-ctlplane",
|
||||
"dnsmasq_range": ["192.0.2.4", "192.0.2.4"],
|
||||
"enable_tunneling": "False",
|
||||
"fixed_range": {
|
||||
"start": "192.0.2.2",
|
||||
"end": "192.0.2.3"
|
||||
},
|
||||
"network_vlan_ranges": "ctlplane",
|
||||
"physical_bridge": "br-ctlplane",
|
||||
"physical_network": "ctlplane",
|
||||
"network_vlan_ranges": "ctlplane",
|
||||
"bridge_mappings": "ctlplane:br-ctlplane",
|
||||
"fixed_range": "192.0.2.32/29",
|
||||
"ovs_range": "192.0.2.0/24",
|
||||
"tenant_network_type": "vlan",
|
||||
"enable_tunneling": "False"
|
||||
"public_interface": "eth1",
|
||||
"tenant_network_type": "vlan"
|
||||
}
|
||||
},
|
||||
"heat": {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
os-apply-config
|
||||
os-refresh-config
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
# In case this script crashed or was interrupted earlier, flush, unlink and
|
||||
# delete the temp chain.
|
||||
iptables -t nat -F BOOTSTACK_MASQ_NEW || true
|
||||
iptables -t nat -D POSTROUTING -j BOOTSTACK_MASQ_NEW || true
|
||||
iptables -t nat -X BOOTSTACK_MASQ_NEW || true
|
||||
iptables -t nat -N BOOTSTACK_MASQ_NEW
|
||||
# Build the chain we want.
|
||||
{{#bootstack.masquerade_networks}}
|
||||
NETWORK={{.}}
|
||||
# Workaround iptables not permitting two -d parameters in one call.
|
||||
iptables -t nat -A BOOTSTACK_MASQ_NEW -s $NETWORK -d 192.168.122.1 -j RETURN
|
||||
iptables -t nat -A BOOTSTACK_MASQ_NEW -s $NETWORK ! -d $NETWORK -j MASQUERADE
|
||||
{{/bootstack.masquerade_networks}}
|
||||
# Link it in.
|
||||
iptables -t nat -I POSTROUTING -j BOOTSTACK_MASQ_NEW
|
||||
# Delete the old chain if present.
|
||||
iptables -t nat -F BOOTSTACK_MASQ || true
|
||||
iptables -t nat -D POSTROUTING -j BOOTSTACK_MASQ || true
|
||||
iptables -t nat -X BOOTSTACK_MASQ || true
|
||||
# Rename the new chain into permanence.
|
||||
iptables -t nat -E BOOTSTACK_MASQ_NEW BOOTSTACK_MASQ
|
||||
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
set -eux
|
||||
|
||||
RULES_SCRIPT=/var/opt/seed-stack/masquerade
|
||||
|
||||
. $RULES_SCRIPT
|
||||
Loading…
Reference in New Issue