Add procedures/scripts as starting point of multi-DC deployment

Change-Id: If455358e9164aa51b99ed3a06dd49ab2cf03d69b
This commit is contained in:
Changbin Liu 2014-07-23 19:55:11 -04:00
parent cf46b6f9fc
commit 8233e63e95
10 changed files with 399 additions and 41 deletions

View File

@ -0,0 +1,10 @@
#!/bin/bash
CONTROLLERS=$@
docker.io rm -f `docker.io ps -a | grep changbl | awk '{print $1}'`
for i in `ovs-vsctl show | grep Port | grep pl | awk '{print $2}' | cut -d '"' -f 2`
do
ovs-vsctl del-port obr2 $i
done

View File

@ -14,7 +14,7 @@ sudo apt-get install -y opscode-keyring # permanent upgradeable keyring
sudo apt-get install -y debconf-utils
sudo apt-get -y upgrade
sudo apt-get install -y libgnumail-java ruby-addressable libextlib-ruby jsvc \
sudo apt-get install -y libgnumail-java ruby-addressable libextlib-ruby jsvc \
libdb5.1-java-gcj erlang-eunit libjaxp1.3-java libcommons-pool-java \
libdb-je-java ruby-mixlib-config gcj-4.6-jre-lib libjson-ruby1.8 \
libbcel-java erlang-crypto libgeronimo-jta-1.1-spec-java libecj-java \

View File

@ -47,20 +47,3 @@ cd libvirt
# --with-xen=yes to the command
make -j
sudo make install
# enable libvirt for VM live migration
sudo sed -i /etc/libvirt/libvirtd.conf \
-e 's/#listen_tls = 0/listen_tls = 0/g' \
-e 's/#listen_tcp = 1/listen_tcp = 1/g' \
-e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
sudo sed -i /etc/init/libvirt-bin.conf \
-e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
sudo sed -i /etc/default/libvirt-bin \
-e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
# restart libvirt
sudo service libvirt-bin restart
# Remove the default network created by libvirt
sudo virsh net-destroy default
sudo virsh net-undefine default

58
bin/install_zookeeper.sh Normal file
View File

@ -0,0 +1,58 @@
#!/bin/bash
## Install ZooKeeper and OpenJDK 7
VERSION=3.4.6
sudo apt-get -y install openjdk-7-jdk
sudo mkdir -p /opt/zookeeper
sudo chmod 00777 /opt/zookeeper
sudo chown root:root /opt/zookeeper
cd /opt/zookeeper
wget http://mirrors.gigenet.com/apache/zookeeper/zookeeper-${VERSION}/zookeeper-${VERSION}.tar.gz
tar xzvf zookeeper-${VERSION}.tar.gz
mv zookeeper-${VERSION} zookeeper
cd zookeeper
sudo mkdir -p /mnt/zookeeper
sudo chmod 00777 /mnt/zookeeper
sudo chown root:root /mnt/zookeeper
echo "1" | tee /mnt/zookeeper/myid
echo "# The number of milliseconds of each tick
tickTime=2000
# The number of ticks that the initial
# synchronization phase can take
initLimit=5
# The number of ticks that can pass between
# sending a request and getting an acknowledgement
syncLimit=2
# the directory where the snapshot is stored.
# do not use /tmp for storage, /tmp here is just
# example sakes.
dataDir=/mnt/zookeeper
# the port at which the clients will connect
clientPort=2181
server.1=<zk_host_1>:2888:3888
server.2=<zk_host_2>:2888:3888
server.3=<zk_host_3>:2888:3888
# the directory where the log is stored
# dataLogDir=/mnt/zookeeper
#
# Be sure to read the maintenance section of the
# administrator guide before turning on autopurge.
#
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
#
# The number of snapshots to retain in dataDir
#autopurge.snapRetainCount=3
# Purge task interval in hours
# Set to 0 to disable auto purge feature
#autopurge.purgeInterval=1
" | tee /opt/zookeeper/zookeeper/conf/zoo.cfg
# Start the service
# /opt/zookeeper/zookeeper/bin/zkServer.sh start

View File

@ -0,0 +1,81 @@
#!/usr/bin/env python
import sys
import os
BRIDGE = "obr2"
# BRIDGE = "br2"
GATEWAY = "10.2.2.10"
NET_PREFIX = "10.2"
MAC_ADDR_PREFIX = "52:54:00:2d:"
IMAGE = "changbl/u1404-3"
def int_to_hex(n):
if not 1 <= n <= 254:
raise KeyError('Wrong value: number not within [1, 254]')
s = hex(n)[2:]
if len(s) == 2:
return s
else:
return '0' + s
def launch(subnet, begin, end):
for i in range(begin, end):
name = "%s.%s" % (subnet, i)
mac_addr = (MAC_ADDR_PREFIX + "%s:%s" % (
int_to_hex(subnet), int_to_hex(i)))
# cmd = """/root/pipework %s -i eth0 $(\
# docker.io run --privileged=true -n=false --name=%s -d %s /usr/sbin/sshd -D\
# ) %s.%s.%s/16@%s %s""" % (
# BRIDGE, name, IMAGE, NET_PREFIX, subnet, i, GATEWAY, mac_addr)
cmd = """/root/pipework %s -i eth0 $(\
docker.io run -n=false --privileged=true --name=%s -d %s /usr/sbin/sshd -D\
) dhcp %s""" % (BRIDGE, name, IMAGE, mac_addr)
print cmd
os.system(cmd)
"""
docker.io run --privileged=true -d -n=false \
-lxc-conf="lxc.network.type = veth" \
-lxc-conf="lxc.network.link = br2" \
-lxc-conf="lxc.network.flags = up" \
-lxc-conf="lxc.network.name = eth0" \
-lxc-conf="lxc.network.ipv4 = 10.2.101.3/16" \
-lxc-conf="lxc.network.hwaddr=52:54:00:2d:65:03" \
-lxc-conf="lxc.network.ipv4.gateway = 10.2.1.10" \
--name=101.3 changbl/u1401-1 /usr/sbin/sshd -D
ID=$(docker.io run -n=false --name=test4 -d changbl/u1404-2 /usr/sbin/sshd -D)
./pipework obr2 -i eth0 $ID dhcp 52:54:00:2d:c9:04
./pipework obr2 -i eth0 $ID 10.2.201.8/16@10.2.2.10 52:54:00:2d:c9:08
docker.io inspect da1fbd5421f7 | grep ID
lxc-attach -n \
da1fbd5421f75ef1a640019d4659489ee53faf4135f4e6feeb8872580f74549a -- /bin/bash
"""
# cmd = """docker.io run --privileged=true -d -n=false \
# -lxc-conf="lxc.network.type = veth" \
# -lxc-conf="lxc.network.link = %s" \
# -lxc-conf="lxc.network.flags = up" \
# -lxc-conf="lxc.network.name = eth0" \
# -lxc-conf="lxc.network.ipv4 = %s.%s.%s/16" \
# -lxc-conf="lxc.network.hwaddr=%s" \
# -lxc-conf="lxc.network.ipv4.gateway = %s" \
# --name=%s \
# %s /usr/sbin/sshd -D""" % (
# BRIDGE, NET_PREFIX, subnet, i, mac_addr, GATEWAY, name, IMAGE)
#
if __name__ == "__main__":
subnet = int(sys.argv[1])
begin = int(sys.argv[2])
end = int(sys.argv[3])
launch(subnet, begin, end)

View File

@ -0,0 +1,94 @@
#!/usr/bin/env python
import sys
import os
import uuid
XML_TEMPLATE = """<domain type='kvm'>
<name>%s</name>
<uuid>%s</uuid>
<memory unit='KiB'>262144</memory>
<vcpu placement='static'>1</vcpu>
<os>
<type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
<boot dev='hd'/>
</os>
<features>
<acpi/>
</features>
<clock offset='utc'/>
<on_poweroff>destroy</on_poweroff>
<on_reboot>restart</on_reboot>
<on_crash>destroy</on_crash>
<devices>
<emulator>/usr/bin/kvm</emulator>
<disk type='file' device='disk'>
<driver name='qemu' type='qcow2'/>
<source file='/var/lib/libvirt/images/%s/image.qcow2'/>
<target dev='vda' bus='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
</disk>
<controller type='usb' index='0'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
</controller>
<controller type='pci' index='0' model='pci-root'/>
<interface type='bridge'>
<mac address='52:54:00:2d:%s:%s'/>
<source bridge='obr2'/>
<virtualport type='openvswitch'/>
<model type='virtio'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
</interface>
<input type='mouse' bus='ps2'/>
<input type='keyboard' bus='ps2'/>
<graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'>
<listen type='address' address='0.0.0.0'/>
</graphics>
<video>
<model type='cirrus' vram='9216' heads='1'/>
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
</video>
<memballoon model='virtio'>
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
</memballoon>
</devices>
</domain>
"""
def int_to_hex(n):
if not 1 <= n <= 254:
raise KeyError('Wrong value: number not within [1, 254]')
s = hex(n)[2:]
if len(s) == 2:
return s
else:
return '0' + s
def gen_libvirt_xml(subnet, begin, end):
for i in range(begin, end):
name = "%s.%s" % (subnet, i)
xml = XML_TEMPLATE % (name, str(uuid.uuid4()), name,
int_to_hex(subnet), int_to_hex(i))
print xml
fout = open('/tmp/%s.%s.xml' % (subnet, i), 'w')
fout.write(xml)
fout.close()
cmd = "mkdir -p /var/lib/libvirt/images/%s.%s/" % (subnet, i)
print cmd
os.system(cmd)
cmd = ("qemu-img create -b /var/lib/libvirt/images/base.qcow2 "
"-f qcow2 /var/lib/libvirt/images/%s.%s/image.qcow2" %
(subnet, i))
print cmd
os.system(cmd)
cmd = "virsh define /tmp/%s.%s.xml" % (subnet, i)
print cmd
os.system(cmd)
if __name__ == "__main__":
subnet = int(sys.argv[1])
begin = int(sys.argv[2])
end = int(sys.argv[3])
gen_libvirt_xml(subnet, begin, end)

126
bin/procedures Normal file
View File

@ -0,0 +1,126 @@
(*) Physical machine
# check CPU auto scaling
grep -E '^model name|^cpu MHz' /proc/cpuinfo
# disable CPU auto scaling if enabled
for CPUFREQ in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
[ -f $CPUFREQ ] || continue
echo -n performance > $CPUFREQ
done
sudo sed -i \
-e 's/echo -n ondemand > $CPUFREQ/echo -n performance > $CPUFREQ/' \
/etc/init.d/ondemand;
(*) OS image
# switch_kernel.sh (deprecated)
sudo apt-get -y update && sudo apt-get -y upgrade
(*) Basic software and configuration
sudo apt-get -y install \
emacs \
htop \
iftop \
iotop \
sysstat \
arping \
cpu-checker \
build-essential \
git \
python-dev
# enabel IP forwarding
sudo sed -i 's/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/' \
/etc/sysctl.conf && sudo sysctl net.ipv4.ip_forward=1
# enable fast ssh login
sudo sed -i.orig \
-e 's/#GSSAPIAuthentication no/GSSAPIAuthentication no\nUseDNS no/' \
/etc/ssh/sshd_config && sudo service ssh restart
# shorten failsafe sleep time
sudo sed -i \
-e 's/sleep 20/sleep 1/' \
-e 's/sleep 40/sleep 1/' \
-e 's/sleep 59/sleep 1/' \
/etc/init/failsafe.conf
(*) Chef server (following steps can be done either manually, or via chef-client)
# install_chefserver_deps.sh (skip-able)
install_chefserver.sh
configure_knife.sh
setup_chef_repo.sh
(*) Open vSwitch
# install_openvswitch.sh (deprecated)
sudo apt-get -y install openvswitch-switch
(*) libvirt, kvm
# install_libvirt.sh (deprecated)
sudo apt-get -y install libvirt-bin kvm qemu
sudo virsh net-destroy default
sudo virsh net-undefine default
# enable libvirt for VM live migration
sudo sed -i.orig /etc/libvirt/libvirtd.conf \
-e 's/#listen_tls = 0/listen_tls = 0/g' \
-e 's/#listen_tcp = 1/listen_tcp = 1/g' \
-e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
sudo sed -i.orig /etc/init/libvirt-bin.conf \
-e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
sudo sed -i.orig /etc/default/libvirt-bin \
-e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
sudo service libvirt-bin restart
# /etc/nova/nova.conf
# live_migration_flag=VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE
# libvirt_cpu_mode=custom
# libvirt_cpu_model=kvm64
(*) ZooKeeper
install_zookeeper.sh
(*) docker
sudo apt-get -y install docker.io lxc udhcpc arping
# enable lxc as backend driver
sudo sed -i /etc/default/docker.io \
-e 's/#DOCKER_OPTS="-dns 8.8.8.8 -dns 8.8.4.4"/DOCKER_OPTS="-e lxc"/'
sudo service docker.io restart
# download and setup pipework
wget https://raw.githubusercontent.com/jpetazzo/pipework/master/pipework -P /root/
sed -i.orig -e 's/docker/docker.io/' /root/pipework
chmod +x /root/pipework
(*) VXLAN tunnels
ovs-vsctl add-br obr1; ovs-vsctl add-br obr2; ovs-vsctl add-br obr3; ovs-vsctl set Bridge obr1 stp_enable=true; ovs-vsctl set Bridge obr2 stp_enable=true; ovs-vsctl set Bridge obr3 stp_enable=true; ovs-vsctl add-port obr1 eth1p; ovs-vsctl add-port obr2 eth2p; ovs-vsctl add-port obr3 eth3p
prefix_ip=10.255; postfix_ip_1=100; postfix_ip_2=64; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} obr${i}_${postfix_ip_1}_${postfix_ip_2} -- set interface obr${i}_${postfix_ip_1}_${postfix_ip_2} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
DC=2; prefix_ip=135.207; postfix_ip_1=223; postfix_ip_2=152; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} gateway${i}_${DC} -- set interface gateway${i}_${DC} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
(*) Ryu WL2 controller
sudo pip install kazoo oslo.config netaddr eventlet webob routes
wget https://github.com/att/ryu.git
(*) Launch guest VMs (or use OpenStack)
launch_libvirt_instance.py
launch_docker_instance.py

View File

@ -1,7 +1,6 @@
#!/bin/bash
## Switch kernel from virtual to generic, for vanilla Ubuntu 12.04
## image launched instance.
## Switch kernel from virtual to generic to use nested virtualization
# routine
sudo apt-get -y update

View File

@ -20,26 +20,33 @@ setup(
"python-novaclient>=2.13.0",
"IPython>=0.13.2",
],
data_files=[('bin', ['bin/configure_knife.sh',
'bin/install_chefserver.sh',
'bin/pre_install_chefserver.sh',
'bin/pre_install_ovs.sh',
'bin/pre_switch_kernel.sh',
'bin/setup_chef_repo.sh',
'bin/userdata.sh.template',
]),
('inception/webui/templates/inception',
['inception/webui/templates/inception/detail.html',
'inception/webui/templates/inception/_detail_log.html',
'inception/webui/templates/inception/_detail_overview.html',
'inception/webui/templates/inception/_flavors_and_quotas.html',
'inception/webui/templates/inception/index.html',
'inception/webui/templates/inception/_launch_customize_help.html',
'inception/webui/templates/inception/_launch_details_help.html',
'inception/webui/templates/inception/_launch_network_help.html',
'inception/webui/templates/inception/_launch_volumes_help.html',
'inception/webui/templates/inception/_update_networks.html',
]),
],
data_files=[
('bin', [
# 'configure_knife.sh',
# 'delete_docker_instances.sh',
# 'install_chefserver_deps.sh',
# 'install_chefserver.sh',
# 'install_libvirt.sh',
# 'install_openvswitch.sh',
# 'install_zookeeper.sh',
# 'launch_docker_instance.py',
# 'launch_libvirt_instance.py',
# 'setup_chef_repo.sh',
# 'switch_kernel.sh',
# 'userdata.sh.template',
]),
('inception/webui/templates/inception', [
# 'detail.html',
# '_detail_log.html',
# '_detail_overview.html',
# '_flavors_and_quotas.html',
# 'index.html',
# '_launch_customize_help.html',
# '_launch_details_help.html',
# '_launch_network_help.html',
# '_launch_volumes_help.html',
# '_update_networks.html',
]),
],
scripts=['bin/orchestrator'],
)