Add procedures/scripts as starting point of multi-DC deployment
Change-Id: If455358e9164aa51b99ed3a06dd49ab2cf03d69b
This commit is contained in:
parent
cf46b6f9fc
commit
7136b3ffd0
|
@ -0,0 +1,10 @@
|
|||
#!/bin/bash
|
||||
|
||||
CONTROLLERS=$@
|
||||
|
||||
docker.io rm -f `docker.io ps -a | grep changbl | awk '{print $1}'`
|
||||
|
||||
for i in `ovs-vsctl show | grep Port | grep pl | awk '{print $2}' | cut -d '"' -f 2`
|
||||
do
|
||||
ovs-vsctl del-port obr2 $i
|
||||
done
|
|
@ -14,7 +14,7 @@ sudo apt-get install -y opscode-keyring # permanent upgradeable keyring
|
|||
sudo apt-get install -y debconf-utils
|
||||
sudo apt-get -y upgrade
|
||||
|
||||
sudo apt-get install -y libgnumail-java ruby-addressable libextlib-ruby jsvc \
|
||||
sudo apt-get install -y libgnumail-java ruby-addressable libextlib-ruby jsvc \
|
||||
libdb5.1-java-gcj erlang-eunit libjaxp1.3-java libcommons-pool-java \
|
||||
libdb-je-java ruby-mixlib-config gcj-4.6-jre-lib libjson-ruby1.8 \
|
||||
libbcel-java erlang-crypto libgeronimo-jta-1.1-spec-java libecj-java \
|
|
@ -47,20 +47,3 @@ cd libvirt
|
|||
# --with-xen=yes to the command
|
||||
make -j
|
||||
sudo make install
|
||||
|
||||
# enable libvirt for VM live migration
|
||||
sudo sed -i /etc/libvirt/libvirtd.conf \
|
||||
-e 's/#listen_tls = 0/listen_tls = 0/g' \
|
||||
-e 's/#listen_tcp = 1/listen_tcp = 1/g' \
|
||||
-e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
|
||||
sudo sed -i /etc/init/libvirt-bin.conf \
|
||||
-e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
|
||||
sudo sed -i /etc/default/libvirt-bin \
|
||||
-e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
|
||||
|
||||
# restart libvirt
|
||||
sudo service libvirt-bin restart
|
||||
|
||||
# Remove the default network created by libvirt
|
||||
sudo virsh net-destroy default
|
||||
sudo virsh net-undefine default
|
|
@ -0,0 +1,58 @@
|
|||
#!/bin/bash
|
||||
|
||||
## Install ZooKeeper and OpenJDK 7
|
||||
|
||||
VERSION=3.4.6
|
||||
|
||||
sudo apt-get -y install openjdk-7-jdk
|
||||
|
||||
sudo mkdir -p /opt/zookeeper
|
||||
sudo chmod 00777 /opt/zookeeper
|
||||
sudo chown root:root /opt/zookeeper
|
||||
|
||||
cd /opt/zookeeper
|
||||
wget http://mirrors.gigenet.com/apache/zookeeper/zookeeper-${VERSION}/zookeeper-${VERSION}.tar.gz
|
||||
tar xzvf zookeeper-${VERSION}.tar.gz
|
||||
mv zookeeper-${VERSION} zookeeper
|
||||
cd zookeeper
|
||||
|
||||
sudo mkdir -p /mnt/zookeeper
|
||||
sudo chmod 00777 /mnt/zookeeper
|
||||
sudo chown root:root /mnt/zookeeper
|
||||
|
||||
echo "1" | tee /mnt/zookeeper/myid
|
||||
|
||||
echo "# The number of milliseconds of each tick
|
||||
tickTime=2000
|
||||
# The number of ticks that the initial
|
||||
# synchronization phase can take
|
||||
initLimit=5
|
||||
# The number of ticks that can pass between
|
||||
# sending a request and getting an acknowledgement
|
||||
syncLimit=2
|
||||
# the directory where the snapshot is stored.
|
||||
# do not use /tmp for storage, /tmp here is just
|
||||
# example sakes.
|
||||
dataDir=/mnt/zookeeper
|
||||
# the port at which the clients will connect
|
||||
clientPort=2181
|
||||
server.1=<zk_host_1>:2888:3888
|
||||
server.2=<zk_host_2>:2888:3888
|
||||
server.3=<zk_host_3>:2888:3888
|
||||
# the directory where the log is stored
|
||||
# dataLogDir=/mnt/zookeeper
|
||||
#
|
||||
# Be sure to read the maintenance section of the
|
||||
# administrator guide before turning on autopurge.
|
||||
#
|
||||
# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
|
||||
#
|
||||
# The number of snapshots to retain in dataDir
|
||||
#autopurge.snapRetainCount=3
|
||||
# Purge task interval in hours
|
||||
# Set to 0 to disable auto purge feature
|
||||
#autopurge.purgeInterval=1
|
||||
" | tee /opt/zookeeper/zookeeper/conf/zoo.cfg
|
||||
|
||||
# Start the service
|
||||
# /opt/zookeeper/zookeeper/bin/zkServer.sh start
|
|
@ -0,0 +1,81 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
BRIDGE = "obr2"
|
||||
# BRIDGE = "br2"
|
||||
GATEWAY = "10.2.2.10"
|
||||
NET_PREFIX = "10.2"
|
||||
MAC_ADDR_PREFIX = "52:54:00:2d:"
|
||||
IMAGE = "changbl/u1404-3"
|
||||
|
||||
|
||||
def int_to_hex(n):
|
||||
if not 1 <= n <= 254:
|
||||
raise KeyError('Wrong value: number not within [1, 254]')
|
||||
s = hex(n)[2:]
|
||||
if len(s) == 2:
|
||||
return s
|
||||
else:
|
||||
return '0' + s
|
||||
|
||||
|
||||
def launch(subnet, begin, end):
|
||||
for i in range(begin, end):
|
||||
name = "%s.%s" % (subnet, i)
|
||||
mac_addr = (MAC_ADDR_PREFIX + "%s:%s" % (
|
||||
int_to_hex(subnet), int_to_hex(i)))
|
||||
|
||||
# cmd = """/root/pipework %s -i eth0 $(\
|
||||
# docker.io run --privileged=true -n=false --name=%s -d %s /usr/sbin/sshd -D\
|
||||
# ) %s.%s.%s/16@%s %s""" % (
|
||||
# BRIDGE, name, IMAGE, NET_PREFIX, subnet, i, GATEWAY, mac_addr)
|
||||
|
||||
cmd = """/root/pipework %s -i eth0 $(\
|
||||
docker.io run -n=false --privileged=true --name=%s -d %s /usr/sbin/sshd -D\
|
||||
) dhcp %s""" % (BRIDGE, name, IMAGE, mac_addr)
|
||||
|
||||
print cmd
|
||||
os.system(cmd)
|
||||
|
||||
"""
|
||||
docker.io run --privileged=true -d -n=false \
|
||||
-lxc-conf="lxc.network.type = veth" \
|
||||
-lxc-conf="lxc.network.link = br2" \
|
||||
-lxc-conf="lxc.network.flags = up" \
|
||||
-lxc-conf="lxc.network.name = eth0" \
|
||||
-lxc-conf="lxc.network.ipv4 = 10.2.101.3/16" \
|
||||
-lxc-conf="lxc.network.hwaddr=52:54:00:2d:65:03" \
|
||||
-lxc-conf="lxc.network.ipv4.gateway = 10.2.1.10" \
|
||||
--name=101.3 changbl/u1401-1 /usr/sbin/sshd -D
|
||||
|
||||
ID=$(docker.io run -n=false --name=test4 -d changbl/u1404-2 /usr/sbin/sshd -D)
|
||||
|
||||
./pipework obr2 -i eth0 $ID dhcp 52:54:00:2d:c9:04
|
||||
|
||||
./pipework obr2 -i eth0 $ID 10.2.201.8/16@10.2.2.10 52:54:00:2d:c9:08
|
||||
|
||||
docker.io inspect da1fbd5421f7 | grep ID
|
||||
|
||||
lxc-attach -n \
|
||||
da1fbd5421f75ef1a640019d4659489ee53faf4135f4e6feeb8872580f74549a -- /bin/bash
|
||||
"""
|
||||
# cmd = """docker.io run --privileged=true -d -n=false \
|
||||
# -lxc-conf="lxc.network.type = veth" \
|
||||
# -lxc-conf="lxc.network.link = %s" \
|
||||
# -lxc-conf="lxc.network.flags = up" \
|
||||
# -lxc-conf="lxc.network.name = eth0" \
|
||||
# -lxc-conf="lxc.network.ipv4 = %s.%s.%s/16" \
|
||||
# -lxc-conf="lxc.network.hwaddr=%s" \
|
||||
# -lxc-conf="lxc.network.ipv4.gateway = %s" \
|
||||
# --name=%s \
|
||||
# %s /usr/sbin/sshd -D""" % (
|
||||
# BRIDGE, NET_PREFIX, subnet, i, mac_addr, GATEWAY, name, IMAGE)
|
||||
#
|
||||
|
||||
if __name__ == "__main__":
|
||||
subnet = int(sys.argv[1])
|
||||
begin = int(sys.argv[2])
|
||||
end = int(sys.argv[3])
|
||||
launch(subnet, begin, end)
|
|
@ -0,0 +1,94 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import sys
|
||||
import os
|
||||
import uuid
|
||||
|
||||
XML_TEMPLATE = """<domain type='kvm'>
|
||||
<name>%s</name>
|
||||
<uuid>%s</uuid>
|
||||
<memory unit='KiB'>262144</memory>
|
||||
<vcpu placement='static'>1</vcpu>
|
||||
<os>
|
||||
<type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
|
||||
<boot dev='hd'/>
|
||||
</os>
|
||||
<features>
|
||||
<acpi/>
|
||||
</features>
|
||||
<clock offset='utc'/>
|
||||
<on_poweroff>destroy</on_poweroff>
|
||||
<on_reboot>restart</on_reboot>
|
||||
<on_crash>destroy</on_crash>
|
||||
<devices>
|
||||
<emulator>/usr/bin/kvm</emulator>
|
||||
<disk type='file' device='disk'>
|
||||
<driver name='qemu' type='qcow2'/>
|
||||
<source file='/var/lib/libvirt/images/%s/image.qcow2'/>
|
||||
<target dev='vda' bus='virtio'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
|
||||
</disk>
|
||||
<controller type='usb' index='0'>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
|
||||
</controller>
|
||||
<controller type='pci' index='0' model='pci-root'/>
|
||||
<interface type='bridge'>
|
||||
<mac address='52:54:00:2d:%s:%s'/>
|
||||
<source bridge='obr2'/>
|
||||
<virtualport type='openvswitch'/>
|
||||
<model type='virtio'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
|
||||
</interface>
|
||||
<input type='mouse' bus='ps2'/>
|
||||
<input type='keyboard' bus='ps2'/>
|
||||
<graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'>
|
||||
<listen type='address' address='0.0.0.0'/>
|
||||
</graphics>
|
||||
<video>
|
||||
<model type='cirrus' vram='9216' heads='1'/>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
|
||||
</video>
|
||||
<memballoon model='virtio'>
|
||||
<address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
|
||||
</memballoon>
|
||||
</devices>
|
||||
</domain>
|
||||
"""
|
||||
|
||||
|
||||
def int_to_hex(n):
|
||||
if not 1 <= n <= 254:
|
||||
raise KeyError('Wrong value: number not within [1, 254]')
|
||||
s = hex(n)[2:]
|
||||
if len(s) == 2:
|
||||
return s
|
||||
else:
|
||||
return '0' + s
|
||||
|
||||
|
||||
def gen_libvirt_xml(subnet, begin, end):
|
||||
for i in range(begin, end):
|
||||
name = "%s.%s" % (subnet, i)
|
||||
xml = XML_TEMPLATE % (name, str(uuid.uuid4()), name,
|
||||
int_to_hex(subnet), int_to_hex(i))
|
||||
print xml
|
||||
fout = open('/tmp/%s.%s.xml' % (subnet, i), 'w')
|
||||
fout.write(xml)
|
||||
fout.close()
|
||||
cmd = "mkdir -p /var/lib/libvirt/images/%s.%s/" % (subnet, i)
|
||||
print cmd
|
||||
os.system(cmd)
|
||||
cmd = ("qemu-img create -b /var/lib/libvirt/images/base.qcow2 "
|
||||
"-f qcow2 /var/lib/libvirt/images/%s.%s/image.qcow2" %
|
||||
(subnet, i))
|
||||
print cmd
|
||||
os.system(cmd)
|
||||
cmd = "virsh define /tmp/%s.%s.xml" % (subnet, i)
|
||||
print cmd
|
||||
os.system(cmd)
|
||||
|
||||
if __name__ == "__main__":
|
||||
subnet = int(sys.argv[1])
|
||||
begin = int(sys.argv[2])
|
||||
end = int(sys.argv[3])
|
||||
gen_libvirt_xml(subnet, begin, end)
|
|
@ -0,0 +1,126 @@
|
|||
(*) Physical machine
|
||||
|
||||
# check CPU auto scaling
|
||||
grep -E '^model name|^cpu MHz' /proc/cpuinfo
|
||||
|
||||
# disable CPU auto scaling if enabled
|
||||
for CPUFREQ in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
|
||||
[ -f $CPUFREQ ] || continue
|
||||
echo -n performance > $CPUFREQ
|
||||
done
|
||||
|
||||
sudo sed -i \
|
||||
-e 's/echo -n ondemand > $CPUFREQ/echo -n performance > $CPUFREQ/' \
|
||||
/etc/init.d/ondemand;
|
||||
|
||||
(*) OS image
|
||||
|
||||
# switch_kernel.sh (deprecated)
|
||||
|
||||
sudo apt-get -y update && sudo apt-get -y upgrade
|
||||
|
||||
(*) Basic software and configuration
|
||||
|
||||
sudo apt-get -y install \
|
||||
emacs \
|
||||
htop \
|
||||
iftop \
|
||||
iotop \
|
||||
sysstat \
|
||||
arping \
|
||||
cpu-checker \
|
||||
build-essential \
|
||||
git \
|
||||
python-dev
|
||||
|
||||
# enabel IP forwarding
|
||||
sudo sed -i 's/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/' \
|
||||
/etc/sysctl.conf && sudo sysctl net.ipv4.ip_forward=1
|
||||
|
||||
# enable fast ssh login
|
||||
sudo sed -i.orig \
|
||||
-e 's/#GSSAPIAuthentication no/GSSAPIAuthentication no\nUseDNS no/' \
|
||||
/etc/ssh/sshd_config && sudo service ssh restart
|
||||
|
||||
# shorten failsafe sleep time
|
||||
sudo sed -i \
|
||||
-e 's/sleep 20/sleep 1/' \
|
||||
-e 's/sleep 40/sleep 1/' \
|
||||
-e 's/sleep 59/sleep 1/' \
|
||||
/etc/init/failsafe.conf
|
||||
|
||||
(*) Chef server (following steps can be done either manually, or via chef-client)
|
||||
|
||||
# install_chefserver_deps.sh (skip-able)
|
||||
install_chefserver.sh
|
||||
configure_knife.sh
|
||||
setup_chef_repo.sh
|
||||
|
||||
(*) Open vSwitch
|
||||
|
||||
# install_openvswitch.sh (deprecated)
|
||||
|
||||
sudo apt-get -y install openvswitch-switch
|
||||
|
||||
(*) libvirt, kvm
|
||||
|
||||
# install_libvirt.sh (deprecated)
|
||||
|
||||
sudo apt-get -y install libvirt-bin kvm qemu
|
||||
|
||||
sudo virsh net-destroy default
|
||||
sudo virsh net-undefine default
|
||||
|
||||
# enable libvirt for VM live migration
|
||||
sudo sed -i.orig /etc/libvirt/libvirtd.conf \
|
||||
-e 's/#listen_tls = 0/listen_tls = 0/g' \
|
||||
-e 's/#listen_tcp = 1/listen_tcp = 1/g' \
|
||||
-e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
|
||||
sudo sed -i.orig /etc/init/libvirt-bin.conf \
|
||||
-e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
|
||||
sudo sed -i.orig /etc/default/libvirt-bin \
|
||||
-e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
|
||||
sudo service libvirt-bin restart
|
||||
|
||||
# /etc/nova/nova.conf
|
||||
# live_migration_flag=VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE
|
||||
# libvirt_cpu_mode=custom
|
||||
# libvirt_cpu_model=kvm64
|
||||
|
||||
(*) ZooKeeper
|
||||
|
||||
install_zookeeper.sh
|
||||
|
||||
(*) docker
|
||||
|
||||
sudo apt-get -y install docker.io lxc udhcpc arping
|
||||
|
||||
# enable lxc as backend driver
|
||||
sudo sed -i /etc/default/docker.io \
|
||||
-e 's/#DOCKER_OPTS="-dns 8.8.8.8 -dns 8.8.4.4"/DOCKER_OPTS="-e lxc"/'
|
||||
sudo service docker.io restart
|
||||
|
||||
# download and setup pipework
|
||||
wget https://raw.githubusercontent.com/jpetazzo/pipework/master/pipework -P /root/
|
||||
sed -i.orig -e 's/docker/docker.io/' /root/pipework
|
||||
chmod +x /root/pipework
|
||||
|
||||
(*) VXLAN tunnels
|
||||
|
||||
ovs-vsctl add-br obr1; ovs-vsctl add-br obr2; ovs-vsctl add-br obr3; ovs-vsctl set Bridge obr1 stp_enable=true; ovs-vsctl set Bridge obr2 stp_enable=true; ovs-vsctl set Bridge obr3 stp_enable=true; ovs-vsctl add-port obr1 eth1p; ovs-vsctl add-port obr2 eth2p; ovs-vsctl add-port obr3 eth3p
|
||||
|
||||
prefix_ip=10.255; postfix_ip_1=100; postfix_ip_2=64; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} obr${i}_${postfix_ip_1}_${postfix_ip_2} -- set interface obr${i}_${postfix_ip_1}_${postfix_ip_2} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
|
||||
|
||||
DC=2; prefix_ip=135.207; postfix_ip_1=223; postfix_ip_2=152; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} gateway${i}_${DC} -- set interface gateway${i}_${DC} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
|
||||
|
||||
(*) Ryu WL2 controller
|
||||
|
||||
sudo pip install kazoo oslo.config netaddr eventlet webob routes
|
||||
|
||||
wget https://github.com/att/ryu.git
|
||||
|
||||
(*) Launch guest VMs (or use OpenStack)
|
||||
|
||||
launch_libvirt_instance.py
|
||||
|
||||
launch_docker_instance.py
|
|
@ -1,7 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
## Switch kernel from virtual to generic, for vanilla Ubuntu 12.04
|
||||
## image launched instance.
|
||||
## Switch kernel from virtual to generic to use nested virtualization
|
||||
|
||||
# routine
|
||||
sudo apt-get -y update
|
11
setup.py
11
setup.py
|
@ -21,11 +21,16 @@ setup(
|
|||
"IPython>=0.13.2",
|
||||
],
|
||||
data_files=[('bin', ['bin/configure_knife.sh',
|
||||
'bin/delete_docker_instances.sh',
|
||||
'bin/install_chefserver_deps.sh',
|
||||
'bin/install_chefserver.sh',
|
||||
'bin/pre_install_chefserver.sh',
|
||||
'bin/pre_install_ovs.sh',
|
||||
'bin/pre_switch_kernel.sh',
|
||||
'bin/install_libvirt.sh',
|
||||
'bin/install_openvswitch.sh',
|
||||
'bin/install_zookeeper.sh',
|
||||
'bin/launch_docker_instance.py',
|
||||
'bin/launch_libvirt_instance.py',
|
||||
'bin/setup_chef_repo.sh',
|
||||
'bin/switch_kernel.sh',
|
||||
'bin/userdata.sh.template',
|
||||
]),
|
||||
('inception/webui/templates/inception',
|
||||
|
|
Loading…
Reference in New Issue