From 8233e63e95641ffb708053065fdde16416442d35 Mon Sep 17 00:00:00 2001
From: Changbin Liu <changbin.liu@gmail.com>
Date: Wed, 23 Jul 2014 19:55:11 -0400
Subject: [PATCH] Add procedures/scripts as starting point of multi-DC
 deployment

Change-Id: If455358e9164aa51b99ed3a06dd49ab2cf03d69b
---
 bin/delete_docker_instances.sh                |  10 ++
 ...efserver.sh => install_chefserver_deps.sh} |   2 +-
 ..._install_libvirt.sh => install_libvirt.sh} |  17 ---
 ..._install_ovs.sh => install_openvswitch.sh} |   0
 bin/install_zookeeper.sh                      |  58 ++++++++
 bin/launch_docker_instance.py                 |  81 +++++++++++
 bin/launch_libvirt_instance.py                |  94 +++++++++++++
 bin/procedures                                | 126 ++++++++++++++++++
 ...{pre_switch_kernel.sh => switch_kernel.sh} |   3 +-
 setup.py                                      |  49 ++++---
 10 files changed, 399 insertions(+), 41 deletions(-)
 create mode 100644 bin/delete_docker_instances.sh
 rename bin/{pre_install_chefserver.sh => install_chefserver_deps.sh} (97%)
 rename bin/{pre_install_libvirt.sh => install_libvirt.sh} (68%)
 rename bin/{pre_install_ovs.sh => install_openvswitch.sh} (100%)
 create mode 100644 bin/install_zookeeper.sh
 create mode 100644 bin/launch_docker_instance.py
 create mode 100644 bin/launch_libvirt_instance.py
 create mode 100644 bin/procedures
 rename bin/{pre_switch_kernel.sh => switch_kernel.sh} (79%)

diff --git a/bin/delete_docker_instances.sh b/bin/delete_docker_instances.sh
new file mode 100644
index 0000000..939e34a
--- /dev/null
+++ b/bin/delete_docker_instances.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+CONTROLLERS=$@
+
+docker.io rm -f `docker.io ps -a | grep changbl | awk '{print $1}'`
+
+for i in `ovs-vsctl show | grep Port | grep pl | awk '{print $2}' | cut -d '"' -f 2`
+do
+    ovs-vsctl del-port obr2 $i
+done
diff --git a/bin/pre_install_chefserver.sh b/bin/install_chefserver_deps.sh
similarity index 97%
rename from bin/pre_install_chefserver.sh
rename to bin/install_chefserver_deps.sh
index e432c13..582aefa 100755
--- a/bin/pre_install_chefserver.sh
+++ b/bin/install_chefserver_deps.sh
@@ -14,7 +14,7 @@ sudo apt-get install -y opscode-keyring # permanent upgradeable keyring
 sudo apt-get install -y debconf-utils
 sudo apt-get -y upgrade
 
-sudo apt-get install -y  libgnumail-java ruby-addressable libextlib-ruby jsvc \
+sudo apt-get install -y libgnumail-java ruby-addressable libextlib-ruby jsvc \
   libdb5.1-java-gcj erlang-eunit libjaxp1.3-java libcommons-pool-java \
   libdb-je-java ruby-mixlib-config gcj-4.6-jre-lib libjson-ruby1.8 \
   libbcel-java erlang-crypto libgeronimo-jta-1.1-spec-java libecj-java \
diff --git a/bin/pre_install_libvirt.sh b/bin/install_libvirt.sh
similarity index 68%
rename from bin/pre_install_libvirt.sh
rename to bin/install_libvirt.sh
index c5cb9ea..99fbe2e 100644
--- a/bin/pre_install_libvirt.sh
+++ b/bin/install_libvirt.sh
@@ -47,20 +47,3 @@ cd libvirt
 		   # --with-xen=yes to the command
 make -j
 sudo make install
-
-# enable libvirt for VM live migration
-sudo sed -i /etc/libvirt/libvirtd.conf \
-    -e 's/#listen_tls = 0/listen_tls = 0/g' \
-    -e 's/#listen_tcp = 1/listen_tcp = 1/g' \
-    -e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
-sudo sed -i /etc/init/libvirt-bin.conf \
-    -e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
-sudo sed -i /etc/default/libvirt-bin \
-    -e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
-
-# restart libvirt
-sudo service libvirt-bin restart
-
-# Remove the default network created by libvirt
-sudo virsh net-destroy default
-sudo virsh net-undefine default
diff --git a/bin/pre_install_ovs.sh b/bin/install_openvswitch.sh
similarity index 100%
rename from bin/pre_install_ovs.sh
rename to bin/install_openvswitch.sh
diff --git a/bin/install_zookeeper.sh b/bin/install_zookeeper.sh
new file mode 100644
index 0000000..c17bd01
--- /dev/null
+++ b/bin/install_zookeeper.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+
+## Install ZooKeeper and OpenJDK 7
+
+VERSION=3.4.6
+
+sudo apt-get -y install openjdk-7-jdk
+
+sudo mkdir -p /opt/zookeeper
+sudo chmod 00777 /opt/zookeeper
+sudo chown root:root /opt/zookeeper
+
+cd /opt/zookeeper
+wget http://mirrors.gigenet.com/apache/zookeeper/zookeeper-${VERSION}/zookeeper-${VERSION}.tar.gz
+tar xzvf zookeeper-${VERSION}.tar.gz
+mv zookeeper-${VERSION} zookeeper
+cd zookeeper
+
+sudo mkdir -p /mnt/zookeeper
+sudo chmod 00777 /mnt/zookeeper
+sudo chown root:root /mnt/zookeeper
+
+echo "1" | tee /mnt/zookeeper/myid
+
+echo "# The number of milliseconds of each tick
+tickTime=2000
+# The number of ticks that the initial
+# synchronization phase can take
+initLimit=5
+# The number of ticks that can pass between
+# sending a request and getting an acknowledgement
+syncLimit=2
+# the directory where the snapshot is stored.
+# do not use /tmp for storage, /tmp here is just
+# example sakes.
+dataDir=/mnt/zookeeper
+# the port at which the clients will connect
+clientPort=2181
+server.1=<zk_host_1>:2888:3888
+server.2=<zk_host_2>:2888:3888
+server.3=<zk_host_3>:2888:3888
+# the directory where the log is stored
+# dataLogDir=/mnt/zookeeper
+#
+# Be sure to read the maintenance section of the
+# administrator guide before turning on autopurge.
+#
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
+#
+# The number of snapshots to retain in dataDir
+#autopurge.snapRetainCount=3
+# Purge task interval in hours
+# Set to 0 to disable auto purge feature
+#autopurge.purgeInterval=1
+" | tee /opt/zookeeper/zookeeper/conf/zoo.cfg
+
+# Start the service
+# /opt/zookeeper/zookeeper/bin/zkServer.sh start
diff --git a/bin/launch_docker_instance.py b/bin/launch_docker_instance.py
new file mode 100644
index 0000000..b45e89e
--- /dev/null
+++ b/bin/launch_docker_instance.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+import sys
+import os
+
+BRIDGE = "obr2"
+# BRIDGE = "br2"
+GATEWAY = "10.2.2.10"
+NET_PREFIX = "10.2"
+MAC_ADDR_PREFIX = "52:54:00:2d:"
+IMAGE = "changbl/u1404-3"
+
+
+def int_to_hex(n):
+    if not 1 <= n <= 254:
+        raise KeyError('Wrong value: number not within [1, 254]')
+    s = hex(n)[2:]
+    if len(s) == 2:
+        return s
+    else:
+        return '0' + s
+
+
+def launch(subnet, begin, end):
+    for i in range(begin, end):
+        name = "%s.%s" % (subnet, i)
+        mac_addr = (MAC_ADDR_PREFIX + "%s:%s" % (
+            int_to_hex(subnet), int_to_hex(i)))
+
+#         cmd = """/root/pipework %s -i eth0 $(\
+# docker.io run --privileged=true -n=false --name=%s -d %s /usr/sbin/sshd -D\
+# ) %s.%s.%s/16@%s %s""" % (
+#     BRIDGE, name, IMAGE, NET_PREFIX, subnet, i, GATEWAY, mac_addr)
+
+        cmd = """/root/pipework %s -i eth0 $(\
+docker.io run -n=false --privileged=true --name=%s -d %s /usr/sbin/sshd -D\
+) dhcp %s""" % (BRIDGE, name, IMAGE, mac_addr)
+
+        print cmd
+        os.system(cmd)
+
+"""
+docker.io run --privileged=true -d -n=false \
+    -lxc-conf="lxc.network.type = veth" \
+    -lxc-conf="lxc.network.link = br2" \
+    -lxc-conf="lxc.network.flags = up" \
+    -lxc-conf="lxc.network.name = eth0" \
+    -lxc-conf="lxc.network.ipv4 = 10.2.101.3/16" \
+    -lxc-conf="lxc.network.hwaddr=52:54:00:2d:65:03" \
+    -lxc-conf="lxc.network.ipv4.gateway = 10.2.1.10" \
+    --name=101.3 changbl/u1401-1 /usr/sbin/sshd -D
+
+ID=$(docker.io run -n=false --name=test4 -d changbl/u1404-2 /usr/sbin/sshd -D)
+
+./pipework obr2 -i eth0 $ID dhcp 52:54:00:2d:c9:04
+
+./pipework obr2 -i eth0 $ID 10.2.201.8/16@10.2.2.10 52:54:00:2d:c9:08
+
+docker.io inspect da1fbd5421f7 | grep ID
+
+lxc-attach -n \
+da1fbd5421f75ef1a640019d4659489ee53faf4135f4e6feeb8872580f74549a -- /bin/bash
+"""
+#         cmd = """docker.io run --privileged=true -d -n=false \
+# -lxc-conf="lxc.network.type = veth" \
+# -lxc-conf="lxc.network.link = %s" \
+# -lxc-conf="lxc.network.flags = up" \
+# -lxc-conf="lxc.network.name = eth0" \
+# -lxc-conf="lxc.network.ipv4 = %s.%s.%s/16" \
+# -lxc-conf="lxc.network.hwaddr=%s" \
+# -lxc-conf="lxc.network.ipv4.gateway = %s" \
+# --name=%s \
+# %s /usr/sbin/sshd -D""" % (
+#     BRIDGE, NET_PREFIX, subnet, i, mac_addr, GATEWAY, name, IMAGE)
+#
+
+if __name__ == "__main__":
+    subnet = int(sys.argv[1])
+    begin = int(sys.argv[2])
+    end = int(sys.argv[3])
+    launch(subnet, begin, end)
diff --git a/bin/launch_libvirt_instance.py b/bin/launch_libvirt_instance.py
new file mode 100644
index 0000000..c43a100
--- /dev/null
+++ b/bin/launch_libvirt_instance.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import uuid
+
+XML_TEMPLATE = """<domain type='kvm'>
+  <name>%s</name>
+  <uuid>%s</uuid>
+  <memory unit='KiB'>262144</memory>
+  <vcpu placement='static'>1</vcpu>
+  <os>
+    <type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
+    <boot dev='hd'/>
+  </os>
+  <features>
+    <acpi/>
+  </features>
+  <clock offset='utc'/>
+  <on_poweroff>destroy</on_poweroff>
+  <on_reboot>restart</on_reboot>
+  <on_crash>destroy</on_crash>
+  <devices>
+    <emulator>/usr/bin/kvm</emulator>
+    <disk type='file' device='disk'>
+      <driver name='qemu' type='qcow2'/>
+      <source file='/var/lib/libvirt/images/%s/image.qcow2'/>
+      <target dev='vda' bus='virtio'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
+    </disk>
+    <controller type='usb' index='0'>
+    <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
+    </controller>
+    <controller type='pci' index='0' model='pci-root'/>
+    <interface type='bridge'>
+      <mac address='52:54:00:2d:%s:%s'/>
+      <source bridge='obr2'/>
+      <virtualport type='openvswitch'/>
+      <model type='virtio'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
+    </interface>
+    <input type='mouse' bus='ps2'/>
+    <input type='keyboard' bus='ps2'/>
+    <graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'>
+      <listen type='address' address='0.0.0.0'/>
+    </graphics>
+    <video>
+      <model type='cirrus' vram='9216' heads='1'/>
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
+    </video>
+    <memballoon model='virtio'>
+    <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
+    </memballoon>
+  </devices>
+</domain>
+"""
+
+
+def int_to_hex(n):
+    if not 1 <= n <= 254:
+        raise KeyError('Wrong value: number not within [1, 254]')
+    s = hex(n)[2:]
+    if len(s) == 2:
+        return s
+    else:
+        return '0' + s
+
+
+def gen_libvirt_xml(subnet, begin, end):
+    for i in range(begin, end):
+        name = "%s.%s" % (subnet, i)
+        xml = XML_TEMPLATE % (name, str(uuid.uuid4()), name,
+                              int_to_hex(subnet), int_to_hex(i))
+        print xml
+        fout = open('/tmp/%s.%s.xml' % (subnet, i), 'w')
+        fout.write(xml)
+        fout.close()
+        cmd = "mkdir -p /var/lib/libvirt/images/%s.%s/" % (subnet, i)
+        print cmd
+        os.system(cmd)
+        cmd = ("qemu-img create -b /var/lib/libvirt/images/base.qcow2 "
+               "-f qcow2 /var/lib/libvirt/images/%s.%s/image.qcow2" %
+               (subnet, i))
+        print cmd
+        os.system(cmd)
+        cmd = "virsh define /tmp/%s.%s.xml" % (subnet, i)
+        print cmd
+        os.system(cmd)
+
+if __name__ == "__main__":
+    subnet = int(sys.argv[1])
+    begin = int(sys.argv[2])
+    end = int(sys.argv[3])
+    gen_libvirt_xml(subnet, begin, end)
diff --git a/bin/procedures b/bin/procedures
new file mode 100644
index 0000000..9ccbf49
--- /dev/null
+++ b/bin/procedures
@@ -0,0 +1,126 @@
+(*) Physical machine
+
+# check CPU auto scaling
+grep -E '^model name|^cpu MHz' /proc/cpuinfo
+
+# disable CPU auto scaling if enabled
+for CPUFREQ in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do 
+    [ -f $CPUFREQ ] || continue
+    echo -n performance > $CPUFREQ
+done
+
+sudo sed -i \
+    -e 's/echo -n ondemand > $CPUFREQ/echo -n performance > $CPUFREQ/' \
+    /etc/init.d/ondemand; 
+
+(*) OS image 
+
+# switch_kernel.sh (deprecated)
+
+sudo apt-get -y update && sudo apt-get -y upgrade
+
+(*) Basic software and configuration
+
+sudo apt-get -y install \
+    emacs \
+    htop \
+    iftop \
+    iotop \
+    sysstat \
+    arping \
+    cpu-checker \
+    build-essential \
+    git \
+    python-dev
+
+# enabel IP forwarding
+sudo sed -i 's/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/' \
+    /etc/sysctl.conf && sudo sysctl net.ipv4.ip_forward=1
+
+# enable fast ssh login
+sudo sed -i.orig \
+    -e 's/#GSSAPIAuthentication no/GSSAPIAuthentication no\nUseDNS no/' \
+    /etc/ssh/sshd_config && sudo service ssh restart
+
+# shorten failsafe sleep time
+sudo sed -i \
+    -e 's/sleep 20/sleep 1/' \
+    -e 's/sleep 40/sleep 1/' \
+    -e 's/sleep 59/sleep 1/' \
+    /etc/init/failsafe.conf
+
+(*) Chef server (following steps can be done either manually, or via chef-client)
+
+# install_chefserver_deps.sh (skip-able)
+install_chefserver.sh
+configure_knife.sh
+setup_chef_repo.sh
+
+(*) Open vSwitch
+
+# install_openvswitch.sh (deprecated)
+
+sudo apt-get -y install openvswitch-switch
+
+(*) libvirt, kvm
+
+# install_libvirt.sh (deprecated)
+
+sudo apt-get -y install libvirt-bin kvm qemu
+
+sudo virsh net-destroy default
+sudo virsh net-undefine default
+
+# enable libvirt for VM live migration
+sudo sed -i.orig /etc/libvirt/libvirtd.conf \
+    -e 's/#listen_tls = 0/listen_tls = 0/g' \
+    -e 's/#listen_tcp = 1/listen_tcp = 1/g' \
+    -e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
+sudo sed -i.orig /etc/init/libvirt-bin.conf \
+    -e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
+sudo sed -i.orig /etc/default/libvirt-bin \
+    -e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
+sudo service libvirt-bin restart
+
+# /etc/nova/nova.conf
+# live_migration_flag=VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE
+# libvirt_cpu_mode=custom
+# libvirt_cpu_model=kvm64
+
+(*) ZooKeeper
+
+install_zookeeper.sh
+
+(*) docker
+
+sudo apt-get -y install docker.io lxc udhcpc arping
+
+# enable lxc as backend driver
+sudo sed -i     /etc/default/docker.io \
+    -e 's/#DOCKER_OPTS="-dns 8.8.8.8 -dns 8.8.4.4"/DOCKER_OPTS="-e lxc"/'
+sudo service docker.io restart
+
+# download and setup pipework
+wget https://raw.githubusercontent.com/jpetazzo/pipework/master/pipework -P /root/
+sed -i.orig -e 's/docker/docker.io/' /root/pipework
+chmod +x /root/pipework
+
+(*) VXLAN tunnels
+
+ovs-vsctl add-br obr1; ovs-vsctl add-br obr2; ovs-vsctl add-br obr3; ovs-vsctl set Bridge obr1 stp_enable=true; ovs-vsctl set Bridge obr2 stp_enable=true; ovs-vsctl set Bridge obr3 stp_enable=true; ovs-vsctl add-port obr1 eth1p; ovs-vsctl add-port obr2 eth2p; ovs-vsctl add-port obr3 eth3p
+
+prefix_ip=10.255; postfix_ip_1=100; postfix_ip_2=64; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} obr${i}_${postfix_ip_1}_${postfix_ip_2} -- set interface obr${i}_${postfix_ip_1}_${postfix_ip_2} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
+
+DC=2; prefix_ip=135.207; postfix_ip_1=223; postfix_ip_2=152; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} gateway${i}_${DC} -- set interface gateway${i}_${DC} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
+
+(*) Ryu WL2 controller
+
+sudo pip install kazoo oslo.config netaddr eventlet webob routes
+
+wget https://github.com/att/ryu.git
+
+(*) Launch guest VMs (or use OpenStack)
+
+launch_libvirt_instance.py
+
+launch_docker_instance.py
diff --git a/bin/pre_switch_kernel.sh b/bin/switch_kernel.sh
similarity index 79%
rename from bin/pre_switch_kernel.sh
rename to bin/switch_kernel.sh
index 790140d..d6e8e78 100755
--- a/bin/pre_switch_kernel.sh
+++ b/bin/switch_kernel.sh
@@ -1,7 +1,6 @@
 #!/bin/bash
 
-## Switch kernel from virtual to generic, for vanilla Ubuntu 12.04
-## image launched instance.
+## Switch kernel from virtual to generic to use nested virtualization
 
 # routine
 sudo apt-get -y update
diff --git a/setup.py b/setup.py
index 5393750..2e33814 100644
--- a/setup.py
+++ b/setup.py
@@ -20,26 +20,33 @@ setup(
         "python-novaclient>=2.13.0",
         "IPython>=0.13.2",
     ],
-    data_files=[('bin', ['bin/configure_knife.sh',
-                         'bin/install_chefserver.sh',
-                         'bin/pre_install_chefserver.sh',
-                         'bin/pre_install_ovs.sh',
-                         'bin/pre_switch_kernel.sh',
-                         'bin/setup_chef_repo.sh',
-                         'bin/userdata.sh.template',
-                         ]),
-                ('inception/webui/templates/inception',
-                 ['inception/webui/templates/inception/detail.html',
-                 'inception/webui/templates/inception/_detail_log.html',
-                 'inception/webui/templates/inception/_detail_overview.html',
-                 'inception/webui/templates/inception/_flavors_and_quotas.html',
-                 'inception/webui/templates/inception/index.html',
-                 'inception/webui/templates/inception/_launch_customize_help.html',
-                 'inception/webui/templates/inception/_launch_details_help.html',
-                 'inception/webui/templates/inception/_launch_network_help.html',
-                 'inception/webui/templates/inception/_launch_volumes_help.html',
-                 'inception/webui/templates/inception/_update_networks.html',
-                 ]),
-               ],
+    data_files=[
+        ('bin', [
+            # 'configure_knife.sh',
+            # 'delete_docker_instances.sh',
+            # 'install_chefserver_deps.sh',
+            # 'install_chefserver.sh',
+            # 'install_libvirt.sh',
+            # 'install_openvswitch.sh',
+            # 'install_zookeeper.sh',
+            # 'launch_docker_instance.py',
+            # 'launch_libvirt_instance.py',
+            # 'setup_chef_repo.sh',
+            # 'switch_kernel.sh',
+            # 'userdata.sh.template',
+        ]),
+        ('inception/webui/templates/inception', [
+            # 'detail.html',
+            # '_detail_log.html',
+            # '_detail_overview.html',
+            # '_flavors_and_quotas.html',
+            # 'index.html',
+            # '_launch_customize_help.html',
+            # '_launch_details_help.html',
+            # '_launch_network_help.html',
+            # '_launch_volumes_help.html',
+            # '_update_networks.html',
+        ]),
+    ],
     scripts=['bin/orchestrator'],
 )