summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChangbin Liu <changbin.liu@gmail.com>2014-07-23 19:55:11 -0400
committerChangbin Liu <changbin.liu@gmail.com>2014-07-25 10:23:06 -0400
commit8233e63e95641ffb708053065fdde16416442d35 (patch)
tree4b4143012bd222e3bcf5989a30dfb0fbdd5922f6
parentcf46b6f9fcbdf986a5f988f4349a49cadab4812b (diff)
Add procedures/scripts as starting point of multi-DC deploymentHEADmaster
Notes
Notes (review): Verified+2: Jenkins Code-Review+2: Changbin Liu <changbin.liu@gmail.com> Workflow+1: Changbin Liu <changbin.liu@gmail.com> Submitted-by: Jenkins Submitted-at: Fri, 25 Jul 2014 14:29:29 +0000 Reviewed-on: https://review.openstack.org/109155 Project: stackforge/inception Branch: refs/heads/master
-rw-r--r--bin/delete_docker_instances.sh10
-rwxr-xr-xbin/install_chefserver_deps.sh (renamed from bin/pre_install_chefserver.sh)2
-rw-r--r--bin/install_libvirt.sh (renamed from bin/pre_install_libvirt.sh)17
-rwxr-xr-xbin/install_openvswitch.sh (renamed from bin/pre_install_ovs.sh)0
-rw-r--r--bin/install_zookeeper.sh58
-rw-r--r--bin/launch_docker_instance.py81
-rw-r--r--bin/launch_libvirt_instance.py94
-rw-r--r--bin/procedures126
-rwxr-xr-xbin/switch_kernel.sh (renamed from bin/pre_switch_kernel.sh)3
-rw-r--r--setup.py49
10 files changed, 399 insertions, 41 deletions
diff --git a/bin/delete_docker_instances.sh b/bin/delete_docker_instances.sh
new file mode 100644
index 0000000..939e34a
--- /dev/null
+++ b/bin/delete_docker_instances.sh
@@ -0,0 +1,10 @@
1#!/bin/bash
2
3CONTROLLERS=$@
4
5docker.io rm -f `docker.io ps -a | grep changbl | awk '{print $1}'`
6
7for i in `ovs-vsctl show | grep Port | grep pl | awk '{print $2}' | cut -d '"' -f 2`
8do
9 ovs-vsctl del-port obr2 $i
10done
diff --git a/bin/pre_install_chefserver.sh b/bin/install_chefserver_deps.sh
index e432c13..582aefa 100755
--- a/bin/pre_install_chefserver.sh
+++ b/bin/install_chefserver_deps.sh
@@ -14,7 +14,7 @@ sudo apt-get install -y opscode-keyring # permanent upgradeable keyring
14sudo apt-get install -y debconf-utils 14sudo apt-get install -y debconf-utils
15sudo apt-get -y upgrade 15sudo apt-get -y upgrade
16 16
17sudo apt-get install -y libgnumail-java ruby-addressable libextlib-ruby jsvc \ 17sudo apt-get install -y libgnumail-java ruby-addressable libextlib-ruby jsvc \
18 libdb5.1-java-gcj erlang-eunit libjaxp1.3-java libcommons-pool-java \ 18 libdb5.1-java-gcj erlang-eunit libjaxp1.3-java libcommons-pool-java \
19 libdb-je-java ruby-mixlib-config gcj-4.6-jre-lib libjson-ruby1.8 \ 19 libdb-je-java ruby-mixlib-config gcj-4.6-jre-lib libjson-ruby1.8 \
20 libbcel-java erlang-crypto libgeronimo-jta-1.1-spec-java libecj-java \ 20 libbcel-java erlang-crypto libgeronimo-jta-1.1-spec-java libecj-java \
diff --git a/bin/pre_install_libvirt.sh b/bin/install_libvirt.sh
index c5cb9ea..99fbe2e 100644
--- a/bin/pre_install_libvirt.sh
+++ b/bin/install_libvirt.sh
@@ -47,20 +47,3 @@ cd libvirt
47 # --with-xen=yes to the command 47 # --with-xen=yes to the command
48make -j 48make -j
49sudo make install 49sudo make install
50
51# enable libvirt for VM live migration
52sudo sed -i /etc/libvirt/libvirtd.conf \
53 -e 's/#listen_tls = 0/listen_tls = 0/g' \
54 -e 's/#listen_tcp = 1/listen_tcp = 1/g' \
55 -e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
56sudo sed -i /etc/init/libvirt-bin.conf \
57 -e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
58sudo sed -i /etc/default/libvirt-bin \
59 -e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
60
61# restart libvirt
62sudo service libvirt-bin restart
63
64# Remove the default network created by libvirt
65sudo virsh net-destroy default
66sudo virsh net-undefine default
diff --git a/bin/pre_install_ovs.sh b/bin/install_openvswitch.sh
index 0c054db..0c054db 100755
--- a/bin/pre_install_ovs.sh
+++ b/bin/install_openvswitch.sh
diff --git a/bin/install_zookeeper.sh b/bin/install_zookeeper.sh
new file mode 100644
index 0000000..c17bd01
--- /dev/null
+++ b/bin/install_zookeeper.sh
@@ -0,0 +1,58 @@
1#!/bin/bash
2
3## Install ZooKeeper and OpenJDK 7
4
5VERSION=3.4.6
6
7sudo apt-get -y install openjdk-7-jdk
8
9sudo mkdir -p /opt/zookeeper
10sudo chmod 00777 /opt/zookeeper
11sudo chown root:root /opt/zookeeper
12
13cd /opt/zookeeper
14wget http://mirrors.gigenet.com/apache/zookeeper/zookeeper-${VERSION}/zookeeper-${VERSION}.tar.gz
15tar xzvf zookeeper-${VERSION}.tar.gz
16mv zookeeper-${VERSION} zookeeper
17cd zookeeper
18
19sudo mkdir -p /mnt/zookeeper
20sudo chmod 00777 /mnt/zookeeper
21sudo chown root:root /mnt/zookeeper
22
23echo "1" | tee /mnt/zookeeper/myid
24
25echo "# The number of milliseconds of each tick
26tickTime=2000
27# The number of ticks that the initial
28# synchronization phase can take
29initLimit=5
30# The number of ticks that can pass between
31# sending a request and getting an acknowledgement
32syncLimit=2
33# the directory where the snapshot is stored.
34# do not use /tmp for storage, /tmp here is just
35# example sakes.
36dataDir=/mnt/zookeeper
37# the port at which the clients will connect
38clientPort=2181
39server.1=<zk_host_1>:2888:3888
40server.2=<zk_host_2>:2888:3888
41server.3=<zk_host_3>:2888:3888
42# the directory where the log is stored
43# dataLogDir=/mnt/zookeeper
44#
45# Be sure to read the maintenance section of the
46# administrator guide before turning on autopurge.
47#
48# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
49#
50# The number of snapshots to retain in dataDir
51#autopurge.snapRetainCount=3
52# Purge task interval in hours
53# Set to 0 to disable auto purge feature
54#autopurge.purgeInterval=1
55" | tee /opt/zookeeper/zookeeper/conf/zoo.cfg
56
57# Start the service
58# /opt/zookeeper/zookeeper/bin/zkServer.sh start
diff --git a/bin/launch_docker_instance.py b/bin/launch_docker_instance.py
new file mode 100644
index 0000000..b45e89e
--- /dev/null
+++ b/bin/launch_docker_instance.py
@@ -0,0 +1,81 @@
1#!/usr/bin/env python
2
3import sys
4import os
5
6BRIDGE = "obr2"
7# BRIDGE = "br2"
8GATEWAY = "10.2.2.10"
9NET_PREFIX = "10.2"
10MAC_ADDR_PREFIX = "52:54:00:2d:"
11IMAGE = "changbl/u1404-3"
12
13
14def int_to_hex(n):
15 if not 1 <= n <= 254:
16 raise KeyError('Wrong value: number not within [1, 254]')
17 s = hex(n)[2:]
18 if len(s) == 2:
19 return s
20 else:
21 return '0' + s
22
23
24def launch(subnet, begin, end):
25 for i in range(begin, end):
26 name = "%s.%s" % (subnet, i)
27 mac_addr = (MAC_ADDR_PREFIX + "%s:%s" % (
28 int_to_hex(subnet), int_to_hex(i)))
29
30# cmd = """/root/pipework %s -i eth0 $(\
31# docker.io run --privileged=true -n=false --name=%s -d %s /usr/sbin/sshd -D\
32# ) %s.%s.%s/16@%s %s""" % (
33# BRIDGE, name, IMAGE, NET_PREFIX, subnet, i, GATEWAY, mac_addr)
34
35 cmd = """/root/pipework %s -i eth0 $(\
36docker.io run -n=false --privileged=true --name=%s -d %s /usr/sbin/sshd -D\
37) dhcp %s""" % (BRIDGE, name, IMAGE, mac_addr)
38
39 print cmd
40 os.system(cmd)
41
42"""
43docker.io run --privileged=true -d -n=false \
44 -lxc-conf="lxc.network.type = veth" \
45 -lxc-conf="lxc.network.link = br2" \
46 -lxc-conf="lxc.network.flags = up" \
47 -lxc-conf="lxc.network.name = eth0" \
48 -lxc-conf="lxc.network.ipv4 = 10.2.101.3/16" \
49 -lxc-conf="lxc.network.hwaddr=52:54:00:2d:65:03" \
50 -lxc-conf="lxc.network.ipv4.gateway = 10.2.1.10" \
51 --name=101.3 changbl/u1401-1 /usr/sbin/sshd -D
52
53ID=$(docker.io run -n=false --name=test4 -d changbl/u1404-2 /usr/sbin/sshd -D)
54
55./pipework obr2 -i eth0 $ID dhcp 52:54:00:2d:c9:04
56
57./pipework obr2 -i eth0 $ID 10.2.201.8/16@10.2.2.10 52:54:00:2d:c9:08
58
59docker.io inspect da1fbd5421f7 | grep ID
60
61lxc-attach -n \
62da1fbd5421f75ef1a640019d4659489ee53faf4135f4e6feeb8872580f74549a -- /bin/bash
63"""
64# cmd = """docker.io run --privileged=true -d -n=false \
65# -lxc-conf="lxc.network.type = veth" \
66# -lxc-conf="lxc.network.link = %s" \
67# -lxc-conf="lxc.network.flags = up" \
68# -lxc-conf="lxc.network.name = eth0" \
69# -lxc-conf="lxc.network.ipv4 = %s.%s.%s/16" \
70# -lxc-conf="lxc.network.hwaddr=%s" \
71# -lxc-conf="lxc.network.ipv4.gateway = %s" \
72# --name=%s \
73# %s /usr/sbin/sshd -D""" % (
74# BRIDGE, NET_PREFIX, subnet, i, mac_addr, GATEWAY, name, IMAGE)
75#
76
77if __name__ == "__main__":
78 subnet = int(sys.argv[1])
79 begin = int(sys.argv[2])
80 end = int(sys.argv[3])
81 launch(subnet, begin, end)
diff --git a/bin/launch_libvirt_instance.py b/bin/launch_libvirt_instance.py
new file mode 100644
index 0000000..c43a100
--- /dev/null
+++ b/bin/launch_libvirt_instance.py
@@ -0,0 +1,94 @@
1#!/usr/bin/env python
2
3import sys
4import os
5import uuid
6
7XML_TEMPLATE = """<domain type='kvm'>
8 <name>%s</name>
9 <uuid>%s</uuid>
10 <memory unit='KiB'>262144</memory>
11 <vcpu placement='static'>1</vcpu>
12 <os>
13 <type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
14 <boot dev='hd'/>
15 </os>
16 <features>
17 <acpi/>
18 </features>
19 <clock offset='utc'/>
20 <on_poweroff>destroy</on_poweroff>
21 <on_reboot>restart</on_reboot>
22 <on_crash>destroy</on_crash>
23 <devices>
24 <emulator>/usr/bin/kvm</emulator>
25 <disk type='file' device='disk'>
26 <driver name='qemu' type='qcow2'/>
27 <source file='/var/lib/libvirt/images/%s/image.qcow2'/>
28 <target dev='vda' bus='virtio'/>
29 <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
30 </disk>
31 <controller type='usb' index='0'>
32 <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
33 </controller>
34 <controller type='pci' index='0' model='pci-root'/>
35 <interface type='bridge'>
36 <mac address='52:54:00:2d:%s:%s'/>
37 <source bridge='obr2'/>
38 <virtualport type='openvswitch'/>
39 <model type='virtio'/>
40 <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
41 </interface>
42 <input type='mouse' bus='ps2'/>
43 <input type='keyboard' bus='ps2'/>
44 <graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'>
45 <listen type='address' address='0.0.0.0'/>
46 </graphics>
47 <video>
48 <model type='cirrus' vram='9216' heads='1'/>
49 <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
50 </video>
51 <memballoon model='virtio'>
52 <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
53 </memballoon>
54 </devices>
55</domain>
56"""
57
58
59def int_to_hex(n):
60 if not 1 <= n <= 254:
61 raise KeyError('Wrong value: number not within [1, 254]')
62 s = hex(n)[2:]
63 if len(s) == 2:
64 return s
65 else:
66 return '0' + s
67
68
69def gen_libvirt_xml(subnet, begin, end):
70 for i in range(begin, end):
71 name = "%s.%s" % (subnet, i)
72 xml = XML_TEMPLATE % (name, str(uuid.uuid4()), name,
73 int_to_hex(subnet), int_to_hex(i))
74 print xml
75 fout = open('/tmp/%s.%s.xml' % (subnet, i), 'w')
76 fout.write(xml)
77 fout.close()
78 cmd = "mkdir -p /var/lib/libvirt/images/%s.%s/" % (subnet, i)
79 print cmd
80 os.system(cmd)
81 cmd = ("qemu-img create -b /var/lib/libvirt/images/base.qcow2 "
82 "-f qcow2 /var/lib/libvirt/images/%s.%s/image.qcow2" %
83 (subnet, i))
84 print cmd
85 os.system(cmd)
86 cmd = "virsh define /tmp/%s.%s.xml" % (subnet, i)
87 print cmd
88 os.system(cmd)
89
90if __name__ == "__main__":
91 subnet = int(sys.argv[1])
92 begin = int(sys.argv[2])
93 end = int(sys.argv[3])
94 gen_libvirt_xml(subnet, begin, end)
diff --git a/bin/procedures b/bin/procedures
new file mode 100644
index 0000000..9ccbf49
--- /dev/null
+++ b/bin/procedures
@@ -0,0 +1,126 @@
1(*) Physical machine
2
3# check CPU auto scaling
4grep -E '^model name|^cpu MHz' /proc/cpuinfo
5
6# disable CPU auto scaling if enabled
7for CPUFREQ in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do
8 [ -f $CPUFREQ ] || continue
9 echo -n performance > $CPUFREQ
10done
11
12sudo sed -i \
13 -e 's/echo -n ondemand > $CPUFREQ/echo -n performance > $CPUFREQ/' \
14 /etc/init.d/ondemand;
15
16(*) OS image
17
18# switch_kernel.sh (deprecated)
19
20sudo apt-get -y update && sudo apt-get -y upgrade
21
22(*) Basic software and configuration
23
24sudo apt-get -y install \
25 emacs \
26 htop \
27 iftop \
28 iotop \
29 sysstat \
30 arping \
31 cpu-checker \
32 build-essential \
33 git \
34 python-dev
35
36# enabel IP forwarding
37sudo sed -i 's/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/' \
38 /etc/sysctl.conf && sudo sysctl net.ipv4.ip_forward=1
39
40# enable fast ssh login
41sudo sed -i.orig \
42 -e 's/#GSSAPIAuthentication no/GSSAPIAuthentication no\nUseDNS no/' \
43 /etc/ssh/sshd_config && sudo service ssh restart
44
45# shorten failsafe sleep time
46sudo sed -i \
47 -e 's/sleep 20/sleep 1/' \
48 -e 's/sleep 40/sleep 1/' \
49 -e 's/sleep 59/sleep 1/' \
50 /etc/init/failsafe.conf
51
52(*) Chef server (following steps can be done either manually, or via chef-client)
53
54# install_chefserver_deps.sh (skip-able)
55install_chefserver.sh
56configure_knife.sh
57setup_chef_repo.sh
58
59(*) Open vSwitch
60
61# install_openvswitch.sh (deprecated)
62
63sudo apt-get -y install openvswitch-switch
64
65(*) libvirt, kvm
66
67# install_libvirt.sh (deprecated)
68
69sudo apt-get -y install libvirt-bin kvm qemu
70
71sudo virsh net-destroy default
72sudo virsh net-undefine default
73
74# enable libvirt for VM live migration
75sudo sed -i.orig /etc/libvirt/libvirtd.conf \
76 -e 's/#listen_tls = 0/listen_tls = 0/g' \
77 -e 's/#listen_tcp = 1/listen_tcp = 1/g' \
78 -e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
79sudo sed -i.orig /etc/init/libvirt-bin.conf \
80 -e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
81sudo sed -i.orig /etc/default/libvirt-bin \
82 -e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
83sudo service libvirt-bin restart
84
85# /etc/nova/nova.conf
86# live_migration_flag=VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE
87# libvirt_cpu_mode=custom
88# libvirt_cpu_model=kvm64
89
90(*) ZooKeeper
91
92install_zookeeper.sh
93
94(*) docker
95
96sudo apt-get -y install docker.io lxc udhcpc arping
97
98# enable lxc as backend driver
99sudo sed -i /etc/default/docker.io \
100 -e 's/#DOCKER_OPTS="-dns 8.8.8.8 -dns 8.8.4.4"/DOCKER_OPTS="-e lxc"/'
101sudo service docker.io restart
102
103# download and setup pipework
104wget https://raw.githubusercontent.com/jpetazzo/pipework/master/pipework -P /root/
105sed -i.orig -e 's/docker/docker.io/' /root/pipework
106chmod +x /root/pipework
107
108(*) VXLAN tunnels
109
110ovs-vsctl add-br obr1; ovs-vsctl add-br obr2; ovs-vsctl add-br obr3; ovs-vsctl set Bridge obr1 stp_enable=true; ovs-vsctl set Bridge obr2 stp_enable=true; ovs-vsctl set Bridge obr3 stp_enable=true; ovs-vsctl add-port obr1 eth1p; ovs-vsctl add-port obr2 eth2p; ovs-vsctl add-port obr3 eth3p
111
112prefix_ip=10.255; postfix_ip_1=100; postfix_ip_2=64; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} obr${i}_${postfix_ip_1}_${postfix_ip_2} -- set interface obr${i}_${postfix_ip_1}_${postfix_ip_2} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
113
114DC=2; prefix_ip=135.207; postfix_ip_1=223; postfix_ip_2=152; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} gateway${i}_${DC} -- set interface gateway${i}_${DC} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
115
116(*) Ryu WL2 controller
117
118sudo pip install kazoo oslo.config netaddr eventlet webob routes
119
120wget https://github.com/att/ryu.git
121
122(*) Launch guest VMs (or use OpenStack)
123
124launch_libvirt_instance.py
125
126launch_docker_instance.py
diff --git a/bin/pre_switch_kernel.sh b/bin/switch_kernel.sh
index 790140d..d6e8e78 100755
--- a/bin/pre_switch_kernel.sh
+++ b/bin/switch_kernel.sh
@@ -1,7 +1,6 @@
1#!/bin/bash 1#!/bin/bash
2 2
3## Switch kernel from virtual to generic, for vanilla Ubuntu 12.04 3## Switch kernel from virtual to generic to use nested virtualization
4## image launched instance.
5 4
6# routine 5# routine
7sudo apt-get -y update 6sudo apt-get -y update
diff --git a/setup.py b/setup.py
index 5393750..2e33814 100644
--- a/setup.py
+++ b/setup.py
@@ -20,26 +20,33 @@ setup(
20 "python-novaclient>=2.13.0", 20 "python-novaclient>=2.13.0",
21 "IPython>=0.13.2", 21 "IPython>=0.13.2",
22 ], 22 ],
23 data_files=[('bin', ['bin/configure_knife.sh', 23 data_files=[
24 'bin/install_chefserver.sh', 24 ('bin', [
25 'bin/pre_install_chefserver.sh', 25 # 'configure_knife.sh',
26 'bin/pre_install_ovs.sh', 26 # 'delete_docker_instances.sh',
27 'bin/pre_switch_kernel.sh', 27 # 'install_chefserver_deps.sh',
28 'bin/setup_chef_repo.sh', 28 # 'install_chefserver.sh',
29 'bin/userdata.sh.template', 29 # 'install_libvirt.sh',
30 ]), 30 # 'install_openvswitch.sh',
31 ('inception/webui/templates/inception', 31 # 'install_zookeeper.sh',
32 ['inception/webui/templates/inception/detail.html', 32 # 'launch_docker_instance.py',
33 'inception/webui/templates/inception/_detail_log.html', 33 # 'launch_libvirt_instance.py',
34 'inception/webui/templates/inception/_detail_overview.html', 34 # 'setup_chef_repo.sh',
35 'inception/webui/templates/inception/_flavors_and_quotas.html', 35 # 'switch_kernel.sh',
36 'inception/webui/templates/inception/index.html', 36 # 'userdata.sh.template',
37 'inception/webui/templates/inception/_launch_customize_help.html', 37 ]),
38 'inception/webui/templates/inception/_launch_details_help.html', 38 ('inception/webui/templates/inception', [
39 'inception/webui/templates/inception/_launch_network_help.html', 39 # 'detail.html',
40 'inception/webui/templates/inception/_launch_volumes_help.html', 40 # '_detail_log.html',
41 'inception/webui/templates/inception/_update_networks.html', 41 # '_detail_overview.html',
42 ]), 42 # '_flavors_and_quotas.html',
43 ], 43 # 'index.html',
44 # '_launch_customize_help.html',
45 # '_launch_details_help.html',
46 # '_launch_network_help.html',
47 # '_launch_volumes_help.html',
48 # '_update_networks.html',
49 ]),
50 ],
44 scripts=['bin/orchestrator'], 51 scripts=['bin/orchestrator'],
45) 52)