SuperHAProxy
SuperHAProxy is a script around HAProxy to add support for live migration and easy handling of multiple instances. Live migration is added using the CRIU system. Very lightweight containers are used to isolate the proxy instances from each other to enable live migration to efficiently transfer the proxies and to ensure instances can be running different versions to support proper rolling upgrades. This script can be used stand alone, or as a building block to provide multiserver proxy pools that are high performance and support a workflow where the servers can be rolling upgraded live without connection loss. Change-Id: Id3150c5d5de6c3f5552f27f79cbcf0debe8adb1f
This commit is contained in:
Kevin Fox
parent
529e1db899
commit
bd711d693c
|
|
@ -0,0 +1,346 @@
|
|||
#!/bin/bash
|
||||
|
||||
#Document the bridge setup....
|
||||
#ovs-vsctl set bridge shabr stp_enable=false
|
||||
|
||||
#FIXME not all of them work... hardcoding for now.
|
||||
#mirror=$(curl -s http://nl.alpinelinux.org/alpine/MIRRORS.txt | shuf | head -n 1)
|
||||
mirror="http://dl-6.alpinelinux.org/alpine/"
|
||||
#FIXME write some logic to detect this.
|
||||
version=2.6.5-r1
|
||||
statedir=/var/lib/superhaproxy
|
||||
wrapperurl='http://git.haproxy.org/?p=haproxy-1.6.git;a=blob_plain;f=src/haproxy-systemd-wrapper.c;hb=HEAD'
|
||||
#FIXME make this configurable
|
||||
bridge=shabr
|
||||
|
||||
function init_config {
|
||||
name="$1"
|
||||
ip=$(crudini --get "$statedir/containers/$name/container.ini" superhaproxy ip)
|
||||
subnet=$(crudini --get "$statedir/containers/$name/container.ini" superhaproxy subnet)
|
||||
gateway=$(crudini --get "$statedir/containers/$name/container.ini" superhaproxy gateway)
|
||||
mtu=$(crudini --get "$statedir/containers/$name/container.ini" superhaproxy mtu)
|
||||
}
|
||||
|
||||
function get_pid_file {
|
||||
echo "$statedir/containers/$1/container.pid"
|
||||
}
|
||||
|
||||
function get_pid {
|
||||
echo "$(< "$statedir/containers/$1/container.pid")"
|
||||
}
|
||||
|
||||
function get_dump_dir {
|
||||
echo "$statedir/dumps/$1"
|
||||
}
|
||||
|
||||
function get_container_dir {
|
||||
echo "$statedir/containers/$1"
|
||||
}
|
||||
|
||||
if [ "x$1" == "x" ]
|
||||
then
|
||||
echo "Usage:"
|
||||
echo " init"
|
||||
echo " list"
|
||||
echo " create"
|
||||
echo " show"
|
||||
echo " start"
|
||||
echo " stop"
|
||||
echo " reload"
|
||||
echo " pid"
|
||||
echo " pstree"
|
||||
echo " shell"
|
||||
echo " hatop"
|
||||
echo " dump local"
|
||||
echo " restore local"
|
||||
exit -1
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xinit" ]
|
||||
then
|
||||
mkdir -p $statedir
|
||||
if [ ! -d $statedir/alpine-tools ]
|
||||
then
|
||||
mkdir -p $statedir/alpine-tools
|
||||
pushd $statedir/alpine-tools
|
||||
curl ${mirror}/latest-stable/main/x86_64/apk-tools-static-${version}.apk | tar -zxf -
|
||||
popd
|
||||
fi
|
||||
if [ ! -d $statedir/rootimg ]
|
||||
then
|
||||
mkdir -p $statedir/rootimg
|
||||
$statedir/alpine-tools/sbin/apk.static -X ${mirror}/latest-stable/main -U --allow-untrusted --root $statedir/rootimg --initdb add alpine-base haproxy
|
||||
#FIXME this makes way too big a binary. Remove once alpine provides the wrapper
|
||||
curl -s "$wrapperurl" -o $statedir/wrapper.c
|
||||
gcc --static -o $statedir/rootimg/usr/sbin/haproxy-systemd-wrapper $statedir/wrapper.c
|
||||
#FIXME criu doesn't support checkpinting the chroot yet.
|
||||
sed -i '/chroot/d' $statedir/rootimg/etc/haproxy/haproxy.cfg
|
||||
fi
|
||||
mkdir -p $statedir/containers
|
||||
mkdir -p $statedir/dumps
|
||||
mkdir -p $statedir/action-scripts
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xlist" ]
|
||||
then
|
||||
ls $statedir/containers/ | cat
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xcreate" ]
|
||||
then
|
||||
shift
|
||||
ip=""
|
||||
name=""
|
||||
subnet="255.255.255.0"
|
||||
gateway=""
|
||||
mtu=9000
|
||||
while getopts ":i:m:n:s:g:" opt; do
|
||||
case ${opt} in
|
||||
i )
|
||||
ip="$OPTARG"
|
||||
;;
|
||||
m )
|
||||
mtu="$OPTARG"
|
||||
;;
|
||||
s )
|
||||
subnet="$OPTARG"
|
||||
;;
|
||||
n )
|
||||
name="$OPTARG"
|
||||
;;
|
||||
\? ) echo "Usage: superhaproxy create [-m mtu] [-s subnetmask] [-g gatewayip] -i ip_address -n name"
|
||||
exit -1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name with -n"
|
||||
exit -1
|
||||
fi
|
||||
if [ "x$ip" == "x" ]
|
||||
then
|
||||
echo "You must specify an ip with -i"
|
||||
exit -1
|
||||
fi
|
||||
cp -a $statedir/rootimg "$statedir/containers/$name"
|
||||
touch "$statedir/containers/$name/container.ini"
|
||||
crudini --set "$statedir/containers/$name/container.ini" superhaproxy ip "$ip"
|
||||
crudini --set "$statedir/containers/$name/container.ini" superhaproxy mtu "$mtu"
|
||||
crudini --set "$statedir/containers/$name/container.ini" superhaproxy subnet "$subnet"
|
||||
crudini --set "$statedir/containers/$name/container.ini" superhaproxy gateway "$gateway"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xshow" ]
|
||||
then
|
||||
name="$2"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
init_config "$name"
|
||||
echo "IP: $ip"
|
||||
echo "Subnet Mask: $subnet"
|
||||
if [ "x$gateay" != "x" ]
|
||||
then
|
||||
echo "Gateway: $gateway"
|
||||
fi
|
||||
echo "MTU: $mtu"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xstart" ]
|
||||
then
|
||||
name="$2"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
init_config "$name"
|
||||
container="$(get_container_dir "$name")"
|
||||
#FIXME ensure escaping is correct.
|
||||
unshare --net --mount --pid --fork -- bash -c "/usr/bin/setsid -- /bin/bash -c 'mount --make-rprivate /; mount --bind $container /tmp; cd /tmp; mkdir -p old; pivot_root . old; mount --bind /old/dev /dev; mount /proc /proc -t proc; umount -l old; exec /usr/sbin/haproxy-systemd-wrapper -f /etc/haproxy/haproxy.cfg -p /run/haproxy.pid </dev/null >/dev/null 2>&1'" &
|
||||
sleep 1
|
||||
awk '{print $1}' /proc/$!/task/$!/children > "$container/container.pid"
|
||||
P="$(get_pid "$name")"
|
||||
ovs-vsctl del-port $bridge "sha$(get_pid "$name")" > /dev/null 2>&1
|
||||
ip link add sha$P type veth peer name shai$P
|
||||
ip link set dev sha$P mtu "$mtu" up
|
||||
ip link set shai$P netns $P name eth0
|
||||
nsenter -t $P -n ip addr add "$ip/$subnet" dev eth0
|
||||
nsenter -t $P -n ip link set dev eth0 mtu "$mtu" up
|
||||
ovs-vsctl add-port $bridge sha$P
|
||||
exit $?
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xpid" ]
|
||||
then
|
||||
name="$2"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
get_pid $name
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xpstree" ]
|
||||
then
|
||||
name="$2"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
pstree -p $(get_pid "$name")
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xstop" ]
|
||||
then
|
||||
name="$2"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
kill $(get_pid "$name")
|
||||
ovs-vsctl del-port $bridge "sha$(get_pid "$name")"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xshell" ]
|
||||
then
|
||||
name="$2"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
nsenter -n -m -p -t $(get_pid "$name") /bin/busybox sh
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xhatop" ]
|
||||
then
|
||||
name="$2"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
hatop -s "$(get_container_dir "$name")/var/lib/haproxy/stats"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xreload" ]
|
||||
then
|
||||
name="$2"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
kill -USR2 $(get_pid "$name")
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xdump" ]
|
||||
then
|
||||
subcmd="$2"
|
||||
if [ "x$subcmd" != "xlocal" ]
|
||||
then
|
||||
echo "only local is supported at the moment"
|
||||
exit -1
|
||||
fi
|
||||
name="$3"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
if [ "x$subcmd" == "xlocal" ]
|
||||
then
|
||||
dumpdir=$(get_dump_dir "$name")
|
||||
rm -rf "$dumpdir"
|
||||
mkdir -p "$dumpdir"
|
||||
criu dump -D "$dumpdir" -t "$(get_pid "$name")" --tcp-established --shell-job --ext-mount-map /dev:dev
|
||||
exit $?
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "x$1" == "xrestore" ]
|
||||
then
|
||||
subcmd="$2"
|
||||
if [ "x$subcmd" != "xlocal" ]
|
||||
then
|
||||
echo "only local is supported at the moment"
|
||||
exit -1
|
||||
fi
|
||||
name="$3"
|
||||
if [ "x$name" == "x" ]
|
||||
then
|
||||
echo "You must specify a name"
|
||||
exit -1
|
||||
fi
|
||||
if [ "x$subcmd" == "xlocal" ]
|
||||
then
|
||||
tmpid=$$
|
||||
pidfile=$(get_pid_file "$name")
|
||||
as="$statedir/action-scripts/$name.sh"
|
||||
cat > "$as" <<EOF
|
||||
#!/bin/bash
|
||||
if [ "x\${CRTOOLS_SCRIPT_ACTION}" == "xpost-restore" ]
|
||||
then
|
||||
P=\$(cat "$pidfile")
|
||||
ip link set dev sha$tmpid name "sha\$P"
|
||||
ip link set dev "sha\$P" mtu 9000 up
|
||||
ovs-vsctl add-port $bridge "sha\$P"
|
||||
fi
|
||||
EOF
|
||||
chmod +x "$as"
|
||||
dumpdir=$(get_dump_dir "$name")
|
||||
container="$(get_container_dir "$name")"
|
||||
if [ ! -d "$dumpdir" ]
|
||||
then
|
||||
echo "Dump does not exist"
|
||||
exit -1
|
||||
fi
|
||||
rm -f "$(get_pid_file "$name")"
|
||||
ovs-vsctl del-port $bridge "sha$(get_pid "$name")" > /dev/null 2>&1
|
||||
mount --bind "$container" "$container"
|
||||
criu restore -d -D "$dumpdir" --shell-job --tcp-established --ext-mount-map dev:/dev --root "$container" --veth-pair eth0="sha$tmpid" --action-script "$as" --pidfile "$(get_pid_file "$name")"
|
||||
res=$?
|
||||
umount "$container"
|
||||
exit $res
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
#migrate
|
||||
#rsync -avz --delete -e ssh /var/lib/superhaproxy/containers/foo 192.168.0.20:/var/lib/superhaproxy/containers/
|
||||
# procedure:
|
||||
# * initial rsync of container
|
||||
# * dump on local host
|
||||
# * second rsync of container
|
||||
# * rsync of images
|
||||
# * restore on remote host
|
||||
# * On success
|
||||
# * rm container and dump on localhost
|
||||
# * On failure
|
||||
# * If autofailback
|
||||
# * Restore container local
|
||||
# * on restore failure
|
||||
# * Try starting remote, if works, remove local container/images all done.
|
||||
# * If failed to start remote, try and start local
|
||||
# * If state all still local, remove remote data.
|
||||
|
||||
echo "Unknown command: $1"
|
||||
exit -1
|
||||
Loading…
Reference in New Issue