From 2f82bb7ebedd432c13148b286ace2023c1136969 Mon Sep 17 00:00:00 2001 From: Antoni Segura Puimedon Date: Fri, 16 Sep 2016 12:45:11 +0200 Subject: [PATCH] devstack: First version of kuryr-kubernetes plugin At the moment, since the repo doesn't have CNI, and the event processing pipeline is just a stub, it won't do practically anything, but it should already be laying all the environment that we'll use to develop the missing pieces and we can use it for the fullstack tests as well. Partially implements: blueprint kuryr-k8s-integration Change-Id: I7ca43ec7d0379d73273609078f5377ea6f55c363 Signed-off-by: Antoni Segura Puimedon --- devstack/local.conf.sample | 131 ++++++ devstack/plugin.sh | 392 ++++++++++++++++++ devstack/settings | 31 ++ scripts/run_server.py | 17 + .../kubelet-plugins/net/exec/kuryr.conf | 4 + 5 files changed, 575 insertions(+) create mode 100644 devstack/local.conf.sample create mode 100644 devstack/plugin.sh create mode 100644 devstack/settings create mode 100644 scripts/run_server.py create mode 100644 usr/libexec/kubernetes/kubelet-plugins/net/exec/kuryr.conf diff --git a/devstack/local.conf.sample b/devstack/local.conf.sample new file mode 100644 index 000000000..4e2330de3 --- /dev/null +++ b/devstack/local.conf.sample @@ -0,0 +1,131 @@ +[[local|localrc]] + +# At the moment kuryr-kubernetes only works with Python35, thus, you should +# read https://wiki.openstack.org/wiki/Python3#Enable_Python_3_in_DevStack +# This devstack plugin should work with Ubuntu 16.04+ and Fedora 24+ +# Make sure you have python3.5 and it's development package installed +USE_PYTHON3=True +PYTHON3_VERSION=3.5 + +enable_plugin kuryr-kubernetes \ + https://git.openstack.org/openstack/kuryr-kubernetes + +# If you do not want stacking to clone new versions of the enabled services, +# like for example when you did local modifications and need to ./unstack.sh +# and ./stack.sh again, uncomment the following +# RECLONE="no" + +# Log settings for better readability +LOGFILE=devstack.log +LOG_COLOR=False +# If you want the screen tabs logged in a specific location, you can use: +# SCREEN_LOGDIR="${HOME}/devstack_logs" + + +# Credentials +ADMIN_PASSWORD=pass +DATABASE_PASSWORD=pass +RABBIT_PASSWORD=pass +SERVICE_PASSWORD=pass +SERVICE_TOKEN=pass +# Enable Keystone v3 +IDENTITY_API_VERSION=3 + +# In pro of speed and being lightweight, we will be explicit in regards to +# which services we enable +ENABLED_SERVICES="" + +# Neutron services +enable_service neutron +enable_service q-agt +enable_service q-dhcp +enable_service q-l3 +enable_service q-svc + +# Keystone +enable_service key + +# dependencies +enable_service mysql +enable_service rabbit + +# By default use all the services from the kuryr-kubernetes plugin + +# Docker +# ====== +# If you already have docker configured, running and with its socket writable +# by the stack user, you can omit the following line. +enable_service docker + +# Etcd +# ==== +# The default is for devstack to run etcd for you. You can select the image and +# version of it by uncommenting and modifying the following defaults. +# KURYR_ETCD_IMAGE="quay.io/coreos/etcd" +# KURYR_ETCD_VERSION="v3.0.8" +# +# You can select the listening and advertising client and peering Etcd +# addresses by uncommenting and changing from the following defaults: +# KURYR_ETCD_ADVERTISE_CLIENT_URL=http://my_host_ip:2379} +# KURYR_ETCD_ADVERTISE_PEER_URL=http://my_host_ip:2380} +# KURYR_ETCD_LISTEN_CLIENT_URL=http://0.0.0.0:2379} +# KURYR_ETCD_LISTEN_PEER_URL=http://0.0.0.0:2380} +# +# If you already have etcd configured and running, you can just comment out +enable_service etcd +# then uncomment and set the following line: +# KURYR_ETCD_CLIENT_URL="http://etcd_ip:etcd_client_port" + +# Kubernetes +# ========== +# +# Kubernetes is run from the hyperkube docker image +# If you already have a Kubernetes deployment, you can use it instead and omit +# enabling the Kubernetes service (except Kubelet, which must be run by +# devstack so that it uses our development CNI driver. +# +# The default is, again, for devstack to run the Kubernetes services: +enable_service kubernetes-api +enable_service kubernetes-controller-manager +enable_service kubernetes-scheduler + +# We use hyperkube to run the services. You can select the hyperkube image and/ +# or version by uncommenting and setting the following ENV vars different +# to the following defaults: +# KURYR_HYPERKUBE_IMAGE="gcr.io/google_containers/hyperkube-amd64" +# KURYR_HYPERKUBE_VERSION="v1.3.7" +# +# If you have the 8080 port already bound to another service, you will need to +# have kubernetes API server bind to another port. In order to do that, +# uncomment and set a different port number in: +# KURYR_K8S_API_PORT="8080" +# +# If you want to test with a different range for the Cluster IPs uncomment and +# set the following ENV var to a different CIDR +# KURYR_K8S_CLUSTER_IP_RANGE="10.0.0.0/24" +# +# If, however, you are reusing an existing deployment, you should uncomment and +# set an ENV var so that the Kubelet devstack runs can find the API server: +# KURYR_K8S_API_URL="http://k8s_api_ip:k8s_api_port" +# + +# Kubelet +# ======= +# +# Kubelet should almost invariably be run by devstack +enable_service kubelet + +# You can specify a different location for the hyperkube binary that will be +# extracted from the hyperkube container into the Host filesystem: +# KURYR_HYPERKUBE_BINARY=/usr/local/bin/hyperkube +# +# NOTE: KURYR_HYPERKUBE_IMAGE, KURYR_HYPERKUBE_VERSION also affect which +# the selected binary for the Kubelet. + +# Kuryr watcher +# ============= +# +# Just like the Kubelet, you'll want to have the watcher enabled. It is the +# part of the codebase that connects to the Kubernetes API server to read the +# resource events and convert them to Neutron actions +enable_service kuryr-kubernetes diff --git a/devstack/plugin.sh b/devstack/plugin.sh new file mode 100644 index 000000000..df0dfda58 --- /dev/null +++ b/devstack/plugin.sh @@ -0,0 +1,392 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set +o xtrace + +function run_container { + # Runs a detached container and uses devstack's run process to monitor + # its logs + local name + + name="$1" + shift + + docker run --name "$name" --detach "$@" + + run_process "$name" \ + "docker logs -f $name" +} + +function stop_container { + local name + + name="$1" + + docker kill "$name" + docker rm "$name" + stop_process "$name" +} + +function create_kuryr_account { + if is_service_enabled kuryr-kubernetes; then + create_service_user "kuryr" "admin" + get_or_create_service "kuryr-kubernetes" "kuryr-kubernetes" \ + "Kuryr-Kubernetes Service" + fi +} + +function create_kuryr_cache_dir { + # Create cache directory + sudo install -d -o "$STACK_USER" "$KURYR_AUTH_CACHE_DIR" + if [[ ! "$KURYR_AUTH_CACHE_DIR" == "" ]]; then + rm -f "$KURYR_AUTH_CACHE_DIR"/* + fi +} + +function get_distutils_data_path { + cat << EOF | python - +from __future__ import print_function +import distutils.dist +import distutils.command.install + +inst = distutils.command.install.install(distutils.dist.Distribution()) +inst.finalize_options() + +print(inst.install_data) +EOF +} + +function configure_kuryr { + sudo install -d -o "$STACK_USER" "$KURYR_CONFIG_DIR" + # TODO(apuimedo): remove when we have config generation + # (cd "$KURYR_HOME" && exec ./tools/generate_config_file_samples.sh) + # cp "$KURYR_HOME/etc/kuryr.conf.sample" "$KURYR_CONFIG" + # iniset -sudo ${KURYR_CONFIG} DEFAULT bindir \ + # "$(get_distutils_data_path)/libexec/kuryr" + + create_kuryr_cache_dir + + # Neutron API server & Neutron plugin + if is_service_enabled kuryr-kubernetes; then + configure_auth_token_middleware "$KURYR_CONFIG" kuryr \ + "$KURYR_AUTH_CACHE_DIR" neutron + fi +} + +function check_docker { + if is_ubuntu; then + dpkg -s docker-engine > /dev/null 2>&1 + else + rpm -q docker-engine > /dev/null 2>&1 + fi +} + +function get_container { + local image + local image_name + local version + image_name="$1" + version="${2:-latest}" + + if [ "$image_name" == "" ]; then + return 0 + fi + + image="${image}:${version}" + if [ -z "$(docker images -q "$image")" ]; then + docker pull "$image" + fi +} + + +function prepare_etcd { + # Make Etcd data directory + sudo install -d -o "$STACK_USER" "$KURYR_ETCD_DATA_DIR" + + # Get Etcd container + get_container "$KURYR_ETCD_IMAGE" "$KURYR_ETCD_VERSION" +} + +function run_etcd { + run_container etcd \ + --net host \ + --volume="${KURYR_ETCD_DATA_DIR}:/var/etcd:rw" \ + "${KURYR_ETCD_IMAGE}:${KURYR_ETCD_VERSION}" \ + /usr/local/bin/etcd \ + --name devstack \ + --data-dir /var/etcd/data \ + --initial-advertise-peer-urls "$KURYR_ETCD_ADVERTISE_PEER_URL" \ + --listen-peer-urls "$KURYR_ETCD_LISTEN_PEER_URL" \ + --listen-client-urls "$KURYR_ETCD_LISTEN_CLIENT_URL" \ + --advertise-client-urls "$KURYR_ETCD_ADVERTISE_CLIENT_URL" \ + --initial-cluster-token etcd-cluster-1 \ + --initial-cluster "devstack=$KURYR_ETCD_PEER_URL" \ + --initial-cluster-state new +} + +function prepare_docker { + curl http://get.docker.com | sudo bash + + # After an ./unstack it will be stopped. So it is OK if it returns + # exit-code == 1 + sudo service docker stop || true + + # Make sure there's no leftover Docker process and pidfile + sudo kill -s SIGTERM "$(cat /var/run/docker.pid)" +} + +function run_docker { + run_process docker \ + "sudo /usr/bin/docker daemon --debug=true \ + -H unix://$KURYR_DOCKER_ENGINE_SOCKET_FILE" + + # We put the stack user as owner of the socket so we do not need to + # run the Docker commands with sudo when developing. + echo -n "Waiting for Docker to create its socket file" + while [ ! -e "$KURYR_DOCKER_ENGINE_SOCKET_FILE" ]; do + echo -n "." + sleep 1 + done + echo "" + sudo chown "$STACK_USER":docker "$KURYR_DOCKER_ENGINE_SOCKET_FILE" +} + +function stop_docker { + stop_process docker + + # Stop process does not handle well Docker 1.12+ new multi process + # split and doesn't kill them all. Let's leverage Docker's own pidfile + sudo kill -s SIGTERM "$(cat /var/run/docker.pid)" +} + +function prepare_kubernetes_files { + # Sets up the base configuration for the Kubernetes API Server and the + # Controller Manager. + docker run \ + --name devstack-k8s-setup-files \ + --detach \ + --volume "$KURYR_HYPERKUBE_DATA_DIR:/data:rw" \ + "${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \ + /setup-files.sh \ + "IP:${HOST_IP},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local" +} + +function wait_for { + local name + local url + name="$1" + url="$2" + + echo -n "Waiting for $name to respond" + + until curl -o /dev/null -sIf "$url"; do + echo -n "." + sleep 1 + done + echo "" +} + +function run_k8s_api { + # Runs Hyperkube's Kubernetes API Server + wait_for "etcd" "${KURYR_ETCD_CLIENT_URL}/v2/machines" + + run_container kubernetes-api \ + --net host \ + --volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \ + "${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \ + /hyperkube apiserver \ + --service-cluster-ip-range="${KURYR_K8S_CLUSTER_IP_RANGE}" \ + --insecure-bind-address=0.0.0.0 \ + --insecure-port="${KURYR_K8S_API_PORT}" \ + --etcd-servers="${KURYR_ETCD_CLIENT_URL}" \ + --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota \ + --client-ca-file=/srv/kubernetes/ca.crt \ + --basic-auth-file=/srv/kubernetes/basic_auth.csv \ + --min-request-timeout=300 \ + --tls-cert-file=/srv/kubernetes/server.cert \ + --tls-private-key-file=/srv/kubernetes/server.key \ + --token-auth-file=/srv/kubernetes/known_tokens.csv \ + --allow-privileged=true \ + --v=2 \ + --logtostderr=true +} + +function run_k8s_controller_manager { + # Runs Hyperkube's Kubernetes controller manager + wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL" + + run_container kubernetes-controller-manager \ + --net host \ + --volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \ + "${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \ + /hyperkube controller-manager \ + --master="$KURYR_K8S_API_URL" \ + --service-account-private-key-file=/srv/kubernetes/server.key \ + --root-ca-file=/srv/kubernetes/ca.crt \ + --min-resync-period=3m \ + --v=2 \ + --logtostderr=true +} + +function run_k8s_scheduler { + # Runs Hyperkube's Kubernetes scheduler + wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL" + + run_container kubernetes-scheduler \ + --net host \ + --volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \ + "${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \ + /hyperkube scheduler \ + --master="$KURYR_K8S_API_URL" \ + --v=2 \ + --logtostderr=true +} + +function extract_hyperkube { + local hyperkube_container + local tmp_hyperkube_path + + tmp_hyperkube_path="/tmp/hyperkube" + hyperkube_container="$(docker ps -aq \ + -f ancestor="${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" | \ + head -1)" + docker cp "$hyperkube_container:/hyperkube" /tmp/hyperkube + sudo install -o "$STACK_USER" -m 0555 "$tmp_hyperkube_path" \ + "$KURYR_HYPERKUBE_BINARY" +} + +function prepare_kubelet { + local kubelet_plugin_dir + kubelet_plugin_dir="/usr/libexec/kubernetes/kubelet-plugins/net/exec" + + sudo install -o "$STACK_USER" -m 0664 -D \ + "${KURYR_HOME}${kubelet_plugin_dir}/kuryr.conf" \ + "${kubelet_plugin_dir}/kuryr.conf" +} + +function run_k8s_kubelet { + # Runs Hyperkube's Kubernetes kubelet from the extracted binary + # + # The reason for extracting the binary and running it in from the Host + # filesystem is so that we can leverage the binding utilities that network + # vendor devstack plugins may have installed (like ovs-vsctl). Also, it + # saves us from the arduous task of setting up mounts to the official image + # adding Python and all our CNI/binding dependencies. + local command + + command="$KURYR_HYPERKUBE_BINARY kubelet\ + --allow-privileged=true \ + --api-servers=$KURYR_K8S_API_URL \ + --v=2 \ + --address='0.0.0.0' \ + --enable-server \ + network-plugin=cni" + wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL" + run_process kubelet "$command" +} + +# main loop +if is_service_enabled kuryr-kubernetes; then + if [[ "$1" == "stack" && "$2" == "install" ]]; then + setup_develop "$KURYR_HOME" + + elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then + create_kuryr_account + configure_kuryr + + if is_service_enabled docker; then + check_docker || prepare_docker + run_docker + fi + + if is_service_enabled etcd; then + prepare_etcd + run_etcd + fi + + get_container "$KURYR_HYPERKUBE_IMAGE" "$KURYR_HYPERKUBE_VERSION" + prepare_kubernetes_files + if is_service_enabled kubernetes-api; then + run_k8s_api + fi + if is_service_enabled kubernetes-controller-manager; then + run_k8s_controller_manager + fi + if is_service_enabled kubernetes-scheduler; then + run_k8s_scheduler + fi + + if is_service_enabled kubelet; then + prepare_kubelet + extract_hyperkube + run_k8s_kubelet + fi + fi + + if [[ "$1" == "stack" && "$2" == "extra" ]]; then + # FIXME(limao): When Kuryr start up, it need to detect if neutron + # support tag plugin. + # + # Kuryr will call neutron extension API to verify if neutron support + # tag. So Kuryr need to start after neutron-server finish load tag + # plugin. The process of devstack is: + # ... + # run_phase "stack" "post-config" + # ... + # start neutron-server + # ... + # run_phase "stack" "extra" + # + # If Kuryr start up in "post-config" phase, there is no way to make + # sure Kuryr can start before neutron-server, so Kuryr start in "extra" + # phase. Bug: https://bugs.launchpad.net/kuryr/+bug/1587522 + run_process kuryr-kubernetes \ + "python3 ${KURYR_HOME}/scripts/run_server.py \ + --config-file $KURYR_CONFIG" + fi + + if [[ "$1" == "unstack" ]]; then + stop_process kuryr-kubernetes + docker kill devstack-k8s-setup-files + docker rm devstack-k8s-setup-files + + if is_service_enabled kubernetes-controller-manager; then + stop_container kubernetes-controller-manager + fi + if is_service_enabled kubernetes-scheduler; then + stop_container kubernetes-scheduler + fi + if is_service_enabled kubelet; then + stop_process kubelet + fi + if is_service_enabled kubernetes-api; then + stop_container kubernetes-api + fi + if is_service_enabled etcd; then + stop_container etcd + fi + stop_docker + fi + + if [[ "$1" == "clean" ]]; then + if is_service_enabled etcd; then + # Cleanup Etcd for the next stacking + sudo rm -rf "$KURYR_ETCD_DATA_DIR" + fi + fi +fi + +# Restore xtrace +$XTRACE diff --git a/devstack/settings b/devstack/settings new file mode 100644 index 000000000..58bacd0ea --- /dev/null +++ b/devstack/settings @@ -0,0 +1,31 @@ +KURYR_HOME=${KURYR_HOME:-$DEST/kuryr-kubernetes} + +KURYR_CONFIG_DIR=${KURYR_CONFIG_DIR:-/etc/kuryr} +KURYR_CONFIG=${KURYR_CONFIG:-${KURYR_CONFIG_DIR}/kuryr.conf} +KURYR_AUTH_CACHE_DIR=${KURYR_AUTH_CACHE_DIR:-/var/cache/kuryr} + +KURYR_POOL_PREFIX=${KURYR_POOL_PREFIX:-10.10.0.0/16} +KURYR_POOL_PREFIX_LEN=${KURYR_POOL_PREFIX_LEN:-24} + +KURYR_DOCKER_ENGINE_PORT=${KURYR_DOCKER_ENGINE_PORT:-2375} +KURYR_DOCKER_ENGINE_SOCKET_FILE=${KURYR_DOCKER_ENGINE_SOCKET_FILE:-/var/run/docker.sock} + +# Etcd +KURYR_ETCD_IMAGE=${KURYR_ETCD_IMAGE:-quay.io/coreos/etcd} +KURYR_ETCD_VERSION=${KURYR_ETCD_VERSION:-v3.0.8} +KURYR_ETCD_DATA_DIR=${KURYR_ETCD_DATA_DIR:-/var/lib/etcd} +KURYR_ETCD_ADVERTISE_CLIENT_URL=${KURYR_ETCD_ADVERTISE_CLIENT_URL:-http://${HOST_IP}:2379} +KURYR_ETCD_ADVERTISE_PEER_URL=${KURYR_ETCD_ADVERTISE_PEER_URL:-http://${HOST_IP}:2380} +KURYR_ETCD_LISTEN_CLIENT_URL=${KURYR_ETCD_LISTEN_CLIENT_URL:-http://0.0.0.0:2379} +KURYR_ETCD_LISTEN_PEER_URL=${KURYR_ETCD_LISTEN_PEER_URL:-http://0.0.0.0:2380} + +# HYPERKUBE +KURYR_HYPERKUBE_IMAGE=${KURYR_HYPERKUBE_IMAGE:-gcr.io/google_containers/hyperkube-amd64} +KURYR_HYPERKUBE_VERSION=${KURYR_HYPERKUBE_VERSION:-v1.3.7} +KURYR_HYPERKUBE_DATA_DIR=${KURYR_HYPERKUBE_DATA_DIR:-/var/lib/hyperkube} +KURYR_HYPERKUBE_BINARY=${KURYR_HYPERKUBE_BINARY:-/usr/local/bin/hyperkube} + +# Kubernetes +KURYR_K8S_CLUSTER_IP_RANGE=${KURYR_K8S_CLUSTER_IP_RANGE:-10.0.0.0/24} +KURYR_K8S_API_PORT=${KURYR_K8S_API_PORT:-8080} +KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-http://${HOST_IP}:${KURYR_K8S_API_PORT}} diff --git a/scripts/run_server.py b/scripts/run_server.py new file mode 100644 index 000000000..ccf40c1f0 --- /dev/null +++ b/scripts/run_server.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from kuryr_kubernetes import server + +server.start() diff --git a/usr/libexec/kubernetes/kubelet-plugins/net/exec/kuryr.conf b/usr/libexec/kubernetes/kubelet-plugins/net/exec/kuryr.conf new file mode 100644 index 000000000..a86a92735 --- /dev/null +++ b/usr/libexec/kubernetes/kubelet-plugins/net/exec/kuryr.conf @@ -0,0 +1,4 @@ +{ + "name": "kuryr", + "type": "kuryr" +}