diff --git a/devstack/local.conf.sample b/devstack/local.conf.sample new file mode 100644 index 000000000..4e2330de3 --- /dev/null +++ b/devstack/local.conf.sample @@ -0,0 +1,131 @@ +[[local|localrc]] + +# At the moment kuryr-kubernetes only works with Python35, thus, you should +# read https://wiki.openstack.org/wiki/Python3#Enable_Python_3_in_DevStack +# This devstack plugin should work with Ubuntu 16.04+ and Fedora 24+ +# Make sure you have python3.5 and it's development package installed +USE_PYTHON3=True +PYTHON3_VERSION=3.5 + +enable_plugin kuryr-kubernetes \ + https://git.openstack.org/openstack/kuryr-kubernetes + +# If you do not want stacking to clone new versions of the enabled services, +# like for example when you did local modifications and need to ./unstack.sh +# and ./stack.sh again, uncomment the following +# RECLONE="no" + +# Log settings for better readability +LOGFILE=devstack.log +LOG_COLOR=False +# If you want the screen tabs logged in a specific location, you can use: +# SCREEN_LOGDIR="${HOME}/devstack_logs" + + +# Credentials +ADMIN_PASSWORD=pass +DATABASE_PASSWORD=pass +RABBIT_PASSWORD=pass +SERVICE_PASSWORD=pass +SERVICE_TOKEN=pass +# Enable Keystone v3 +IDENTITY_API_VERSION=3 + +# In pro of speed and being lightweight, we will be explicit in regards to +# which services we enable +ENABLED_SERVICES="" + +# Neutron services +enable_service neutron +enable_service q-agt +enable_service q-dhcp +enable_service q-l3 +enable_service q-svc + +# Keystone +enable_service key + +# dependencies +enable_service mysql +enable_service rabbit + +# By default use all the services from the kuryr-kubernetes plugin + +# Docker +# ====== +# If you already have docker configured, running and with its socket writable +# by the stack user, you can omit the following line. +enable_service docker + +# Etcd +# ==== +# The default is for devstack to run etcd for you. You can select the image and +# version of it by uncommenting and modifying the following defaults. +# KURYR_ETCD_IMAGE="quay.io/coreos/etcd" +# KURYR_ETCD_VERSION="v3.0.8" +# +# You can select the listening and advertising client and peering Etcd +# addresses by uncommenting and changing from the following defaults: +# KURYR_ETCD_ADVERTISE_CLIENT_URL=http://my_host_ip:2379} +# KURYR_ETCD_ADVERTISE_PEER_URL=http://my_host_ip:2380} +# KURYR_ETCD_LISTEN_CLIENT_URL=http://0.0.0.0:2379} +# KURYR_ETCD_LISTEN_PEER_URL=http://0.0.0.0:2380} +# +# If you already have etcd configured and running, you can just comment out +enable_service etcd +# then uncomment and set the following line: +# KURYR_ETCD_CLIENT_URL="http://etcd_ip:etcd_client_port" + +# Kubernetes +# ========== +# +# Kubernetes is run from the hyperkube docker image +# If you already have a Kubernetes deployment, you can use it instead and omit +# enabling the Kubernetes service (except Kubelet, which must be run by +# devstack so that it uses our development CNI driver. +# +# The default is, again, for devstack to run the Kubernetes services: +enable_service kubernetes-api +enable_service kubernetes-controller-manager +enable_service kubernetes-scheduler + +# We use hyperkube to run the services. You can select the hyperkube image and/ +# or version by uncommenting and setting the following ENV vars different +# to the following defaults: +# KURYR_HYPERKUBE_IMAGE="gcr.io/google_containers/hyperkube-amd64" +# KURYR_HYPERKUBE_VERSION="v1.3.7" +# +# If you have the 8080 port already bound to another service, you will need to +# have kubernetes API server bind to another port. In order to do that, +# uncomment and set a different port number in: +# KURYR_K8S_API_PORT="8080" +# +# If you want to test with a different range for the Cluster IPs uncomment and +# set the following ENV var to a different CIDR +# KURYR_K8S_CLUSTER_IP_RANGE="10.0.0.0/24" +# +# If, however, you are reusing an existing deployment, you should uncomment and +# set an ENV var so that the Kubelet devstack runs can find the API server: +# KURYR_K8S_API_URL="http://k8s_api_ip:k8s_api_port" +# + +# Kubelet +# ======= +# +# Kubelet should almost invariably be run by devstack +enable_service kubelet + +# You can specify a different location for the hyperkube binary that will be +# extracted from the hyperkube container into the Host filesystem: +# KURYR_HYPERKUBE_BINARY=/usr/local/bin/hyperkube +# +# NOTE: KURYR_HYPERKUBE_IMAGE, KURYR_HYPERKUBE_VERSION also affect which +# the selected binary for the Kubelet. + +# Kuryr watcher +# ============= +# +# Just like the Kubelet, you'll want to have the watcher enabled. It is the +# part of the codebase that connects to the Kubernetes API server to read the +# resource events and convert them to Neutron actions +enable_service kuryr-kubernetes diff --git a/devstack/plugin.sh b/devstack/plugin.sh new file mode 100644 index 000000000..df0dfda58 --- /dev/null +++ b/devstack/plugin.sh @@ -0,0 +1,392 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Save trace setting +XTRACE=$(set +o | grep xtrace) +set +o xtrace + +function run_container { + # Runs a detached container and uses devstack's run process to monitor + # its logs + local name + + name="$1" + shift + + docker run --name "$name" --detach "$@" + + run_process "$name" \ + "docker logs -f $name" +} + +function stop_container { + local name + + name="$1" + + docker kill "$name" + docker rm "$name" + stop_process "$name" +} + +function create_kuryr_account { + if is_service_enabled kuryr-kubernetes; then + create_service_user "kuryr" "admin" + get_or_create_service "kuryr-kubernetes" "kuryr-kubernetes" \ + "Kuryr-Kubernetes Service" + fi +} + +function create_kuryr_cache_dir { + # Create cache directory + sudo install -d -o "$STACK_USER" "$KURYR_AUTH_CACHE_DIR" + if [[ ! "$KURYR_AUTH_CACHE_DIR" == "" ]]; then + rm -f "$KURYR_AUTH_CACHE_DIR"/* + fi +} + +function get_distutils_data_path { + cat << EOF | python - +from __future__ import print_function +import distutils.dist +import distutils.command.install + +inst = distutils.command.install.install(distutils.dist.Distribution()) +inst.finalize_options() + +print(inst.install_data) +EOF +} + +function configure_kuryr { + sudo install -d -o "$STACK_USER" "$KURYR_CONFIG_DIR" + # TODO(apuimedo): remove when we have config generation + # (cd "$KURYR_HOME" && exec ./tools/generate_config_file_samples.sh) + # cp "$KURYR_HOME/etc/kuryr.conf.sample" "$KURYR_CONFIG" + # iniset -sudo ${KURYR_CONFIG} DEFAULT bindir \ + # "$(get_distutils_data_path)/libexec/kuryr" + + create_kuryr_cache_dir + + # Neutron API server & Neutron plugin + if is_service_enabled kuryr-kubernetes; then + configure_auth_token_middleware "$KURYR_CONFIG" kuryr \ + "$KURYR_AUTH_CACHE_DIR" neutron + fi +} + +function check_docker { + if is_ubuntu; then + dpkg -s docker-engine > /dev/null 2>&1 + else + rpm -q docker-engine > /dev/null 2>&1 + fi +} + +function get_container { + local image + local image_name + local version + image_name="$1" + version="${2:-latest}" + + if [ "$image_name" == "" ]; then + return 0 + fi + + image="${image}:${version}" + if [ -z "$(docker images -q "$image")" ]; then + docker pull "$image" + fi +} + + +function prepare_etcd { + # Make Etcd data directory + sudo install -d -o "$STACK_USER" "$KURYR_ETCD_DATA_DIR" + + # Get Etcd container + get_container "$KURYR_ETCD_IMAGE" "$KURYR_ETCD_VERSION" +} + +function run_etcd { + run_container etcd \ + --net host \ + --volume="${KURYR_ETCD_DATA_DIR}:/var/etcd:rw" \ + "${KURYR_ETCD_IMAGE}:${KURYR_ETCD_VERSION}" \ + /usr/local/bin/etcd \ + --name devstack \ + --data-dir /var/etcd/data \ + --initial-advertise-peer-urls "$KURYR_ETCD_ADVERTISE_PEER_URL" \ + --listen-peer-urls "$KURYR_ETCD_LISTEN_PEER_URL" \ + --listen-client-urls "$KURYR_ETCD_LISTEN_CLIENT_URL" \ + --advertise-client-urls "$KURYR_ETCD_ADVERTISE_CLIENT_URL" \ + --initial-cluster-token etcd-cluster-1 \ + --initial-cluster "devstack=$KURYR_ETCD_PEER_URL" \ + --initial-cluster-state new +} + +function prepare_docker { + curl http://get.docker.com | sudo bash + + # After an ./unstack it will be stopped. So it is OK if it returns + # exit-code == 1 + sudo service docker stop || true + + # Make sure there's no leftover Docker process and pidfile + sudo kill -s SIGTERM "$(cat /var/run/docker.pid)" +} + +function run_docker { + run_process docker \ + "sudo /usr/bin/docker daemon --debug=true \ + -H unix://$KURYR_DOCKER_ENGINE_SOCKET_FILE" + + # We put the stack user as owner of the socket so we do not need to + # run the Docker commands with sudo when developing. + echo -n "Waiting for Docker to create its socket file" + while [ ! -e "$KURYR_DOCKER_ENGINE_SOCKET_FILE" ]; do + echo -n "." + sleep 1 + done + echo "" + sudo chown "$STACK_USER":docker "$KURYR_DOCKER_ENGINE_SOCKET_FILE" +} + +function stop_docker { + stop_process docker + + # Stop process does not handle well Docker 1.12+ new multi process + # split and doesn't kill them all. Let's leverage Docker's own pidfile + sudo kill -s SIGTERM "$(cat /var/run/docker.pid)" +} + +function prepare_kubernetes_files { + # Sets up the base configuration for the Kubernetes API Server and the + # Controller Manager. + docker run \ + --name devstack-k8s-setup-files \ + --detach \ + --volume "$KURYR_HYPERKUBE_DATA_DIR:/data:rw" \ + "${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \ + /setup-files.sh \ + "IP:${HOST_IP},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local" +} + +function wait_for { + local name + local url + name="$1" + url="$2" + + echo -n "Waiting for $name to respond" + + until curl -o /dev/null -sIf "$url"; do + echo -n "." + sleep 1 + done + echo "" +} + +function run_k8s_api { + # Runs Hyperkube's Kubernetes API Server + wait_for "etcd" "${KURYR_ETCD_CLIENT_URL}/v2/machines" + + run_container kubernetes-api \ + --net host \ + --volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \ + "${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \ + /hyperkube apiserver \ + --service-cluster-ip-range="${KURYR_K8S_CLUSTER_IP_RANGE}" \ + --insecure-bind-address=0.0.0.0 \ + --insecure-port="${KURYR_K8S_API_PORT}" \ + --etcd-servers="${KURYR_ETCD_CLIENT_URL}" \ + --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota \ + --client-ca-file=/srv/kubernetes/ca.crt \ + --basic-auth-file=/srv/kubernetes/basic_auth.csv \ + --min-request-timeout=300 \ + --tls-cert-file=/srv/kubernetes/server.cert \ + --tls-private-key-file=/srv/kubernetes/server.key \ + --token-auth-file=/srv/kubernetes/known_tokens.csv \ + --allow-privileged=true \ + --v=2 \ + --logtostderr=true +} + +function run_k8s_controller_manager { + # Runs Hyperkube's Kubernetes controller manager + wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL" + + run_container kubernetes-controller-manager \ + --net host \ + --volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \ + "${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \ + /hyperkube controller-manager \ + --master="$KURYR_K8S_API_URL" \ + --service-account-private-key-file=/srv/kubernetes/server.key \ + --root-ca-file=/srv/kubernetes/ca.crt \ + --min-resync-period=3m \ + --v=2 \ + --logtostderr=true +} + +function run_k8s_scheduler { + # Runs Hyperkube's Kubernetes scheduler + wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL" + + run_container kubernetes-scheduler \ + --net host \ + --volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \ + "${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \ + /hyperkube scheduler \ + --master="$KURYR_K8S_API_URL" \ + --v=2 \ + --logtostderr=true +} + +function extract_hyperkube { + local hyperkube_container + local tmp_hyperkube_path + + tmp_hyperkube_path="/tmp/hyperkube" + hyperkube_container="$(docker ps -aq \ + -f ancestor="${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" | \ + head -1)" + docker cp "$hyperkube_container:/hyperkube" /tmp/hyperkube + sudo install -o "$STACK_USER" -m 0555 "$tmp_hyperkube_path" \ + "$KURYR_HYPERKUBE_BINARY" +} + +function prepare_kubelet { + local kubelet_plugin_dir + kubelet_plugin_dir="/usr/libexec/kubernetes/kubelet-plugins/net/exec" + + sudo install -o "$STACK_USER" -m 0664 -D \ + "${KURYR_HOME}${kubelet_plugin_dir}/kuryr.conf" \ + "${kubelet_plugin_dir}/kuryr.conf" +} + +function run_k8s_kubelet { + # Runs Hyperkube's Kubernetes kubelet from the extracted binary + # + # The reason for extracting the binary and running it in from the Host + # filesystem is so that we can leverage the binding utilities that network + # vendor devstack plugins may have installed (like ovs-vsctl). Also, it + # saves us from the arduous task of setting up mounts to the official image + # adding Python and all our CNI/binding dependencies. + local command + + command="$KURYR_HYPERKUBE_BINARY kubelet\ + --allow-privileged=true \ + --api-servers=$KURYR_K8S_API_URL \ + --v=2 \ + --address='0.0.0.0' \ + --enable-server \ + network-plugin=cni" + wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL" + run_process kubelet "$command" +} + +# main loop +if is_service_enabled kuryr-kubernetes; then + if [[ "$1" == "stack" && "$2" == "install" ]]; then + setup_develop "$KURYR_HOME" + + elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then + create_kuryr_account + configure_kuryr + + if is_service_enabled docker; then + check_docker || prepare_docker + run_docker + fi + + if is_service_enabled etcd; then + prepare_etcd + run_etcd + fi + + get_container "$KURYR_HYPERKUBE_IMAGE" "$KURYR_HYPERKUBE_VERSION" + prepare_kubernetes_files + if is_service_enabled kubernetes-api; then + run_k8s_api + fi + if is_service_enabled kubernetes-controller-manager; then + run_k8s_controller_manager + fi + if is_service_enabled kubernetes-scheduler; then + run_k8s_scheduler + fi + + if is_service_enabled kubelet; then + prepare_kubelet + extract_hyperkube + run_k8s_kubelet + fi + fi + + if [[ "$1" == "stack" && "$2" == "extra" ]]; then + # FIXME(limao): When Kuryr start up, it need to detect if neutron + # support tag plugin. + # + # Kuryr will call neutron extension API to verify if neutron support + # tag. So Kuryr need to start after neutron-server finish load tag + # plugin. The process of devstack is: + # ... + # run_phase "stack" "post-config" + # ... + # start neutron-server + # ... + # run_phase "stack" "extra" + # + # If Kuryr start up in "post-config" phase, there is no way to make + # sure Kuryr can start before neutron-server, so Kuryr start in "extra" + # phase. Bug: https://bugs.launchpad.net/kuryr/+bug/1587522 + run_process kuryr-kubernetes \ + "python3 ${KURYR_HOME}/scripts/run_server.py \ + --config-file $KURYR_CONFIG" + fi + + if [[ "$1" == "unstack" ]]; then + stop_process kuryr-kubernetes + docker kill devstack-k8s-setup-files + docker rm devstack-k8s-setup-files + + if is_service_enabled kubernetes-controller-manager; then + stop_container kubernetes-controller-manager + fi + if is_service_enabled kubernetes-scheduler; then + stop_container kubernetes-scheduler + fi + if is_service_enabled kubelet; then + stop_process kubelet + fi + if is_service_enabled kubernetes-api; then + stop_container kubernetes-api + fi + if is_service_enabled etcd; then + stop_container etcd + fi + stop_docker + fi + + if [[ "$1" == "clean" ]]; then + if is_service_enabled etcd; then + # Cleanup Etcd for the next stacking + sudo rm -rf "$KURYR_ETCD_DATA_DIR" + fi + fi +fi + +# Restore xtrace +$XTRACE diff --git a/devstack/settings b/devstack/settings new file mode 100644 index 000000000..58bacd0ea --- /dev/null +++ b/devstack/settings @@ -0,0 +1,31 @@ +KURYR_HOME=${KURYR_HOME:-$DEST/kuryr-kubernetes} + +KURYR_CONFIG_DIR=${KURYR_CONFIG_DIR:-/etc/kuryr} +KURYR_CONFIG=${KURYR_CONFIG:-${KURYR_CONFIG_DIR}/kuryr.conf} +KURYR_AUTH_CACHE_DIR=${KURYR_AUTH_CACHE_DIR:-/var/cache/kuryr} + +KURYR_POOL_PREFIX=${KURYR_POOL_PREFIX:-10.10.0.0/16} +KURYR_POOL_PREFIX_LEN=${KURYR_POOL_PREFIX_LEN:-24} + +KURYR_DOCKER_ENGINE_PORT=${KURYR_DOCKER_ENGINE_PORT:-2375} +KURYR_DOCKER_ENGINE_SOCKET_FILE=${KURYR_DOCKER_ENGINE_SOCKET_FILE:-/var/run/docker.sock} + +# Etcd +KURYR_ETCD_IMAGE=${KURYR_ETCD_IMAGE:-quay.io/coreos/etcd} +KURYR_ETCD_VERSION=${KURYR_ETCD_VERSION:-v3.0.8} +KURYR_ETCD_DATA_DIR=${KURYR_ETCD_DATA_DIR:-/var/lib/etcd} +KURYR_ETCD_ADVERTISE_CLIENT_URL=${KURYR_ETCD_ADVERTISE_CLIENT_URL:-http://${HOST_IP}:2379} +KURYR_ETCD_ADVERTISE_PEER_URL=${KURYR_ETCD_ADVERTISE_PEER_URL:-http://${HOST_IP}:2380} +KURYR_ETCD_LISTEN_CLIENT_URL=${KURYR_ETCD_LISTEN_CLIENT_URL:-http://0.0.0.0:2379} +KURYR_ETCD_LISTEN_PEER_URL=${KURYR_ETCD_LISTEN_PEER_URL:-http://0.0.0.0:2380} + +# HYPERKUBE +KURYR_HYPERKUBE_IMAGE=${KURYR_HYPERKUBE_IMAGE:-gcr.io/google_containers/hyperkube-amd64} +KURYR_HYPERKUBE_VERSION=${KURYR_HYPERKUBE_VERSION:-v1.3.7} +KURYR_HYPERKUBE_DATA_DIR=${KURYR_HYPERKUBE_DATA_DIR:-/var/lib/hyperkube} +KURYR_HYPERKUBE_BINARY=${KURYR_HYPERKUBE_BINARY:-/usr/local/bin/hyperkube} + +# Kubernetes +KURYR_K8S_CLUSTER_IP_RANGE=${KURYR_K8S_CLUSTER_IP_RANGE:-10.0.0.0/24} +KURYR_K8S_API_PORT=${KURYR_K8S_API_PORT:-8080} +KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-http://${HOST_IP}:${KURYR_K8S_API_PORT}} diff --git a/scripts/run_server.py b/scripts/run_server.py new file mode 100644 index 000000000..ccf40c1f0 --- /dev/null +++ b/scripts/run_server.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from kuryr_kubernetes import server + +server.start() diff --git a/usr/libexec/kubernetes/kubelet-plugins/net/exec/kuryr.conf b/usr/libexec/kubernetes/kubelet-plugins/net/exec/kuryr.conf new file mode 100644 index 000000000..a86a92735 --- /dev/null +++ b/usr/libexec/kubernetes/kubelet-plugins/net/exec/kuryr.conf @@ -0,0 +1,4 @@ +{ + "name": "kuryr", + "type": "kuryr" +}