Use K8s API LB IP instead of HOST_IP in DevStack

This commit switches Kuryr configuration to use the IP of load balancer
created for Kubernetes API instead of hardcoding HOST_IP there. This
should be useful e.g. for multinode.

Change-Id: I4535986fdac4d7d6b2eda0debdcaf6122a4f5b9f
Closes-Bug: 1771801
This commit is contained in:
Michał Dulko 2018-05-17 13:40:15 +02:00
parent 719cc1cc28
commit ce3305b9ef
4 changed files with 56 additions and 26 deletions

View File

@ -748,7 +748,6 @@ function run_openshift_master {
"--write-config=${OPENSHIFT_DATA_DIR}"
# Reconfigure Kuryr-Kubernetes to use the certs generated
iniset "$KURYR_CONFIG" kubernetes api_root "$OPENSHIFT_API_URL"
iniset "$KURYR_CONFIG" kubernetes ssl_client_crt_file "${OPENSHIFT_DATA_DIR}/admin.crt"
iniset "$KURYR_CONFIG" kubernetes ssl_client_key_file "${OPENSHIFT_DATA_DIR}/admin.key"
iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file "${OPENSHIFT_DATA_DIR}/ca.crt"

View File

@ -52,7 +52,6 @@ function configure_kuryr {
sudo install -o "$STACK_USER" -m 640 -D "${KURYR_HOME}/etc/kuryr.conf.sample" \
"$KURYR_CONFIG"
iniset "$KURYR_CONFIG" kubernetes api_root "$KURYR_K8S_API_URL"
if [ "$KURYR_K8S_API_CERT" ]; then
iniset "$KURYR_CONFIG" kubernetes ssl_client_crt_file "$KURYR_K8S_API_CERT"
fi
@ -61,6 +60,7 @@ function configure_kuryr {
fi
if [ "$KURYR_K8S_API_CACERT" ]; then
iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file "$KURYR_K8S_API_CACERT"
iniset "$KURYR_CONFIG" kubernetes ssl_verify_server_crt True
fi
# REVISIT(ivc): 'use_stderr' is required for current CNI driver. Once a
# daemon-based CNI driver is implemented, this could be removed.
@ -73,6 +73,9 @@ function configure_kuryr {
iniset "$KURYR_CONFIG" kubernetes pod_subnets_driver "$KURYR_SUBNET_DRIVER"
iniset "$KURYR_CONFIG" kubernetes enabled_handlers "$KURYR_ENABLED_HANDLERS"
# Let Kuryr retry connections to K8s API for 20 minutes.
iniset "$KURYR_CONFIG" kubernetes watch_retry_timeout 1200
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
# This works around the issue of being unable to set oslo.privsep mode
@ -237,9 +240,9 @@ function create_k8s_api_service {
create_load_balancer "$lb_name" "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET"\
"$project_id" "$k8s_api_clusterip"
create_load_balancer_listener default/kubernetes:443 HTTPS 443 "$lb_name" "$project_id" 3600000
create_load_balancer_pool default/kubernetes:443 HTTPS ROUND_ROBIN \
default/kubernetes:443 "$project_id" "$lb_name"
create_load_balancer_listener default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ${KURYR_K8S_API_LB_PORT} "$lb_name" "$project_id" 3600000
create_load_balancer_pool default/kubernetes:${KURYR_K8S_API_LB_PORT} HTTPS ROUND_ROBIN \
default/kubernetes:${KURYR_K8S_API_LB_PORT} "$project_id" "$lb_name"
local api_port
if is_service_enabled openshift-master; then
@ -260,10 +263,10 @@ function create_k8s_api_service {
if [[ "$use_octavia" == "True" && \
"$KURYR_K8S_OCTAVIA_MEMBER_MODE" == "L2" ]]; then
create_load_balancer_member "$(hostname)" "$address" "$api_port" \
default/kubernetes:443 $KURYR_NEUTRON_DEFAULT_POD_SUBNET "$lb_name" "$project_id"
default/kubernetes:${KURYR_K8S_API_LB_PORT} $KURYR_NEUTRON_DEFAULT_POD_SUBNET "$lb_name" "$project_id"
else
create_load_balancer_member "$(hostname)" "$address" "$api_port" \
default/kubernetes:443 public-subnet "$lb_name" "$project_id"
default/kubernetes:${KURYR_K8S_API_LB_PORT} public-subnet "$lb_name" "$project_id"
fi
}
@ -357,6 +360,22 @@ function configure_neutron_defaults {
sg_ids+=",${octavia_pod_access_sg_id}"
fi
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
local service_cidr
local k8s_api_clusterip
service_cidr=$(openstack --os-cloud devstack-admin \
--os-region "$REGION_NAME" \
subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \
-c cidr -f value)
k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1)
# NOTE(dulek): KURYR_K8S_API_LB_URL will be a global to be used by next
# deployment phases.
KURYR_K8S_API_LB_URL="https://${k8s_api_clusterip}:${KURYR_K8S_API_LB_PORT}"
iniset "$KURYR_CONFIG" kubernetes api_root ${KURYR_K8S_API_LB_URL}
else
iniset "$KURYR_CONFIG" kubernetes api_root ""
fi
iniset "$KURYR_CONFIG" neutron_defaults project "$project_id"
iniset "$KURYR_CONFIG" neutron_defaults pod_subnet "$pod_subnet_id"
iniset "$KURYR_CONFIG" neutron_defaults pod_security_groups "$sg_ids"
@ -435,6 +454,11 @@ function prepare_kubernetes_files {
sudo bash -c "echo '$(create_token),kubelet,kubelet' >> ${KURYR_HYPERKUBE_DATA_DIR}/known_tokens.csv"
sudo bash -c "echo '$(create_token),kube_proxy,kube_proxy' >> ${KURYR_HYPERKUBE_DATA_DIR}/known_tokens.csv"
# Copy certs for Kuryr services to use
sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/kubecfg.crt" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr.crt"
sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/kubecfg.key" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr.key"
sudo install -m 644 "${KURYR_HYPERKUBE_DATA_DIR}/ca.crt" "${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt"
# FIXME(ivc): replace 'sleep' with a strict check (e.g. wait_for_files)
# 'kubernetes-api' fails if started before files are generated.
# this is a workaround to prevent races.
@ -456,7 +480,7 @@ function wait_for {
extra_flags=${cacert_path:+"--cacert ${cacert_path}"}
local start_time=$(date +%s)
until curl -o /dev/null -sf $extra_flags "$url"; do
until curl -o /dev/null -s $extra_flags "$url"; do
echo -n "."
local curr_time=$(date +%s)
local time_diff=$(($curr_time - $start_time))
@ -631,13 +655,14 @@ function run_k8s_kubelet {
function run_kuryr_kubernetes {
local python_bin=$(which python)
if is_service_enabled openshift-master; then
wait_for "OpenShift API Server" "$OPENSHIFT_API_URL" \
"${OPENSHIFT_DATA_DIR}/ca.crt"
wait_for "OpenShift API Server" "$KURYR_K8S_API_LB_URL" \
"${OPENSHIFT_DATA_DIR}/ca.crt" 1200
else
wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
wait_for "Kubernetes API Server" "$KURYR_K8S_API_LB_URL" \
"${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt" 1200
fi
run_process kuryr-kubernetes \
"$python_bin ${KURYR_HOME}/scripts/run_server.py \
--config-file $KURYR_CONFIG"
@ -791,12 +816,6 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
run_k8s_scheduler
fi
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then
# If running in containerized mode, we'll run the daemon as DaemonSet.
run_kuryr_daemon
fi
if is_service_enabled kubelet; then
prepare_kubelet
extract_hyperkube
@ -813,6 +832,7 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
configure_k8s_pod_sg_rules
fi
KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT)
if is_service_enabled kuryr-kubernetes; then
/usr/local/bin/kubectl apply -f ${KURYR_HOME}/kubernetes_crds/kuryrnet.yaml
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
@ -831,7 +851,6 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
if is_service_enabled kuryr-kubernetes; then
# NOTE(dulek): This is so late, because Devstack's Octavia is unable
# to create loadbalancers until test-config phase.
local use_octavia
use_octavia=$(trueorfalse True KURYR_K8S_LBAAS_USE_OCTAVIA)
if [[ "$use_octavia" == "False" ]]; then
create_k8s_router_fake_service
@ -846,7 +865,7 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
# FIXME(dulek): This is a very late phase to start Kuryr services.
# We're doing it here because we need K8s API LB to be
# created in order to run kuryr-cni container. Thing is
# created in order to run kuryr services. Thing is
# Octavia is unable to create LB until test-config phase.
# We can revisit this once Octavia's DevStack plugin will
# get improved.
@ -854,6 +873,7 @@ elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then
run_containerized_kuryr_resources
else
run_kuryr_kubernetes
run_kuryr_daemon
fi
fi
if is_service_enabled tempest && [[ "$KURYR_USE_PORT_POOLS" == "True" ]]; then

View File

@ -40,10 +40,12 @@ KURYR_HYPERKUBE_BINARY=${KURYR_HYPERKUBE_BINARY:-/usr/local/bin/hyperkube}
# Kubernetes
KURYR_K8S_API_PORT=${KURYR_K8S_API_PORT:-8080}
KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-http://${HOST_IP}:${KURYR_K8S_API_PORT}}
KURYR_K8S_API_CERT=${KURYR_K8S_API_CERT:-}
KURYR_K8S_API_KEY=${KURYR_K8S_API_KEY:-}
KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-}
# NOTE(dulek): [kubernetes]api_root option will use LB IP instead.
KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-"http://${HOST_IP}:${KURYR_K8S_API_PORT}"}
KURYR_K8S_API_CERT=${KURYR_K8S_API_CERT:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr.crt"}
KURYR_K8S_API_KEY=${KURYR_K8S_API_KEY:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr.key"}
KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt"}
KURYR_K8S_API_LB_PORT=${KURYR_K8S_API_LB_PORT:-443}
KURYR_PORT_DEBUG=${KURYR_PORT_DEBUG:-True}
KURYR_SUBNET_DRIVER=${KURYR_SUBNET_DRIVER:-default}
KURYR_ENABLED_HANDLERS=${KURYR_ENABLED_HANDLERS:-vif,lb,lbaasspec}

View File

@ -13,6 +13,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import os
from kuryr.lib import utils
from kuryr_kubernetes import config
@ -68,5 +70,12 @@ def setup_loadbalancer_client():
def setup_kubernetes_client():
_clients[_KUBERNETES_CLIENT] = k8s_client.K8sClient(
config.CONF.kubernetes.api_root)
if config.CONF.kubernetes.api_root:
api_root = config.CONF.kubernetes.api_root
else:
# NOTE(dulek): This is for containerized deployments, i.e. running in
# K8s Pods.
host = os.environ['KUBERNETES_SERVICE_HOST']
port = os.environ['KUBERNETES_SERVICE_PORT_HTTPS']
api_root = "https://%s:%s" % (host, port)
_clients[_KUBERNETES_CLIENT] = k8s_client.K8sClient(api_root)