diff --git a/cni.Dockerfile b/cni.Dockerfile index 699a740bd..4c226268d 100644 --- a/cni.Dockerfile +++ b/cni.Dockerfile @@ -4,16 +4,13 @@ LABEL authors="Antoni Segura Puimedon, Vikas Choudhary /kuryr-cni << EOF -#!/bin/bash -${CNI_BIN_DIR_PATH}/kuryr-venv/bin/kuryr-cni +#!/bin/bash -x +envs=(\$(env | grep ^CNI_)) +docker exec \${envs[@]/#/--env } -i "${CONTAINERID}" kuryr-cni --config-file /etc/kuryr/kuryr.conf EOF + + # Copy the script into the designated location cp /kuryr-cni "/opt/cni/bin/kuryr-cni" chmod +x /opt/cni/bin/kuryr-cni - cat /tmp/kuryr/* > /etc/kuryr/kuryr.conf + cp /opt/kuryr-kubernetes/etc/cni/net.d/* /etc/cni/net.d } cleanup @@ -31,7 +48,7 @@ deploy # Start CNI daemon if required if [ "$CNI_DAEMON" == "True" ]; then - /kuryr-kubernetes/bin/kuryr-daemon --config-file /etc/kuryr/kuryr.conf + kuryr-daemon --config-file /etc/kuryr/kuryr.conf else sleep infinity fi diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes index 017c0b429..d37fb05a2 100644 --- a/devstack/lib/kuryr_kubernetes +++ b/devstack/lib/kuryr_kubernetes @@ -543,6 +543,10 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: KURYR_CNI_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name securityContext: privileged: true volumeMounts: @@ -551,10 +555,8 @@ spec: - name: net-conf mountPath: /etc/cni/net.d - name: config-volume - mountPath: /tmp/kuryr/kuryr.conf + mountPath: /etc/kuryr/kuryr.conf subPath: kuryr-cni.conf - - name: etc - mountPath: /etc - name: proc mountPath: /host_proc - name: openvswitch @@ -567,13 +569,13 @@ EOF path: /ready port: ${cni_health_server_port} scheme: HTTP - initialDelaySeconds: 15 - timeoutSeconds: 5 + initialDelaySeconds: 60 + timeoutSeconds: 10 livenessProbe: httpGet: path: /alive port: ${cni_health_server_port} - initialDelaySeconds: 15 + initialDelaySeconds: 60 EOF fi cat >> "${output_dir}/cni_ds.yml" << EOF @@ -587,9 +589,6 @@ EOF - name: config-volume configMap: name: kuryr-config - - name: etc - hostPath: - path: /etc - name: proc hostPath: path: /proc diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 7ace921de..0ee0b8e7c 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -79,17 +79,17 @@ function configure_kuryr { # process. iniset "$KURYR_CONFIG" vif_plug_ovs_privileged helper_command privsep-helper iniset "$KURYR_CONFIG" vif_plug_linux_bridge_privileged helper_command privsep-helper + + # When running kuryr-daemon or CNI in container we need to set up + # some configs. + iniset "$KURYR_CONFIG" cni_daemon docker_mode True + iniset "$KURYR_CONFIG" cni_daemon netns_proc_dir "/host_proc" fi if is_service_enabled kuryr-daemon; then iniset "$KURYR_CONFIG" oslo_concurrency lock_path "$KURYR_LOCK_DIR" create_kuryr_lock_dir - if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then - # When running kuryr-daemon in container we need to set up some - # configs. - iniset "$KURYR_CONFIG" cni_daemon docker_mode True - iniset "$KURYR_CONFIG" cni_daemon netns_proc_dir "/host_proc" - else + if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then iniset "$KURYR_CONFIG" cni_health_server cg_path \ "/system.slice/system-devstack.slice/devstack@kuryr-daemon.service" fi @@ -132,20 +132,15 @@ function generate_containerized_kuryr_resources { inicomment "$KURYR_CONFIG" kubernetes ssl_client_crt_file inicomment "$KURYR_CONFIG" kubernetes ssl_client_key_file - # kuryr-controller and kuryr-cni will have tokens in different dirs. - KURYR_CNI_CONFIG=${KURYR_CONFIG}-cni - cp $KURYR_CONFIG $KURYR_CNI_CONFIG # NOTE(dulek): In the container the CA bundle will be mounted in a standard # directory, so we need to modify that. iniset "$KURYR_CONFIG" neutron cafile /etc/ssl/certs/kuryr-ca-bundle.crt iniset "$KURYR_CONFIG" kubernetes token_file /var/run/secrets/kubernetes.io/serviceaccount/token iniset "$KURYR_CONFIG" kubernetes ssl_ca_crt_file /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - iniset "$KURYR_CNI_CONFIG" kubernetes token_file /etc/kuryr/token - iniset "$KURYR_CNI_CONFIG" kubernetes ssl_ca_crt_file /etc/kuryr/ca.crt # Generate kuryr resources in k8s formats. local output_dir="${DATA_DIR}/kuryr-kubernetes" - generate_kuryr_configmap $output_dir $KURYR_CONFIG $KURYR_CNI_CONFIG + generate_kuryr_configmap $output_dir $KURYR_CONFIG $KURYR_CONFIG generate_kuryr_certificates_secret $output_dir $SSL_BUNDLE_FILE generate_kuryr_service_account $output_dir generate_controller_deployment $output_dir $KURYR_HEALTH_SERVER_PORT @@ -741,14 +736,25 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then build_kuryr_containers $CNI_BIN_DIR $CNI_CONF_DIR False generate_containerized_kuryr_resources False fi - run_containerized_kuryr_resources fi fi elif [[ "$1" == "stack" && "$2" == "test-config" ]]; then if is_service_enabled kuryr-kubernetes; then + # NOTE(dulek): This is so late, because Devstack's Octavia is unable + # to create loadbalancers until test-config phase. create_k8s_router_fake_service create_k8s_api_service + + # FIXME(dulek): This is a very late phase to start Kuryr services. + # We're doing it here because we need K8s API LB to be + # created in order to run kuryr-cni container. Thing is + # Octavia is unable to create LB until test-config phase. + # We can revisit this once Octavia's DevStack plugin will + # get improved. + if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then + run_containerized_kuryr_resources + fi fi if is_service_enabled tempest && [[ "$KURYR_USE_PORT_POOLS" == "True" ]]; then iniset $TEMPEST_CONFIG kuryr_kubernetes port_pool_enabled True @@ -761,19 +767,17 @@ fi if [[ "$1" == "unstack" ]]; then KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT) if is_service_enabled kuryr-kubernetes; then - if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then - stop_process kuryr-kubernetes - else + if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then $KURYR_HYPERKUBE_BINARY kubectl delete deploy/kuryr-controller fi + stop_process kuryr-kubernetes elif is_service_enabled kubelet; then $KURYR_HYPERKUBE_BINARY kubectl delete nodes ${HOSTNAME} fi - if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then - stop_process kuryr-daemon - else + if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then $KURYR_HYPERKUBE_BINARY kubectl delete ds/kuryr-cni-ds fi + stop_process kuryr-daemon if is_service_enabled kubernetes-controller-manager; then stop_container kubernetes-controller-manager