diff --git a/.zuul.d/multinode.yaml b/.zuul.d/multinode.yaml index d3d653126..7cb764544 100644 --- a/.zuul.d/multinode.yaml +++ b/.zuul.d/multinode.yaml @@ -35,3 +35,30 @@ devstack_services: tls-proxy: false voting: false + +- job: + name: kuryr-kubernetes-tempest-multinode-daemon-octavia-ha + parent: kuryr-kubernetes-tempest-multinode-octavia + description: | + Kuryr-Kubernetes tempest multinode job using octavia and running + containerized in HA + voting: false + timeout: 7800 + vars: + devstack_localrc: + KURYR_CONTROLLER_REPLICAS: 2 + KURYR_K8S_CONTAINERIZED_DEPLOYMENT: true + KURYR_K8S_SERIAL_TESTS: true + devstack_services: + kuryr-daemon: true + tempest_concurrency: 1 + group-vars: + subnode: + devstack_plugins: + devstack-plugin-container: https://git.openstack.org/openstack/devstack-plugin-container + kuryr-kubernetes: https://git.openstack.org/openstack/kuryr + devstack_services: + kubelet: true + devstack_localrc: + KURYR_FORCE_IMAGE_BUILD: true + KURYR_K8S_CONTAINERIZED_DEPLOYMENT: true diff --git a/.zuul.d/project.yaml b/.zuul.d/project.yaml index 4db260d7c..0246b1b2a 100644 --- a/.zuul.d/project.yaml +++ b/.zuul.d/project.yaml @@ -53,3 +53,4 @@ - kuryr-kubernetes-tempest-daemon-ovn - kuryr-kubernetes-tempest-daemon-openshift-octavia-ingress - kuryr-kubernetes-tempest-daemon-openshift-octavia-multi-vif + - kuryr-kubernetes-tempest-multinode-daemon-octavia-ha diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes index aba3fd075..3e6858377 100644 --- a/devstack/lib/kuryr_kubernetes +++ b/devstack/lib/kuryr_kubernetes @@ -631,6 +631,8 @@ spec: valueFrom: fieldRef: fieldPath: metadata.name + - name: CNI_DAEMON + value: "${cni_daemon}" securityContext: privileged: true volumeMounts: diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 45c6a63c8..0863f217f 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -390,22 +390,6 @@ function configure_neutron_defaults { fi fi - KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT) - if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then - local service_cidr - local k8s_api_clusterip - service_cidr=$(openstack --os-cloud devstack-admin \ - --os-region "$REGION_NAME" \ - subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \ - -c cidr -f value) - k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1) - # NOTE(dulek): KURYR_K8S_API_LB_URL will be a global to be used by next - # deployment phases. - KURYR_K8S_API_LB_URL="https://${k8s_api_clusterip}:${KURYR_K8S_API_LB_PORT}" - iniset "$KURYR_CONFIG" kubernetes api_root ${KURYR_K8S_API_LB_URL} - else - iniset "$KURYR_CONFIG" kubernetes api_root '""' - fi iniset "$KURYR_CONFIG" neutron_defaults project "$project_id" iniset "$KURYR_CONFIG" neutron_defaults pod_subnet "$pod_subnet_id" iniset "$KURYR_CONFIG" neutron_defaults pod_security_groups "$sg_ids" @@ -858,6 +842,9 @@ function update_tempest_conf_file { if [[ "$use_octavia" == "True" ]]; then iniset $TEMPEST_CONFIG kuryr_kubernetes test_udp_services True fi + if [[ "$KURYR_CONTROLLER_HA" == "True" ]]; then + iniset $TEMPEST_CONFIG kuryr_kubernetes ap_ha True + fi } source $DEST/kuryr-kubernetes/devstack/lib/kuryr_kubernetes @@ -878,12 +865,28 @@ elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then fi if [[ "$1" == "stack" && "$2" == "extra" ]]; then - if is_service_enabled kuryr-kubernetes; then + if [ "$KURYR_CONFIGURE_NEUTRON_DEFAULTS" == "True" ]; then KURYR_CONFIGURE_NEUTRON_DEFAULTS=$(trueorfalse True KURYR_CONFIGURE_NEUTRON_DEFAULTS) - if [ "$KURYR_CONFIGURE_NEUTRON_DEFAULTS" == "True" ]; then + if is_service_enabled kuryr-kubernetes; then configure_neutron_defaults fi + + KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT) + if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "False" ]; then + service_cidr=$(openstack --os-cloud devstack-admin \ + --os-region "$REGION_NAME" \ + subnet show "$KURYR_NEUTRON_DEFAULT_SERVICE_SUBNET" \ + -c cidr -f value) + k8s_api_clusterip=$(_cidr_range "$service_cidr" | cut -f1) + # NOTE(dulek): KURYR_K8S_API_LB_URL will be a global to be used by next + # deployment phases. + KURYR_K8S_API_LB_URL="https://${k8s_api_clusterip}:${KURYR_K8S_API_LB_PORT}" + iniset "$KURYR_CONFIG" kubernetes api_root ${KURYR_K8S_API_LB_URL} + else + iniset "$KURYR_CONFIG" kubernetes api_root '""' + fi fi + # FIXME(limao): When Kuryr start up, it need to detect if neutron # support tag plugin. # @@ -967,15 +970,24 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then fi KURYR_K8S_CONTAINERIZED_DEPLOYMENT=$(trueorfalse False KURYR_K8S_CONTAINERIZED_DEPLOYMENT) + KURYR_FORCE_IMAGE_BUILD=$(trueorfalse False KURYR_FORCE_IMAGE_BUILD) + if is_service_enabled kuryr-kubernetes || [[ ${KURYR_FORCE_IMAGE_BUILD} == "True" ]]; then + if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then + if is_service_enabled kuryr-daemon; then + build_kuryr_containers $CNI_BIN_DIR $CNI_CONF_DIR True + else + build_kuryr_containers $CNI_BIN_DIR $CNI_CONF_DIR False + fi + fi + fi + if is_service_enabled kuryr-kubernetes; then /usr/local/bin/kubectl apply -f ${KURYR_HOME}/kubernetes_crds/kuryrnet.yaml /usr/local/bin/kubectl apply -f ${KURYR_HOME}/kubernetes_crds/kuryrnetpolicy.yaml if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then if is_service_enabled kuryr-daemon; then - build_kuryr_containers $CNI_BIN_DIR $CNI_CONF_DIR True generate_containerized_kuryr_resources True else - build_kuryr_containers $CNI_BIN_DIR $CNI_CONF_DIR False generate_containerized_kuryr_resources False fi fi diff --git a/devstack/settings b/devstack/settings index df7877852..f393b48f6 100644 --- a/devstack/settings +++ b/devstack/settings @@ -35,7 +35,7 @@ KURYR_HYPERKUBE_BINARY=${KURYR_HYPERKUBE_BINARY:-/usr/local/bin/hyperkube} # Kubernetes KURYR_K8S_API_PORT=${KURYR_K8S_API_PORT:-8080} # NOTE(dulek): [kubernetes]api_root option will use LB IP instead. -KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-"http://${HOST_IP}:${KURYR_K8S_API_PORT}"} +KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-"http://${SERVICE_HOST}:${KURYR_K8S_API_PORT}"} KURYR_K8S_API_CERT=${KURYR_K8S_API_CERT:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr.crt"} KURYR_K8S_API_KEY=${KURYR_K8S_API_KEY:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr.key"} KURYR_K8S_API_CACERT=${KURYR_K8S_API_CACERT:-"${KURYR_HYPERKUBE_DATA_DIR}/kuryr-ca.crt"} @@ -93,6 +93,7 @@ KURYR_CNI_HEALTH_SERVER_PORT=${KURYR_CNI_HEALTH_SERVER_PORT:-8090} # High availability of controller KURYR_CONTROLLER_HA_PORT=${KURYR_CONTROLLER_HA_PORT:-16401} KURYR_CONTROLLER_REPLICAS=${KURYR_CONTROLLER_REPLICAS:-1} +KURYR_FORCE_IMAGE_BUILD=${KURYR_FORCE_IMAGE_BUILD:-False} # Kuryr ingress enable KURYR_ENABLE_INGRESS=${KURYR_ENABLE_INGRESS:-False} diff --git a/tools/gate/copy_k8s_logs.sh b/tools/gate/copy_k8s_logs.sh index d454904f4..830439a31 100755 --- a/tools/gate/copy_k8s_logs.sh +++ b/tools/gate/copy_k8s_logs.sh @@ -41,7 +41,11 @@ while read -r line do name=$(echo ${line} | cut -f1 -d " ") namespace=$(echo ${line} | cut -f2 -d " ") - /usr/local/bin/kubectl --kubeconfig=/opt/stack/.kube/config logs -n ${namespace} ${name} >> ${K8S_LOG_DIR}/pod_logs/${namespace}-${name}.txt + containers=`/usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config -n ${namespace} get pods ${name} -o jsonpath="{.spec.containers[*].name}"` + for container in ${containers} + do + /usr/local/bin/kubectl --kubeconfig=${HOME}/.kube/config logs -n ${namespace} -c ${container} ${name} >> ${K8S_LOG_DIR}/pod_logs/${namespace}-${name}-${container}.txt + done done < <(/usr/local/bin/kubectl get pods -o=custom-columns=NAME:.metadata.name,NAMESPACE:.metadata.namespace --all-namespaces | tail -n +2) sudo chown -R zuul:zuul ${K8S_LOG_DIR}