From 90029de5146d4ac69fa8ae34d097c68d5eda4f6c Mon Sep 17 00:00:00 2001
From: hyunsikYang <yangun@dcn.ssu.ac.kr>
Date: Wed, 23 Jan 2019 00:40:45 +0900
Subject: [PATCH] Kubernetes VIM installation error

Fix Kubernetes VIM installation

Closes-Bug: #1812766
Related-Bug: #1812878
Change-Id: I8288e172a0ae62cfd4d9525cda03dca596bdecc1
---
 devstack/lib/kubernetes_vim                   | 41 -----------
 devstack/local.conf.example                   |  7 --
 devstack/local.conf.kubernetes                | 70 +++++++++++++++++++
 devstack/plugin.sh                            |  7 --
 devstack/settings                             | 36 ++++------
 .../install/kubernetes_vim_installation.rst   | 42 ++++++-----
 6 files changed, 107 insertions(+), 96 deletions(-)
 delete mode 100644 devstack/lib/kubernetes_vim
 create mode 100644 devstack/local.conf.kubernetes

diff --git a/devstack/lib/kubernetes_vim b/devstack/lib/kubernetes_vim
deleted file mode 100644
index d310bfb01..000000000
--- a/devstack/lib/kubernetes_vim
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/bin/bash
-#
-# lib/kubernetes_vim
-# functions - functions specific to kubernetes_vim
-
-function configure_k8s_vim {
-    iniset "/$Q_PLUGIN_CONF_FILE" ml2_type_flat flat_networks $PUBLIC_PHYSICAL_NETWORK,$MGMT_PHYS_NET,$K8S_PHYS_NET
-    iniset "/$Q_PLUGIN_CONF_FILE" ovs bridge_mappings $PUBLIC_PHYSICAL_NETWORK:$PUBLIC_BRIDGE,$MGMT_PHYS_NET:$BR_MGMT,$K8S_PHYS_NET:$BR_K8S
-
-    echo "Creating Kubernetes bridge"
-    sudo ovs-vsctl --may-exist add-br ${BR_K8S}
-    iniset $TACKER_CONF k8s_vim use_barbican True
-}
-
-function tacker_create_initial_k8s_network {
-    SUBNETPOOL_V4_ID=$(openstack subnet pool create ${SUBNETPOOL_NAME_V4} --default-prefix-length ${SUBNETPOOL_SIZE_V4} --pool-prefix ${FIXED_RANGE_K8S} --share --default -f value -c id)
-    NET_K8S_ID=$(openstack network create --provider-network-type flat --provider-physical-network ${K8S_PHYS_NET} --share ${NET_K8S} | awk '/ id /{print $4}')
-    SUBNET_K8S_ID=$(openstack subnet create ${SUBNET_K8S} --ip-version 4 --gateway ${NETWORK_GATEWAY_K8S} --network ${NET_K8S_ID} --subnet-pool ${SUBNETPOOL_V4_ID} | awk '/ id /{print $4}')
-    SUBNET_K8S_CIDR=$(openstack subnet show ${SUBNET_K8S_ID} -c cidr -f value)
-
-    echo "Assign ip address to ${BR_K8S}"
-    sudo ip link set ${BR_K8S} up
-    sudo ip -4 address flush dev ${BR_K8S}
-    sudo ip address add ${NETWORK_GATEWAY_K8S_IP} dev ${BR_K8S}
-
-    echo "Create router to connect VM, Pod and Service networks"
-    openstack router create ${Q_ROUTER_NAME} | grep ' id ' | get_field 2
-    ROUTER_K8S_PORT_IP=$(_cidr_range "${SUBNET_K8S_CIDR}" | cut -f2)
-    ROUTER_K8S_PORT_ID=$(openstack port create --network ${NET_K8S_ID} --fixed-ip subnet=${SUBNET_K8S_ID},ip-address=${ROUTER_K8S_PORT_IP} port-router -f value -c id)
-    openstack router add port ${Q_ROUTER_NAME} ${ROUTER_K8S_PORT_ID}
-    openstack subnet set --host-route destination=${FIXED_RANGE_K8S},gateway=${ROUTER_K8S_PORT_IP} ${SUBNET_K8S_ID}
-}
-
-function _cidr_range {
-    python - <<EOF "$1"
-import sys
-from netaddr import IPAddress, IPNetwork
-n = IPNetwork(sys.argv[1])
-print("%s\\t%s" % (IPAddress(n.first + 1), IPAddress(n.last - 1)))
-EOF
-}
diff --git a/devstack/local.conf.example b/devstack/local.conf.example
index 0584a09f7..b6be8dd6d 100644
--- a/devstack/local.conf.example
+++ b/devstack/local.conf.example
@@ -54,13 +54,6 @@ enable_service n-cauth
 
 disable_service tempest
 
-# Enable Kubernetes and kuryr-kubernetes
-KUBERNETES_VIM=True
-NEUTRON_CREATE_INITIAL_NETWORKS=False
-enable_plugin kuryr-kubernetes https://git.openstack.org/openstack/kuryr-kubernetes master
-enable_plugin neutron-lbaas git://git.openstack.org/openstack/neutron-lbaas master
-enable_plugin devstack-plugin-container https://git.openstack.org/openstack/devstack-plugin-container master
-
 [[post-config|/etc/neutron/dhcp_agent.ini]]
 [DEFAULT]
 enable_isolated_metadata = True
diff --git a/devstack/local.conf.kubernetes b/devstack/local.conf.kubernetes
new file mode 100644
index 000000000..ff56d15f9
--- /dev/null
+++ b/devstack/local.conf.kubernetes
@@ -0,0 +1,70 @@
+[[local|localrc]]
+############################################################
+# Customize the following HOST_IP based on your installation
+############################################################
+HOST_IP=127.0.0.1
+
+ADMIN_PASSWORD=devstack
+MYSQL_PASSWORD=devstack
+RABBIT_PASSWORD=devstack
+SERVICE_PASSWORD=$ADMIN_PASSWORD
+SERVICE_TOKEN=devstack
+
+############################################################
+# Customize the following section based on your installation
+############################################################
+
+# Pip
+PIP_USE_MIRRORS=False
+USE_GET_PIP=1
+
+#OFFLINE=False
+#RECLONE=True
+
+# Logging
+LOGFILE=$DEST/logs/stack.sh.log
+VERBOSE=True
+ENABLE_DEBUG_LOG_LEVEL=True
+ENABLE_VERBOSE_LOG_LEVEL=True
+
+# Neutron ML2 with OpenVSwitch
+Q_PLUGIN=ml2
+Q_AGENT=openvswitch
+
+# Disable security groups
+Q_USE_SECGROUP=False
+LIBVIRT_FIREWALL_DRIVER=nova.virt.firewall.NoopFirewallDriver
+
+# Enable heat, networking-sfc, barbican and mistral
+enable_plugin heat https://git.openstack.org/openstack/heat master
+enable_plugin networking-sfc git://git.openstack.org/openstack/networking-sfc master
+enable_plugin barbican https://git.openstack.org/openstack/barbican master
+enable_plugin mistral https://git.openstack.org/openstack/mistral master
+
+# Ceilometer
+#CEILOMETER_PIPELINE_INTERVAL=300
+enable_plugin ceilometer https://git.openstack.org/openstack/ceilometer master
+enable_plugin aodh https://git.openstack.org/openstack/aodh master
+
+# Tacker
+enable_plugin tacker https://git.openstack.org/openstack/tacker master
+
+enable_service n-novnc
+enable_service n-cauth
+
+disable_service tempest
+
+# Enable kuryr-kubernetes, docker, octavia
+KUBERNETES_VIM=True
+enable_plugin kuryr-kubernetes https://git.openstack.org/openstack/kuryr-kubernetes master
+enable_plugin octavia https://git.openstack.org/openstack/octavia master
+enable_plugin devstack-plugin-container https://git.openstack.org/openstack/devstack-plugin-container master
+#KURYR_K8S_CLUSTER_IP_RANGE="10.0.0.0/24"
+
+[[post-config|/etc/neutron/dhcp_agent.ini]]
+[DEFAULT]
+enable_isolated_metadata = True
+
+[[post-config|$OCTAVIA_CONF]]
+[controller_worker]
+amp_active_retries=9999
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 264267fd5..9669bddd4 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -6,7 +6,6 @@ set -o xtrace
 
 echo_summary "tacker's plugin.sh was called with args $1 and $2 ..."
 . $DEST/tacker/devstack/lib/tacker
-. $DEST/tacker/devstack/lib/kubernetes_vim
 (set -o posix; set)
 
 # check for service enabled
@@ -20,9 +19,6 @@ if is_service_enabled tacker; then
         # Configure after the other layer 1 and 2 services have been configured
         echo_summary "Configuring Tacker"
         configure_tacker
-        if [ "${KUBERNETES_VIM}" == "True" ]; then
-            configure_k8s_vim
-        fi
         create_tacker_accounts
 
     elif [[ "$1" == "stack" && "$2" == "extra" ]]; then
@@ -41,9 +37,6 @@ if is_service_enabled tacker; then
             modify_heat_flavor_policy_rule
             echo_summary "Setup initial tacker network"
             tacker_create_initial_network
-            if [ "${KUBERNETES_VIM}" == "True" ]; then
-                tacker_create_initial_k8s_network
-            fi
             echo_summary "Check and download images for tacker initial"
             tacker_check_and_download_images
             echo_summary "Registering default VIM"
diff --git a/devstack/settings b/devstack/settings
index 942450d92..fdbd8b47c 100644
--- a/devstack/settings
+++ b/devstack/settings
@@ -42,29 +42,21 @@ if [ "${TACKER_MODE}" == "all" ]; then
     NETWORK_GATEWAY1=${NETWORK_GATEWAY1:-10.10.1.1}
 
     if [ "${KUBERNETES_VIM}" == "True" ]; then
-        K8S_PHYS_NET=${K8S_PHYS_NET:-"k8s-physnet"}
-        BR_K8S=${BR_K8S:-"br-k8s0"}
-        NET_K8S=${NET_K8S:-"k8s-public-net"}
-        SUBNET_K8S=${SUBNET_K8S:-"public-subnet"}
-        FIXED_RANGE_K8S=${FIXED_RANGE_K8S:-192.168.28.0/22}
-        NETWORK_GATEWAY_K8S=${NETWORK_GATEWAY_K8S:-192.168.28.1}
-        NETWORK_GATEWAY_K8S_IP=${NETWORK_GATEWAY_K8S_IP:-192.168.28.1/24}
+        KURYR_NEUTRON_DEFAULT_PROJECT="default"
+        # Octavia LBaaSv2
+        LIBS_FROM_GIT+=python-octaviaclient
+        TACKER_OCTAVIA_AGENTS=${TACKER_OCTAVIA_AGENTS:-octavia,o-api,o-cw,o-hm,o-hk}
+        for i in $(echo $TACKER_OCTAVIA_AGENTS | sed 's/,/ /g')
+        do
+            enable_service $i
+        done
 
-        KURYR_NEUTRON_DEFAULT_PROJECT="admin"
-        Q_ROUTER_NAME="route-k8s"
-        KURYR_NEUTRON_DEFAULT_ROUTER=${KURYR_NEUTRON_DEFAULT_ROUTER:-$Q_ROUTER_NAME}
-        SUBNETPOOL_NAME_V4=${SUBNETPOOL_NAME:-"shared-default-subnetpool-v4"}
-        SUBNETPOOL_SIZE_V4=${SUBNETPOOL_SIZE_V4:-26}
-        SUBNETPOOL_V4_ID=${SUBNETPOOL_V4_ID:-} #nqa
-
-        enable_service q-lbaasv2
-        enable_service etcd3
-        enable_service kubernetes-api
-        enable_service kubernetes-controller-manager
-        enable_service kubernetes-scheduler
-        enable_service kubelet
-        enable_service kuryr-kubernetes
-        NEUTRON_LBAAS_SERVICE_PROVIDERV2="LOADBALANCERV2:Haproxy:neutron_lbaas.drivers.haproxy.plugin_driver.HaproxyOnHostPluginDriver:default"
+        # enable kuryr-kubernetes services
+        KURYR_KUBERNETES_AGENTS=${KURYR_KUBERNETES_AGENTS:-etcd3,kubernetes-api,kubernetes-controller-manager,kubernetes-scheduler,kubelet,kuryr-kubernetes,kuryr-daemon}
+        for i in $(echo $KURYR_KUBERNETES_AGENTS | sed 's/,/ /g')
+        do
+            enable_service $i
+        done
     fi
 elif [ "${TACKER_MODE}" == "standalone" ]; then
     # set the enabled services here. This will need tacker devstack plugin put as the last one in local.conf
diff --git a/doc/source/install/kubernetes_vim_installation.rst b/doc/source/install/kubernetes_vim_installation.rst
index d7682519d..6bb08b418 100644
--- a/doc/source/install/kubernetes_vim_installation.rst
+++ b/doc/source/install/kubernetes_vim_installation.rst
@@ -31,13 +31,17 @@ bring VMs and Pods (and other Kubernetes resources) on the same network.
 
 .. code-block:: console
 
+  # Enable kuryr-kubernetes, docker, octavia
   KUBERNETES_VIM=True
-  NEUTRON_CREATE_INITIAL_NETWORKS=False
   enable_plugin kuryr-kubernetes https://git.openstack.org/openstack/kuryr-kubernetes master
-  enable_plugin neutron-lbaas git://git.openstack.org/openstack/neutron-lbaas master
+  enable_plugin octavia https://git.openstack.org/openstack/octavia master
   enable_plugin devstack-plugin-container https://git.openstack.org/openstack/devstack-plugin-container master
+  KURYR_K8S_CLUSTER_IP_RANGE="10.0.0.0/24"
 
-You can also see the same examples in [#first]_ and [#second]_.
+The public network will be used to launched LoadBalancer for Services in
+Kubernetes. The example for setting public subnet is described in [#first]_
+
+For more details, users also see the same examples in [#second]_ and [#third]_.
 
 2. Run stack.sh
 
@@ -60,7 +64,7 @@ information for authenticating to Kubernetes cluster.
 
 .. code-block:: console
 
-  $ cat /opt/stack/data/hyperkube/ca.crt
+  $ sudo cat /opt/stack/data/hyperkube/ca.crt
   -----BEGIN CERTIFICATE-----
   MIIDUzCCAjugAwIBAgIJAI+laRsxtQQMMA0GCSqGSIb3DQEBCwUAMCAxHjAcBgNV
   BAMMFTE3Mi4xNy4wLjJAMTUwNzU1NTc4MzAeFw0xNzEwMDkxMzI5NDNaFw0yNzEw
@@ -132,7 +136,7 @@ the project k8s:
   +--------------------------------------+-----------------+--------------------------------------+
   | ID                                   | Name            | Subnets                              |
   +--------------------------------------+-----------------+--------------------------------------+
-  | 28361f77-1875-4070-b0dc-014e26c48aeb | k8s-public-net  | 28c51d19-d437-46e8-9b0e-00bc392c57d6 |
+  | 28361f77-1875-4070-b0dc-014e26c48aeb | public          | 28c51d19-d437-46e8-9b0e-00bc392c57d6 |
   | 71c20650-6295-4462-9219-e0007120e64b | k8s-service-net | f2835c3a-f567-44f6-b006-a6f7c52f2396 |
   | 97c12aef-54f3-41dc-8b80-7f07c34f2972 | k8s-pod-net     | 7759453f-6e8a-4660-b845-964eca537c44 |
   | 9935fff9-f60c-4fe8-aa77-39ba7ac10417 | net0            | 92b2bd7b-3c14-4d32-8de3-9d3cc4d204cb |
@@ -141,7 +145,7 @@ the project k8s:
   +--------------------------------------+-----------------+--------------------------------------+
 
 To check Kubernetes cluster works well, please see some tests in
-kuryr-kubernetes to get more information [#third]_.
+kuryr-kubernetes to get more information [#fourth]_.
 
 5. Register Kubernetes VIM
 
@@ -218,10 +222,10 @@ support multi tenant on Kubernetes in the future.
   type: "kubernetes"
 
 User can change the authentication like username, password, etc. Please see
-Kubernetes document [#fourth]_ to read more information about Kubernetes
+Kubernetes document [#fifth]_ to read more information about Kubernetes
 authentication.
 
-* Run OpenStackClient command for register vim:
+* Run Tacker command for register vim:
 
 .. code-block:: console
 
@@ -229,14 +233,13 @@ authentication.
 
   $ openstack vim list
   +--------------------------------------+----------------------------------+----------------+------------+------------+------------------------------------------------------------+-----------+
-  | ID                                   | Tenant ID                        | Name           | Type       | Is Default | Placement attribution                                      | Status    |
+  | id                                   | tenant_id                        | name           | type       | is_default | placement_attr                                             | status    |
   +--------------------------------------+----------------------------------+----------------+------------+------------+------------------------------------------------------------+-----------+
   | 45456bde-6179-409c-86a1-d8cd93bd0c6d | a6f9b4bc9a4d439faa91518416ec0999 | vim-kubernetes | kubernetes | False      | {u'regions': [u'default', u'kube-public', u'kube-system']} | REACHABLE |
   +--------------------------------------+----------------------------------+----------------+------------+------------+------------------------------------------------------------+-----------+
 
-In ``Placement attribution``, there are three regions: 'default',
-'kube-public', 'kube-system', that map to ``namespace`` in Kubernetes
-environment.
+In ``placement_attr``, there are three regions: 'default', 'kube-public',
+'kube-system', that map to ``namespace`` in Kubernetes environment.
 
 * Other related commands to Kubernetes VIM
 
@@ -250,9 +253,9 @@ environment.
   type: "kubernetes"
 
 
-  $ openstack vim set vim-kubernetes --config-file kubernetes-VIM-update.yaml
-  $ openstack vim show vim-kubernetes
-  $ openstack vim delete vim-kubernetes
+  $ tacker vim-update vim-kubernetes --config-file kubernetes-VIM-update.yaml
+  $ tacker vim-show vim-kubernetes
+  $ tacker vim-delete vim-kubernetes
 
 When update Kubernetes VIM, user can update VIM information (such as username,
 password, bearer_token and ssl_ca_cert) except auth_url and type of VIM.
@@ -260,7 +263,8 @@ password, bearer_token and ssl_ca_cert) except auth_url and type of VIM.
 
 References
 ==========
-.. [#first] https://github.com/openstack/tacker/blob/master/doc/source/install/devstack.rst
-.. [#second] https://github.com/openstack/tacker/blob/master/devstack/local.conf.example
-.. [#third] https://github.com/openstack/kuryr-kubernetes/blob/master/doc/source/installation/testing_connectivity.rst
-.. [#fourth] https://kubernetes.io/docs/admin/authentication
+.. [#first] https://github.com/openstack-dev/devstack/blob/master/doc/source/networking.rst#shared-guest-interface
+.. [#second] https://github.com/openstack/tacker/blob/master/doc/source/install/devstack.rst
+.. [#third] https://github.com/openstack/tacker/blob/master/devstack/local.conf.kubernetes
+.. [#fourth] https://github.com/openstack/kuryr-kubernetes/blob/master/doc/source/installation/testing_connectivity.rst
+.. [#fifth] https://kubernetes.io/docs/admin/authentication