From 46f750edae74225a4dfbcce1f70990e1122a9613 Mon Sep 17 00:00:00 2001
From: Antoni Segura Puimedon <celebdor@passthrough.home.sepu.cz>
Date: Fri, 29 Jun 2018 17:44:41 +0200
Subject: [PATCH] devstack support for openshift-dns

This adds a new devstack service to provide cluster local DNS
for Pods. It uses dnsmasq to still allow access to the upstream
nameserver, so an extra devstack service openshift-dnsmasq is
also created.

Change-Id: I43a6b5423bd32a564511ea10ae620922bbad2d2a
Signed-off-by: Antoni Segura Puimedon <asegurap@redhat.com>
---
 .zuul.d/octavia.yaml                 |   2 +
 devstack/lib/kuryr_kubernetes        | 142 +++++++++++++++++++++++++++
 devstack/local.conf.openshift.sample |   2 +
 devstack/plugin.sh                   |  13 +++
 4 files changed, 159 insertions(+)

diff --git a/.zuul.d/octavia.yaml b/.zuul.d/octavia.yaml
index 0294e5c90..7adc62ee9 100644
--- a/.zuul.d/octavia.yaml
+++ b/.zuul.d/octavia.yaml
@@ -93,6 +93,8 @@
         kubelet: false
         openshift-master: true
         openshift-node: true
+        openshift-dnsmasq: true
+        openshift-dns: true
 
 - job:
     name: kuryr-kubernetes-tempest-daemon-openshift-octavia
diff --git a/devstack/lib/kuryr_kubernetes b/devstack/lib/kuryr_kubernetes
index 59698717c..3d4eec965 100644
--- a/devstack/lib/kuryr_kubernetes
+++ b/devstack/lib/kuryr_kubernetes
@@ -1057,3 +1057,145 @@ function get_loadbalancer_attribute {
         neutron lbaas-loadbalancer-show "$lb_name" -c "$lb_attr" -f value
     fi
 }
+
+# openshift_node_set_dns_config
+#   Description: Configures Openshift node's DNS section atomically
+#   Params:
+#       node_conf_path: path_to_node_config
+#       upstream_dns_ip: IP of the upstream DNS
+function openshift_node_set_dns_config {
+    local openshift_dnsmasq_recursive_resolv
+    local upstream_dns_ip
+    openshift_dnsmasq_recursive_resolv="${OPENSHIFT_DATA_DIR}/node/resolv.conf"
+
+    upstream_dns_ip="$2"
+    cat > "$openshift_dnsmasq_recursive_resolv" << EOF
+nameserver $upstream_dns_ip
+EOF
+
+    python - <<EOF "$@"
+import os
+import sys
+import tempfile
+import yaml
+
+if len(sys.argv) < 3:
+    sys.exit(1)
+node_conf_path = sys.argv[1]
+conf_dir = os.path.dirname(node_conf_path)
+
+def dns_configure_copy(conf):
+    new_conf = conf.copy()
+    # 127.0.0.1 is used by unbound in gates, let's use another localshost addr
+    new_conf['dnsBindAddress'] = '127.0.0.11:53'
+    new_conf['dnsDomain'] = 'cluster.local'
+    new_conf['dnsIP'] = '0.0.0.0'
+    new_conf['dnsRecursiveResolvConf'] = '${openshift_dnsmasq_recursive_resolv}'
+    return new_conf
+
+old_config = {}
+while True:
+    tp = tempfile.NamedTemporaryFile(dir=conf_dir, delete=False)
+    try:
+        with open(node_conf_path) as node_conf:
+            current_conf = yaml.load(node_conf.read())
+            if current_conf == old_config:
+                tp.write(yaml.dump(new_conf, default_flow_style=False))
+                tp.flush()
+                os.fsync(tp.fileno())
+                tp.close()
+                os.rename(tp.name, node_conf_path)
+                break
+            else:
+                new_conf = dns_configure_copy(current_conf)
+                old_config = current_conf
+                tp.close()
+                os.unlink(tp.name)
+    except Exception:
+        tp.close()
+        os.unlink(tp.name)
+EOF
+}
+
+# run_openshift_dnsmasq
+#   Description: Configures and runs a dnsmasq instance to be run as the node
+#       DNS server that will choose between openshift's DNS and the
+#       upstream DNS depending on the domain
+#   Params:
+#       upstream_dns_ip: IP of the upstream DNS
+function run_openshift_dnsmasq {
+    local dnmasq_binary
+    local cmd
+    local upstream_dns_ip
+    local openshift_dnsmasq_conf_path
+    local search_domains
+
+    upstream_dns_ip="$1"
+    openshift_dnsmasq_conf_path="${OPENSHIFT_DATA_DIR}/node/node_dnsmasq.conf"
+    install_package dnsmasq
+    cat > "$openshift_dnsmasq_conf_path" << EOF
+server=${upstream_dns_ip}
+no-resolv
+domain-needed
+no-negcache
+max-cache-ttl=1
+# Enable dbus so openshift dns can use it to set cluster.local rules
+enable-dbus
+dns-forward-max=10000
+cache-size=10000
+bind-dynamic
+# Do not bind to localhost addresses 127.0.0.1/8 (where skydns binds)
+except-interface=lo
+EOF
+
+    #Open port 53 so pods can reach the DNS server
+    sudo iptables -I INPUT 1 -p udp -m udp --dport 53 -j ACCEPT
+
+    dnsmasq_binary="$(command -v dnsmasq)"
+    cmd="${dnsmasq_binary} -k -C ${openshift_dnsmasq_conf_path}"
+    if [[ "$USE_SYSTEMD" = "True" ]]; then
+        # If systemd is being used, proceed as normal
+        run_process openshift-dnsmasq "$cmd" root root
+    else
+        # If screen is being used, there is a possibility that the devstack
+        # environment is on a stable branch. Older versions of run_process have
+        # a different signature. Sudo is used as a workaround that works in
+        # both older and newer versions of devstack.
+        run_process openshift-dnsmasq "sudo $cmd"
+    fi
+
+    sudo cp /etc/resolv.conf /etc/resolv.conf.orig
+    search_domains=$(awk '/search/ {for (i=2; i<NF; i++) printf $i " "; print $NF}' /etc/resolv.conf.orig)
+    search_domains="cluster.local ${search_domains}"
+    echo "search ${search_domains}" | sudo tee /etc/resolv.conf.openshift_devstack
+    echo "nameserver ${HOST_IP}" | sudo tee --append /etc/resolv.conf.openshift_devstack
+    grep "nameserver" /etc/resolv.conf.orig | sudo tee --append  /etc/resolv.conf.openshift_devstack
+    sudo mv /etc/resolv.conf.openshift_devstack /etc/resolv.conf
+}
+
+function reinstate_old_dns_config {
+    sudo mv /etc/resolv.conf.orig /etc/resolv.conf
+}
+
+
+# run_openshift_dns
+#   Description: Starts openshift's DNS
+function run_openshift_dns {
+    local command
+
+    command="/usr/local/bin/openshift start network \
+        --enable=dns \
+        --config=${OPENSHIFT_DATA_DIR}/node/node-config.yaml \
+        --kubeconfig=${OPENSHIFT_DATA_DIR}/node/node.kubeconfig"
+
+    if [[ "$USE_SYSTEMD" = "True" ]]; then
+        # If systemd is being used, proceed as normal
+        run_process openshift-dns "$command" root root
+    else
+        # If screen is being used, there is a possibility that the devstack
+        # environment is on a stable branch. Older versions of run_process have
+        # a different signature. Sudo is used as a workaround that works in
+        # both older and newer versions of devstack.
+        run_process openshift-dns "sudo $command"
+    fi
+}
diff --git a/devstack/local.conf.openshift.sample b/devstack/local.conf.openshift.sample
index 9e00771de..9cbfe2807 100644
--- a/devstack/local.conf.openshift.sample
+++ b/devstack/local.conf.openshift.sample
@@ -117,6 +117,8 @@ enable_service etcd3
 # OpenShift is run from the binaries conained in a binary release tarball
 enable_service openshift-master
 enable_service openshift-node
+enable_service openshift-dnsmasq
+enable_service openshift-dns
 
 # OpenShift node uses systemd as its cgroup driver. Thus we need Docker to
 # use the same.
diff --git a/devstack/plugin.sh b/devstack/plugin.sh
index 7dc953594..3ae8cdda0 100644
--- a/devstack/plugin.sh
+++ b/devstack/plugin.sh
@@ -755,6 +755,14 @@ if [[ "$1" == "stack" && "$2" == "extra" ]]; then
     if is_service_enabled openshift-node; then
         prepare_kubelet
         run_openshift_node
+        if is_service_enabled openshift-dns; then
+            FIRST_NAMESERVER=$(grep nameserver /etc/resolv.conf | awk '{print $2; exit}')
+            openshift_node_set_dns_config "${OPENSHIFT_DATA_DIR}/node/node-config.yaml" \
+                "$FIRST_NAMESERVER"
+            run_openshift_dnsmasq "$FIRST_NAMESERVER"
+            run_openshift_dns
+       fi
+
         KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE=$(trueorfalse True KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE)
         if [[ "$KURYR_CONFIGURE_BAREMETAL_KUBELET_IFACE" == "True" ]]; then
             ovs_bind_for_kubelet "$KURYR_NEUTRON_DEFAULT_PROJECT" ${OPENSHIFT_API_PORT}
@@ -889,6 +897,11 @@ if [[ "$1" == "unstack" ]]; then
     fi
     if is_service_enabled openshift-node; then
         stop_process openshift-node
+        if is_service_enabled openshift-dns; then
+            reinstate_old_dns_config
+            stop_process openshift-dns
+            stop_process openshift-dnsmasq
+        fi
         # NOTE(dulek): We need to clean up the configuration as well, otherwise
         # when doing stack.sh again, openshift-node will use old certificates.
         sudo rm -rf ${OPENSHIFT_DATA_DIR}