[k8s] Add KubeDns addon to K8s Cluster

This patch introduce kubeDns addon support in murano k8s app. The docs of how kubeDns works and how it can be checked are here: https://github.com/kubernetes/kubernetes/blob/master/build/kube-dns/README.md#how-do-i-test-if-it-is-working Co-Authored-By: ddovbii <ddovbii@mirantis.com> Change-Id: I77b400ffd059a326b8fa051e038706370cbf6aaf
2016-09-11 13:09:33 -04:00 · 2016-09-11 13:09:33 -04:00 · 6b27f02c55
parent df281eb66c
commit 6b27f02c55
10 changed files with 282 additions and 30 deletions
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesCluster.yaml
@ -51,6 +51,10 @@ Properties:
    Contract: $.bool().notNull()
    Default: false

+  enableKubeDns:
+    Contract: $.bool().notNull()
+    Default: true
+
  dockerRegistry:
    Contract: $.string()

@ -144,6 +148,10 @@ Methods:
              - $.minionNodes.take($.nodeCount).pselect($.setupNode())
              - $.gatewayNodes.take($.gatewayCount).pselect($.setupNode())

+          - If: $.enableKubeDns
+            Then:
+              $._deployDns()
+
          - $._environment.stack.push()
          - $._updateServicePublicIps()
          - $.setAttr(lastNodeCount, $.nodeCount)
@ -157,6 +165,40 @@ Methods:
      Return: $.masterNode.getIp()


+  _deployDns:
+    Body:
+      - If: not $.getAttr(dnsDeployed, false)
+        Then:
+          - $securityGroupIngress:
+              - ToPort: 8001
+                FromPort: 8001
+                IpProtocol: tcp
+                External: false
+              - ToPort: 10053
+                FromPort: 10053
+                IpProtocol: udp
+                External: false
+              - ToPort: 10053
+                FromPort: 10053
+                IpProtocol: tcp
+                External: false
+              - ToPort: 53
+                FromPort: 53
+                IpProtocol: udp
+                External: false
+              - ToPort: 53
+                FromPort: 53
+                IpProtocol: tcp
+                External: false
+
+          - $._environment.securityGroupManager.addGroupIngress($securityGroupIngress)
+
+          - $resources: new(sys:Resources)
+          - $template: $resources.yaml('DeployKubeDns.template')
+          - $.masterNode.instance.agent.call($template, $resources)
+          - $.setAttr(dnsDeployed, true)
+
+
  _deployContainersNetwork:
    Body:
      - If: $.useFlannel
@ -529,7 +571,6 @@ Methods:
          - $._environment.reporter.report($this, 'No gateway nodes that can be removed')


-
  scaleRc:
    Arguments:
      - rcName:
--- a/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml
+++ b/Kubernetes/KubernetesCluster/package/Classes/KubernetesMinionNode.yaml
@ -103,7 +103,8 @@ Methods:
                dockerRegistry => $._cluster.dockerRegistry,
                dockerMirror => $._cluster.dockerMirror,
                gcloudKey => $._cluster.gcloudKey,
-                useFlannel => $._cluster.useFlannel
+                useFlannel => $._cluster.useFlannel,
+                enableKubeDns => $._cluster.enableKubeDns
              ))
          - $.instance.agent.call($template, $resources)
          - $.setAttr(nodeConfigured, true)
--- a/Kubernetes/KubernetesCluster/package/Resources/DeployKubeDns.template
+++ b/Kubernetes/KubernetesCluster/package/Resources/DeployKubeDns.template
@ -0,0 +1,32 @@
+#  Licensed under the Apache License, Version 2.0 (the "License"); you may
+#  not use this file except in compliance with the License. You may obtain
+#  a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#  License for the specific language governing permissions and limitations
+#  under the License.
+
+FormatVersion: 2.0.0
+Version: 1.0.0
+Name: Deploy KubeDNS
+
+Parameters:
+
+Body: |
+  return deploy().stdout
+
+Scripts:
+  deploy:
+    Type: Application
+    Version: 1.0.0
+    EntryPoint: deployKubeDns.sh
+    Files:
+      - addons/kube-dns-addon.yaml
+    Options:
+      captureStdout: true
+      captureStderr: true
+
--- a/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template
+++ b/Kubernetes/KubernetesCluster/package/Resources/KubeMinionSetup.template
@ -22,6 +22,7 @@ Parameters:
  dockerMirror: $dockerMirror
  gcloudKey: $gcloudKey
  useFlannel: $useFlannel
+  enableKubeDns: $enableKubeDns

 Body: |
  if args.dockerRegistry:
@ -32,7 +33,7 @@ Body: |
    loginToGoogleRegistry("'{0}'".format(args.gcloudKey))

  restartDocker()
-  setup('{0} {1} {2} {3}'.format(args.name, args.ip, args.masterIp, args.useFlannel))
+  setup('{0} {1} {2} {3} {4}'.format(args.name, args.ip, args.masterIp, args.useFlannel, args.enableKubeDns))

 Scripts:
  setup:
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/addons/kube-dns-addon.yaml
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/addons/kube-dns-addon.yaml
@ -0,0 +1,152 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    kubernetes.io/name: "KubeDNS"
+spec:
+  selector:
+    k8s-app: kube-dns
+  clusterIP: 10.32.0.10
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+
+
+---
+
+
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: kube-dns-v20
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    version: v20
+    kubernetes.io/cluster-service: "true"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      k8s-app: kube-dns
+      version: v20
+  template:
+    metadata:
+      labels:
+        k8s-app: kube-dns
+        version: v20
+        kubernetes.io/cluster-service: "true"
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: '[{"key":"CriticalAddonsOnly", "operator":"Exists"}]'
+    spec:
+      volumes:
+      - name: kubeconfig
+        hostPath:
+          path: /etc/kubernetes/kubeconfig.yaml
+      containers:
+      - name: kubedns
+        image: gcr.io/google_containers/kubedns-amd64:1.8
+        resources:
+          # TODO: Set memory limits when we've profiled the container for
+          # large
+          # clusters, then set request = limit to keep this container in
+          # guaranteed class. Currently, this container falls into the
+          # "burstable" category so the kubelet doesn't backoff from
+          # restarting it.
+          limits:
+            memory: 170Mi
+          requests:
+            cpu: 100m
+            memory: 70Mi
+        livenessProbe:
+          httpGet:
+            path: /healthz-kubedns
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        readinessProbe:
+          httpGet:
+            path: /readiness
+            port: 8081
+            scheme: HTTP
+          # we poll on pod startup for the Kubernetes master service and
+          # only setup the /readiness HTTP server once that's available.
+          initialDelaySeconds: 3
+          timeoutSeconds: 5
+        args:
+        # command = "/kube-dns"
+        - --domain=kubernetes.local
+        - --dns-port=10053
+        - --kubecfg-file=/etc/kubernetes/kubeconfig.yaml
+        volumeMounts:
+        - mountPath: /etc/kubernetes/kubeconfig.yaml
+          name: kubeconfig
+          readOnly: true
+        ports:
+        - containerPort: 10053
+          name: dns-local
+          protocol: UDP
+        - containerPort: 10053
+          name: dns-tcp-local
+          protocol: TCP
+      - name: dnsmasq
+        image: gcr.io/google_containers/kube-dnsmasq-amd64:1.4
+        livenessProbe:
+          httpGet:
+            path: /healthz-dnsmasq
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+          successThreshold: 1
+          failureThreshold: 5
+        args:
+        - --cache-size=1000
+        - --no-resolv
+        - --server=127.0.0.1#10053
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+      - name: healthz
+        image: gcr.io/google_containers/exechealthz-amd64:1.2
+        resources:
+          # keep request = limit to keep this container in guaranteed class
+          limits:
+            memory: 50Mi
+          requests:
+            cpu: 10m
+            # Note that this container shouldn't really need 50Mi of memory.
+            # The
+            # limits are set higher than expected pending investigation on
+            # #29688.
+            # The extra memory was stolen from the kubedns container to keep
+            # the
+            # net memory requested by the pod constant.
+            memory: 50Mi
+        args:
+        - --cmd=nslookup kubernetes.default.svc.kubernetes.local 127.0.0.1 >/dev/null
+        - --url=/healthz-dnsmasq
+        - --cmd=nslookup kubernetes.default.svc.kubernetes.local 127.0.0.1:10053 >/dev/null
+        - --url=/healthz-kubedns
+        - --port=8080
+        - --quiet
+        ports:
+        - containerPort: 8080
+          protocol: TCP
+      dnsPolicy: Default  # Don't use cluster DNS.
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/deployKubeDns.sh
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/deployKubeDns.sh
@ -0,0 +1,5 @@
+#!/bin/bash
+
+cp -f addons/kube-dns-addon.yaml /etc/kubernetes/addons
+
+/opt/bin/kubectl create -f /etc/kubernetes/addons/kube-dns-addon.yaml >> /tmp/murano-kube.log
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/master-kube-setup.sh
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/master-kube-setup.sh
@ -14,8 +14,6 @@ if [[ $(which systemctl) ]]; then
  sed -i.bak "s/%%MASTER_IP%%/$2/g" environ/apiserver
  sed -i.bak "s/%%MASTER_IP%%/$2/g" default_scripts/kube-apiserver

-  mkdir -p /etc/kubernetes/
-
  cp -f environ/apiserver /etc/kubernetes/apiserver
  cp -f environ/kube-config /etc/kubernetes/config

--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/minion-kube-setup.sh
@ -4,6 +4,7 @@
 # $2 - IP
 # $3 - MASTER_IP
 # $4 - USE_CALICO
+# $5 - ENABLE_DNS

 mkdir -p /var/run/murano-kubernetes
 mkdir -p /etc/kubernetes/
@ -18,6 +19,15 @@ if [[ $(which systemctl) ]]; then
    echo KUBELET_ARGS=\"--network-plugin=cni --network-plugin-dir=/etc/cni/net.d\" >> environ/kubelet
  fi

+  if [ "$5" == "True" ]; then
+    echo KUBELET_CLUSTER_DNS=\"--cluster-dns=10.32.0.10\" >> environ/kubelet
+    echo KUBELET_DNS_DOMAIN=\"--cluster-domain=kubernetes.local\" >> environ/kubelet
+
+    if [ "$4" == "False" ]; then
+      echo KUBE_PROXY_ARGS=\"--proxy-mode=iptables --masquerade-all=true\" > /etc/kubernetes/proxy
+    fi
+  fi
+
  mkdir -p /etc/kubernetes/

  cp -f environ/kubelet /etc/kubernetes/
--- a/Kubernetes/KubernetesCluster/package/Resources/scripts/systemd/kubelet.service
+++ b/Kubernetes/KubernetesCluster/package/Resources/scripts/systemd/kubelet.service
@ -15,6 +15,8 @@ ExecStart=/opt/bin/hyperkube kubelet \
            $KUBELET_PORT \
            $KUBELET_HOSTNAME \
            $KUBE_ALLOW_PRIV \
+            $KUBELET_CLUSTER_DNS \
+            $KUBELET_DNS_DOMAIN \
            $KUBELET_ARGS
 Restart=on-failure
 KillMode=process
--- a/Kubernetes/KubernetesCluster/package/UI/ui.yaml
+++ b/Kubernetes/KubernetesCluster/package/UI/ui.yaml
@ -18,10 +18,10 @@ Templates:
    instance:
      ?:
        type: io.murano.resources.LinuxMuranoInstance
-      name: generateHostname($.appConfiguration.unitNamingPattern, 1)
+      name: generateHostname($.nodesConfiguration.unitNamingPattern, 1)
      flavor: $.instanceConfiguration.flavor
      image: $.instanceConfiguration.image
-      assignFloatingIp: $.appConfiguration.assignFloatingIP
+      assignFloatingIp: $.nodesConfiguration.assignFloatingIP
      keyname: $.instanceConfiguration.keyPair
      availabilityZone: $.instanceConfiguration.availabilityZone

@ -31,13 +31,13 @@ Templates:
    instance:
      ?:
        type: io.murano.resources.LinuxMuranoInstance
-      name: generateHostname($.appConfiguration.unitNamingPattern, $index + 1)
+      name: generateHostname($.nodesConfiguration.unitNamingPattern, $index + 1)
      flavor: $.instanceConfiguration.flavor
      image: $.instanceConfiguration.image
-      assignFloatingIp: $.appConfiguration.assignFloatingIP
+      assignFloatingIp: $.nodesConfiguration.assignFloatingIP
      keyname: $.instanceConfiguration.keyPair
      availabilityZone: $.instanceConfiguration.availabilityZone
-    exposeCAdvisor: $.appConfiguration.exposeCAdvisor
+    exposeCAdvisor: $.nodesConfiguration.exposeCAdvisor


  gatewayNode:
@ -46,30 +46,30 @@ Templates:
    instance:
      ?:
        type: io.murano.resources.LinuxMuranoInstance
-      name: generateHostname($.appConfiguration.gatewayNamingPattern, $index)
+      name: generateHostname($.nodesConfiguration.gatewayNamingPattern, $index)
      flavor: $.instanceConfiguration.flavor
      image: $.instanceConfiguration.image
-      assignFloatingIp: $.appConfiguration.assignGatewayFloatingIP
+      assignFloatingIp: $.nodesConfiguration.assignGatewayFloatingIP
      keyname: $.instanceConfiguration.keyPair
      availabilityZone: $.instanceConfiguration.availabilityZone

 Application:
  ?:
    type: com.mirantis.docker.kubernetes.KubernetesCluster
-  name: $.appConfiguration.name
+  name: $.nodesConfiguration.name
  masterNode: $masterNode
-  minionNodes: repeat($minionNode, $.appConfiguration.maxMinionCount)
-  nodeCount: $.appConfiguration.minionCount
-  gatewayCount: $.appConfiguration.gatewayCount
-  useFlannel: $.appConfiguration.useFlannel
-  gatewayNodes: repeat($gatewayNode, $.appConfiguration.maxGatewayCount)
-  dockerRegistry: $.appConfiguration.dockerRegistry
-  dockerMirror: $.appConfiguration.dockerMirror
-  gcloudKey: $.appConfiguration.gcloudKey
-
+  minionNodes: repeat($minionNode, $.nodesConfiguration.maxMinionCount)
+  nodeCount: $.nodesConfiguration.minionCount
+  gatewayCount: $.nodesConfiguration.gatewayCount
+  useFlannel: $.kubeNetConfiguration.useFlannel
+  gatewayNodes: repeat($gatewayNode, $.nodesConfiguration.maxGatewayCount)
+  dockerRegistry: $.kubeNetConfiguration.dockerRegistry
+  dockerMirror: $.kubeNetConfiguration.dockerMirror
+  gcloudKey: $.kubeNetConfiguration.gcloudKey
+  enableKubeDns: $.kubeNetConfiguration.enableKubeDns

 Forms:
-  - appConfiguration:
+  - nodesConfiguration:
      fields:
        - name: license
          type: string
@ -106,13 +106,6 @@ Forms:
          description: >-
             Check to assign floating IP to Kubernetes nodes
          required: false
-        - name: useFlannel
-          type: boolean
-          initial: false
-          label: Use Flannel networking instead of Calico
-          description: >-
-             Check, if you are going use the Flannel networking instead of Calico
-          required: false
        - name: unitNamingPattern
          type: string
          initial: kube-#
@ -170,6 +163,23 @@ Forms:
          helpText: "# expands to gateway sequence number"
          description: >-
             Check to assign floating IP to gateway nodes
+
+  - kubeNetConfiguration:
+      fields:
+        - name: useFlannel
+          type: boolean
+          initial: false
+          label: Use Flannel networking instead of Calico
+          description: >-
+             Check, if you are going to use the Flannel networking instead of Calico
+          required: false
+        - name: enableKubeDns
+          type: boolean
+          initial: true
+          label: Enable KubeDNS addon
+          description: >-
+             Check, if you are going to use KubeDNS feature in your cluster
+          required: false
        - name: dockerRegistry
          type: string
          label: Custom Docker registry URL