summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRick Cano <ricardo.cano@lexisnexisrisk.com>2018-06-28 17:56:46 -0400
committerRick Cano <ricardo.cano@lexisnexisrisk.com>2018-08-22 12:58:27 -0400
commit419a2285032b8fe209c95f518c32d6e5e191f45a (patch)
tree4c7b9c63a0088383cf1dccc72742509d9cb48343
parent1eb1f35a75671659298395023d351fc0a7216866 (diff)
Fixing CoreOS driver
Decoding ca on nodes Change-Id: I4a30a348c1c0a62cb1a7b429b05878f321db92ed
Notes
Notes (review): Code-Review+2: Spyros Trigazis <strigazi@gmail.com> Workflow+1: Spyros Trigazis <strigazi@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Tue, 04 Sep 2018 11:08:33 +0000 Reviewed-on: https://review.openstack.org/579026 Project: openstack/magnum Branch: refs/heads/master
-rw-r--r--magnum/drivers/heat/k8s_coreos_template_def.py132
-rw-r--r--magnum/drivers/k8s_coreos_v1/template_def.py23
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/add-ext-ca-certs.yaml10
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/configure-docker.yaml15
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/enable-docker-mount.yaml52
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml2
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml1
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/wc-notify.yaml6
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml2
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml2
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/write-kubeconfig.yaml1
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/fragments/write-master-kubeconfig.yaml21
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml416
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml309
-rw-r--r--magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml197
-rw-r--r--magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py20
16 files changed, 999 insertions, 210 deletions
diff --git a/magnum/drivers/heat/k8s_coreos_template_def.py b/magnum/drivers/heat/k8s_coreos_template_def.py
new file mode 100644
index 0000000..5eb1a80
--- /dev/null
+++ b/magnum/drivers/heat/k8s_coreos_template_def.py
@@ -0,0 +1,132 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13import base64
14from oslo_log import log as logging
15from oslo_utils import strutils
16
17from magnum.common import utils
18from magnum.common.x509 import operations as x509
19from magnum.conductor.handlers.common import cert_manager
20from magnum.drivers.heat import k8s_template_def
21from magnum.drivers.heat import template_def
22from oslo_config import cfg
23
24CONF = cfg.CONF
25
26LOG = logging.getLogger(__name__)
27
28
29class ServerAddressOutputMapping(template_def.OutputMapping):
30
31 public_ip_output_key = None
32 private_ip_output_key = None
33
34 def __init__(self, dummy_arg, cluster_attr=None):
35 self.cluster_attr = cluster_attr
36 self.heat_output = self.public_ip_output_key
37
38 def set_output(self, stack, cluster_template, cluster):
39 if not cluster_template.floating_ip_enabled:
40 self.heat_output = self.private_ip_output_key
41
42 LOG.debug("Using heat_output: %s", self.heat_output)
43 super(ServerAddressOutputMapping,
44 self).set_output(stack, cluster_template, cluster)
45
46
47class MasterAddressOutputMapping(ServerAddressOutputMapping):
48 public_ip_output_key = 'kube_masters'
49 private_ip_output_key = 'kube_masters_private'
50
51
52class NodeAddressOutputMapping(ServerAddressOutputMapping):
53 public_ip_output_key = 'kube_minions'
54 private_ip_output_key = 'kube_minions_private'
55
56
57class CoreOSK8sTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
58 """Kubernetes template for a CoreOS."""
59
60 def __init__(self):
61 super(CoreOSK8sTemplateDefinition, self).__init__()
62 self.add_parameter('docker_volume_size',
63 cluster_attr='docker_volume_size')
64 self.add_parameter('docker_storage_driver',
65 cluster_template_attr='docker_storage_driver')
66 self.add_output('kube_minions',
67 cluster_attr='node_addresses',
68 mapping_type=NodeAddressOutputMapping)
69 self.add_output('kube_masters',
70 cluster_attr='master_addresses',
71 mapping_type=MasterAddressOutputMapping)
72
73 def get_params(self, context, cluster_template, cluster, **kwargs):
74 extra_params = kwargs.pop('extra_params', {})
75
76 extra_params['username'] = context.user_name
77 osc = self.get_osc(context)
78 extra_params['region_name'] = osc.cinder_region_name()
79
80 # set docker_volume_type
81 # use the configuration default if None provided
82 docker_volume_type = cluster.labels.get(
83 'docker_volume_type', CONF.cinder.default_docker_volume_type)
84 extra_params['docker_volume_type'] = docker_volume_type
85
86 extra_params['nodes_affinity_policy'] = \
87 CONF.cluster.nodes_affinity_policy
88
89 if cluster_template.network_driver == 'flannel':
90 extra_params["pods_network_cidr"] = \
91 cluster.labels.get('flannel_network_cidr', '10.100.0.0/16')
92 if cluster_template.network_driver == 'calico':
93 extra_params["pods_network_cidr"] = \
94 cluster.labels.get('calico_ipv4pool', '192.168.0.0/16')
95
96 label_list = ['kube_tag', 'container_infra_prefix',
97 'availability_zone',
98 'calico_tag', 'calico_cni_tag',
99 'calico_kube_controllers_tag', 'calico_ipv4pool',
100 'etcd_tag', 'flannel_tag']
101 for label in label_list:
102 label_value = cluster.labels.get(label)
103 if label_value:
104 extra_params[label] = label_value
105
106 cert_manager_api = cluster.labels.get('cert_manager_api')
107 if strutils.bool_from_string(cert_manager_api):
108 extra_params['cert_manager_api'] = cert_manager_api
109 ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
110 extra_params['ca_key'] = x509.decrypt_key(
111 ca_cert.get_private_key(),
112 ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
113
114 plain_openstack_ca = utils.get_openstack_ca()
115 encoded_openstack_ca = base64.b64encode(plain_openstack_ca.encode())
116 extra_params['openstack_ca_coreos'] = encoded_openstack_ca.decode()
117
118 return super(CoreOSK8sTemplateDefinition,
119 self).get_params(context, cluster_template, cluster,
120 extra_params=extra_params,
121 **kwargs)
122
123 def get_env_files(self, cluster_template, cluster):
124 env_files = []
125
126 template_def.add_priv_net_env_file(env_files, cluster_template)
127 template_def.add_etcd_volume_env_file(env_files, cluster_template)
128 template_def.add_volume_env_file(env_files, cluster)
129 template_def.add_lb_env_file(env_files, cluster_template)
130 template_def.add_fip_env_file(env_files, cluster_template)
131
132 return env_files
diff --git a/magnum/drivers/k8s_coreos_v1/template_def.py b/magnum/drivers/k8s_coreos_v1/template_def.py
index 448dcc7..194e746 100644
--- a/magnum/drivers/k8s_coreos_v1/template_def.py
+++ b/magnum/drivers/k8s_coreos_v1/template_def.py
@@ -14,30 +14,13 @@
14import os 14import os
15 15
16import magnum.conf 16import magnum.conf
17from magnum.drivers.heat import k8s_template_def 17from magnum.drivers.heat import k8s_coreos_template_def as kctd
18from magnum.drivers.heat import template_def
19 18
20CONF = magnum.conf.CONF 19CONF = magnum.conf.CONF
21 20
22 21
23class CoreOSK8sTemplateDefinition(k8s_template_def.K8sTemplateDefinition): 22class CoreOSK8sTemplateDefinition(kctd.CoreOSK8sTemplateDefinition):
24 """Kubernetes template for CoreOS VM.""" 23 """Kubernetes template for a CoreOS Atomic VM."""
25
26 def __init__(self):
27 super(CoreOSK8sTemplateDefinition, self).__init__()
28 self.add_output('kube_minions',
29 cluster_attr='node_addresses')
30 self.add_output('kube_masters',
31 cluster_attr='master_addresses')
32
33 def get_env_files(self, cluster_template, cluster):
34 env_files = []
35
36 template_def.add_priv_net_env_file(env_files, cluster_template)
37 template_def.add_lb_env_file(env_files, cluster_template)
38 template_def.add_fip_env_file(env_files, cluster_template)
39
40 return env_files
41 24
42 @property 25 @property
43 def driver_module_path(self): 26 def driver_module_path(self):
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/add-ext-ca-certs.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/add-ext-ca-certs.yaml
index 9cb8e30..1f2d865 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/add-ext-ca-certs.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/add-ext-ca-certs.yaml
@@ -15,6 +15,13 @@ write_files:
15 [Install] 15 [Install]
16 WantedBy=multi-user.target 16 WantedBy=multi-user.target
17 17
18 - path: /etc/ssl/certs/openstack-ca.pem
19 owner: "root:root"
20 permissions: "0644"
21 encoding: b64
22 content: |
23 $OPENSTACK_CA
24
18 - path: /etc/sysconfig/add-ext-ca-certs.sh 25 - path: /etc/sysconfig/add-ext-ca-certs.sh
19 owner: "root:root" 26 owner: "root:root"
20 permissions: "0755" 27 permissions: "0755"
@@ -22,9 +29,8 @@ write_files:
22 #!/bin/sh 29 #!/bin/sh
23 30
24 CERT_FILE=/etc/ssl/certs/openstack-ca.pem 31 CERT_FILE=/etc/ssl/certs/openstack-ca.pem
25 if [ -n "$OPENSTACK_CA" ] 32 if [ -f "$CERT_FILE" ]
26 then 33 then
27 echo -ne "$OPENSTACK_CA" | tee -a ${CERT_FILE}
28 34
29 chmod 0644 ${CERT_FILE} 35 chmod 0644 ${CERT_FILE}
30 chown root:root ${CERT_FILE} 36 chown root:root ${CERT_FILE}
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/configure-docker.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/configure-docker.yaml
index 09600fa..c7b1767 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/configure-docker.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/configure-docker.yaml
@@ -1,5 +1,20 @@
1#cloud-config 1#cloud-config
2write_files: 2write_files:
3 - path: /etc/systemd/system/var-lib-docker.mount
4 owner: "root:root"
5 permissions: "0644"
6 content: |
7 [Unit]
8 Description=Mount ephemeral to /var/lib/docker
9
10 [Mount]
11 What=/dev/vdb
12 Where=/var/lib/docker
13 Type=ext4
14
15 [Install]
16 WantedBy=local-fs.target
17
3 - path: /etc/systemd/system/configure-docker.service 18 - path: /etc/systemd/system/configure-docker.service
4 owner: "root:root" 19 owner: "root:root"
5 permissions: "0644" 20 permissions: "0644"
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-docker-mount.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-docker-mount.yaml
new file mode 100644
index 0000000..f5e4d42
--- /dev/null
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-docker-mount.yaml
@@ -0,0 +1,52 @@
1#cloud-config
2write_files:
3 - path: /etc/sytemd/system/var-lib-docker.mount
4 owner: "root:root"
5 permissions: "0644"
6 content: |
7 [Unit]
8 Description=Mount ephemeral to /var/lib/docker
9
10 [Mount]
11 What=/dev/vdb
12 Where=/var/lib/docker
13 Type=ext4
14
15 [Install]
16 WantedBy=local-fs.target
17
18 - path: /etc/sysconfig/enable-docker-mount.sh
19 owner: "root:root"
20 permissions: "0755"
21 content: |
22 #!/bin/sh
23 if [ -n "$DOCKER_VOLUME_SIZE" ] && [ "$DOCKER_VOLUME_SIZE" -gt 0 ]; then
24 if [[ $(blkid -o value -s TYPE /dev/vdb) ]]; then
25 systemctl daemon-reload
26 systemctl start var-lib-docker.mount
27 systemctl enable var-lib-docker.mount
28 else
29 mkfs -t ext4 /dev/vdb
30 systemctl daemon-reload
31 systemctl start var-lib-docker.mount
32 systemctl enable var-lib-docker.mount
33 fi
34 fi
35
36 - path: /etc/systemd/system/enable-docker-mount.service
37 owner: "root:root"
38 permissions: "0644"
39 content: |
40 [Unit]
41 Description=Mount docker volume
42
43 [Service]
44 Type=oneshot
45 EnvironmentFile=/etc/sysconfig/heat-params
46 ExecStart=/etc/sysconfig/enable-docker-mount.sh
47
48 [Install]
49 RequiredBy=multi-user.target
50
51
52
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml
index 868c2f4..5365595 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-master.yaml
@@ -56,7 +56,7 @@ write_files:
56 ExecStartPre=/usr/bin/mkdir -p /var/log/containers 56 ExecStartPre=/usr/bin/mkdir -p /var/log/containers
57 ExecStartPre=-/usr/bin/rkt rm --uuid-file=${uuid_file} 57 ExecStartPre=-/usr/bin/rkt rm --uuid-file=${uuid_file}
58 ExecStart=/usr/lib/coreos/kubelet-wrapper \ 58 ExecStart=/usr/lib/coreos/kubelet-wrapper \
59 --api-servers=http://127.0.0.1:8080 \ 59 --kubeconfig=/etc/kubernetes/master-kubeconfig.yaml \
60 --cni-conf-dir=/etc/kubernetes/cni/net.d \ 60 --cni-conf-dir=/etc/kubernetes/cni/net.d \
61 --network-plugin=cni \ 61 --network-plugin=cni \
62 --hostname-override=${HOSTNAME_OVERRIDE} \ 62 --hostname-override=${HOSTNAME_OVERRIDE} \
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml
index 486ebd6..6070215 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/enable-kubelet-minion.yaml
@@ -68,7 +68,6 @@ write_files:
68 ExecStartPre=/usr/bin/mkdir -p /var/log/containers 68 ExecStartPre=/usr/bin/mkdir -p /var/log/containers
69 ExecStartPre=-/usr/bin/rkt rm --uuid-file=${uuid_file} 69 ExecStartPre=-/usr/bin/rkt rm --uuid-file=${uuid_file}
70 ExecStart=/usr/lib/coreos/kubelet-wrapper \ 70 ExecStart=/usr/lib/coreos/kubelet-wrapper \
71 --api-servers=${KUBE_MASTER_URI} \
72 --cni-conf-dir=/etc/kubernetes/cni/net.d \ 71 --cni-conf-dir=/etc/kubernetes/cni/net.d \
73 --network-plugin=cni \ 72 --network-plugin=cni \
74 --hostname-override=${HOSTNAME_OVERRIDE} \ 73 --hostname-override=${HOSTNAME_OVERRIDE} \
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/wc-notify.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/wc-notify.yaml
index 7857bd7..6315bb3 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/wc-notify.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/wc-notify.yaml
@@ -20,5 +20,11 @@ write_files:
20 permissions: "0755" 20 permissions: "0755"
21 content: | 21 content: |
22 #!/bin/bash -v 22 #!/bin/bash -v
23 if [ "$VERIFY_CA" == "True" ]; then
24 VERIFY_CA=""
25 else
26 VERIFY_CA="-k"
27 fi
28
23 command="$WAIT_CURL $VERIFY_CA --data-binary '{\"status\": \"SUCCESS\"}'" 29 command="$WAIT_CURL $VERIFY_CA --data-binary '{\"status\": \"SUCCESS\"}'"
24 eval $(echo "$command") 30 eval $(echo "$command")
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml
index fd379d5..7b16fce 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params-master.yaml
@@ -12,6 +12,7 @@ write_files:
12 KUBE_NODE_IP="$KUBE_NODE_IP" 12 KUBE_NODE_IP="$KUBE_NODE_IP"
13 KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV" 13 KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
14 DOCKER_VOLUME="$DOCKER_VOLUME" 14 DOCKER_VOLUME="$DOCKER_VOLUME"
15 DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE"
15 DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" 16 DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
16 NETWORK_DRIVER="$NETWORK_DRIVER" 17 NETWORK_DRIVER="$NETWORK_DRIVER"
17 FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR" 18 FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
@@ -49,4 +50,3 @@ write_files:
49 KUBE_DASHBOARD_VERSION="$KUBE_DASHBOARD_VERSION" 50 KUBE_DASHBOARD_VERSION="$KUBE_DASHBOARD_VERSION"
50 DNS_SERVICE_IP="$DNS_SERVICE_IP" 51 DNS_SERVICE_IP="$DNS_SERVICE_IP"
51 DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN" 52 DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
52 OCTAVIA_ENABLED="$OCTAVIA_ENABLED"
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml
index 3a40a76..8a376f4 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-heat-params.yaml
@@ -12,6 +12,7 @@ write_files:
12 KUBE_NODE_IP="$KUBE_NODE_IP" 12 KUBE_NODE_IP="$KUBE_NODE_IP"
13 ETCD_SERVER_IP="$ETCD_SERVER_IP" 13 ETCD_SERVER_IP="$ETCD_SERVER_IP"
14 DOCKER_VOLUME="$DOCKER_VOLUME" 14 DOCKER_VOLUME="$DOCKER_VOLUME"
15 DOCKER_VOLUME_SIZE="$DOCKER_VOLUME_SIZE"
15 DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER" 16 DOCKER_STORAGE_DRIVER="$DOCKER_STORAGE_DRIVER"
16 NETWORK_DRIVER="$NETWORK_DRIVER" 17 NETWORK_DRIVER="$NETWORK_DRIVER"
17 REGISTRY_ENABLED="$REGISTRY_ENABLED" 18 REGISTRY_ENABLED="$REGISTRY_ENABLED"
@@ -47,4 +48,3 @@ write_files:
47 CONTAINER_RUNTIME="$CONTAINER_RUNTIME" 48 CONTAINER_RUNTIME="$CONTAINER_RUNTIME"
48 DNS_SERVICE_IP="$DNS_SERVICE_IP" 49 DNS_SERVICE_IP="$DNS_SERVICE_IP"
49 DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN" 50 DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
50 OCTAVIA_ENABLED="$OCTAVIA_ENABLED"
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-kubeconfig.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-kubeconfig.yaml
index c6661bb..f7cc1b1 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-kubeconfig.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-kubeconfig.yaml
@@ -10,6 +10,7 @@ write_files:
10 clusters: 10 clusters:
11 - name: local 11 - name: local
12 cluster: 12 cluster:
13 server: https://$KUBE_MASTER_IP:$KUBE_API_PORT
13 certificate-authority: /etc/kubernetes/ssl/ca.pem 14 certificate-authority: /etc/kubernetes/ssl/ca.pem
14 users: 15 users:
15 - name: kubelet 16 - name: kubelet
diff --git a/magnum/drivers/k8s_coreos_v1/templates/fragments/write-master-kubeconfig.yaml b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-master-kubeconfig.yaml
new file mode 100644
index 0000000..25e71e6
--- /dev/null
+++ b/magnum/drivers/k8s_coreos_v1/templates/fragments/write-master-kubeconfig.yaml
@@ -0,0 +1,21 @@
1#cloud-config
2merge_how: dict(recurse_array)+list(append)
3write_files:
4 - path: /etc/kubernetes/master-kubeconfig.yaml
5 owner: "root:root"
6 permissions: "0644"
7 content: |
8 apiVersion: v1
9 kind: Config
10 clusters:
11 - name: local
12 cluster:
13 server: http://127.0.0.1:8080
14 users:
15 - name: kubelet
16 contexts:
17 - context:
18 cluster: local
19 user: kubelet
20 name: kubelet-context
21 current-context: kubelet-context
diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml
index a8ddfd8..4c8b6d5 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/kubecluster.yaml
@@ -1,15 +1,19 @@
1heat_template_version: 2014-10-16 1heat_template_version: 2014-10-16
2 2
3description: > 3description: >
4 This template will boot a coreos cluster with one or more minions (as 4 This template will boot a Kubernetes cluster with one or more
5 specified by the number_of_minions parameter, which defaults to 1) and one 5 minions (as specified by the number_of_minions parameter, which
6 master node. Allowing multiple masters is a work in progress. 6 defaults to 1).
7 7
8parameters: 8parameters:
9 9
10 octavia_enabled:
11 type: string
12 default: true
13
10 ssh_key_name: 14 ssh_key_name:
11 type: string 15 type: string
12 description: name of ssh key to be provisioned on the servers 16 description: name of ssh key to be provisioned on our server
13 17
14 external_network: 18 external_network:
15 type: string 19 type: string
@@ -28,18 +32,17 @@ parameters:
28 32
29 server_image: 33 server_image:
30 type: string 34 type: string
31 default: CoreOS 35 description: glance image used to boot the server
32 description: glance image used to boot the servers
33 36
34 master_flavor: 37 master_flavor:
35 type: string 38 type: string
36 default: m1.small 39 default: m1.small
37 description: flavor to use when booting the server for master node 40 description: flavor to use when booting the server for master nodes
38 41
39 minion_flavor: 42 minion_flavor:
40 type: string 43 type: string
41 default: m1.small 44 default: m1.small
42 description: flavor to use when booting the servers for minions 45 description: flavor to use when booting the server for minions
43 46
44 prometheus_monitoring: 47 prometheus_monitoring:
45 type: boolean 48 type: boolean
@@ -54,14 +57,9 @@ parameters:
54 description: > 57 description: >
55 admin user password for the Grafana monitoring interface 58 admin user password for the Grafana monitoring interface
56 59
57 discovery_url:
58 type: string
59 description: >
60 Discovery URL used for bootstrapping the etcd cluster.
61
62 dns_nameserver: 60 dns_nameserver:
63 type: string 61 type: string
64 description: address of a dns nameserver reachable in your environment 62 description: address of a DNS nameserver reachable in your environment
65 default: 8.8.8.8 63 default: 8.8.8.8
66 64
67 number_of_masters: 65 number_of_masters:
@@ -85,6 +83,11 @@ parameters:
85 address range used by kubernetes for service portals 83 address range used by kubernetes for service portals
86 default: 10.254.0.0/16 84 default: 10.254.0.0/16
87 85
86 network_driver:
87 type: string
88 description: network driver to use for instantiating container networks
89 default: flannel
90
88 flannel_network_cidr: 91 flannel_network_cidr:
89 type: string 92 type: string
90 description: network range for flannel overlay network 93 description: network range for flannel overlay network
@@ -99,7 +102,7 @@ parameters:
99 type: string 102 type: string
100 description: > 103 description: >
101 specify the backend for flannel, default udp backend 104 specify the backend for flannel, default udp backend
102 default: "host-gw" 105 default: "udp"
103 constraints: 106 constraints:
104 - allowed_values: ["udp", "vxlan", "host-gw"] 107 - allowed_values: ["udp", "vxlan", "host-gw"]
105 108
@@ -131,19 +134,115 @@ parameters:
131 constraints: 134 constraints:
132 - allowed_values: ["true", "false"] 135 - allowed_values: ["true", "false"]
133 136
137 etcd_volume_size:
138 type: number
139 description: >
140 size of the cinder volume for etcd storage
141 default: 0
142
143 docker_volume_size:
144 type: number
145 description: >
146 size of a cinder volume to allocate to docker for container/image
147 storage
148 default: 0
149
150 docker_volume_type:
151 type: string
152 description: >
153 type of a cinder volume to allocate to docker for container/image
154 storage
155
156 docker_storage_driver:
157 type: string
158 description: docker storage driver name
159 default: "devicemapper"
160
161 wait_condition_timeout:
162 type: number
163 description: >
164 timeout for the Wait Conditions
165 default: 6000
166
134 minions_to_remove: 167 minions_to_remove:
135 type: comma_delimited_list 168 type: comma_delimited_list
136 description: > 169 description: >
137 List of minions to be removed when doing an update. Individual minion may 170 List of minions to be removed when doing an update. Individual minion may
138 be referenced several ways: (1) The resource name (e.g. ['1', '3']), 171 be referenced several ways: (1) The resource name (e.g. ['1', '3']),
139 (2) The private IP address ['10.0.0.4', '10.0.0.6']. Note: the list should 172 (2) The private IP address ['10.0.0.4', '10.0.0.6']. Note: the list should
140 be empty when doing a create. 173 be empty when doing an create.
141 default: [] 174 default: []
142 175
143 network_driver: 176 discovery_url:
144 type: string 177 type: string
145 description: network driver to use for instantiating container networks 178 description: >
146 default: flannel 179 Discovery URL used for bootstrapping the etcd cluster.
180
181 registry_enabled:
182 type: boolean
183 description: >
184 Indicates whether the docker registry is enabled.
185 default: false
186
187 registry_port:
188 type: number
189 description: port of registry service
190 default: 5000
191
192 swift_region:
193 type: string
194 description: region of swift service
195 default: ""
196
197 registry_container:
198 type: string
199 description: >
200 name of swift container which docker registry stores images in
201 default: "container"
202
203 registry_insecure:
204 type: boolean
205 description: >
206 indicates whether to skip TLS verification between registry and backend storage
207 default: true
208
209 registry_chunksize:
210 type: number
211 description: >
212 size fo the data segments for the swift dynamic large objects
213 default: 5242880
214
215 volume_driver:
216 type: string
217 description: volume driver to use for container storage
218 default: ""
219
220 region_name:
221 type: string
222 description: A logically separate section of the cluster
223
224 username:
225 type: string
226 description: >
227 user account
228
229 password:
230 type: string
231 description: >
232 user password, not set in current implementation, only used to
233 fill in for Kubernetes config file
234 default:
235 ChangeMe
236 hidden: true
237
238 loadbalancing_protocol:
239 type: string
240 description: >
241 The protocol which is used for load balancing. If you want to change
242 tls_disabled option to 'True', please change this to "HTTP".
243 default: TCP
244 constraints:
245 - allowed_values: ["TCP", "HTTP"]
147 246
148 tls_disabled: 247 tls_disabled:
149 type: boolean 248 type: boolean
@@ -152,7 +251,7 @@ parameters:
152 251
153 kube_dashboard_enabled: 252 kube_dashboard_enabled:
154 type: boolean 253 type: boolean
155 description: whether or not to disable kubernetes dashboard 254 description: whether or not to enable kubernetes dashboard
156 default: True 255 default: True
157 256
158 influx_grafana_dashboard_enabled: 257 influx_grafana_dashboard_enabled:
@@ -164,15 +263,6 @@ parameters:
164 type: boolean 263 type: boolean
165 description: whether or not to validate certificate authority 264 description: whether or not to validate certificate authority
166 265
167 loadbalancing_protocol:
168 type: string
169 description: >
170 The protocol which is used for load balancing. If you want to change
171 tls_disabled option to 'True', please change this to "HTTP".
172 default: TCP
173 constraints:
174 - allowed_values: ["TCP", "HTTP"]
175
176 kubernetes_port: 266 kubernetes_port:
177 type: number 267 type: number
178 description: > 268 description: >
@@ -206,43 +296,53 @@ parameters:
206 trustee_domain_id: 296 trustee_domain_id:
207 type: string 297 type: string
208 description: domain id of the trustee 298 description: domain id of the trustee
209 default: ""
210 299
211 trustee_user_id: 300 trustee_user_id:
212 type: string 301 type: string
213 description: user id of the trustee 302 description: user id of the trustee
214 default: ""
215 303
216 trustee_username: 304 trustee_username:
217 type: string 305 type: string
218 description: username of the trustee 306 description: username of the trustee
219 default: ""
220 307
221 trustee_password: 308 trustee_password:
222 type: string 309 type: string
223 description: password of the trustee 310 description: password of the trustee
224 default: ""
225 hidden: true 311 hidden: true
226 312
227 trust_id: 313 trust_id:
228 type: string 314 type: string
229 description: id of the trust which is used by the trustee 315 description: id of the trust which is used by the trustee
230 default: ""
231 hidden: true 316 hidden: true
232 317
233 auth_url: 318 auth_url:
234 type: string 319 type: string
235 description: url for keystone 320 description: url for keystone
236 321
322 kube_tag:
323 type: string
324 description: tag of the k8s containers used to provision the kubernetes cluster
325 default: v1.9.3
326
327 etcd_tag:
328 type: string
329 description: tag of the etcd system container
330 default: v3.2.7
331
332 flannel_tag:
333 type: string
334 description: tag of the flannel system containers
335 default: v0.9.0
336
237 kube_version: 337 kube_version:
238 type: string 338 type: string
239 description: version of kubernetes used for kubernetes cluster 339 description: version of kubernetes used for kubernetes cluster
240 default: v1.6.2_coreos.0 340 default: v1.10.3_coreos.0
241 341
242 kube_dashboard_version: 342 kube_dashboard_version:
243 type: string 343 type: string
244 description: version of kubernetes dashboard used for kubernetes cluster 344 description: version of kubernetes dashboard used for kubernetes cluster
245 default: v1.5.1 345 default: v1.8.3
246 346
247 hyperkube_image: 347 hyperkube_image:
248 type: string 348 type: string
@@ -250,37 +350,19 @@ parameters:
250 Docker registry used for hyperkube image 350 Docker registry used for hyperkube image
251 default: quay.io/coreos/hyperkube 351 default: quay.io/coreos/hyperkube
252 352
253 registry_enabled:
254 type: boolean
255 description: >
256 Indicates whether the docker registry is enabled.
257 default: false
258
259 registry_port:
260 type: number
261 description: port of registry service
262 default: 5000
263
264 wait_condition_timeout:
265 type: number
266 description: >
267 timeout for the Wait Conditions
268 default: 6000
269
270 insecure_registry_url: 353 insecure_registry_url:
271 type: string 354 type: string
272 description: insecure registry url 355 description: insecure registry url
273 constraints:
274 - allowed_pattern: "^$|.*/"
275 default: "" 356 default: ""
276 357
277 container_runtime: 358 container_infra_prefix:
278 type: string 359 type: string
279 description: > 360 description: >
280 Container runtime to use with Kubernetes. 361 prefix of container images used in the cluster, kubernetes components,
281 default: "docker" 362 kubernetes-dashboard, coredns etc
282 constraints: 363 constraints:
283 - allowed_values: ["docker"] 364 - allowed_pattern: "^$|.*/"
365 default: ""
284 366
285 dns_service_ip: 367 dns_service_ip:
286 type: string 368 type: string
@@ -299,6 +381,11 @@ parameters:
299 hidden: true 381 hidden: true
300 description: The OpenStack CA certificate to install on the node. 382 description: The OpenStack CA certificate to install on the node.
301 383
384 openstack_ca_coreos:
385 type: string
386 hidden: true
387 description: The OpenStack CA certificate to install on the node.
388
302 nodes_affinity_policy: 389 nodes_affinity_policy:
303 type: string 390 type: string
304 description: > 391 description: >
@@ -307,17 +394,104 @@ parameters:
307 - allowed_values: ["affinity", "anti-affinity", "soft-affinity", 394 - allowed_values: ["affinity", "anti-affinity", "soft-affinity",
308 "soft-anti-affinity"] 395 "soft-anti-affinity"]
309 396
310 octavia_enabled: 397 availability_zone:
398 type: string
399 description: >
400 availability zone for master and nodes
401 default: ""
402
403 cert_manager_api:
311 type: boolean 404 type: boolean
405 description: true if the kubernetes cert api manager should be enabled
406 default: false
407
408 ca_key:
409 type: string
410 description: key of internal ca for the kube certificate api manager
411 default: ""
412 hidden: true
413
414 calico_tag:
415 type: string
416 description: tag of the calico containers used to provision the calico node
417 default: v2.6.7
418
419 calico_cni_tag:
420 type: string
421 description: tag of the cni used to provision the calico node
422 default: v1.11.2
423
424 calico_kube_controllers_tag:
425 type: string
426 description: tag of the kube_controllers used to provision the calico node
427 default: v1.0.3
428
429 calico_ipv4pool:
430 type: string
431 description: Configure the IP pool from which Pod IPs will be chosen
432 default: "192.168.0.0/16"
433
434 pods_network_cidr:
435 type: string
436 description: Configure the IP pool/range from which pod IPs will be chosen
437
438 ingress_controller:
439 type: string
312 description: > 440 description: >
313 whether or not to use Octavia for LoadBalancer type service. 441 ingress controller backend to use
314 default: False 442 default: ""
443
444 ingress_controller_role:
445 type: string
446 description: >
447 node role where the ingress controller backend should run
448 default: "ingress"
449
450 kubelet_options:
451 type: string
452 description: >
453 additional options to be passed to the kubelet
454 default: ""
455
456 kubeapi_options:
457 type: string
458 description: >
459 additional options to be passed to the api
460 default: ""
461
462 kubecontroller_options:
463 type: string
464 description: >
465 additional options to be passed to the controller manager
466 default: ""
467
468 kubeproxy_options:
469 type: string
470 description: >
471 additional options to be passed to the kube proxy
472 default: ""
473
474 kubescheduler_options:
475 type: string
476 description: >
477 additional options to be passed to the scheduler
478 default: ""
479
480 container_runtime:
481 type: string
482 description: >
483 Container runtime to use with Kubernetes.
484 default: "docker"
485 constraints:
486 - allowed_values: ["docker"]
487
488
315 489
316resources: 490resources:
317 491
318 ###################################################################### 492 ######################################################################
319 # 493 #
320 # network resources. allocate a network and router for our server. 494 # network resources. allocate a network and router for our server.
321 # Important: the Load Balancer feature in Kubernetes requires that 495 # Important: the Load Balancer feature in Kubernetes requires that
322 # the name for the fixed_network must be "private" for the 496 # the name for the fixed_network must be "private" for the
323 # address lookup in Kubernetes to work properly 497 # address lookup in Kubernetes to work properly
@@ -349,13 +523,13 @@ resources:
349 protocol: {get_param: loadbalancing_protocol} 523 protocol: {get_param: loadbalancing_protocol}
350 port: 2379 524 port: 2379
351 525
352 ###################################################################### 526 ######################################################################
353 # 527 #
354 # security groups. we need to permit network traffic of various 528 # security groups. we need to permit network traffic of various
355 # sorts. 529 # sorts.
356 # 530 #
357 531
358 secgroup_master: 532 secgroup_kube_master:
359 type: OS::Neutron::SecurityGroup 533 type: OS::Neutron::SecurityGroup
360 properties: 534 properties:
361 rules: 535 rules:
@@ -378,8 +552,11 @@ resources:
378 - protocol: tcp 552 - protocol: tcp
379 port_range_min: 6443 553 port_range_min: 6443
380 port_range_max: 6443 554 port_range_max: 6443
555 - protocol: tcp
556 port_range_min: 30000
557 port_range_max: 32767
381 558
382 secgroup_minion_all_open: 559 secgroup_kube_minion:
383 type: OS::Neutron::SecurityGroup 560 type: OS::Neutron::SecurityGroup
384 properties: 561 properties:
385 rules: 562 rules:
@@ -433,7 +610,7 @@ resources:
433 ###################################################################### 610 ######################################################################
434 # 611 #
435 # kubernetes masters. This is a resource group that will create 612 # kubernetes masters. This is a resource group that will create
436 # <number_of_masters> master. 613 # <number_of_masters> masters.
437 # 614 #
438 615
439 kube_masters: 616 kube_masters:
@@ -449,6 +626,8 @@ resources:
449 list_join: 626 list_join:
450 - '-' 627 - '-'
451 - [{ get_param: 'OS::stack_name' }, 'master', '%index%'] 628 - [{ get_param: 'OS::stack_name' }, 'master', '%index%']
629 prometheus_monitoring: {get_param: prometheus_monitoring}
630 grafana_admin_passwd: {get_param: grafana_admin_passwd}
452 api_public_address: {get_attr: [api_lb, floating_address]} 631 api_public_address: {get_attr: [api_lb, floating_address]}
453 api_private_address: {get_attr: [api_lb, address]} 632 api_private_address: {get_attr: [api_lb, address]}
454 ssh_key_name: {get_param: ssh_key_name} 633 ssh_key_name: {get_param: ssh_key_name}
@@ -456,6 +635,12 @@ resources:
456 master_flavor: {get_param: master_flavor} 635 master_flavor: {get_param: master_flavor}
457 external_network: {get_param: external_network} 636 external_network: {get_param: external_network}
458 kube_allow_priv: {get_param: kube_allow_priv} 637 kube_allow_priv: {get_param: kube_allow_priv}
638 etcd_volume_size: {get_param: etcd_volume_size}
639 docker_volume_size: {get_param: docker_volume_size}
640 docker_volume_type: {get_param: docker_volume_type}
641 docker_storage_driver: {get_param: docker_storage_driver}
642 wait_condition_timeout: {get_param: wait_condition_timeout}
643 network_driver: {get_param: network_driver}
459 flannel_network_cidr: {get_param: flannel_network_cidr} 644 flannel_network_cidr: {get_param: flannel_network_cidr}
460 flannel_network_subnetlen: {get_param: flannel_network_subnetlen} 645 flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
461 flannel_backend: {get_param: flannel_backend} 646 flannel_backend: {get_param: flannel_backend}
@@ -463,26 +648,29 @@ resources:
463 system_pods_timeout: {get_param: system_pods_timeout} 648 system_pods_timeout: {get_param: system_pods_timeout}
464 portal_network_cidr: {get_param: portal_network_cidr} 649 portal_network_cidr: {get_param: portal_network_cidr}
465 admission_control_list: {get_param: admission_control_list} 650 admission_control_list: {get_param: admission_control_list}
651 discovery_url: {get_param: discovery_url}
652 cluster_uuid: {get_param: cluster_uuid}
653 magnum_url: {get_param: magnum_url}
654 volume_driver: {get_param: volume_driver}
466 fixed_network: {get_attr: [network, fixed_network]} 655 fixed_network: {get_attr: [network, fixed_network]}
467 fixed_subnet: {get_attr: [network, fixed_subnet]} 656 fixed_subnet: {get_attr: [network, fixed_subnet]}
468 discovery_url: {get_param: discovery_url} 657 api_pool_id: {get_attr: [api_lb, pool_id]}
469 network_driver: {get_param: network_driver} 658 etcd_pool_id: {get_attr: [etcd_lb, pool_id]}
659 username: {get_param: username}
660 password: {get_param: password}
470 kubernetes_port: {get_param: kubernetes_port} 661 kubernetes_port: {get_param: kubernetes_port}
471 tls_disabled: {get_param: tls_disabled} 662 tls_disabled: {get_param: tls_disabled}
472 kube_dashboard_enabled: {get_param: kube_dashboard_enabled} 663 kube_dashboard_enabled: {get_param: kube_dashboard_enabled}
473 influx_grafana_dashboard_enabled: {get_param: influx_grafana_dashboard_enabled} 664 influx_grafana_dashboard_enabled: {get_param: influx_grafana_dashboard_enabled}
474 verify_ca: {get_param: verify_ca} 665 verify_ca: {get_param: verify_ca}
475 secgroup_kube_master_id: {get_resource: secgroup_master} 666 secgroup_kube_master_id: {get_resource: secgroup_kube_master}
476 http_proxy: {get_param: http_proxy} 667 http_proxy: {get_param: http_proxy}
477 https_proxy: {get_param: https_proxy} 668 https_proxy: {get_param: https_proxy}
478 no_proxy: {get_param: no_proxy} 669 no_proxy: {get_param: no_proxy}
670 kube_tag: {get_param: kube_tag}
479 kube_version: {get_param: kube_version} 671 kube_version: {get_param: kube_version}
672 etcd_tag: {get_param: etcd_tag}
480 kube_dashboard_version: {get_param: kube_dashboard_version} 673 kube_dashboard_version: {get_param: kube_dashboard_version}
481 wait_condition_timeout: {get_param: wait_condition_timeout}
482 cluster_uuid: {get_param: cluster_uuid}
483 api_pool_id: {get_attr: [api_lb, pool_id]}
484 etcd_pool_id: {get_attr: [etcd_lb, pool_id]}
485 magnum_url: {get_param: magnum_url}
486 trustee_user_id: {get_param: trustee_user_id} 674 trustee_user_id: {get_param: trustee_user_id}
487 trustee_password: {get_param: trustee_password} 675 trustee_password: {get_param: trustee_password}
488 trust_id: {get_param: trust_id} 676 trust_id: {get_param: trust_id}
@@ -490,18 +678,31 @@ resources:
490 hyperkube_image: {get_param: hyperkube_image} 678 hyperkube_image: {get_param: hyperkube_image}
491 insecure_registry_url: {get_param: insecure_registry_url} 679 insecure_registry_url: {get_param: insecure_registry_url}
492 container_runtime: {get_param: container_runtime} 680 container_runtime: {get_param: container_runtime}
493 prometheus_monitoring: {get_param: prometheus_monitoring} 681 container_infra_prefix: {get_param: container_infra_prefix}
494 grafana_admin_passwd: {get_param: grafana_admin_passwd}
495 etcd_lb_vip: {get_attr: [etcd_lb, address]} 682 etcd_lb_vip: {get_attr: [etcd_lb, address]}
496 dns_service_ip: {get_param: dns_service_ip} 683 dns_service_ip: {get_param: dns_service_ip}
497 dns_cluster_domain: {get_param: dns_cluster_domain} 684 dns_cluster_domain: {get_param: dns_cluster_domain}
498 openstack_ca: {get_param: openstack_ca} 685 openstack_ca: {get_param: openstack_ca_coreos}
499 nodes_server_group_id: {get_resource: nodes_server_group} 686 nodes_server_group_id: {get_resource: nodes_server_group}
500 octavia_enabled: {get_param: octavia_enabled} 687 availability_zone: {get_param: availability_zone}
688 ca_key: {get_param: ca_key}
689 cert_manager_api: {get_param: cert_manager_api}
690 calico_tag: {get_param: calico_tag}
691 calico_cni_tag: {get_param: calico_cni_tag}
692 calico_kube_controllers_tag: {get_param: calico_kube_controllers_tag}
693 calico_ipv4pool: {get_param: calico_ipv4pool}
694 pods_network_cidr: {get_param: pods_network_cidr}
695 ingress_controller: {get_param: ingress_controller}
696 ingress_controller_role: {get_param: ingress_controller_role}
697 kubelet_options: {get_param: kubelet_options}
698 kubeapi_options: {get_param: kubeapi_options}
699 kubeproxy_options: {get_param: kubeproxy_options}
700 kubecontroller_options: {get_param: kubecontroller_options}
701 kubescheduler_options: {get_param: kubescheduler_options}
501 702
502 ###################################################################### 703 ######################################################################
503 # 704 #
504 # kubernetes minions. This is a resource group that will initially 705 # kubernetes minions. This is an resource group that will initially
505 # create <number_of_minions> minions, and needs to be manually scaled. 706 # create <number_of_minions> minions, and needs to be manually scaled.
506 # 707 #
507 708
@@ -509,7 +710,6 @@ resources:
509 type: OS::Heat::ResourceGroup 710 type: OS::Heat::ResourceGroup
510 depends_on: 711 depends_on:
511 - network 712 - network
512 - kube_masters
513 properties: 713 properties:
514 count: {get_param: number_of_minions} 714 count: {get_param: number_of_minions}
515 removal_policies: [{resource_list: {get_param: minions_to_remove}}] 715 removal_policies: [{resource_list: {get_param: minions_to_remove}}]
@@ -520,41 +720,62 @@ resources:
520 list_join: 720 list_join:
521 - '-' 721 - '-'
522 - [{ get_param: 'OS::stack_name' }, 'minion', '%index%'] 722 - [{ get_param: 'OS::stack_name' }, 'minion', '%index%']
723 prometheus_monitoring: {get_param: prometheus_monitoring}
523 ssh_key_name: {get_param: ssh_key_name} 724 ssh_key_name: {get_param: ssh_key_name}
524 server_image: {get_param: server_image} 725 server_image: {get_param: server_image}
525 minion_flavor: {get_param: minion_flavor} 726 minion_flavor: {get_param: minion_flavor}
526 fixed_network: {get_attr: [network, fixed_network]} 727 fixed_network: {get_attr: [network, fixed_network]}
527 fixed_subnet: {get_attr: [network, fixed_subnet]} 728 fixed_subnet: {get_attr: [network, fixed_subnet]}
729 network_driver: {get_param: network_driver}
528 flannel_network_cidr: {get_param: flannel_network_cidr} 730 flannel_network_cidr: {get_param: flannel_network_cidr}
529 kube_master_ip: {get_attr: [api_address_lb_switch, private_ip]} 731 kube_master_ip: {get_attr: [api_address_lb_switch, private_ip]}
530 etcd_server_ip: {get_attr: [etcd_address_lb_switch, private_ip]} 732 etcd_server_ip: {get_attr: [etcd_address_lb_switch, private_ip]}
531 external_network: {get_param: external_network} 733 external_network: {get_param: external_network}
532 kube_allow_priv: {get_param: kube_allow_priv} 734 kube_allow_priv: {get_param: kube_allow_priv}
533 network_driver: {get_param: network_driver} 735 docker_volume_size: {get_param: docker_volume_size}
736 docker_volume_type: {get_param: docker_volume_type}
737 docker_storage_driver: {get_param: docker_storage_driver}
738 wait_condition_timeout: {get_param: wait_condition_timeout}
739 registry_enabled: {get_param: registry_enabled}
740 registry_port: {get_param: registry_port}
741 swift_region: {get_param: swift_region}
742 registry_container: {get_param: registry_container}
743 registry_insecure: {get_param: registry_insecure}
744 registry_chunksize: {get_param: registry_chunksize}
745 cluster_uuid: {get_param: cluster_uuid}
746 magnum_url: {get_param: magnum_url}
747 volume_driver: {get_param: volume_driver}
748 region_name: {get_param: region_name}
749 auth_url: {get_param: auth_url}
750 hyperkube_image: {get_param: hyperkube_image}
751 username: {get_param: username}
752 password: {get_param: password}
534 kubernetes_port: {get_param: kubernetes_port} 753 kubernetes_port: {get_param: kubernetes_port}
535 tls_disabled: {get_param: tls_disabled} 754 tls_disabled: {get_param: tls_disabled}
536 verify_ca: {get_param: verify_ca} 755 verify_ca: {get_param: verify_ca}
537 secgroup_kube_minion_id: {get_resource: secgroup_minion_all_open} 756 secgroup_kube_minion_id: {get_resource: secgroup_kube_minion}
538 http_proxy: {get_param: http_proxy} 757 http_proxy: {get_param: http_proxy}
539 https_proxy: {get_param: https_proxy} 758 https_proxy: {get_param: https_proxy}
540 no_proxy: {get_param: no_proxy} 759 no_proxy: {get_param: no_proxy}
760 kube_tag: {get_param: kube_tag}
541 kube_version: {get_param: kube_version} 761 kube_version: {get_param: kube_version}
542 wait_condition_timeout: {get_param: wait_condition_timeout} 762 flannel_tag: {get_param: flannel_tag}
543 cluster_uuid: {get_param: cluster_uuid}
544 magnum_url: {get_param: magnum_url}
545 trustee_user_id: {get_param: trustee_user_id} 763 trustee_user_id: {get_param: trustee_user_id}
764 trustee_username: {get_param: trustee_username}
546 trustee_password: {get_param: trustee_password} 765 trustee_password: {get_param: trustee_password}
766 trustee_domain_id: {get_param: trustee_domain_id}
547 trust_id: {get_param: trust_id} 767 trust_id: {get_param: trust_id}
548 auth_url: {get_param: auth_url}
549 hyperkube_image: {get_param: hyperkube_image}
550 insecure_registry_url: {get_param: insecure_registry_url} 768 insecure_registry_url: {get_param: insecure_registry_url}
551 container_runtime: {get_param: container_runtime} 769 container_runtime: {get_param: container_runtime}
552 prometheus_monitoring: {get_param: prometheus_monitoring} 770 container_infra_prefix: {get_param: container_infra_prefix}
553 dns_service_ip: {get_param: dns_service_ip} 771 dns_service_ip: {get_param: dns_service_ip}
554 dns_cluster_domain: {get_param: dns_cluster_domain} 772 dns_cluster_domain: {get_param: dns_cluster_domain}
555 openstack_ca: {get_param: openstack_ca} 773 openstack_ca: {get_param: openstack_ca_coreos}
556 nodes_server_group_id: {get_resource: nodes_server_group} 774 nodes_server_group_id: {get_resource: nodes_server_group}
557 octavia_enabled: {get_param: octavia_enabled} 775 availability_zone: {get_param: availability_zone}
776 pods_network_cidr: {get_param: pods_network_cidr}
777 kubelet_options: {get_param: kubelet_options}
778 kubeproxy_options: {get_param: kubeproxy_options}
558 779
559outputs: 780outputs:
560 781
@@ -568,6 +789,16 @@ outputs:
568 This is the API endpoint of the Kubernetes cluster. Use this to access 789 This is the API endpoint of the Kubernetes cluster. Use this to access
569 the Kubernetes API. 790 the Kubernetes API.
570 791
792 registry_address:
793 value:
794 str_replace:
795 template: localhost:port
796 params:
797 port: {get_param: registry_port}
798 description:
799 This is the url of docker registry server where you can store docker
800 images.
801
571 kube_masters_private: 802 kube_masters_private:
572 value: {get_attr: [kube_masters, kube_master_ip]} 803 value: {get_attr: [kube_masters, kube_master_ip]}
573 description: > 804 description: >
@@ -577,8 +808,7 @@ outputs:
577 value: {get_attr: [kube_masters, kube_master_external_ip]} 808 value: {get_attr: [kube_masters, kube_master_external_ip]}
578 description: > 809 description: >
579 This is a list of the "public" IP addresses of all the Kubernetes masters. 810 This is a list of the "public" IP addresses of all the Kubernetes masters.
580 Use these IP addresses to log in to the Kubernetes masters via ssh or to access 811 Use these IP addresses to log in to the Kubernetes masters via ssh.
581 the Kubernetes API.
582 812
583 kube_minions_private: 813 kube_minions_private:
584 value: {get_attr: [kube_minions, kube_minion_ip]} 814 value: {get_attr: [kube_minions, kube_minion_ip]}
diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml
index a25a1d5..cc7dba0 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/kubemaster.yaml
@@ -1,9 +1,9 @@
1heat_template_version: 2014-10-16 1heat_template_version: 2014-10-16
2 2
3description: > 3description: >
4 This is a nested stack that defines a Kubernetes master. This stack is 4 This is a nested stack that defines a single Kubernetes master, This stack is
5 included by an ResourceGroup resource in the parent template 5 included by an ResourceGroup resource in the parent template
6 (kubeclusters.yaml). 6 (kubecluster.yaml).
7 7
8parameters: 8parameters:
9 9
@@ -27,19 +27,6 @@ parameters:
27 type: string 27 type: string
28 description: uuid/name of a network to use for floating ip addresses 28 description: uuid/name of a network to use for floating ip addresses
29 29
30 discovery_url:
31 type: string
32 description: >
33 Discovery URL used for bootstrapping the etcd cluster.
34
35 api_pool_id:
36 type: string
37 description: ID of the load balancer pool of k8s API server.
38
39 etcd_pool_id:
40 type: string
41 description: ID of the load balancer pool of etcd server.
42
43 portal_network_cidr: 30 portal_network_cidr:
44 type: string 31 type: string
45 description: > 32 description: >
@@ -52,6 +39,32 @@ parameters:
52 constraints: 39 constraints:
53 - allowed_values: ["true", "false"] 40 - allowed_values: ["true", "false"]
54 41
42 etcd_volume_size:
43 type: number
44 description: >
45 size of a cinder volume to allocate for etcd storage
46
47 docker_volume_size:
48 type: number
49 description: >
50 size of a cinder volume to allocate to docker for container/image
51 storage
52
53 docker_volume_type:
54 type: string
55 description: >
56 type of a cinder volume to allocate to docker for container/image
57 storage
58
59 docker_storage_driver:
60 type: string
61 description: docker storage driver name
62 default: "devicemapper"
63
64 volume_driver:
65 type: string
66 description: volume driver to use for container storage
67
55 flannel_network_cidr: 68 flannel_network_cidr:
56 type: string 69 type: string
57 description: network range for flannel overlay network 70 description: network range for flannel overlay network
@@ -86,26 +99,10 @@ parameters:
86 description: > 99 description: >
87 List of admission control plugins to activate 100 List of admission control plugins to activate
88 101
89 fixed_network: 102 discovery_url:
90 type: string
91 description: Network from which to allocate fixed addresses.
92
93 fixed_subnet:
94 type: string
95 description: Subnet from which to allocate fixed addresses.
96
97 wait_condition_timeout:
98 type: number
99 description : >
100 timeout for the Wait Conditions
101
102 secgroup_kube_master_id:
103 type: string
104 description: ID of the security group for kubernetes master.
105
106 network_driver:
107 type: string 103 type: string
108 description: network driver to use for instantiating container networks 104 description: >
105 Discovery URL used for bootstrapping the etcd cluster.
109 106
110 tls_disabled: 107 tls_disabled:
111 type: boolean 108 type: boolean
@@ -117,7 +114,7 @@ parameters:
117 114
118 influx_grafana_dashboard_enabled: 115 influx_grafana_dashboard_enabled:
119 type: boolean 116 type: boolean
120 description: whether or not to disable kubernetes dashboard 117 description: Enable influxdb with grafana dashboard for data from heapster
121 118
122 verify_ca: 119 verify_ca:
123 type: boolean 120 type: boolean
@@ -128,25 +125,15 @@ parameters:
128 description: > 125 description: >
129 The port which are used by kube-apiserver to provide Kubernetes 126 The port which are used by kube-apiserver to provide Kubernetes
130 service. 127 service.
131 default: 6443
132
133 kube_version:
134 type: string
135 description: version of kubernetes used for kubernetes cluster
136
137 kube_dashboard_version:
138 type: string
139 description: version of kubernetes dashboard used for kubernetes cluster
140
141 hyperkube_image:
142 type: string
143 description: >
144 Docker registry used for hyperkube image
145 128
146 cluster_uuid: 129 cluster_uuid:
147 type: string 130 type: string
148 description: identifier for the cluster this template is generating 131 description: identifier for the cluster this template is generating
149 132
133 magnum_url:
134 type: string
135 description: endpoint to retrieve TLS certs from
136
150 prometheus_monitoring: 137 prometheus_monitoring:
151 type: boolean 138 type: boolean
152 description: > 139 description: >
@@ -158,10 +145,6 @@ parameters:
158 description: > 145 description: >
159 admin user password for the Grafana monitoring interface 146 admin user password for the Grafana monitoring interface
160 147
161 magnum_url:
162 type: string
163 description: endpoint to retrieve TLS certs from
164
165 api_public_address: 148 api_public_address:
166 type: string 149 type: string
167 description: Public IP address of the Kubernetes master server. 150 description: Public IP address of the Kubernetes master server.
@@ -172,6 +155,50 @@ parameters:
172 description: Private IP address of the Kubernetes master server. 155 description: Private IP address of the Kubernetes master server.
173 default: "" 156 default: ""
174 157
158 fixed_network:
159 type: string
160 description: Network from which to allocate fixed addresses.
161
162 fixed_subnet:
163 type: string
164 description: Subnet from which to allocate fixed addresses.
165
166 network_driver:
167 type: string
168 description: network driver to use for instantiating container networks
169
170 wait_condition_timeout:
171 type: number
172 description : >
173 timeout for the Wait Conditions
174
175 secgroup_kube_master_id:
176 type: string
177 description: ID of the security group for kubernetes master.
178
179 api_pool_id:
180 type: string
181 description: ID of the load balancer pool of k8s API server.
182
183 etcd_pool_id:
184 type: string
185 description: ID of the load balancer pool of etcd server.
186
187 auth_url:
188 type: string
189 description: >
190 url for kubernetes to authenticate
191
192 username:
193 type: string
194 description: >
195 user account
196
197 password:
198 type: string
199 description: >
200 user password
201
175 http_proxy: 202 http_proxy:
176 type: string 203 type: string
177 description: http proxy address for docker 204 description: http proxy address for docker
@@ -184,35 +211,45 @@ parameters:
184 type: string 211 type: string
185 description: no proxies for docker 212 description: no proxies for docker
186 213
214 kube_tag:
215 type: string
216 description: tag of the k8s containers used to provision the kubernetes cluster
217
218 etcd_tag:
219 type: string
220 description: tag of the etcd system container
221
222 kube_version:
223 type: string
224 description: version of kubernetes used for kubernetes cluster
225
226 kube_dashboard_version:
227 type: string
228 description: version of kubernetes dashboard used for kubernetes cluster
229
187 trustee_user_id: 230 trustee_user_id:
188 type: string 231 type: string
189 description: user id of the trustee 232 description: user id of the trustee
190 default: ""
191 233
192 trustee_password: 234 trustee_password:
193 type: string 235 type: string
194 description: password of the trustee 236 description: password of the trustee
195 default: ""
196 hidden: true 237 hidden: true
197 238
198 trust_id: 239 trust_id:
199 type: string 240 type: string
200 description: id of the trust which is used by the trustee 241 description: id of the trust which is used by the trustee
201 default: ""
202 hidden: true 242 hidden: true
203 243
204 auth_url:
205 type: string
206 description: url for keystone
207
208 insecure_registry_url: 244 insecure_registry_url:
209 type: string 245 type: string
210 description: insecure registry url 246 description: insecure registry url
211 247
212 container_runtime: 248 container_infra_prefix:
213 type: string 249 type: string
214 description: > 250 description: >
215 Container runtime to use with Kubernetes. 251 prefix of container images used in the cluster, kubernetes components,
252 kubernetes-dashboard, coredns etc
216 253
217 etcd_lb_vip: 254 etcd_lb_vip:
218 type: string 255 type: string
@@ -233,18 +270,101 @@ parameters:
233 openstack_ca: 270 openstack_ca:
234 type: string 271 type: string
235 description: The OpenStack CA certificate to install on the node. 272 description: The OpenStack CA certificate to install on the node.
273
236 nodes_server_group_id: 274 nodes_server_group_id:
237 type: string 275 type: string
238 description: ID of the server group for kubernetes cluster nodes. 276 description: ID of the server group for kubernetes cluster nodes.
239 277
278 availability_zone:
279 type: string
280 description: >
281 availability zone for master and nodes
282 default: ""
283
284 ca_key:
285 type: string
286 description: key of internal ca for the kube certificate api manager
287 hidden: true
288
289 cert_manager_api:
290 type: boolean
291 description: true if the kubernetes cert api manager should be enabled
292 default: false
293
294 calico_tag:
295 type: string
296 description: tag of the calico containers used to provision the calico node
297
298 calico_cni_tag:
299 type: string
300 description: tag of the cni used to provision the calico node
301
302 calico_kube_controllers_tag:
303 type: string
304 description: tag of the kube_controllers used to provision the calico node
305
306 calico_ipv4pool:
307 type: string
308 description: Configure the IP pool from which Pod IPs will be chosen
309
310 pods_network_cidr:
311 type: string
312 description: Configure the IP pool/range from which pod IPs will be chosen
313
314 ingress_controller:
315 type: string
316 description: >
317 ingress controller backend to use
318
319 ingress_controller_role:
320 type: string
321 description: >
322 node role where the ingress controller should run
323
324 kubelet_options:
325 type: string
326 description: >
327 additional options to be passed to the kubelet
328
329 kubeapi_options:
330 type: string
331 description: >
332 additional options to be passed to the api
333
334 kubecontroller_options:
335 type: string
336 description: >
337 additional options to be passed to the controller manager
338
339 kubeproxy_options:
340 type: string
341 description: >
342 additional options to be passed to the kube proxy
343
344 kubescheduler_options:
345 type: string
346 description: >
347 additional options to be passed to the scheduler
348
240 octavia_enabled: 349 octavia_enabled:
241 type: boolean 350 type: boolean
242 description: > 351 description: >
243 whether or not to use Octavia for LoadBalancer type service. 352 whether or not to use Octavia for LoadBalancer type service.
244 default: False 353 default: False
245 354
355 container_runtime:
356 type: string
357 description: >
358 Container runtime to use with Kubernetes.
359
360 hyperkube_image:
361 type: string
362 description: >
363 Docker registry used for hyperkube image
364
246resources: 365resources:
247 366
367
248 master_wait_handle: 368 master_wait_handle:
249 type: OS::Heat::WaitConditionHandle 369 type: OS::Heat::WaitConditionHandle
250 370
@@ -288,6 +408,10 @@ resources:
288 "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]} 408 "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_master_floating, floating_ip_address]}
289 "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]} 409 "$KUBE_NODE_IP": {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
290 "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} 410 "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
411 "$ETCD_VOLUME": {get_resource: etcd_volume}
412 "$ETCD_VOLUME_SIZE": {get_param: etcd_volume_size}
413 "$DOCKER_VOLUME": {get_resource: docker_volume}
414 "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size}
291 "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr} 415 "$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
292 "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen} 416 "$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
293 "$FLANNEL_BACKEND": {get_param: flannel_backend} 417 "$FLANNEL_BACKEND": {get_param: flannel_backend}
@@ -303,7 +427,7 @@ resources:
303 "$TLS_DISABLED": {get_param: tls_disabled} 427 "$TLS_DISABLED": {get_param: tls_disabled}
304 "$VERIFY_CA": {get_param: verify_ca} 428 "$VERIFY_CA": {get_param: verify_ca}
305 "$KUBE_DASHBOARD_ENABLED": {get_param: kube_dashboard_enabled} 429 "$KUBE_DASHBOARD_ENABLED": {get_param: kube_dashboard_enabled}
306 "$INFLUX_GRAFANA_DASHBOARD_ENABLED": {get_param: enable_influx_grafana_dashboard} 430 "$INFLUX_GRAFANA_DASHBOARD_ENABLED": {get_param: influx_grafana_dashboard_enabled}
307 "$KUBE_VERSION": {get_param: kube_version} 431 "$KUBE_VERSION": {get_param: kube_version}
308 "$KUBE_DASHBOARD_VERSION": {get_param: kube_dashboard_version} 432 "$KUBE_DASHBOARD_VERSION": {get_param: kube_dashboard_version}
309 "$CLUSTER_UUID": {get_param: cluster_uuid} 433 "$CLUSTER_UUID": {get_param: cluster_uuid}
@@ -330,6 +454,19 @@ resources:
330 "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain} 454 "$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}
331 "$OCTAVIA_ENABLED": {get_param: octavia_enabled} 455 "$OCTAVIA_ENABLED": {get_param: octavia_enabled}
332 456
457 write_kubeconfig:
458 type: OS::Heat::SoftwareConfig
459 properties:
460 group: ungrouped
461 config: {get_file: fragments/write-master-kubeconfig.yaml}
462
463 enable_docker_mount:
464 type: OS::Heat::SoftwareConfig
465 properties:
466 group: ungrouped
467 config: {get_file: fragments/enable-docker-mount.yaml}
468
469
333 add_ext_ca_certs: 470 add_ext_ca_certs:
334 type: OS::Heat::SoftwareConfig 471 type: OS::Heat::SoftwareConfig
335 properties: 472 properties:
@@ -439,6 +576,8 @@ resources:
439 template: | 576 template: |
440 $add_ext_ca_certs 577 $add_ext_ca_certs
441 $write_heat_params 578 $write_heat_params
579 $write_kubeconfig
580 $enable_docker_mount
442 $make_cert 581 $make_cert
443 $configure_docker 582 $configure_docker
444 $add_proxy 583 $add_proxy
@@ -460,6 +599,8 @@ resources:
460 command: "start" 599 command: "start"
461 - name: "make-cert.service" 600 - name: "make-cert.service"
462 command: "start" 601 command: "start"
602 - name: "enable-docker-mount.service"
603 command: "start"
463 - name: "configure-docker.service" 604 - name: "configure-docker.service"
464 command: "start" 605 command: "start"
465 - name: "add-proxy.service" 606 - name: "add-proxy.service"
@@ -491,6 +632,8 @@ resources:
491 params: 632 params:
492 "$add_ext_ca_certs": {get_attr: [add_ext_ca_certs, config]} 633 "$add_ext_ca_certs": {get_attr: [add_ext_ca_certs, config]}
493 "$write_heat_params": {get_attr: [write_heat_params, config]} 634 "$write_heat_params": {get_attr: [write_heat_params, config]}
635 "$write_kubeconfig": {get_attr: [write_kubeconfig, config]}
636 "$enable_docker_mount": {get_attr: [enable_docker_mount, config]}
494 "$make_cert": {get_attr: [make_cert, config]} 637 "$make_cert": {get_attr: [make_cert, config]}
495 "$configure_docker": {get_attr: [configure_docker, config]} 638 "$configure_docker": {get_attr: [configure_docker, config]}
496 "$add_proxy": {get_attr: [add_proxy, config]} 639 "$add_proxy": {get_attr: [add_proxy, config]}
@@ -562,6 +705,44 @@ resources:
562 subnet: { get_param: fixed_subnet } 705 subnet: { get_param: fixed_subnet }
563 protocol_port: 2379 706 protocol_port: 2379
564 707
708 ######################################################################
709 #
710 # etcd storage. This allocates a cinder volume and attaches it
711 # to the master.
712 #
713
714 etcd_volume:
715 type: Magnum::Optional::Etcd::Volume
716 properties:
717 size: {get_param: etcd_volume_size}
718
719 etcd_volume_attach:
720 type: Magnum::Optional::Etcd::VolumeAttachment
721 properties:
722 instance_uuid: {get_resource: kube-master}
723 volume_id: {get_resource: etcd_volume}
724 mountpoint: /dev/vdc
725
726 ######################################################################
727 #
728 # docker storage. This allocates a cinder volume and attaches it
729 # to the minion.
730 #
731
732 docker_volume:
733 type: Magnum::Optional::Cinder::Volume
734 properties:
735 size: {get_param: docker_volume_size}
736 volume_type: {get_param: docker_volume_type}
737
738 docker_volume_attach:
739 type: Magnum::Optional::Cinder::VolumeAttachment
740 properties:
741 instance_uuid: {get_resource: kube-master}
742 volume_id: {get_resource: docker_volume}
743 mountpoint: /dev/vdb
744
745
565outputs: 746outputs:
566 747
567 kube_master_ip: 748 kube_master_ip:
diff --git a/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml
index 749c00a..2bbea99 100644
--- a/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml
+++ b/magnum/drivers/k8s_coreos_v1/templates/kubeminion.yaml
@@ -1,9 +1,9 @@
1heat_template_version: 2014-10-16 1heat_template_version: 2014-10-16
2 2
3description: > 3description: >
4 This is a nested stack that defines a single Kubernetes minion, 4 This is a nested stack that defines a single Kubernetes minion, This stack is
5 based on a CoreOS cloud image. This stack is included by a ResourceGroup 5 included by an AutoScalingGroup resource in the parent template
6 resource in the parent template (kubecluster.yaml). 6 (kubecluster.yaml).
7 7
8parameters: 8parameters:
9 9
@@ -34,9 +34,22 @@ parameters:
34 constraints: 34 constraints:
35 - allowed_values: ["true", "false"] 35 - allowed_values: ["true", "false"]
36 36
37 network_driver: 37 docker_volume_size:
38 type: number
39 description: >
40 size of a cinder volume to allocate to docker for container/image
41 storage
42
43 docker_volume_type:
38 type: string 44 type: string
39 description: network driver to use for instantiating container networks 45 description: >
46 type of a cinder volume to allocate to docker for container/image
47 storage
48
49 docker_storage_driver:
50 type: string
51 description: docker storage driver name
52 default: "devicemapper"
40 53
41 tls_disabled: 54 tls_disabled:
42 type: boolean 55 type: boolean
@@ -51,7 +64,6 @@ parameters:
51 description: > 64 description: >
52 The port which are used by kube-apiserver to provide Kubernetes 65 The port which are used by kube-apiserver to provide Kubernetes
53 service. 66 service.
54 default: 6443
55 67
56 cluster_uuid: 68 cluster_uuid:
57 type: string 69 type: string
@@ -61,14 +73,10 @@ parameters:
61 type: string 73 type: string
62 description: endpoint to retrieve TLS certs from 74 description: endpoint to retrieve TLS certs from
63 75
64 kube_version: 76 prometheus_monitoring:
65 type: string 77 type: boolean
66 description: version of kubernetes used for kubernetes cluster
67
68 hyperkube_image:
69 type: string
70 description: > 78 description: >
71 Docker registry used for hyperkube image 79 whether or not to have the node-exporter running on the node
72 80
73 kube_master_ip: 81 kube_master_ip:
74 type: string 82 type: string
@@ -86,19 +94,71 @@ parameters:
86 type: string 94 type: string
87 description: Subnet from which to allocate fixed addresses. 95 description: Subnet from which to allocate fixed addresses.
88 96
97 network_driver:
98 type: string
99 description: network driver to use for instantiating container networks
100
89 flannel_network_cidr: 101 flannel_network_cidr:
90 type: string 102 type: string
91 description: network range for flannel overlay network 103 description: network range for flannel overlay network
92 104
93 wait_condition_timeout: 105 wait_condition_timeout:
94 type: number 106 type: number
95 description: > 107 description : >
96 timeout for the Wait Conditions 108 timeout for the Wait Conditions
97 109
110 registry_enabled:
111 type: boolean
112 description: >
113 Indicates whether the docker registry is enabled.
114
115 registry_port:
116 type: number
117 description: port of registry service
118
119 swift_region:
120 type: string
121 description: region of swift service
122
123 registry_container:
124 type: string
125 description: >
126 name of swift container which docker registry stores images in
127
128 registry_insecure:
129 type: boolean
130 description: >
131 indicates whether to skip TLS verification between registry and backend storage
132
133 registry_chunksize:
134 type: number
135 description: >
136 size fo the data segments for the swift dynamic large objects
137
98 secgroup_kube_minion_id: 138 secgroup_kube_minion_id:
99 type: string 139 type: string
100 description: ID of the security group for kubernetes minion. 140 description: ID of the security group for kubernetes minion.
101 141
142 volume_driver:
143 type: string
144 description: volume driver to use for container storage
145
146 region_name:
147 type: string
148 description: A logically separate section of the cluster
149
150 username:
151 type: string
152 description: >
153 user account
154
155 password:
156 type: string
157 description: >
158 user password, not set in current implementation, only used to
159 fill in for Kubernetes config file
160 hidden: true
161
102 http_proxy: 162 http_proxy:
103 type: string 163 type: string
104 description: http proxy address for docker 164 description: http proxy address for docker
@@ -111,40 +171,55 @@ parameters:
111 type: string 171 type: string
112 description: no proxies for docker 172 description: no proxies for docker
113 173
174 kube_tag:
175 type: string
176 description: tag of the k8s containers used to provision the kubernetes cluster
177
178 flannel_tag:
179 type: string
180 description: tag of the flannel system containers
181
182 kube_version:
183 type: string
184 description: version of kubernetes used for kubernetes cluster
185
186 trustee_domain_id:
187 type: string
188 description: domain id of the trustee
189
114 trustee_user_id: 190 trustee_user_id:
115 type: string 191 type: string
116 description: user id of the trustee 192 description: user id of the trustee
117 default: "" 193
194 trustee_username:
195 type: string
196 description: username of the trustee
118 197
119 trustee_password: 198 trustee_password:
120 type: string 199 type: string
121 description: password of the trustee 200 description: password of the trustee
122 default: ""
123 hidden: true 201 hidden: true
124 202
125 trust_id: 203 trust_id:
126 type: string 204 type: string
127 description: id of the trust which is used by the trustee 205 description: id of the trust which is used by the trustee
128 default: ""
129 hidden: true 206 hidden: true
130 207
131 auth_url: 208 auth_url:
132 type: string 209 type: string
133 description: url for keystone 210 description: >
211 url for keystone, must be v2 since k8s backend only support v2
212 at this point
134 213
135 insecure_registry_url: 214 insecure_registry_url:
136 type: string 215 type: string
137 description: insecure registry url 216 description: insecure registry url
138 217
139 container_runtime: 218 container_infra_prefix:
140 type: string 219 type: string
141 description: > 220 description: >
142 Container runtime to use with Kubernetes. 221 prefix of container images used in the cluster, kubernetes components,
143 222 kubernetes-dashboard, coredns etc
144 prometheus_monitoring:
145 type: boolean
146 description: >
147 whether or not to have the node-exporter running on the node
148 223
149 dns_service_ip: 224 dns_service_ip:
150 type: string 225 type: string
@@ -164,14 +239,45 @@ parameters:
164 type: string 239 type: string
165 description: ID of the server group for kubernetes cluster nodes. 240 description: ID of the server group for kubernetes cluster nodes.
166 241
242 availability_zone:
243 type: string
244 description: >
245 availability zone for master and nodes
246 default: ""
247
248 pods_network_cidr:
249 type: string
250 description: Configure the IP pool/range from which pod IPs will be chosen
251
252 kubelet_options:
253 type: string
254 description: >
255 additional options to be passed to the kubelet
256
257 kubeproxy_options:
258 type: string
259 description: >
260 additional options to be passed to the kube proxy
261
167 octavia_enabled: 262 octavia_enabled:
168 type: boolean 263 type: boolean
169 description: > 264 description: >
170 whether or not to use Octavia for LoadBalancer type service. 265 whether or not to use Octavia for LoadBalancer type service.
171 default: False 266 default: False
172 267
268 container_runtime:
269 type: string
270 description: >
271 Container runtime to use with Kubernetes.
272
273 hyperkube_image:
274 type: string
275 description: >
276 Docker registry used for hyperkube image
277
173resources: 278resources:
174 279
280
175 minion_wait_handle: 281 minion_wait_handle:
176 type: OS::Heat::WaitConditionHandle 282 type: OS::Heat::WaitConditionHandle
177 283
@@ -197,6 +303,8 @@ resources:
197 template: {get_file: fragments/write-heat-params.yaml} 303 template: {get_file: fragments/write-heat-params.yaml}
198 params: 304 params:
199 "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv} 305 "$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
306 "$DOCKER_VOLUME": {get_resource: docker_volume}
307 "$DOCKER_VOLUME_SIZE": {get_param: docker_volume_size}
200 "$KUBE_MASTER_IP": {get_param: kube_master_ip} 308 "$KUBE_MASTER_IP": {get_param: kube_master_ip}
201 "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_minion_floating, floating_ip_address]} 309 "$KUBE_NODE_PUBLIC_IP": {get_attr: [kube_minion_floating, floating_ip_address]}
202 "$KUBE_NODE_IP": {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} 310 "$KUBE_NODE_IP": {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
@@ -240,11 +348,23 @@ resources:
240 $OPENSTACK_CA: {get_param: openstack_ca} 348 $OPENSTACK_CA: {get_param: openstack_ca}
241 template: {get_file: fragments/add-ext-ca-certs.yaml} 349 template: {get_file: fragments/add-ext-ca-certs.yaml}
242 350
351 enable_docker_mount:
352 type: OS::Heat::SoftwareConfig
353 properties:
354 group: ungrouped
355 config: {get_file: fragments/enable-docker-mount.yaml}
356
243 write_kubeconfig: 357 write_kubeconfig:
244 type: OS::Heat::SoftwareConfig 358 type: OS::Heat::SoftwareConfig
245 properties: 359 properties:
246 group: ungrouped 360 group: ungrouped
247 config: {get_file: fragments/write-kubeconfig.yaml} 361 config:
362 str_replace:
363 template: {get_file: fragments/write-kubeconfig.yaml}
364 params:
365 "$KUBE_API_PORT": {get_param: kubernetes_port}
366 "$KUBE_MASTER_IP": {get_param: kube_master_ip}
367
248 368
249 make_cert: 369 make_cert:
250 type: OS::Heat::SoftwareConfig 370 type: OS::Heat::SoftwareConfig
@@ -297,6 +417,7 @@ resources:
297 template: | 417 template: |
298 $add_ext_ca_certs 418 $add_ext_ca_certs
299 $write_heat_params 419 $write_heat_params
420 $enable_docker_mount
300 $write_kubeconfig 421 $write_kubeconfig
301 $make_cert 422 $make_cert
302 $configure_docker 423 $configure_docker
@@ -311,6 +432,8 @@ resources:
311 command: "start" 432 command: "start"
312 - name: "make-cert.service" 433 - name: "make-cert.service"
313 command: "start" 434 command: "start"
435 - name: "enable-docker-mount.service"
436 command: "start"
314 - name: "configure-docker.service" 437 - name: "configure-docker.service"
315 command: "start" 438 command: "start"
316 - name: "add-proxy.service" 439 - name: "add-proxy.service"
@@ -327,6 +450,7 @@ resources:
327 "$add_ext_ca_certs": {get_attr: [add_ext_ca_certs, config]} 450 "$add_ext_ca_certs": {get_attr: [add_ext_ca_certs, config]}
328 "$write_heat_params": {get_attr: [write_heat_params, config]} 451 "$write_heat_params": {get_attr: [write_heat_params, config]}
329 "$write_kubeconfig": {get_attr: [write_kubeconfig, config]} 452 "$write_kubeconfig": {get_attr: [write_kubeconfig, config]}
453 "$enable_docker_mount": {get_attr: [enable_docker_mount, config]}
330 "$make_cert": {get_attr: [make_cert, config]} 454 "$make_cert": {get_attr: [make_cert, config]}
331 "$configure_docker": {get_attr: [configure_docker, config]} 455 "$configure_docker": {get_attr: [configure_docker, config]}
332 "$add_proxy": {get_attr: [add_proxy, config]} 456 "$add_proxy": {get_attr: [add_proxy, config]}
@@ -369,12 +493,31 @@ resources:
369 floating_network: {get_param: external_network} 493 floating_network: {get_param: external_network}
370 port_id: {get_resource: kube_minion_eth0} 494 port_id: {get_resource: kube_minion_eth0}
371 495
496 ######################################################################
497 #
498 # docker storage. This allocates a cinder volume and attaches it
499 # to the minion.
500 #
501
502 docker_volume:
503 type: Magnum::Optional::Cinder::Volume
504 properties:
505 size: {get_param: docker_volume_size}
506 volume_type: {get_param: docker_volume_type}
507
508 docker_volume_attach:
509 type: Magnum::Optional::Cinder::VolumeAttachment
510 properties:
511 instance_uuid: {get_resource: kube-minion}
512 volume_id: {get_resource: docker_volume}
513 mountpoint: /dev/vdb
514
372outputs: 515outputs:
373 516
374 kube_minion_ip: 517 kube_minion_ip:
375 value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]} 518 value: {get_attr: [kube_minion_eth0, fixed_ips, 0, ip_address]}
376 description: > 519 description: >
377 This is the "private" IP address of the Kubernetes minion node. 520 This is the "public" IP address of the Kubernetes minion node.
378 521
379 kube_minion_external_ip: 522 kube_minion_external_ip:
380 value: {get_attr: [kube_minion_floating, floating_ip_address]} 523 value: {get_attr: [kube_minion_floating, floating_ip_address]}
diff --git a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
index a793c09..8a6989b 100644
--- a/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
+++ b/magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
@@ -518,7 +518,12 @@ class TestClusterConductorWithK8s(base.TestCase):
518 'external_network': 'external_network_id', 518 'external_network': 'external_network_id',
519 'fixed_network': 'fixed_network', 519 'fixed_network': 'fixed_network',
520 'fixed_subnet': 'fixed_subnet', 520 'fixed_subnet': 'fixed_subnet',
521 'availability_zone': 'az_1',
522 'nodes_affinity_policy': 'soft-anti-affinity',
521 'dns_nameserver': 'dns_nameserver', 523 'dns_nameserver': 'dns_nameserver',
524 'docker_storage_driver': 'devicemapper',
525 'docker_volume_size': 20,
526 'docker_volume_type': 'lvmdriver-1',
522 'server_image': 'image_id', 527 'server_image': 'image_id',
523 'minion_flavor': 'flavor_id', 528 'minion_flavor': 'flavor_id',
524 'master_flavor': 'master_flavor_id', 529 'master_flavor': 'master_flavor_id',
@@ -538,6 +543,7 @@ class TestClusterConductorWithK8s(base.TestCase):
538 'system_pods_timeout': '1', 543 'system_pods_timeout': '1',
539 'admission_control_list': 'fake_list', 544 'admission_control_list': 'fake_list',
540 'prometheus_monitoring': 'False', 545 'prometheus_monitoring': 'False',
546 'region_name': 'RegionOne',
541 'grafana_admin_passwd': 'fake_pwd', 547 'grafana_admin_passwd': 'fake_pwd',
542 'kube_dashboard_enabled': 'True', 548 'kube_dashboard_enabled': 'True',
543 'influx_grafana_dashboard_enabled': 'True', 549 'influx_grafana_dashboard_enabled': 'True',
@@ -547,6 +553,7 @@ class TestClusterConductorWithK8s(base.TestCase):
547 'trustee_username': 'fake_trustee', 553 'trustee_username': 'fake_trustee',
548 'trustee_password': 'fake_trustee_password', 554 'trustee_password': 'fake_trustee_password',
549 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', 555 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
556 'username': 'fake_user',
550 'trust_id': '', 557 'trust_id': '',
551 'auth_url': 'http://192.168.10.10:5000/v3', 558 'auth_url': 'http://192.168.10.10:5000/v3',
552 'cluster_uuid': self.cluster_dict['uuid'], 559 'cluster_uuid': self.cluster_dict['uuid'],
@@ -555,6 +562,7 @@ class TestClusterConductorWithK8s(base.TestCase):
555 'kube_version': 'fake-version', 562 'kube_version': 'fake-version',
556 'verify_ca': True, 563 'verify_ca': True,
557 'openstack_ca': '', 564 'openstack_ca': '',
565 'openstack_ca_coreos': '',
558 'cert_manager_api': 'False', 566 'cert_manager_api': 'False',
559 'ingress_controller': 'i-controller', 567 'ingress_controller': 'i-controller',
560 'ingress_controller_role': 'i-controller-role', 568 'ingress_controller_role': 'i-controller-role',
@@ -568,6 +576,8 @@ class TestClusterConductorWithK8s(base.TestCase):
568 self.assertEqual(expected, definition) 576 self.assertEqual(expected, definition)
569 self.assertEqual( 577 self.assertEqual(
570 ['../../common/templates/environments/no_private_network.yaml', 578 ['../../common/templates/environments/no_private_network.yaml',
579 '../../common/templates/environments/no_etcd_volume.yaml',
580 '../../common/templates/environments/with_volume.yaml',
571 '../../common/templates/environments/no_master_lb.yaml', 581 '../../common/templates/environments/no_master_lb.yaml',
572 '../../common/templates/environments/disable_floating_ip.yaml'], 582 '../../common/templates/environments/disable_floating_ip.yaml'],
573 env_files) 583 env_files)
@@ -599,10 +609,14 @@ class TestClusterConductorWithK8s(base.TestCase):
599 609
600 expected = { 610 expected = {
601 'ssh_key_name': 'keypair_id', 611 'ssh_key_name': 'keypair_id',
612 'availability_zone': 'az_1',
602 'external_network': 'external_network_id', 613 'external_network': 'external_network_id',
603 'fixed_network': 'fixed_network', 614 'fixed_network': 'fixed_network',
604 'fixed_subnet': 'fixed_subnet', 615 'fixed_subnet': 'fixed_subnet',
605 'dns_nameserver': 'dns_nameserver', 616 'dns_nameserver': 'dns_nameserver',
617 'docker_storage_driver': u'devicemapper',
618 'docker_volume_size': 20,
619 'docker_volume_type': u'lvmdriver-1',
606 'server_image': 'image_id', 620 'server_image': 'image_id',
607 'minion_flavor': 'flavor_id', 621 'minion_flavor': 'flavor_id',
608 'master_flavor': 'master_flavor_id', 622 'master_flavor': 'master_flavor_id',
@@ -615,6 +629,7 @@ class TestClusterConductorWithK8s(base.TestCase):
615 'http_proxy': 'http_proxy', 629 'http_proxy': 'http_proxy',
616 'https_proxy': 'https_proxy', 630 'https_proxy': 'https_proxy',
617 'no_proxy': 'no_proxy', 631 'no_proxy': 'no_proxy',
632 'nodes_affinity_policy': 'soft-anti-affinity',
618 'flannel_network_cidr': '10.101.0.0/16', 633 'flannel_network_cidr': '10.101.0.0/16',
619 'flannel_network_subnetlen': '26', 634 'flannel_network_subnetlen': '26',
620 'flannel_backend': 'vxlan', 635 'flannel_backend': 'vxlan',
@@ -622,6 +637,7 @@ class TestClusterConductorWithK8s(base.TestCase):
622 'system_pods_timeout': '1', 637 'system_pods_timeout': '1',
623 'admission_control_list': 'fake_list', 638 'admission_control_list': 'fake_list',
624 'prometheus_monitoring': 'False', 639 'prometheus_monitoring': 'False',
640 'region_name': self.mock_osc.cinder_region_name.return_value,
625 'grafana_admin_passwd': 'fake_pwd', 641 'grafana_admin_passwd': 'fake_pwd',
626 'kube_dashboard_enabled': 'True', 642 'kube_dashboard_enabled': 'True',
627 'influx_grafana_dashboard_enabled': 'True', 643 'influx_grafana_dashboard_enabled': 'True',
@@ -629,6 +645,7 @@ class TestClusterConductorWithK8s(base.TestCase):
629 'registry_enabled': False, 645 'registry_enabled': False,
630 'trustee_domain_id': self.mock_keystone.trustee_domain_id, 646 'trustee_domain_id': self.mock_keystone.trustee_domain_id,
631 'trustee_username': 'fake_trustee', 647 'trustee_username': 'fake_trustee',
648 'username': 'fake_user',
632 'trustee_password': 'fake_trustee_password', 649 'trustee_password': 'fake_trustee_password',
633 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656', 650 'trustee_user_id': '7b489f04-b458-4541-8179-6a48a553e656',
634 'trust_id': '', 651 'trust_id': '',
@@ -639,6 +656,7 @@ class TestClusterConductorWithK8s(base.TestCase):
639 'kube_version': 'fake-version', 656 'kube_version': 'fake-version',
640 'verify_ca': True, 657 'verify_ca': True,
641 'openstack_ca': '', 658 'openstack_ca': '',
659 'openstack_ca_coreos': '',
642 'cert_manager_api': 'False', 660 'cert_manager_api': 'False',
643 'ingress_controller': 'i-controller', 661 'ingress_controller': 'i-controller',
644 'ingress_controller_role': 'i-controller-role', 662 'ingress_controller_role': 'i-controller-role',
@@ -652,6 +670,8 @@ class TestClusterConductorWithK8s(base.TestCase):
652 self.assertEqual(expected, definition) 670 self.assertEqual(expected, definition)
653 self.assertEqual( 671 self.assertEqual(
654 ['../../common/templates/environments/no_private_network.yaml', 672 ['../../common/templates/environments/no_private_network.yaml',
673 '../../common/templates/environments/no_etcd_volume.yaml',
674 '../../common/templates/environments/with_volume.yaml',
655 '../../common/templates/environments/no_master_lb.yaml', 675 '../../common/templates/environments/no_master_lb.yaml',
656 '../../common/templates/environments/disable_floating_ip.yaml'], 676 '../../common/templates/environments/disable_floating_ip.yaml'],
657 env_files) 677 env_files)