openstack
/
magnum
Code Issues Proposed changes

[k8s] allow enabling kubernetes cert manager api 

Add a new label 'cert_manager_api' to kubernetes clusters controlling the
enable/disable of the kubernetes certificate manager api.

The same cluster cert/key pair is used by this api. The heat agent is used
to install the key in the master node(s), as this is required for kubernetes
to later sign new certificate requests.

The master template init order is changed so the heat agent is launched
previous to enabling the services - the controller manager requires the CA key
to be locally available before being launched.

Change-Id: Ibf85147316e3a194d8a3f92cbb4ae9ce8e16c98f
Partial-Bug: #1734318
This commit is contained in:
Ricardo Rocha 2017-12-22 11:07:51 +00:00 committed by Spyros Trigazis (strigazi)
parent c6e7b290ab
commit faa9e90402
12 changed files with 119 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
doc/source/user/index.rst
View File

@ -348,10 +348,11 @@ the table are linked to more details elsewhere in the user guide.
+---------------------------------------+--------------------+---------------+
| `container_infra_prefix`_ | see below | "" |
+---------------------------------------+--------------------+---------------+
+---------------------------------------+--------------------+---------------+
| `availability_zone`_ | AZ for the cluster | "" |
| | nodes | |
+---------------------------------------+--------------------+---------------+
| `cert_manager_api`_ | see below | false |
+---------------------------------------+--------------------+---------------+
Cluster
-------
@ -1113,6 +1114,10 @@ _`kube_dashboard_enabled`
This label triggers the deployment of the kubernetes dashboard.
The default value is 1, meaning it will be enabled.
_`cert_manager_api`
This label enables the kubernetes `certificate manager api
<https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/>`_.
External load balancer for services
-----------------------------------

4
magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
View File

@ -57,6 +57,10 @@ if [ -n "$TRUST_ID" ]; then
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cloud-config=/etc/kubernetes/kube_openstack_config --cloud-provider=openstack"
fi
if [ -n "$CERT_MANAGER_API" ]; then
KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-signing-cert-file=$CERT_DIR/ca.crt --cluster-signing-key-file=$CERT_DIR/ca.key"
fi
sed -i '
/^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/
/^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#

15
magnum/drivers/common/templates/kubernetes/fragments/enable-cert-api-manager Normal file
View File

@ -0,0 +1,15 @@
#!/bin/bash
. /etc/sysconfig/heat-params
if [ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" = "false" ]; then
exit 0
fi
cert_dir=/etc/kubernetes/certs
echo -e "$CA_KEY" > ${cert_dir}/ca.key
chown kube.kube ${cert_dir}/ca.key
chmod 400 ${cert_dir}/ca.key

10
magnum/drivers/common/templates/kubernetes/fragments/enable-services-master.sh
View File

@ -1,8 +1,18 @@
#!/bin/sh
. /etc/sysconfig/heat-params
# make sure we pick up any modified unit files
systemctl daemon-reload
# if the certificate manager api is enabled, wait for the ca key to be handled
# by the heat container agent (required for the controller-manager)
while [ ! -f /etc/kubernetes/certs/ca.key ] && \
[ "$(echo $CERT_MANAGER_API | tr '[:upper:]' '[:lower:]')" == "true" ]; do
echo "waiting for CA to be made available for certificate manager api"
sleep 2
done
echo "starting services"
for service in etcd docker kube-apiserver kube-controller-manager kube-scheduler; do
echo "activating service $service"

2
magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.yaml
View File

@ -52,3 +52,5 @@ write_files:
ETCD_LB_VIP="$ETCD_LB_VIP"
DNS_SERVICE_IP="$DNS_SERVICE_IP"
DNS_CLUSTER_DOMAIN="$DNS_CLUSTER_DOMAIN"
CERT_MANAGER_API="$CERT_MANAGER_API"
CA_KEY="$CA_KEY"

11
magnum/drivers/heat/k8s_fedora_template_def.py
View File

@ -11,7 +11,10 @@
# under the License.
from oslo_log import log as logging
from oslo_utils import strutils
from magnum.common.x509 import operations as x509
from magnum.conductor.handlers.common import cert_manager
from magnum.drivers.heat import k8s_template_def
from magnum.drivers.heat import template_def
from oslo_config import cfg
@ -88,6 +91,14 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
if label_value:
extra_params[label] = label_value
cert_manager_api = cluster.labels.get('cert_manager_api')
if strutils.bool_from_string(cert_manager_api):
extra_params['cert_manager_api'] = cert_manager_api
ca_cert = cert_manager.get_cluster_ca_certificate(cluster)
extra_params['ca_key'] = x509.decrypt_key(
ca_cert.get_private_key(),
ca_cert.get_private_key_passphrase()).replace("\n", "\\n")
return super(K8sFedoraTemplateDefinition,
self).get_params(context, cluster_template, cluster,
extra_params=extra_params,

3
magnum/drivers/heat/k8s_template_def.py
View File

@ -113,7 +113,8 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
'prometheus_monitoring',
'grafana_admin_passwd',
'kube_dashboard_enabled',
'etcd_volume_size']
'etcd_volume_size',
'cert_manager_api']
for label in label_list:
extra_params[label] = cluster.labels.get(label)

13
magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml
View File

@ -370,6 +370,17 @@ parameters:
availability zone for master and nodes
default: ""
cert_manager_api:
type: boolean
description: true if the kubernetes cert api manager should be enabled
default: false
ca_key:
type: string
description: key of internal ca for the kube certificate api manager
default: ""
hidden: true
resources:
######################################################################
@ -564,6 +575,8 @@ resources:
openstack_ca: {get_param: openstack_ca}
nodes_server_group_id: {get_resource: nodes_server_group}
availability_zone: {get_param: availability_zone}
ca_key: {get_param: ca_key}
cert_manager_api: {get_param: cert_manager_api}
######################################################################
#

32
magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml
View File

@ -273,6 +273,16 @@ parameters:
availability zone for master and nodes
default: ""
ca_key:
type: string
description: key of internal ca for the kube certificate api manager
hidden: true
cert_manager_api:
type: boolean
description: true if the kubernetes cert api manager should be enabled
default: false
resources:
master_wait_handle:
@ -359,6 +369,8 @@ resources:
"$ETCD_LB_VIP": {get_param: etcd_lb_vip}
"$DNS_SERVICE_IP": {get_param: dns_service_ip}
"$DNS_CLUSTER_DOMAIN": {get_param: dns_cluster_domain}
"$CERT_MANAGER_API": {get_param: cert_manager_api}
"$CA_KEY": {get_param: ca_key}
install_openstack_ca:
type: OS::Heat::SoftwareConfig
@ -483,6 +495,7 @@ resources:
- config: {get_resource: configure_docker_storage}
- config: {get_resource: configure_kubernetes}
- config: {get_resource: add_proxy}
- config: {get_resource: start_container_agent}
- config: {get_resource: enable_services}
- config: {get_resource: write_network_config}
- config: {get_resource: network_config_service}
@ -490,7 +503,6 @@ resources:
- config: {get_resource: kube_apiserver_to_kubelet_role}
- config: {get_resource: core_dns_service}
- config: {get_resource: kube_ui_service}
- config: {get_resource: start_container_agent}
- config: {get_resource: master_wc_notify}
enable_prometheus_monitoring:
@ -511,6 +523,24 @@ resources:
server: {get_resource: kube-master}
actions: ['CREATE']
enable_cert_manager_api:
type: OS::Heat::SoftwareConfig
properties:
group: script
config:
str_replace:
template: {get_file: ../../common/templates/kubernetes/fragments/enable-cert-api-manager}
params:
"$CA_KEY": {get_param: ca_key}
enable_cert_manager_api_deployment:
type: OS::Heat::SoftwareDeployment
properties:
signal_transport: HEAT_SIGNAL
config: {get_resource: enable_cert_manager_api}
server: {get_resource: kube-master}
actions: ['CREATE']
######################################################################
#
# a single kubernetes master.

24
magnum/tests/unit/conductor/handlers/test_k8s_cluster_conductor.py
View File

@ -97,7 +97,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'grafana_admin_passwd': 'fake_pwd',
'kube_dashboard_enabled': 'True',
'docker_volume_type': 'lvmdriver-1',
'availability_zone': 'az_1'},
'availability_zone': 'az_1',
'cert_manager_api': 'False'},
'master_flavor_id': 'master_flavor_id',
'flavor_id': 'flavor_id',
}
@ -177,7 +178,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'kube_dashboard_enabled': 'True',
'docker_volume_type': 'lvmdriver-1',
'etcd_volume_size': None,
'availability_zone': 'az_1'},
'availability_zone': 'az_1',
'cert_manager_api': 'False'},
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
'no_proxy': 'no_proxy',
@ -233,7 +235,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity",
'availability_zone': 'az_1'
'availability_zone': 'az_1',
'cert_manager_api': 'False',
}
if missing_attr is not None:
expected.pop(mapping[missing_attr], None)
@ -331,7 +334,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity",
'availability_zone': 'az_1'
'availability_zone': 'az_1',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
@ -416,7 +420,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity",
'availability_zone': 'az_1'
'availability_zone': 'az_1',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -494,7 +499,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'insecure_registry_url': '10.0.0.1:5000',
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': ''
'openstack_ca': '',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -567,7 +573,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'insecure_registry_url': '10.0.0.1:5000',
'kube_version': 'fake-version',
'verify_ca': True,
'openstack_ca': ''
'openstack_ca': '',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
self.assertEqual(
@ -741,7 +748,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'verify_ca': True,
'openstack_ca': '',
"nodes_affinity_policy": "soft-anti-affinity",
'availability_zone': 'az_1'
'availability_zone': 'az_1',
'cert_manager_api': 'False',
}
self.assertEqual(expected, definition)
self.assertEqual(

6
magnum/tests/unit/drivers/test_template_definition.py
View File

@ -274,6 +274,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'container_infra_prefix')
availability_zone = mock_cluster.labels.get(
'availability_zone')
cert_manager_api = mock_cluster.labels.get('cert_manager_api')
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
@ -301,7 +302,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'container_infra_prefix': container_infra_prefix,
'nodes_affinity_policy': 'soft-anti-affinity',
'availability_zone': availability_zone,
}}
'cert_manager_api': cert_manager_api}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,
@ -362,6 +363,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'container_infra_prefix')
availability_zone = mock_cluster.labels.get(
'availability_zone')
cert_manager_api = mock_cluster.labels.get('cert_manager_api')
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
@ -391,7 +393,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'container_infra_prefix': container_infra_prefix,
'nodes_affinity_policy': 'soft-anti-affinity',
'availability_zone': availability_zone,
}}
'cert_manager_api': cert_manager_api}}
mock_get_params.assert_called_once_with(mock_context,
mock_cluster_template,
mock_cluster,

5
releasenotes/notes/cert-manager-api-ee0cf7f3b767bb5d.yaml Normal file
View File

@ -0,0 +1,5 @@
---
features:
- |
Add new label 'cert_manager_api' enabling the kubernetes certificate
manager api.