Make Kubernetes pods' health checks configurable

Make Kubernetes' kube-controller-manager and kube-scheduler
health checks configurable as a parameter to the cluster-template
(label).
Set their value higher for all deployments. And set their value
to a high number for tests, for the CI.

Change-Id: I65e2da12487c513419125f0525a4e21bac22210e
Closes-Bug: 1648826
This commit is contained in:
Bertrand NOEL 2016-12-09 15:18:01 +01:00 committed by Spyros Trigazis
parent 48f3638055
commit 2a4ad9aa41
20 changed files with 188 additions and 12 deletions

View File

@ -25,3 +25,5 @@ write_files:
KUBE_VERSION="$KUBE_VERSION"
CLUSTER_UUID="$CLUSTER_UUID"
MAGNUM_URL="$MAGNUM_URL"
SYSTEM_PODS_INITIAL_DELAY="$SYSTEM_PODS_INITIAL_DELAY"
SYSTEM_PODS_TIMEOUT="$SYSTEM_PODS_TIMEOUT"

View File

@ -89,6 +89,20 @@ parameters:
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
kube_allow_priv:
type: string
description: >
@ -499,6 +513,8 @@ resources:
flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
flannel_network_subnet_min: {get_param: flannel_network_subnet_min}
flannel_network_subnet_max: {get_param: flannel_network_subnet_max}
system_pods_initial_delay: {get_param: system_pods_initial_delay}
system_pods_timeout: {get_param: system_pods_timeout}
portal_network_cidr: {get_param: portal_network_cidr}
discovery_url: {get_param: discovery_url}
cluster_uuid: {get_param: cluster_uuid}

View File

@ -66,6 +66,20 @@ parameters:
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
discovery_url:
type: string
description: >
@ -210,6 +224,8 @@ resources:
"$FLANNEL_NETWORK_SUBNET_MIN": {get_param: flannel_network_subnet_min}
"$FLANNEL_NETWORK_SUBNET_MAX": {get_param: flannel_network_subnet_max}
"$FLANNEL_BACKEND": {get_param: flannel_backend}
"$SYSTEM_PODS_INITIAL_DELAY": {get_param: system_pods_initial_delay}
"$SYSTEM_PODS_TIMEOUT": {get_param: system_pods_timeout}
"$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}
"$AUTH_URL": {get_param: auth_url}

View File

@ -56,8 +56,8 @@ $(generate_pod_args " - " $KUBE_LOGTOSTDERR $KUBE_LOG_LEVEL $KUBE_MASTER $KUB
host: 127.0.0.1
path: /healthz
port: 10252
initialDelaySeconds: 15
timeoutSeconds: 1
initialDelaySeconds: ${SYSTEM_PODS_INITIAL_DELAY}
timeoutSeconds: ${SYSTEM_PODS_TIMEOUT}
volumeMounts:
- mountPath: /etc/ssl/certs
name: ssl-certs-host
@ -108,8 +108,8 @@ $(generate_pod_args " - " $KUBE_LOGTOSTDERR $KUBE_LOG_LEVEL $KUBE_MASTER $KUB
host: 127.0.0.1
path: /healthz
port: 10251
initialDelaySeconds: 15
timeoutSeconds: 1
initialDelaySeconds: ${SYSTEM_PODS_INITIAL_DELAY}
timeoutSeconds: ${SYSTEM_PODS_TIMEOUT}
volumeMounts:
- mountPath: /etc/ssl/certs
name: ssl-certs-host

View File

@ -38,3 +38,5 @@ write_files:
TRUST_ID="$TRUST_ID"
AUTH_URL="$AUTH_URL"
INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL"
SYSTEM_PODS_INITIAL_DELAY="$SYSTEM_PODS_INITIAL_DELAY"
SYSTEM_PODS_TIMEOUT="$SYSTEM_PODS_TIMEOUT"

View File

@ -106,7 +106,9 @@ class K8sTemplateDefinition(template_def.BaseTemplateDefinition):
extra_params['kubernetes_port'] = 8080
label_list = ['flannel_network_cidr', 'flannel_backend',
'flannel_network_subnetlen']
'flannel_network_subnetlen',
'system_pods_initial_delay',
'system_pods_timeout']
for label in label_list:
extra_params[label] = cluster_template.labels.get(label)

View File

@ -59,8 +59,8 @@ write_files:
host: 127.0.0.1
path: /healthz
port: 10252
initialDelaySeconds: 15
timeoutSeconds: 1
initialDelaySeconds: ${SYSTEM_PODS_INITIAL_DELAY}
timeoutSeconds: ${SYSTEM_PODS_TIMEOUT}
volumeMounts:
- mountPath: /etc/kubernetes/ssl
name: ssl-certs-kubernetes

View File

@ -46,6 +46,6 @@ write_files:
host: 127.0.0.1
path: /healthz
port: 10251
initialDelaySeconds: 15
timeoutSeconds: 1
initialDelaySeconds: ${SYSTEM_PODS_INITIAL_DELAY}
timeoutSeconds: ${SYSTEM_PODS_TIMEOUT}
EOF

View File

@ -37,3 +37,5 @@ write_files:
TRUST_ID="$TRUST_ID"
AUTH_URL="$AUTH_URL"
INSECURE_REGISTRY_URL="$INSECURE_REGISTRY_URL"
SYSTEM_PODS_INITIAL_DELAY="$SYSTEM_PODS_INITIAL_DELAY"
SYSTEM_PODS_TIMEOUT="$SYSTEM_PODS_TIMEOUT"

View File

@ -90,6 +90,20 @@ parameters:
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
kube_allow_priv:
type: string
description: >
@ -340,6 +354,8 @@ resources:
flannel_network_cidr: {get_param: flannel_network_cidr}
flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
flannel_backend: {get_param: flannel_backend}
system_pods_initial_delay: {get_param: system_pods_initial_delay}
system_pods_timeout: {get_param: system_pods_timeout}
portal_network_cidr: {get_param: portal_network_cidr}
fixed_network: {get_attr: [network, fixed_network]}
fixed_subnet: {get_attr: [network, fixed_subnet]}

View File

@ -63,6 +63,20 @@ parameters:
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
fixed_network:
type: string
description: Network from which to allocate fixed addresses.
@ -199,6 +213,8 @@ resources:
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
"$FLANNEL_BACKEND": {get_param: flannel_backend}
"$SYSTEM_PODS_INITIAL_DELAY": {get_param: system_pods_initial_delay}
"$SYSTEM_PODS_TIMEOUT": {get_param: system_pods_timeout}
"$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
"$CLUSTER_SUBNET": {get_param: fixed_subnet}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}

View File

@ -89,6 +89,20 @@ parameters:
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
kube_allow_priv:
type: string
description: >
@ -411,6 +425,8 @@ resources:
flannel_network_cidr: {get_param: flannel_network_cidr}
flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
flannel_backend: {get_param: flannel_backend}
system_pods_initial_delay: {get_param: system_pods_initial_delay}
system_pods_timeout: {get_param: system_pods_timeout}
portal_network_cidr: {get_param: portal_network_cidr}
discovery_url: {get_param: discovery_url}
cluster_uuid: {get_param: cluster_uuid}

View File

@ -63,6 +63,20 @@ parameters:
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
discovery_url:
type: string
description: >
@ -232,6 +246,8 @@ resources:
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
"$FLANNEL_BACKEND": {get_param: flannel_backend}
"$SYSTEM_PODS_INITIAL_DELAY": {get_param: system_pods_initial_delay}
"$SYSTEM_PODS_TIMEOUT": {get_param: system_pods_timeout}
"$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}
"$AUTH_URL": {get_param: auth_url}

View File

@ -87,6 +87,20 @@ parameters:
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
kube_allow_priv:
type: string
description: >
@ -398,6 +412,8 @@ resources:
flannel_network_cidr: {get_param: flannel_network_cidr}
flannel_network_subnetlen: {get_param: flannel_network_subnetlen}
flannel_backend: {get_param: flannel_backend}
system_pods_initial_delay: {get_param: system_pods_initial_delay}
system_pods_timeout: {get_param: system_pods_timeout}
portal_network_cidr: {get_param: portal_network_cidr}
discovery_url: {get_param: discovery_url}
cluster_uuid: {get_param: cluster_uuid}

View File

@ -63,6 +63,20 @@ parameters:
constraints:
- allowed_values: ["udp", "vxlan", "host-gw"]
system_pods_initial_delay:
type: number
description: >
health check, time to wait for system pods (podmaster, scheduler) to boot
(in seconds)
default: 30
system_pods_timeout:
type: number
description: >
health check, timeout for system pods (podmaster, scheduler) to answer.
(in seconds)
default: 5
discovery_url:
type: string
description: >
@ -224,6 +238,8 @@ resources:
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
"$FLANNEL_NETWORK_SUBNETLEN": {get_param: flannel_network_subnetlen}
"$FLANNEL_BACKEND": {get_param: flannel_backend}
"$SYSTEM_PODS_INITIAL_DELAY": {get_param: system_pods_initial_delay}
"$SYSTEM_PODS_TIMEOUT": {get_param: system_pods_timeout}
"$PORTAL_NETWORK_CIDR": {get_param: portal_network_cidr}
"$ETCD_DISCOVERY_URL": {get_param: discovery_url}
"$AUTH_URL": {get_param: auth_url}

View File

@ -19,4 +19,8 @@ class TestKubernetesAPIs(base.BaseK8sTest):
"network_driver": 'flannel',
"volume_driver": 'cinder',
"docker_storage_driver": 'overlay',
"labels": {
"system_pods_initial_delay": 3600,
"system_pods_timeout": 600
}
}

View File

@ -17,5 +17,9 @@ class TestCoreosKubernetesAPIs(base.BaseK8sTest):
cluster_template_kwargs = {
"tls_disabled": True,
"network_driver": 'flannel',
"volume_driver": None
"volume_driver": None,
"labels": {
"system_pods_initial_delay": 3600,
"system_pods_timeout": 600
}
}

View File

@ -22,4 +22,8 @@ class TestFedoraKubernetesIronicAPIs(base.BaseK8sTest):
"fixed_subnet": 'private-subnet',
"server_type": 'bm',
"docker_storage_driver": 'overlay',
"labels": {
"system_pods_initial_delay": 3600,
"system_pods_timeout": 600
}
}

View File

@ -48,7 +48,9 @@ class TestClusterConductorWithK8s(base.TestCase):
'no_proxy': 'no_proxy',
'labels': {'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'flannel_backend': 'vxlan'},
'flannel_backend': 'vxlan',
'system_pods_initial_delay': '15',
'system_pods_timeout': '1'},
'tls_disabled': False,
'server_type': 'vm',
'registry_enabled': False,
@ -143,7 +145,9 @@ class TestClusterConductorWithK8s(base.TestCase):
'discovery_url': 'discovery_url',
'labels': {'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'flannel_backend': 'vxlan'},
'flannel_backend': 'vxlan',
'system_pods_initial_delay': '15',
'system_pods_timeout': '1'},
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
'no_proxy': 'no_proxy',
@ -171,6 +175,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'flannel_backend': 'vxlan',
'system_pods_initial_delay': '15',
'system_pods_timeout': '1',
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
'no_proxy': 'no_proxy',
@ -245,6 +251,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'flannel_backend': 'vxlan',
'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'system_pods_initial_delay': '15',
'system_pods_timeout': '1',
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
'magnum_url': 'http://127.0.0.1:9511/v1',
@ -325,6 +333,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'flannel_backend': 'vxlan',
'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'system_pods_initial_delay': '15',
'system_pods_timeout': '1',
'insecure_registry_url': '10.0.0.1:5000',
'kube_version': 'fake-version',
'magnum_url': 'http://127.0.0.1:9511/v1',
@ -397,6 +407,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'flannel_backend': 'vxlan',
'system_pods_initial_delay': '15',
'system_pods_timeout': '1',
'tls_disabled': False,
'registry_enabled': False,
'trustee_domain_id': self.mock_keystone.trustee_domain_id,
@ -461,6 +473,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'flannel_backend': 'vxlan',
'system_pods_initial_delay': '15',
'system_pods_timeout': '1',
'tls_disabled': False,
'registry_enabled': False,
'trustee_domain_id': self.mock_keystone.trustee_domain_id,
@ -651,6 +665,8 @@ class TestClusterConductorWithK8s(base.TestCase):
'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'flannel_backend': 'vxlan',
'system_pods_initial_delay': '15',
'system_pods_timeout': '1',
'tenant_name': 'fake_tenant',
'username': 'fake_user',
'cluster_uuid': self.cluster_dict['uuid'],

View File

@ -254,6 +254,10 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
flannel_subnet = mock_cluster_template.labels.get(
'flannel_network_subnetlen')
flannel_backend = mock_cluster_template.labels.get('flannel_backend')
system_pods_initial_delay = mock_cluster_template.labels.get(
'system_pods_initial_delay')
system_pods_timeout = mock_cluster_template.labels.get(
'system_pods_timeout')
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
@ -266,6 +270,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'flannel_network_cidr': flannel_cidr,
'flannel_network_subnetlen': flannel_subnet,
'flannel_backend': flannel_backend,
'system_pods_initial_delay': system_pods_initial_delay,
'system_pods_timeout': system_pods_timeout,
'username': 'fake_user',
'tenant_name': 'fake_tenant',
'magnum_url': mock_osc.magnum_url.return_value,
@ -310,6 +316,10 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
flannel_subnet = mock_cluster_template.labels.get(
'flannel_network_subnetlen')
flannel_backend = mock_cluster_template.labels.get('flannel_backend')
system_pods_initial_delay = mock_cluster_template.labels.get(
'system_pods_initial_delay')
system_pods_timeout = mock_cluster_template.labels.get(
'system_pods_timeout')
k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
@ -322,6 +332,8 @@ class AtomicK8sTemplateDefinitionTestCase(BaseTemplateDefinitionTestCase):
'flannel_network_cidr': flannel_cidr,
'flannel_network_subnetlen': flannel_subnet,
'flannel_backend': flannel_backend,
'system_pods_initial_delay': system_pods_initial_delay,
'system_pods_timeout': system_pods_timeout,
'username': 'fake_user',
'tenant_name': 'fake_tenant',
'magnum_url': mock_osc.magnum_url.return_value,