From 890964eca07a2c2007806278bb41de9f9ce6bd78 Mon Sep 17 00:00:00 2001 From: Aaron Sheffield Date: Fri, 22 Jun 2018 11:54:36 -0500 Subject: [PATCH] Add Additional Liveness Probes for apiserver - Updated apiserver-anchor with a liveness probe. - Changed apiserver liveness probe to query kubectl. This allows the pod to restart if it looses access to etcd. Change-Id: I0ef9cbc941a0533268e4f499a1333e88be3e43a3 --- charts/apiserver/templates/configmap-etc.yaml | 2 + .../templates/etc/_kubeconfig.yaml.tpl | 34 +++++++++++++++ .../etc/_kubernetes-apiserver.yaml.tpl | 15 ++++--- charts/apiserver/templates/rbac.yaml | 42 +++++++++++++++++++ charts/apiserver/values.yaml | 2 + 5 files changed, 89 insertions(+), 6 deletions(-) create mode 100644 charts/apiserver/templates/etc/_kubeconfig.yaml.tpl create mode 100644 charts/apiserver/templates/rbac.yaml diff --git a/charts/apiserver/templates/configmap-etc.yaml b/charts/apiserver/templates/configmap-etc.yaml index 5942bd1f..725f52ac 100644 --- a/charts/apiserver/templates/configmap-etc.yaml +++ b/charts/apiserver/templates/configmap-etc.yaml @@ -25,4 +25,6 @@ metadata: data: kubernetes-apiserver.yaml: |+ {{ tuple "etc/_kubernetes-apiserver.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + kubeconfig.yaml: |+ +{{ tuple "etc/_kubeconfig.yaml.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} {{- end }} diff --git a/charts/apiserver/templates/etc/_kubeconfig.yaml.tpl b/charts/apiserver/templates/etc/_kubeconfig.yaml.tpl new file mode 100644 index 00000000..53810a6f --- /dev/null +++ b/charts/apiserver/templates/etc/_kubeconfig.yaml.tpl @@ -0,0 +1,34 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v1 +clusters: +- cluster: + server: https://127.0.0.1:{{ .Values.network.kubernetes_apiserver.port }} + certificate-authority: pki/cluster-ca.pem + name: kubernetes +contexts: +- context: + cluster: kubernetes + user: apiserver + name: apiserver@kubernetes +current-context: apiserver@kubernetes +kind: Config +preferences: {} +users: +- name: apiserver + user: + client-certificate: pki/apiserver.pem + client-key: pki/apiserver-key.pem diff --git a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl index 9d296ad2..7919503a 100644 --- a/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl +++ b/charts/apiserver/templates/etc/_kubernetes-apiserver.yaml.tpl @@ -33,6 +33,12 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: KUBECONFIG + value: /etc/kubernetes/apiserver/kubeconfig.yaml command: {{- range .Values.command_prefix }} @@ -81,13 +87,10 @@ spec: - /bin/bash - -c - |- - if [ ! -f /etc/kubernetes/apiserver/pki/apiserver-both.pem ]; then - cat /etc/kubernetes/apiserver/pki/apiserver-key.pem /etc/kubernetes/apiserver/pki/apiserver.pem > /etc/kubernetes/apiserver/pki/apiserver-both.pem - fi - echo -e 'GET /healthz HTTP/1.0\r\n' | socat - openssl:localhost:{{ .Values.network.kubernetes_apiserver.port }},cert=/etc/kubernetes/apiserver/pki/apiserver-both.pem,cafile=/etc/kubernetes/apiserver/pki/cluster-ca.pem | grep '200 OK' + kubectl get nodes ${NODENAME} | grep ${NODENAME} exit $? - failureThreshold: 2 - initialDelaySeconds: 15 + failureThreshold: 3 + initialDelaySeconds: 60 periodSeconds: 10 successThreshold: 1 timeoutSeconds: 10 diff --git a/charts/apiserver/templates/rbac.yaml b/charts/apiserver/templates/rbac.yaml new file mode 100644 index 00000000..60e90a83 --- /dev/null +++ b/charts/apiserver/templates/rbac.yaml @@ -0,0 +1,42 @@ +{{/* +Copyright 2018 AT&T Intellectual Property. All other rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: apiserver + namespace: kube-system +rules: + - apiGroups: [""] + resources: + - nodes + verbs: + - get +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: apiserver + namespace: kube-system +subjects: + - kind: User + name: apiserver + apiGroup: rbac.authorization.k8s.io +roleRef: + kind: ClusterRole + name: apiserver + apiGroup: rbac.authorization.k8s.io diff --git a/charts/apiserver/values.yaml b/charts/apiserver/values.yaml index 0a563bfa..986cdbe9 100644 --- a/charts/apiserver/values.yaml +++ b/charts/apiserver/values.yaml @@ -48,6 +48,8 @@ anchor: dest: /etc/kubernetes/apiserver/pki/etcd-client-key.pem - source: /tmp/etc/kubernetes-apiserver.yaml dest: /etc/kubernetes/manifests/kubernetes-apiserver.yaml + - source: /tmp/etc/kubeconfig.yaml + dest: /etc/kubernetes/apiserver/kubeconfig.yaml command_prefix: - /apiserver