From 6538342c26c479988fbf8515260ee76e2809bc08 Mon Sep 17 00:00:00 2001 From: "Hu, Yong" Date: Wed, 18 Dec 2019 02:23:44 +0000 Subject: [PATCH] Update mariadb chart to enable probe overrides Adding probes parameters for armada overriding them in duplex AIO and multi-node deployment. Specifically, there are 2 mariadb-servers in the DB cluster for OpenStack services at duplex or multi-node cases. These 2 mariadb-server pods are placed on Controller-0 and Controller-1 respectively (manipulated by anti-affinity). Whenever one Controller is rebooted on purpose or even worse accidiently shutdown for any reasons mariadb-server pod on that controller is gone together. To keep mariadb cluster still working even with only one instance, we have to adjust the default probe behaviors. Upon this request, we have to export probe parameters for "startupProbe" and "readinessProbe" so that StarlingX Armada application could set these parameters accordingly and thereby mariadb server can still work as expected with even only one pod in the cases of Controller node rebooting or shutdown. Closes-bug: 1855474 Change-Id: I3a8a99edd44d7ac4257ddf79b6baba5c52714324 Signed-off-by: Hu, Yong Co-Authored-By: Zhipeng, Liu --- .../centos/openstack-helm-infra.spec | 2 + ...e-of-mariadb-server-probe-parameters.patch | 87 +++++++++++++++++++ .../manifests/manifest.yaml | 7 ++ 3 files changed, 96 insertions(+) create mode 100644 openstack-helm-infra/files/0011-Enable-override-of-mariadb-server-probe-parameters.patch diff --git a/openstack-helm-infra/centos/openstack-helm-infra.spec b/openstack-helm-infra/centos/openstack-helm-infra.spec index ac390613..dca04e0a 100644 --- a/openstack-helm-infra/centos/openstack-helm-infra.spec +++ b/openstack-helm-infra/centos/openstack-helm-infra.spec @@ -25,6 +25,7 @@ Patch07: 0007-Add-io_thread_pool-for-rabbitmq.patch Patch08: 0008-Enable-override-of-rabbitmq-probe-parameters.patch Patch09: 0009-Fix-ipv6-address-issue-causing-mariadb-ingress-not-ready.patch Patch10: 0010-Fix-rabbitmq-could-not-bind-port-to-ipv6-address-iss.patch +Patch11: 0011-Enable-override-of-mariadb-server-probe-parameters.patch BuildRequires: helm @@ -43,6 +44,7 @@ Openstack Helm Infra charts %patch08 -p1 %patch09 -p1 %patch10 -p1 +%patch11 -p1 %build # initialize helm and build the toolkit diff --git a/openstack-helm-infra/files/0011-Enable-override-of-mariadb-server-probe-parameters.patch b/openstack-helm-infra/files/0011-Enable-override-of-mariadb-server-probe-parameters.patch new file mode 100644 index 00000000..1a4acae5 --- /dev/null +++ b/openstack-helm-infra/files/0011-Enable-override-of-mariadb-server-probe-parameters.patch @@ -0,0 +1,87 @@ +From a8ef0407811373b98a91be7787fcdcca18c76e23 Mon Sep 17 00:00:00 2001 +From: "Hu, Yong" +Date: Tue, 17 Dec 2019 15:11:13 +0000 +Subject: [PATCH] Enable override of mariadb-server probe parameters + +Parameters are exported for startupProbe and readinessProbe. +They are set with the default values, and might be overridden in + some special cases. For example, some cases require more times +of "readinessProbe" failure before the peer mariadb-server node is +recovered. Other cases, in which the regular readiness probe cannot +be performed because only one signle mariadb-server exists, have to +take use of "startupProbe" to ensure the WSREP status are ready. +For all these exceptional cases above, which are different from the +default mariadb cluster with 3 nodes, we have to export the probe +parameters and override the related probe pareameters. + +Signed-off-by: Hu, Yong +Co-Authored-By: Zhipeng, Liu +--- + mariadb/templates/statefulset.yaml | 16 +++++++++------- + mariadb/values.yaml | 16 ++++++++++++++++ + 2 files changed, 25 insertions(+), 7 deletions(-) + +diff --git a/mariadb/templates/statefulset.yaml b/mariadb/templates/statefulset.yaml +index 66d5339..9e3d37d 100644 +--- a/mariadb/templates/statefulset.yaml ++++ b/mariadb/templates/statefulset.yaml +@@ -14,6 +14,13 @@ See the License for the specific language governing permissions and + limitations under the License. + */}} + ++{{- define "mariadbReadinessProbe" }} ++exec: ++ command: ++ - /tmp/readiness.sh ++{{- end }} ++ ++ + {{- if .Values.manifests.statefulset }} + {{- $envAll := . }} + +@@ -170,13 +177,8 @@ spec: + exec: + command: + - /tmp/stop.sh +- readinessProbe: +- initialDelaySeconds: 30 +- periodSeconds: 30 +- timeoutSeconds: 3 +- exec: +- command: +- - /tmp/readiness.sh ++{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "readiness" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} ++{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "startup" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp +diff --git a/mariadb/values.yaml b/mariadb/values.yaml +index b913636..c7a70d2 100644 +--- a/mariadb/values.yaml ++++ b/mariadb/values.yaml +@@ -59,6 +59,22 @@ labels: + node_selector_value: enabled + + pod: ++ probes: ++ server: ++ mariadb: ++ readiness: ++ enabled: true ++ params: ++ initialDelaySeconds: 30 ++ periodSeconds: 30 ++ timeoutSeconds: 15 ++ failureThreshold: 3 ++ startup: ++ enabled: false ++ params: ++ initialDelaySeconds: 30 ++ periodSeconds: 30 ++ failureThreshold: 3 + security_context: + server: + pod: +-- +2.7.4 + diff --git a/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml b/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml index 1b15e235..07599c6b 100644 --- a/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml +++ b/stx-openstack-helm/stx-openstack-helm/manifests/manifest.yaml @@ -194,6 +194,13 @@ data: mariadb_backup: docker.io/openstackhelm/mariadb:10.2.18 scripted_test: null pod: + probes: + server: + mariadb: + readiness: + enabled: true + startup: + enabled: true affinity: anti: type: