[backups] Added staggered backups

This PS adds staggered backups possibility by adding anti-affinity rules to backups cronjobs that can be followed across several namespaces to decrease load on remote backup destination server making sure that at every moment in time there is only one backup upload is in progress. Change-Id: If49791f866a73a08fb98fa0e0b4854042d079c66
2023-12-01 22:09:13 +00:00 · 2023-12-01 22:09:13 +00:00 · 4a95f75b6b
parent 29f2b616cc
commit 4a95f75b6b
17 changed files with 173 additions and 13 deletions
--- a/helm-toolkit/Chart.yaml
+++ b/helm-toolkit/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v1.0.0
 description: OpenStack-Helm Helm-Toolkit
 name: helm-toolkit
-version: 0.2.56
+version: 0.2.57
 home: https://docs.openstack.org/openstack-helm
 icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png
 sources:
--- a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
+++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl
@ -243,7 +243,7 @@ send_to_remote_server() {
      log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values"
      return 2
  fi
-  rm -rf ${REMOTE_FILE}
+  rm -f ${REMOTE_FILE}

  log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully."
  return 0
--- a/mariadb-backup/Chart.yaml
+++ b/mariadb-backup/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.14
 description: OpenStack-Helm MariaDB backups
 name: mariadb-backup
-version: 0.0.1
+version: 0.0.2
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:
--- a/mariadb-backup/templates/cron-job-backup-mariadb.yaml
+++ b/mariadb-backup/templates/cron-job-backup-mariadb.yaml
@ -56,6 +56,12 @@ spec:
 {{ if $envAll.Values.pod.tolerations.mariadb.enabled }}
 {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }}
 {{ end }}
+{{- if $envAll.Values.pod.affinity }}
+{{- if $envAll.Values.pod.affinity.mariadb_backup }}
+          affinity:
+{{  index $envAll.Values.pod.affinity "mariadb_backup"  | toYaml | indent 12}}
+{{- end }}
+{{- end }}
          nodeSelector:
            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
          initContainers:
--- a/mariadb-backup/values_overrides/staggered-backups.yaml
+++ b/mariadb-backup/values_overrides/staggered-backups.yaml
@ -0,0 +1,38 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+pod:
+  labels:
+    backup:
+      staggered_backups: enabled
+  affinity:
+    mariadb_backup:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: status.phase
+                  operator: NotIn
+                  values:
+                    - Running
+                - key: staggered-backups
+                  operator: In
+                  values:
+                    - enabled
+            namespaces:
+              - openstack
+              - osh-infra
+              - ucp
+            topologyKey: kubernetes.io/os
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_mariadb_backup: true
+  secret_backup_restore: true
+...
--- a/mariadb/Chart.yaml
+++ b/mariadb/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v10.6.7
 description: OpenStack-Helm MariaDB
 name: mariadb
-version: 0.2.35
+version: 0.2.36
 home: https://mariadb.com/kb/en/
 icon: http://badges.mariadb.org/mariadb-badge-180x60.png
 sources:
--- a/mariadb/templates/cron-job-backup-mariadb.yaml
+++ b/mariadb/templates/cron-job-backup-mariadb.yaml
@ -53,9 +53,15 @@ spec:
          serviceAccountName: {{ $serviceAccountName }}
          restartPolicy: OnFailure
          shareProcessNamespace: true
-{{ if $envAll.Values.pod.tolerations.mariadb.enabled }}
+{{- if $envAll.Values.pod.tolerations.mariadb.enabled }}
 {{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }}
-{{ end }}
+{{- end }}
+{{- if $envAll.Values.pod.affinity }}
+{{- if $envAll.Values.pod.affinity.mariadb_backup }}
+          affinity:
+{{  index $envAll.Values.pod.affinity "mariadb_backup"  | toYaml | indent 12}}
+{{- end }}
+{{- end }}
          nodeSelector:
            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
          initContainers:
@ -191,9 +197,6 @@ spec:
                  mountPath: /tmp/start_verification_server.sh
                  readOnly: true
                  subPath: start_verification_server.sh
-          restartPolicy: OnFailure
-          serviceAccount: {{ $serviceAccountName }}
-          serviceAccountName: {{ $serviceAccountName }}
          volumes:
            - name: pod-tmp
              emptyDir: {}
--- a/mariadb/values_overrides/backups.yaml
+++ b/mariadb/values_overrides/backups.yaml
@ -0,0 +1,15 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_mariadb_backup: true
+  secret_backup_restore: true
+...
--- a/mariadb/values_overrides/staggered-backups.yaml
+++ b/mariadb/values_overrides/staggered-backups.yaml
@ -0,0 +1,38 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+pod:
+  labels:
+    backup:
+      staggered_backups: enabled
+  affinity:
+    mariadb_backup:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: status.phase
+                  operator: NotIn
+                  values:
+                    - Running
+                - key: staggered-backups
+                  operator: In
+                  values:
+                    - enabled
+            namespaces:
+              - openstack
+              - osh-infra
+              - ucp
+            topologyKey: kubernetes.io/os
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_mariadb_backup: true
+  secret_backup_restore: true
+...
--- a/postgresql/Chart.yaml
+++ b/postgresql/Chart.yaml
@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v14.5
 description: OpenStack-Helm PostgreSQL
 name: postgresql
-version: 0.1.18
+version: 0.1.19
 home: https://www.postgresql.org
 sources:
  - https://github.com/postgres/postgres
--- a/postgresql/templates/cron-job-backup-postgres.yaml
+++ b/postgresql/templates/cron-job-backup-postgres.yaml
@ -52,6 +52,12 @@ spec:
 {{ dict "envAll" $envAll "application" "postgresql_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }}
          serviceAccountName: {{ $serviceAccountName }}
          restartPolicy: OnFailure
+{{- if $envAll.Values.pod.affinity }}
+{{- if $envAll.Values.pod.affinity.postgresql_backup }}
+          affinity:
+{{  index $envAll.Values.pod.affinity "postgresql_backup"  | toYaml | indent 12}}
+{{- end }}
+{{- end }}
          nodeSelector:
            {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }}
          initContainers:
@ -137,9 +143,6 @@ spec:
                  mountPath: /etc/postgresql/admin_user.conf
                  subPath: admin_user.conf
                  readOnly: true
-          restartPolicy: OnFailure
-          serviceAccount: {{ $serviceAccountName }}
-          serviceAccountName: {{ $serviceAccountName }}
          volumes:
            - name: pod-tmp
              emptyDir: {}
--- a/postgresql/values_overrides/backups.yaml
+++ b/postgresql/values_overrides/backups.yaml
@ -0,0 +1,15 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_postgresql_backup: true
+  secret_backup_restore: true
+...
--- a/postgresql/values_overrides/staggered-backups.yaml
+++ b/postgresql/values_overrides/staggered-backups.yaml
@ -0,0 +1,38 @@
+---
+conf:
+  backup:
+    enabled: true
+    remote_backup:
+      enabled: false
+pod:
+  labels:
+    backup:
+      staggered_backups: enabled
+  affinity:
+    postgresql_backup:
+      podAntiAffinity:
+        requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+                - key: status.phase
+                  operator: NotIn
+                  values:
+                    - Running
+                - key: staggered-backups
+                  operator: In
+                  values:
+                    - enabled
+            namespaces:
+              - openstack
+              - osh-infra
+              - ucp
+            topologyKey: kubernetes.io/os
+volume:
+  backup:
+    enabled: true
+manifests:
+  pvc_backup: true
+  job_ks_user: false
+  cron_job_postgresql_backup: true
+  secret_backup_restore: true
+...
--- a/releasenotes/notes/helm-toolkit.yaml
+++ b/releasenotes/notes/helm-toolkit.yaml
@ -63,4 +63,5 @@ helm-toolkit:
  - 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice
  - 0.2.55 Updated deprecated IngressClass annotation
  - 0.2.56 Expose S3 credentials from Rook bucket CRD secret
+  - 0.2.57 Safer file removal
 ...
--- a/releasenotes/notes/mariadb-backup.yaml
+++ b/releasenotes/notes/mariadb-backup.yaml
@ -1,4 +1,5 @@
 ---
 mariadb-backup:
  - 0.0.1 Initial Chart
+  - 0.0.2 Added staggered backups support
 ...
--- a/releasenotes/notes/mariadb.yaml
+++ b/releasenotes/notes/mariadb.yaml
@ -51,4 +51,5 @@ mariadb:
  - 0.2.33 Add 2023.1 Ubuntu Focal overrides
  - 0.2.34 Uplift ingress controller image to 1.8.2
  - 0.2.35 Update apparmor override
+  - 0.2.36 Added staggered backups support
 ...
--- a/releasenotes/notes/postgresql.yaml
+++ b/releasenotes/notes/postgresql.yaml
@ -19,4 +19,5 @@ postgresql:
  - 0.1.16 Added OCI registry authentication
  - 0.1.17 Added empty verify_databases_backup_archives() function implementation to match updated backup_databases() function in helm-toolkit
  - 0.1.18 Updated postgres to 14.5 and replaced deprecated config item wal_keep_segments with wal_keep_size
+  - 0.1.19 Added staggered backups support
 ...