Local CA update playbook improvements

Include some improvements in the playbook:
- CAs cert/keys are verified first, then installed at the end of
  the playbook (this makes the playbook work if the system-local-ca
  secret was deleted - not having the secret prevented installing
  the RCA as trusted in the early steps of playbook).

- Not deleting oidc-auth-apps-certificate Certificate unless the
  application is applied in the system (Certificate is recreated in
  this case).

- Only wait for 'system-openldap-local-certificate' when the
  playbook created it (only standalone or SystemController).

- Included step to reapply old 'system-local-ca' secret if the
  playbook fails in a state where the secret was already deleted.

Test Plan:
PASS: Run update_platform_certificates playbook in DC + SX subcloud.

PASS: Remove system-local-ca secret.
      Run upgrade_platform_certificates playbook.

PASS: Provide wrong field in inventory file.
      Run upgrade_platform_certificates playbook, observe that it
      fails.
      Fix the inventory file.
      Run upgrade_platform_certificates plabook.

PASS: Issue oidc-auth-apps-certificate Certificate, using
      system-local-ca ClusterIssuer.
      Without oidc auth apps applied in the system, run
      upgrade_platform_certificates plabook.
      Observe that the certificate is not deleted.

Story: 2009811
Task: 50080

Change-Id: Ic0213ea739dbb116536f9e4a85d16da0b55cf6ca
Signed-off-by: Marcelo Loebens <Marcelo.DeCastroLoebens@windriver.com>

playbookconfig/src/playbooks/roles/update-platform-certificates/tasks/main.yml

@@ -22,15 +22,11 @@
       include_tasks: check-for-management-alarms.yml
       when: ignore_alarms is undefined or ignore_alarms | bool == False
 
-    - name: Install Root CA certificate as trusted by the platform
+    - name: Verify 'system-local-ca' certs
       include_role:
         name: common/verify-and-install-system-local-ca-certs
       vars:
-        - install_rca: true
-
-    - name: Restart kube-apiserver to pick the new certificate
-      include_role:
-        name: common/restart-kube-apiserver
+        - install_rca: false
 
     - name: Check certificates to be installed
       include_tasks: check-certificates-to-be-installed.yml
@@ -65,6 +61,12 @@
       retries: 3
       delay: 30
 
+    - name: Generate kubernetes yaml for cert-manager resources
+      include_role:
+        name: common/generate-platform-certificates-template
+      vars:
+        destination: "{{ cert_manager_spec_file }}"
+
     - name: Retrieve certificates that may own system-local-ca secret
       shell: >-
         kubectl get certificates -A
@@ -76,6 +78,13 @@
         KUBECONFIG: /etc/kubernetes/admin.conf
       register: cert_to_remove
 
+    - name: Dump system-local-ca secret (to recover if necessary)
+      command: kubectl get secret -n cert-manager system-local-ca -o yaml --ignore-not-found=true
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+      register: system_local_ca_dump
+      no_log: true
+
     - name: Delete certificate that owns the secret 'system-local-ca' if it exists
       include_role:
         name: common/delete-kubernetes-resources
@@ -89,20 +98,20 @@
         - { name: system-local-ca, namespace: cert-manager, type: clusterissuer }
         - { name: system-local-ca, namespace: cert-manager, type: secret }
 
-    - name: Generate kubernetes yaml for cert-manager resources
-      include_role:
-        name: common/generate-platform-certificates-template
-      vars:
-        destination: "{{ cert_manager_spec_file }}"
-
-    - name: Remove default leaf certificates (plus OIDC)
+    - name: Remove default leaf certificates
       include_role:
         name: common/delete-kubernetes-resources
       with_items:
         - { name: system-openldap-local-certificate, namespace: deployment, type: certificate }
         - { name: system-registry-local-certificate, namespace: deployment, type: certificate }
         - { name: system-restapi-gui-certificate, namespace: deployment, type: certificate }
+
+    - name: Remove OIDC certificate if we are recreating it
+      include_role:
+        name: common/delete-kubernetes-resources
+      with_items:
         - { name: oidc-auth-apps-certificate, namespace: kube-system, type: certificate }
+      when: install_oidc_auth_apps_certificate
 
     # This list is composed of other certificates issued by the cluster issuer
     # (i.e. not local REST API/GUI, OpenLDAP, Docker Registry or OIDC)
@@ -128,12 +137,6 @@
       retries: 10
       delay: 30
 
-    - name: Delete kubernetes yaml with certificate spec
-      file:
-        path: "{{ cert_manager_spec_file }}"
-        state: absent
-      become: yes
-
     - name: Force certificate renewals by deleting their secrets
       include_role:
         name: common/delete-kubernetes-resources
@@ -145,6 +148,17 @@
         --for=condition=Ready --timeout=90s
       environment:
         KUBECONFIG: /etc/kubernetes/admin.conf
+      when: install_system_open_ldap_certificate
+
+    - name: Install Root CA certificate as trusted by the platform
+      include_role:
+        name: common/verify-and-install-system-local-ca-certs
+      vars:
+        - install_rca: true
+
+    - name: Restart kube-apiserver to pick the new certificate
+      include_role:
+        name: common/restart-kube-apiserver
 
     - name: Update oidc-auth-apps in order to use new certificate
       include_tasks: reapply-oidc-auth-app.yml
@@ -179,6 +193,7 @@
         copy:
           dest: "{{ item.path }}"
           content: "{{ item.secret | b64decode }}"
+        no_log: true
         loop:
           - path: "{{ root_ca_cert.path }}"
             secret: "{{ system_root_ca_cert }}"
@@ -205,6 +220,25 @@
             - "{{ local_ca_cert.path }}"
 
   rescue:
+    - name: Check if system-local-ca is in place
+      command: >-
+        kubectl get secret -n cert-manager system-local-ca --ignore-not-found=true --no-headers=true
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+      register: system_local_ca_get
+      no_log: true
+
+    - name: Recover previous system-local-ca secret
+      shell: kubectl apply -f <(echo '{{ system_local_ca_dump.stdout }}')
+      environment:
+        KUBECONFIG: /etc/kubernetes/admin.conf
+      register: create_k8_apply_ep
+      until: create_k8_apply_ep is not failed
+      retries: 10
+      delay: 30
+      no_log: true
+      when: system_local_ca_get.stdout == ""
+
     - block:
       - debug:
           msg: >-
@@ -223,6 +257,13 @@
 
       when: backup_directory is defined
 
+  always:
+    - name: Delete kubernetes yaml with certificate spec
+      file:
+        path: "{{ cert_manager_spec_file }}"
+        state: absent
+      become: yes
+
   when: mode == 'update'
 
 # This mode is here to aid in testing,