Report missing CA if cert requests are pending

This sets the charm status to blocked to make it obvious when the CA
cert is missing and there are pending cert requests being blocked.

This also moves the optional interfaces checks down to ensure that they
don't mask the more important status messages.  (E.g., if Vault is
providing certs for Etcd, it's more important to know that Vault is
sealed or missing the CA than to know that Etcd doesn't have its cert
yet.)

This also adds some error checking to gracefully handle the case where
Vault becomes sealed after it was successfully started rather than
having it go into a hook error.

Change-Id: I18a5dbeabc562e14d164f82c041fed207032f52b
Closes-Bug: 1840696

src/reactive/vault_handlers.py

@@ -713,10 +713,6 @@ def _assess_status():
                              missing_interfaces=_missing_interfaces,
                              incomplete_interfaces=_incomplete_interfaces)
 
-    _assess_interface_groups(OPTIONAL_INTERFACES, optional=True,
-                             missing_interfaces=_missing_interfaces,
-                             incomplete_interfaces=_incomplete_interfaces)
-
     if _missing_interfaces or _incomplete_interfaces:
         state = 'blocked' if _missing_interfaces else 'waiting'
         status_set(state, ', '.join(_missing_interfaces +
@@ -760,6 +756,22 @@ def _assess_status():
         status_set('blocked', 'Vault cannot authorize approle')
         return
 
+    has_ca = is_flag_set('charm.vault.ca.ready')
+    has_cert_reqs = is_flag_set('certificates.certs.requested')
+    if has_cert_reqs and not has_ca:
+        status_set('blocked', 'Missing CA cert')
+        return
+
+    _assess_interface_groups(OPTIONAL_INTERFACES, optional=True,
+                             missing_interfaces=_missing_interfaces,
+                             incomplete_interfaces=_incomplete_interfaces)
+
+    if _missing_interfaces or _incomplete_interfaces:
+        state = 'blocked' if _missing_interfaces else 'waiting'
+        status_set(state, ', '.join(_missing_interfaces +
+                                    _incomplete_interfaces))
+        return
+
     mlock_disabled = is_container() or config('disable-mlock')
 
     vault_installed_version = snap.get_installed_version('vault')
@@ -859,7 +871,11 @@ def publish_ca_info():
         log("Unable to publish ca info, service sealed.")
     else:
         tls.set_ca(vault_pki.get_ca())
-        chain = vault_pki.get_chain()
+        try:
+            # this might fail if we were restarted and need to be unsealed
+            chain = vault_pki.get_chain()
+        except vault.hvac.exceptions.VaultDown:
+            chain = None
         if chain:
             tls.set_chain(chain)
 

unit_tests/test_reactive_vault_handlers.py

@@ -462,6 +462,21 @@ class TestHandlers(unit_tests.test_utils.CharmTestCase):
             'New version of vault installed, manual intervention '
             'required to restart the service.')
 
+    @patch.object(handlers, 'leader_get')
+    @patch.object(handlers, 'client_approle_authorized')
+    @patch.object(handlers, '_assess_interface_groups')
+    @patch.object(handlers.vault, 'get_vault_health')
+    def test_assess_status_vault_missing_ca(self, get_vault_health,
+                                            _assess_interface_groups,
+                                            _client_approle_authorized,
+                                            _leader_get):
+        self.is_flag_set.side_effect = lambda f: (
+            True if f == 'certificates.certs.requested' else False
+        )
+        get_vault_health.return_value = self._health_response
+        handlers._assess_status()
+        self.status_set.assert_called_with('blocked', 'Missing CA cert')
+
     def test_assess_interface_groups(self):
         flags = {
             'db.master.available': True,