Policyd override implementation

This patchset implements policy overrides for nova-cloud-controler.

This change includes a charm-helpers sync to bring in the policyd helper
code.

Note there are no functional tests for this feature as the charm still
uses the old style non-zaza amulet framework.  The Related-Bug below is
tracking this issue.

Change-Id: Ia5f3f8189d4a7b7b46a827707d964ebe40740aeb
Closes-Bug: #1741723
Related-Bug: #1845639

README.md

@@ -127,3 +127,46 @@ juju run-action nova-cloud-controller/0 clear-unit-knownhost-cache
 
 In a high-availability setup, the action must be run on all
 `nova-cloud-controller` units.
+
+# Policy Overrides
+
+This feature allows for policy overrides using the `policy.d` directory.  This
+is an **advanced** feature and the policies that the OpenStack service supports
+should be clearly and unambiguously understood before trying to override, or
+add to, the default policies that the service uses.  The charm also has some
+policy defaults.  They should also be understood before being overridden.
+
+> **Caution**: It is possible to break the system (for tenants and other
+  services) if policies are incorrectly applied to the service.
+
+Policy overrides are YAML files that contain rules that will add to, or
+override, existing policy rules in the service.  The `policy.d` directory is
+a place to put the YAML override files.  This charm owns the
+`/etc/keystone/policy.d` directory, and as such, any manual changes to it will
+be overwritten on charm upgrades.
+
+Overrides are provided to the charm using a Juju resource called
+`policyd-override`.  The resource is a ZIP file.  This file, say
+`overrides.zip`, is attached to the charm by:
+
+
+    juju attach-resource nova-cloud-controller policyd-override=overrides.zip
+
+The policy override is enabled in the charm using:
+
+    juju config nova-cloud-controller use-policyd-override=true
+
+When `use-policyd-override` is `True` the status line of the charm will be
+prefixed with `PO:` indicating that policies have been overridden.  If the
+installation of the policy override YAML files failed for any reason then the
+status line will be prefixed with `PO (broken):`.  The log file for the charm
+will indicate the reason.  No policy override files are installed if the `PO
+(broken):` is shown.  The status line indicates that the overrides are broken,
+not that the policy for the service has failed. The policy will be the defaults
+for the charm and service.
+
+Policy overrides on one service may affect the functionality of another
+service. Therefore, it may be necessary to provide policy overrides for
+multiple service charms to achieve a consistent set of policies across the
+OpenStack system.  The charms for the other services that may need overrides
+should be checked to ensure that they support overrides before proceeding.

charmhelpers/contrib/openstack/policyd.py

@@ -299,10 +299,17 @@ def maybe_do_policyd_overrides(openstack_release,
     config = hookenv.config()
     try:
         if not config.get(POLICYD_CONFIG_NAME, False):
-            remove_policy_success_file()
             clean_policyd_dir_for(service, blacklist_paths)
+            if (os.path.isfile(_policy_success_file()) and
+                    restart_handler is not None and
+                    callable(restart_handler)):
+                restart_handler()
+            remove_policy_success_file()
             return
-    except Exception:
+    except Exception as e:
+        print("Exception is: ", str(e))
+        import traceback
+        traceback.print_exc()
         return
     if not is_policyd_override_valid_on_this_release(openstack_release):
         return
@@ -348,8 +355,12 @@ def maybe_do_policyd_overrides_on_config_changed(openstack_release,
     config = hookenv.config()
     try:
         if not config.get(POLICYD_CONFIG_NAME, False):
-            remove_policy_success_file()
             clean_policyd_dir_for(service, blacklist_paths)
+            if (os.path.isfile(_policy_success_file()) and
+                    restart_handler is not None and
+                    callable(restart_handler)):
+                restart_handler()
+            remove_policy_success_file()
             return
     except Exception:
         return
@@ -430,8 +441,13 @@ def _yamlfiles(zipfile):
     """
     l = []
     for infolist_item in zipfile.infolist():
-        if infolist_item.is_dir():
-            continue
+        try:
+            if infolist_item.is_dir():
+                continue
+        except AttributeError:
+            # fallback to "old" way to determine dir entry for pre-py36
+            if infolist_item.filename.endswith('/'):
+                continue
         _, name_ext = os.path.split(infolist_item.filename)
         name, ext = os.path.splitext(name_ext)
         ext = ext.lower()

config.yaml

@@ -583,3 +583,11 @@ options:
       and before
       .
       Possible Values are positive integers or 0 and -1 to disable the quota.
+  use-policyd-override:
+    type: boolean
+    default: False
+    description: |
+      If True then use the resource file named 'policyd-override' to install
+      override YAML files in the service's policy.d directory.  The resource
+      file should be a ZIP file containing at least one yaml file with a .yaml
+      or .yml extension.  If False then remove the overrides.

hooks/nova_cc_hooks.py

@@ -43,6 +43,7 @@ import charmhelpers.contrib.openstack.context as ch_context
 import charmhelpers.contrib.openstack.ha.utils as ch_ha_utils
 import charmhelpers.contrib.openstack.ip as ch_ip
 import charmhelpers.contrib.openstack.neutron as ch_neutron
+import charmhelpers.contrib.openstack.policyd as policyd
 import charmhelpers.contrib.openstack.utils as ch_utils
 import charmhelpers.contrib.peerstorage as ch_peerstorage
 import charmhelpers.core.hookenv as hookenv
@@ -211,6 +212,9 @@ def install():
     else:
         hookenv.log('Unit is in paused state, not issuing stop/pause '
                     'to all services')
+    # call the policy overrides handler which will install any policy overrides
+    policyd.maybe_do_policyd_overrides(
+        ch_utils.os_release('nova-common'), 'nova')
 
 
 @hooks.hook('config-changed')
@@ -290,6 +294,9 @@ def config_changed():
     if (not ch_utils.is_unit_paused_set() and
             ncc_utils.is_console_auth_enabled()):
         ch_host.service_resume('nova-consoleauth')
+    # call the policy overrides handler which will install any policy overrides
+    policyd.maybe_do_policyd_overrides_on_config_changed(
+        ch_utils.os_release('nova-common'), 'nova')
 
 
 @hooks.hook('amqp-relation-joined')
@@ -1124,6 +1131,9 @@ def upgrade_charm():
     leader_init_db_if_ready_allowed_units()
 
     update_nrpe_config()
+    # call the policy overrides handler which will install any policy overrides
+    policyd.maybe_do_policyd_overrides(
+        ch_utils.os_release('nova-common'), 'nova')
 
 
 @hooks.hook('neutron-api-relation-joined')

metadata.yaml

@@ -67,4 +67,8 @@ requires:
 peers:
   cluster:
     interface: nova-ha
-
+resources:
+  policyd-override:
+    type: file
+    filename: policyd-override.zip
+    description: The policy.d overrides file

unit_tests/test_nova_cc_hooks.py

@@ -34,6 +34,9 @@ TO_PATCH = [
     'charmhelpers.contrib.openstack.ha.utils.generate_ha_relation_data',
     'charmhelpers.contrib.openstack.ha.utils.update_dns_ha_resource_params',
     'charmhelpers.contrib.openstack.neutron.network_manager',
+    'charmhelpers.contrib.openstack.policyd.maybe_do_policyd_overrides',
+    'charmhelpers.contrib.openstack.policyd.'
+    'maybe_do_policyd_overrides_on_config_changed',
     'charmhelpers.contrib.openstack.utils.configure_installation_source',
     'charmhelpers.contrib.openstack.utils.config_value_changed',
     'charmhelpers.contrib.openstack.utils.is_unit_paused_set',