Policyd override implementation

This patchset implements policy overrides for glance.  It uses the
code in charmhelpers.

Change-Id: I0586326ff87fdf03f2c88e4c459627f4085c3367
Closed-Bug: #1741723

README.md

@@ -142,3 +142,47 @@ Alternatively, these can also be provided as part of a juju native bundle config
 NOTE: Spaces must be configured in the underlying provider prior to attempting to use them.
 
 NOTE: Existing deployments using os-*-network configuration options will continue to function; these options are preferred over any network space binding provided if set.
+
+Policy Overrides
+================
+
+This feature allows for policy overrides using the `policy.d` directory.  This
+is an **advanced** feature and the policies that the OpenStack service supports
+should be clearly and unambiguously understood before trying to override, or
+add to, the default policies that the service uses.  The charm also has some
+policy defaults.  They should also be understood before being overridden.
+
+> **Caution**: It is possible to break the system (for tenants and other
+  services) if policies are incorrectly applied to the service.
+
+Policy overrides are YAML files that contain rules that will add to, or
+override, existing policy rules in the service.  The `policy.d` directory is
+a place to put the YAML override files.  This charm owns the
+`/etc/keystone/policy.d` directory, and as such, any manual changes to it will
+be overwritten on charm upgrades.
+
+Overrides are provided to the charm using a Juju resource called
+`policyd-override`.  The resource is a ZIP file.  This file, say
+`overrides.zip`, is attached to the charm by:
+
+
+    juju attach-resource glance policyd-override=overrides.zip
+
+The policy override is enabled in the charm using:
+
+    juju config glance use-policyd-override=true
+
+When `use-policyd-override` is `True` the status line of the charm will be
+prefixed with `PO:` indicating that policies have been overridden.  If the
+installation of the policy override YAML files failed for any reason then the
+status line will be prefixed with `PO (broken):`.  The log file for the charm
+will indicate the reason.  No policy override files are installed if the `PO
+(broken):` is shown.  The status line indicates that the overrides are broken,
+not that the policy for the service has failed. The policy will be the defaults
+for the charm and service.
+
+Policy overrides on one service may affect the functionality of another
+service. Therefore, it may be necessary to provide policy overrides for
+multiple service charms to achieve a consistent set of policies across the
+OpenStack system.  The charms for the other services that may need overrides
+should be checked to ensure that they support overrides before proceeding.

charmhelpers/contrib/openstack/policyd.py

@@ -299,10 +299,17 @@ def maybe_do_policyd_overrides(openstack_release,
     config = hookenv.config()
     try:
         if not config.get(POLICYD_CONFIG_NAME, False):
-            remove_policy_success_file()
             clean_policyd_dir_for(service, blacklist_paths)
+            if (os.path.isfile(_policy_success_file()) and
+                    restart_handler is not None and
+                    callable(restart_handler)):
+                restart_handler()
+            remove_policy_success_file()
             return
-    except Exception:
+    except Exception as e:
+        print("Exception is: ", str(e))
+        import traceback
+        traceback.print_exc()
         return
     if not is_policyd_override_valid_on_this_release(openstack_release):
         return
@@ -348,8 +355,12 @@ def maybe_do_policyd_overrides_on_config_changed(openstack_release,
     config = hookenv.config()
     try:
         if not config.get(POLICYD_CONFIG_NAME, False):
-            remove_policy_success_file()
             clean_policyd_dir_for(service, blacklist_paths)
+            if (os.path.isfile(_policy_success_file()) and
+                    restart_handler is not None and
+                    callable(restart_handler)):
+                restart_handler()
+            remove_policy_success_file()
             return
     except Exception:
         return
@@ -430,8 +441,13 @@ def _yamlfiles(zipfile):
     """
     l = []
     for infolist_item in zipfile.infolist():
-        if infolist_item.is_dir():
-            continue
+        try:
+            if infolist_item.is_dir():
+                continue
+        except AttributeError:
+            # fallback to "old" way to determine dir entry for pre-py36
+            if infolist_item.filename.endswith('/'):
+                continue
         _, name_ext = os.path.split(infolist_item.filename)
         name, ext = os.path.splitext(name_ext)
         ext = ext.lower()

config.yaml

@@ -348,3 +348,11 @@ options:
       raised and the filesystem store may not be available for adding new
       images. NOTE: This directory is used only when filesystem store is used
       as a storage backend.
+  use-policyd-override:
+    type: boolean
+    default: False
+    description: |
+      If True then use the resource file named 'policyd-override' to install
+      override YAML files in the service's policy.d directory.  The resource
+      file should be a ZIP file containing at least one yaml file with a .yaml
+      or .yml extension.  If False then remove the overrides.

hooks/glance_relations.py

@@ -132,6 +132,11 @@ from charmhelpers.contrib.openstack.cert_utils import (
     get_certificate_request,
     process_certificates,
 )
+from charmhelpers.contrib.openstack.policyd import (
+    maybe_do_policyd_overrides,
+    maybe_do_policyd_overrides_on_config_changed,
+)
+
 
 hooks = Hooks()
 CONFIGS = register_configs()
@@ -155,6 +160,11 @@ def install_hook():
 
     for service in SERVICES:
         service_stop(service)
+    # call the policy overrides handler which will install any policy overrides
+    maybe_do_policyd_overrides(
+        os_release('glance-common'),
+        'glance',
+        restart_handler=lambda: service_restart('glance-api'))
 
 
 @hooks.hook('shared-db-relation-joined')
@@ -376,6 +386,12 @@ def config_changed():
     ceph_changed()
     update_image_location_policy()
 
+    # call the policy overrides handler which will install any policy overrides
+    maybe_do_policyd_overrides_on_config_changed(
+        os_release('glance-common'),
+        'glance',
+        restart_handler=lambda: service_restart('glance-api'))
+
 
 @hooks.hook('cluster-relation-joined')
 def cluster_joined(relation_id=None):
@@ -417,6 +433,11 @@ def upgrade_charm():
         juju_log("Package purge detected, restarting services", "INFO")
         for s in services():
             service_restart(s)
+    # call the policy overrides handler which will install any policy overrides
+    maybe_do_policyd_overrides(
+        os_release('glance-common'),
+        'glance',
+        restart_handler=lambda: service_restart('glance-api'))
 
 
 @hooks.hook('ha-relation-joined')

metadata.yaml

@@ -51,3 +51,8 @@ requires:
 peers:
   cluster:
     interface: glance-ha
+resources:
+  policyd-override:
+    type: file
+    filename: policyd-override.zip
+    description: The policy.d overrides file

tests/tests.yaml

@@ -21,5 +21,13 @@ gate_bundles:
 dev_bundles:
   - bionic-train
 
+configure:
+  - zaza.openstack.charm_tests.keystone.setup.add_demo_user
+
 tests:
   - zaza.openstack.charm_tests.glance.tests.GlanceTest
+  - zaza.openstack.charm_tests.policyd.tests.GlanceTests
+
+tests_options:
+  policyd:
+    service: glance

unit_tests/test_glance_relations.py

@@ -76,6 +76,9 @@ TO_PATCH = [
     'configure_installation_source',
     'os_release',
     'openstack_upgrade_available',
+    # charmhelpers.contrib.openstack.policyd
+    'maybe_do_policyd_overrides',
+    'maybe_do_policyd_overrides_on_config_changed',
     # charmhelpers.contrib.openstack.ha.utils
     'generate_ha_relation_data',
     'is_clustered',