Use the venv contents instead of a git source for templates

When deploying the base templates for api-paste, policy files
and other files which are included in the service git source,
we now use the venv files instead of requiring access to a
git source and a complex set of lookups and variable
implementations.

This is simpler and more cross-series, and works from Queens
due to the related bug's patches.

Change-Id: I6a4e2514e66b15b2ae227e62b6dc9ae1a50a4fbd
Related-Bug: #1718356

defaults/main.yml

@@ -440,12 +440,13 @@ keystone_paste_default_file_path: "/etc/openstack_deploy/keystone/keystone-paste
 keystone_policy_default_file_path: "/etc/openstack_deploy/keystone/policy.json"
 keystone_sso_callback_file_path: "/etc/openstack_deploy/keystone/sso_callback_template.html"
 
-# If the above-mentioned files do not exist, then these
-# paths will be used to find the files from the git config
-# lookup location.
-keystone_git_config_lookup_location: https://git.openstack.org/cgit/openstack/keystone/plain
-keystone_paste_git_file_path: "etc/keystone-paste.ini?h={{ keystone_git_install_branch }}"
-keystone_sso_callback_git_file_path: "etc/sso_callback_template.html?h={{ keystone_git_install_branch }}"
+# If the above-mentioned files do not exist, then the defaults
+# inside the venvs will be used, but cached at this location
+# on the deployment host. Using the cache makes the re-use
+# of the files faster when deploying, but is also required in
+# order to still be able to apply the config_template override.
+keystone_config_cache_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/keystone"
+keystone_config_cache_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
 
 #: Tunable var-based overrides
 # The contents of these are templated over the default files.

handlers/main.yml

@@ -57,6 +57,12 @@
 # important during a major upgrade. We therefore only put the policy
 # file in place after the service has been stopped.
 #
+- name: Check whether a custom policy file is being used
+  stat:
+    path: "/etc/keystone/policy.json-{{ keystone_venv_tag }}"
+  register: _custom_policy_file
+  listen: "Restart uWSGI"
+
 - name: Copy new policy file into place
   copy:
     src: "/etc/keystone/policy.json-{{ keystone_venv_tag }}"
@@ -65,6 +71,8 @@
     group: "{{ keystone_system_group_name }}"
     mode: "0640"
     remote_src: yes
+  when:
+    - _custom_policy_file['stat']['exists'] | bool
   listen: "Restart uWSGI"
 
 - name: Start uWSGI

releasenotes/notes/git-sourced-config-change-5b445d3ce26d29c1.yaml

@@ -0,0 +1,17 @@
+---
+upgrade:
+  - |
+    In order to collect the default files used for various templates, the
+    implementation has been changed from using a git source to rather
+    using the built-in templates from the venv build based on the setup.cfg
+    file. As such, the following variables have been removed.
+
+    * ``keystone_git_config_lookup_location``
+    * ``keystone_paste_git_file_path``
+    * ``keystone_sso_callback_git_file_path``
+
+    Instead, a location on the deployment host where the venv defaults are
+    stored is now configurable using the variable
+    ``keystone_config_cache_path`` which defaults to ``cache/keystone`` in
+    the deploy user home directory. This location is used as a template
+    source when deploying the file to the target host.

tasks/keystone_post_install.yml

@@ -21,14 +21,34 @@
   with_items: "{{ ansible_play_hosts }}"
   when: "inventory_hostname == ansible_play_hosts[0]"
 
-- name: Retrieve default configuration files
-  uri:
-    url: "{{ item }}"
-    return_content: yes
+- name: Check whether user-provided configuration files are provided
+  stat:
+    path: "{{ item }}"
   with_items:
-    - "{{ keystone_git_config_lookup_location }}/{{ keystone_paste_git_file_path }}"
-    - "{{ keystone_git_config_lookup_location }}/{{ keystone_sso_callback_git_file_path }}"
-  register: _git_file_fetch
+    - "{{ keystone_paste_default_file_path }}"
+    - "{{ keystone_policy_default_file_path }}"
+    - "{{ keystone_sso_callback_file_path }}"
+  register: _user_provided_config_files
+  delegate_to: localhost
+
+- name: Ensure that local config cache path exists on the deploy host
+  file:
+    path: "{{ keystone_config_cache_path }}"
+    state: directory
+    owner: "{{ keystone_config_cache_path_owner }}"
+  delegate_to: localhost
+  run_once: yes
+
+- name: Retrieve default configuration files from venv
+  fetch:
+    src: "{{ keystone_bin | dirname }}/etc/keystone/{{ item }}"
+    dest: "{{ keystone_config_cache_path }}/"
+    flat: yes
+  with_items:
+    - "{{ keystone_paste_default_file_path | basename }}"
+    - "{{ keystone_sso_callback_file_path | basename }}"
+  run_once: yes
+  register: _venv_config_file_fetch
 
 - name: Copy keystone configuration files
   config_template:
@@ -40,19 +60,26 @@
     mode: "0640"
     config_overrides: "{{ item.config_overrides }}"
     config_type: "{{ item.config_type }}"
+  when:
+    - item.condition | default(True)
   with_items:
     - src: "keystone.conf.j2"
       dest: "/etc/keystone/keystone.conf"
       config_overrides: "{{ keystone_keystone_conf_overrides }}"
       config_type: "ini"
-    - dest: "/etc/keystone/keystone-paste.ini"
+    - src: >-
+        {{ (_user_provided_config_files['results'][0]['stat']['exists'] | bool) |
+           ternary(keystone_paste_default_file_path,
+           keystone_config_cache_path ~ '/' ~ keystone_paste_default_file_path | basename) }}
+      dest: "/etc/keystone/keystone-paste.ini"
       config_overrides: "{{ keystone_keystone_paste_ini_overrides }}"
       config_type: "ini"
-      content: "{{ keystone_paste_user_content | default(keystone_paste_default_content, true) }}"
-    - dest: "/etc/keystone/policy.json-{{ keystone_venv_tag }}"
+    - src: "{{ keystone_policy_default_file_path }}"
+      dest: "/etc/keystone/policy.json-{{ keystone_venv_tag }}"
       config_overrides: "{{ keystone_policy_overrides }}"
       config_type: "json"
-      content: "{{ keystone_policy_user_content | default('{}', true) }}"
+      condition: >-
+        {{ _user_provided_config_files['results'][1]['stat']['exists'] | bool }}
   notify:
     - Manage LB
     - Restart uWSGI
@@ -60,7 +87,10 @@
 
 - name: Copy Keystone Federation SP SSO callback template
   copy:
-    content: "{{ keystone_sso_callback_user_content | default(keystone_sso_callback_default_content, true) }}"
+    src: >-
+        {{ (_user_provided_config_files['results'][2]['stat']['exists'] | bool) |
+           ternary(keystone_sso_callback_file_path,
+           keystone_config_cache_path ~ '/' ~ keystone_sso_callback_file_path | basename) }}
     dest: "/etc/keystone/sso_callback_template.html"
     owner: "{{ keystone_system_user_name }}"
     group: "{{ keystone_system_group_name }}"

vars/main.yml

@@ -35,15 +35,3 @@ keystone_package_list: |-
   {%   set _ = packages.extend(keystone_developer_mode_distro_packages) %}
   {% endif %}
   {{ packages }}
-
-# These vars find a file on the deployment node, if it exists - otherwise the result is empty.
-keystone_paste_user_content: "{{ lookup('pipe', 'cat ' ~ keystone_paste_default_file_path ~ ' 2>/dev/null || true') }}"
-keystone_policy_user_content: "{{ lookup('pipe', 'cat ' ~ keystone_policy_default_file_path ~ ' 2>/dev/null || true') }}"
-keystone_sso_callback_user_content: "{{ lookup('pipe', 'cat ' ~  keystone_sso_callback_file_path ~ ' 2>/dev/null || true') }}"
-
-# These vars find the appropriate result content from the with_items loop
-keystone_paste_default_content: |
-  {{ _git_file_fetch.results | selectattr('item', 'equalto', keystone_git_config_lookup_location ~ '/' ~ keystone_paste_git_file_path) | map(attribute='content') | first }}
-
-keystone_sso_callback_default_content: |
-  {{ _git_file_fetch.results | selectattr('item', 'equalto', keystone_git_config_lookup_location ~ '/' ~ keystone_sso_callback_git_file_path) | map(attribute='content') | first }}