Add better support for multi-OS deployments

This change adds a few variables which allow it to consume a repo-server
setup which has releases based on OS and ARCH. The changes require a
deployer to have at least one repo-server per-OS type being deployed.
This means that if a deployer wanted to do a mixed CentOS 7 and Ubuntu
16.04 deployment they'd need at least one repo server in each of OS type
which facilitates the building of wheels for each OS.

Change-Id: Ifd5ff22ae47bc9a2a88c1bbb2632ce48ba5accbf
Closes-Bug: 1641131
Depends-On: I3f36afa307e02a38d73b860fc23ed94a09882c34
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
(cherry picked from commit cacf41028b)

ansible-role-requirements.yml

@@ -133,7 +133,7 @@
 - name: repo_build
   scm: git
   src: https://git.openstack.org/openstack/openstack-ansible-repo_build
-  version: 7ea396c94d923fe788c196c88ab9471b9ed33887
+  version: 00fd4897853589beeeef099d70c9aca282bef66f
 - name: repo_server
   scm: git
   src: https://git.openstack.org/openstack/openstack-ansible-repo_server

playbooks/inventory/group_vars/all.yml

@@ -14,7 +14,7 @@
 # limitations under the License.
 
 ## OpenStack Source Code Release
-openstack_release: 15.0.1
+openstack_release: 15.1.0
 
 ## Verbosity Options
 debug: False
@@ -29,12 +29,26 @@ package_state: "latest"
 # Set "/var/log" to be a bind mount to the physical host.
 default_bind_mount_logs: true
 
+# Set distro variable
+os_distro_version: "{{ ansible_distribution | lower }}-{{ ansible_distribution_version.split('.')[:2] | join('.') }}-{{ ansible_architecture | lower }}"
+
 # Ensure that the package state matches the global setting
 ceph_client_package_state: "{{ package_state }}"
 galera_client_package_state: "{{ package_state }}"
 pip_install_package_state: "{{ package_state }}"
 rsyslog_client_package_state: "{{ package_state }}"
 
+## OpenStack source options
+openstack_repo_url: "http://{{ internal_lb_vip_address }}:{{ repo_server_port }}"
+openstack_repo_git_url: "git://{{ internal_lb_vip_address }}"
+
+# URL for the frozen internal openstack repo.
+repo_server_port: 8181
+repo_pkg_cache_enabled: true
+repo_pkg_cache_port: 3142
+repo_pkg_cache_url: "http://{{ internal_lb_vip_address }}:{{ repo_pkg_cache_port }}"
+repo_release_path: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/{{ os_distro_version }}"
+
 # These are pinned to ensure exactly the same behaviour forever!
 # These pins are updated through the sources-branch-updater script
 pip_packages:
@@ -43,23 +57,14 @@ pip_packages:
   - wheel==0.29.0
 
 pip_links:
-  - { name: "openstack_release", link: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/" }
+  - { name: "openstack_release", link: "{{ repo_release_path }}" }
 pip_lock_to_internal_repo: "{{ (pip_links | length) >= 1 }}"
 
 # The upper constraints to apply to all pip installations
-pip_install_upper_constraints: "{{ openstack_repo_url }}/os-releases/{{ openstack_release }}/requirements_absolute_requirements.txt"
+pip_install_upper_constraints: "{{ repo_release_path }}/requirements_absolute_requirements.txt"
 
 # The URL to retrieve the get-pip.py installation script
-pip_upstream_url: "{{ (pip_offline_install | bool) | ternary('https://bootstrap.pypa.io/get-pip.py', openstack_repo_url ~ '/os-releases/' ~ openstack_release ~ '/get-pip.py') }}"
-
-## OpenStack source options
-# URL for the frozen internal openstack repo.
-repo_server_port: 8181
-repo_pkg_cache_enabled: true
-repo_pkg_cache_port: 3142
-repo_pkg_cache_url: "http://{{ internal_lb_vip_address }}:{{ repo_pkg_cache_port }}"
-openstack_repo_url: "http://{{ internal_lb_vip_address }}:{{ repo_server_port }}"
-openstack_repo_git_url: "git://{{ internal_lb_vip_address }}"
+pip_upstream_url: "{{ (pip_offline_install | bool) | ternary('https://bootstrap.pypa.io/get-pip.py', repo_release_path ~ '/get-pip.py') }}"
 
 ## kernel modules for specific group hosts
 # :param name: name of the kernel module
@@ -117,7 +122,7 @@ service_ldap_backend_enabled: "{{ keystone_ldap is defined and keystone_ldap.Def
 
 ## Base venv configuration
 venv_tag: "{{ openstack_release }}"
-venv_base_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ ansible_distribution | lower }}"
+venv_base_download_url: "{{ openstack_repo_url }}/venvs/{{ openstack_release }}/{{ os_distro_version }}"
 
 ## Aodh
 aodh_service_region: "{{ service_region }}"

playbooks/inventory/group_vars/repo_all.yml

@@ -54,3 +54,7 @@ pip_lock_to_internal_repo: False
 #
 repo_build_pip_no_binary:
   - libvirt-python
+
+# Set the build tag and the repo version
+repo_build_release_tag: "{{ openstack_release }}"
+repo_build_os_distro_version: "{{ os_distro_version }}"

playbooks/repo-build.yml

@@ -29,11 +29,12 @@
     - name: Prepare group of master repo servers
       local_action:
         module: "add_host"
-        name: "{{ groups['repo_servers_' + hostvars[item].ansible_architecture][0] }}"
+        name: "{{ groups['repo_servers_' + (hostvars[item].ansible_distribution_version.split('.')[:2] | join('.')) + hostvars[item].ansible_architecture][0] }}"
         groups: "repo_nodes"
       # Process all nodes that don't match the architecture of repo_all[0]
-      when:
-        - hostvars[item].ansible_architecture != hostvars[groups['repo_all'][0]].ansible_architecture
+      when: >
+        hostvars[item].ansible_architecture != hostvars[groups['repo_all'][0]].ansible_architecture or
+        (hostvars[item].ansible_distribution_version.split('.')[:2] | join('.')) != (hostvars[groups['repo_all'][0]].ansible_distribution_version.split('.')[:2] | join('.'))
       with_items:
         - "{{ groups['repo_all'][1:] }}"
   tags:
@@ -123,7 +124,6 @@
 
   roles:
     - role: "repo_build"
-      repo_build_release_tag: "{{ openstack_release }}"
   vars_files:
     - defaults/repo_packages/openstack_services.yml
   vars:

playbooks/utility-install.yml

@@ -71,7 +71,7 @@
       when: ansible_pkg_mgr == 'yum'
 
     - name: Get list of python clients
-      shell: "curl -s {{ openstack_repo_url }}/os-releases/{{ openstack_release }}/requirements_absolute_requirements.txt  | grep client | cut -d'=' -f1"
+      shell: "curl -s {{ repo_release_path }}/requirements_absolute_requirements.txt  | grep client | cut -d'=' -f1"
       register: client_list
       run_once: true
       changed_when: false

releasenotes/notes/multi-os-wheel-repos-7910b13547ea59ad.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - The repo server file system structure has been updated to
+    allow for multiple Operating systems running multiple
+    architectures to be run at the same time and served from
+    a single server without impacting pools, venvs, wheel
+    archives, and manifests. The new structure follows the
+    following pattern `$RELEASE/$OS_TYPE-$ARCH` and has been
+    applied to *os-releases*, *venvs*, and *pools*.
+upgrade:
+  - The repo server file system structure has been updated to
+    allow for multiple Operating systems running multiple
+    architectures to be run at the same time and served from
+    a single server without impacting pools, venvs, wheel
+    archives, and manifests. The new structure follows the
+    following pattern `$RELEASE/$OS_TYPE-$ARCH` and has been
+    applied to *os-releases*, *venvs*, and *pools*.