From ac5e5e9283e8b263dd23b9b1f1dcea218c9370e5 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Fri, 31 Aug 2018 14:39:18 +0100 Subject: [PATCH] Simplify builds/installs by eliminating venv packaging Python venvs are not particularly portable. To overcome this problem we have a bunch of stuff implemented to fix them after we unpack them. This ranges from relatively simple things like re-implementing python in the venv and changing the shebangs for everything in it, to more complex things like having to package a venv per distro/architecture. All of this can be eliminated by simplifying the mechanism into just creating the venv on the target host, and installing the python packages into it. To help speed up the build, we can simply build wheels before hand and store them on a web server somewhere. This patch implements the changes to: 1. Do away with packaging the venv. 2. Keep it simple. We install into the venv, and that's that. 3. Add a toggle to rebuild the venv if you'd like to. 4. Use import_tasks with tags for each stage so that it's easy to skip a portion of what executes. Change-Id: I708b5cf32e5cce6a18624d0b3be0cd4c828ad389 --- defaults/main.yml | 42 +++---- tasks/main.yml | 36 ++---- tasks/python_venv_build.yml | 164 ------------------------ tasks/python_venv_install.yml | 137 ++++++++------------ tasks/python_venv_preflight.yml | 17 --- tasks/python_venv_set_facts.yml | 9 +- tasks/python_venv_wheel_build.yml | 85 +++++++++++++ tests/test-functional.yml | 170 ------------------------- tests/test.yml | 200 +++++++++++++++++++++++++----- vars/main.yml | 19 --- 10 files changed, 341 insertions(+), 538 deletions(-) delete mode 100644 tasks/python_venv_build.yml create mode 100644 tasks/python_venv_wheel_build.yml delete mode 100644 tests/test-functional.yml delete mode 100644 vars/main.yml diff --git a/defaults/main.yml b/defaults/main.yml index 512ed26..954c079 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -53,33 +53,31 @@ venv_pip_install_args: "" # The python executable to use for creating the venv venv_python_executable: "python2" -# Enable the ability to *only* do the build, so that -# the build host can be pre-populated with the venvs. -venv_build_only: no +# Enable the recreation of the venv from scratch. +# This is useful if you think the venv may be corrupted +# or if you have changed options which means that packages +# should be removed from the venv. +# Under normal circumstances, the installs will be done +# into the existing venv over the top of any previously +# installed packages. +venv_rebuild: no -# Enable the packaging of venvs, so that they can -# be used for installation. -venv_build_package: yes +# Set the host where the wheels will be built. +# If this host is not the same as the target host, then +# python wheels will be built in order to speed up the +# subsequent venv builds on this host and others. When +# this is the same as the target host, then we will not +# bother building wheels. +venv_build_host: "{{ ((groups['repo_all'] is defined) and (groups['repo_all'] | length > 0)) | ternary(groups.get('repo_all')[0], inventory_hostname) }}" -# Enable the use of wheels to speed up the venv -# build process, especially when building multiple -# venvs. -venv_build_wheels: yes +# The path for the wheel build venv. +# This is the path where a venv will be created on the +# build host for the purpose of building the wheels. +venv_build_host_venv_path: "/openstack/venvs/wheel-builder" # The path where the wheels are cached on the build host # for speeding up the build process. -venv_build_wheel_path: "{{ lookup('env', 'HOME') }}/archive/wheels" - -# The path where the venvs are archived on the build host -# to be reused for installations. -venv_build_archive_path: "{{ lookup('env', 'HOME') }}/archive/venvs" - -# The path where a built venv is sourced from when -# installing. If the path set begins with https(s):// -# then the install process will recognise the source -# is a web server. Otherwise the path is assumed to -# be on the deployment host. -venv_install_source_path: "{{ lookup('env', 'HOME') }}/archive/venvs" +venv_build_host_wheel_path: "{{ repo_pypiserver_package_path | default('/var/www/repo/pools') }}" # The facts to set when the venv changes during a # build, or the installation of a venv. diff --git a/tasks/main.yml b/tasks/main.yml index d3268cd..69ef048 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -13,35 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -- include_tasks: "python_venv_preflight.yml" +- import_tasks: "python_venv_preflight.yml" tags: - always -# To ensure that only the first host in every distro/architecture -# group does the build, and the others make use of that build, -# without forcing the use of serialisation, we use a dynamic -# group based on the venv_distro_arch_grouping which combines -# the distribution/architecture and is therefore unique for mixed -# environments. -- name: Add hosts to dynamic inventory group to ensure build/install serialization - group_by: - key: "{{ venv_distro_arch_grouping }}" +- import_tasks: "python_venv_wheel_build.yml" + run_once: yes tags: - - always + - build -- include_tasks: "python_venv_build.yml" - when: - - (venv_build_only | bool) or - ((_venv_source == 'file') and - (not _src_venv_present.stat.exists | bool)) - - inventory_hostname == groups[venv_distro_arch_grouping][0] +- import_tasks: "python_venv_install.yml" + tags: + - install -- include_tasks: "python_venv_install.yml" - when: - - not venv_build_only | bool - - inventory_hostname != groups[venv_distro_arch_grouping][0] - -- include_tasks: "python_venv_set_facts.yml" - when: - - not venv_build_only | bool - - venv_facts_when_changed != [] +- import_tasks: "python_venv_set_facts.yml" + tags: + - install diff --git a/tasks/python_venv_build.yml b/tasks/python_venv_build.yml deleted file mode 100644 index fe38dfe..0000000 --- a/tasks/python_venv_build.yml +++ /dev/null @@ -1,164 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Install distro packages for venv build - package: - name: "{{ venv_build_distro_package_list + venv_install_distro_package_list }}" - state: "{{ venv_distro_package_state }}" - update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" - cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(venv_distro_cache_valid_time, omit) }}" - when: - - (venv_build_distro_package_list | length > 0) or - (venv_install_distro_package_list | length > 0) - register: _install_distro_packages - until: _install_distro_packages is success - retries: 5 - delay: 2 - -- name: Ensure a fresh venv_install_destination_path if venv_build_only is enabled - file: - path: "{{ venv_install_destination_path }}" - state: absent - when: - - venv_build_only | bool - -- name: Create wheel/venv directories on the target host - file: - path: "{{ item }}" - state: directory - with_items: - - "{{ venv_build_archive_path }}" - - "{{ venv_build_wheel_path }}" - -# NOTE(odyssey4me): -# Not using --always-copy for CentOS/SuSE due to -# https://github.com/pypa/virtualenv/issues/565 -- name: Create the virtualenv (if it does not exist) - command: >- - virtualenv - {{ _venv_create_extra_options }} - --python={{ venv_python_executable }} - {{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }} - {{ venv_install_destination_path }} - args: - creates: "{{ venv_install_destination_path }}/bin/activate" - -- name: Upgrade pip/setuptools/wheel to the versions we want - pip: - name: - - pip - - setuptools - - wheel - state: "{{ venv_pip_package_state }}" - virtualenv: "{{ venv_install_destination_path }}" - extra_args: >- - --find-links {{ venv_build_wheel_path }}/ - --log /var/log/python_venv_build.log - {{ venv_pip_install_args }} - register: _update_virtualenv_packages - until: _update_virtualenv_packages is success - retries: 5 - delay: 2 - -- name: Build wheels for the packages to be installed into the venv - command: >- - {{ venv_install_destination_path }}/bin/pip wheel - --wheel-dir {{ venv_build_wheel_path }}/ - --find-links {{ venv_build_wheel_path }}/ - --log /var/log/python_wheel_build.log - {{ venv_pip_install_args }} - {{ venv_pip_packages | join(' ') }} - when: - - venv_build_wheels | bool - -- name: Build venv - pip: - name: "{{ venv_pip_packages }}" - state: "{{ venv_pip_package_state }}" - virtualenv: "{{ venv_install_destination_path }}" - extra_args: >- - --find-links {{ venv_build_wheel_path }}/ - --log /var/log/python_venv_build.log - {{ venv_pip_install_args }} - register: _install_venv_pip_packages - until: _install_venv_pip_packages is success - retries: 5 - delay: 2 - notify: - - venv changed - -- name: Package the venv when venv_build_package is enabled - when: - - venv_build_package | bool - block: - - - name: Clean up the virtualenv before packaging - shell: | - find {{ venv_install_destination_path }}/bin -type f -name '*.pyc' -delete - when: - - _install_venv_pip_packages is mapping - - _install_venv_pip_packages is changed - - # Note(odyssey4me): - # We purposefully use shell instead of the archive module - # here. The archive module's output is far too verbose to - # be practical when debugging. - - name: Package venv - command: | - tar czf '{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.tgz' -C '{{ venv_install_destination_path }}' . - args: - chdir: "{{ venv_install_destination_path }}" - warn: no - register: _venv_package_build - when: - - _install_venv_pip_packages is mapping - - _install_venv_pip_packages is changed - - - name: Prepare checksum for packaged venv - shell: | - sha1sum '{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.tgz' | awk '{print $1}' > '{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.checksum' - args: - executable: /bin/bash - when: - - _venv_package_build is mapping - - _venv_package_build is changed - - - name: Ensure that venv_install_source_path exists on the deployment host - file: - path: "{{ venv_install_source_path }}" - state: directory - owner: "{{ lookup('env', 'USER') }}" - recurse: yes - delegate_to: localhost - run_once: yes - when: - - _venv_source == 'file' - - inventory_hostname != 'localhost' - - - name: Copy the packaged venv and checksum file to the deployment host - fetch: - src: "{{ item.src }}" - dest: "{{ item.dest }}" - flat: yes - with_items: - - src: "{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.tgz" - dest: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.tgz" - - src: "{{ venv_build_archive_path }}/{{ venv_install_destination_path | basename }}.checksum" - dest: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.checksum" - when: - - _venv_source == 'file' - - _venv_package_build is mapping - - _venv_package_build is changed - - inventory_hostname != 'localhost' diff --git a/tasks/python_venv_install.yml b/tasks/python_venv_install.yml index bee1833..0e59930 100644 --- a/tasks/python_venv_install.yml +++ b/tasks/python_venv_install.yml @@ -13,103 +13,72 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Install distro packages for venv install +# TODO(odyssey4me): +# Set a fact for the selective inclusion of the build package list. +# Perhaps do this if the distro/architecture of the target host differs +# from the build host. + +- name: Install distro packages for venv build package: - name: "{{ venv_install_distro_package_list }}" + name: "{{ venv_build_distro_package_list + venv_install_distro_package_list }}" state: "{{ venv_distro_package_state }}" update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(venv_distro_cache_valid_time, omit) }}" when: - - venv_install_distro_package_list | length > 0 + - (venv_build_distro_package_list | length > 0) or + (venv_install_distro_package_list | length > 0) register: _install_distro_packages - until: _install_distro_packages is success + until: _install_distro_packages is success retries: 5 delay: 2 -- name: Create venv parent directories on the target host - file: - path: "{{ venv_install_destination_path | dirname }}" - state: directory - -- name: Copy the venv archive checksum - copy: - src: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.checksum" - dest: "{{ venv_install_destination_path | dirname }}/" - register: _venv_checksum_copy - when: - - _venv_source == 'file' - -- name: Copy the venv archive - copy: - src: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.tgz" - dest: "{{ venv_install_destination_path | dirname }}/" - when: - - _venv_source == 'file' - -- name: Download the venv archive checksum - uri: - url: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.checksum" - dest: "{{ venv_install_destination_path | dirname }}/" - return_content: yes - register: _venv_checksum_download - when: - - _venv_source == 'url' - -- name: Download the venv archive - get_url: - url: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.tgz" - dest: "{{ venv_install_destination_path | dirname }}/" - checksum: "sha1:{{ _venv_checksum_download.content | trim }}" - when: - - _venv_source == 'url' - -- name: Remove existing venv on target host if it is changing +- name: Ensure a fresh venv_install_destination_path if venv_rebuild is enabled file: path: "{{ venv_install_destination_path }}" state: absent when: - - (_venv_checksum_copy is mapping and _venv_checksum_copy | changed) or - (_venv_checksum_download is mapping and _venv_checksum_download | changed) - -- name: Create venv directory on the target host - file: - path: "{{ venv_install_destination_path }}" - state: directory - register: _venv_dir_create - -- name: Unarchive pre-built venv - unarchive: - src: "{{ venv_install_destination_path }}.tgz" - dest: "{{ venv_install_destination_path }}" - remote_src: yes - when: - - (_venv_checksum_copy is mapping and _venv_checksum_copy | changed) or - (_venv_checksum_download is mapping and _venv_checksum_download | changed) or - _venv_dir_create is changed - notify: - - venv changed + - venv_rebuild | bool # NOTE(odyssey4me): -# We reinitialize the venv to ensure that the right -# version of python is in the venv, but we do not -# want virtualenv to also replace pip, setuptools -# and wheel so we tell it not to. -# We do not use --always-copy for CentOS/SuSE due -# to https://github.com/pypa/virtualenv/issues/565 -- name: Update virtualenv python and paths - shell: | - sed -si '1s/^.*python.*$/#!{{ (venv_install_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_install_destination_path }}/bin/* - virtualenv {{ venv_install_destination_path }} \ - {{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }} \ - --no-pip \ - --no-setuptools \ - --no-wheel +# Not using --always-copy for CentOS/SuSE due to +# https://github.com/pypa/virtualenv/issues/565 +- name: Create the virtualenv (if it does not exist) + command: >- + virtualenv + {{ _venv_create_extra_options }} + --python={{ venv_python_executable }} + {{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }} + {{ venv_install_destination_path }} args: - executable: /bin/bash - warn: no - when: - - (_venv_checksum_copy is mapping and _venv_checksum_copy | changed) or - (_venv_checksum_download is mapping and _venv_checksum_download | changed) or - _venv_dir_create is changed - tags: - - skip_ansible_lint + creates: "{{ venv_install_destination_path }}/bin/activate" + +- name: Upgrade pip/setuptools/wheel to the versions we want + pip: + name: + - pip + - setuptools + - wheel + state: "{{ venv_pip_package_state }}" + virtualenv: "{{ venv_install_destination_path }}" + extra_args: >- + --log /var/log/python_venv_build.log + {{ venv_pip_install_args }} + register: _update_virtualenv_packages + until: _update_virtualenv_packages is success + retries: 5 + delay: 2 + +- name: Install python packages into the venv + pip: + name: "{{ venv_pip_packages }}" + state: "{{ venv_pip_package_state }}" + virtualenv: "{{ venv_install_destination_path }}" + extra_args: >- + --log /var/log/python_venv_build.log + {{ venv_pip_install_args }} + register: _install_venv_pip_packages + until: _install_venv_pip_packages is success + retries: 5 + delay: 2 + notify: + - venv changed diff --git a/tasks/python_venv_preflight.yml b/tasks/python_venv_preflight.yml index 5ebc52e..91ebc1b 100644 --- a/tasks/python_venv_preflight.yml +++ b/tasks/python_venv_preflight.yml @@ -45,20 +45,3 @@ _venv_create_extra_options: >- {{ ((_virtualenv_version.stdout | trim) is version_compare('14.0.0', '<')) | ternary('--never-download', '--no-download') }} {{ ((_virtualenv_version.stdout | trim) is version_compare('1.7.0', '<')) | ternary('--no-site-packages', '') }} - -- name: Check whether the venv_install_source_path is a URL or a file path - set_fact: - _venv_source: "{{ ((venv_install_source_path | trim) is match('^https?://.*')) | ternary('url', 'file') }}" - -- name: Check if venv tgz is present on the deployment host - stat: - path: "{{ venv_install_source_path }}/{{ venv_install_destination_path | basename }}.tgz" - get_attributes: no - get_checksum: no - get_md5: no - get_mime: no - register: _src_venv_present - delegate_to: localhost - run_once: yes - when: - - _venv_source == 'file' diff --git a/tasks/python_venv_set_facts.yml b/tasks/python_venv_set_facts.yml index 7e09fe0..627e879 100644 --- a/tasks/python_venv_set_facts.yml +++ b/tasks/python_venv_set_facts.yml @@ -26,12 +26,7 @@ value: "{{ item.value }}" with_items: "{{ venv_facts_when_changed }}" when: - - (_venv_checksum_copy is defined and - _venv_checksum_copy is mapping and - _venv_checksum_copy | changed) or - (_venv_checksum_download is defined and - _venv_checksum_download is mapping and - _venv_checksum_download | changed) or - (_install_venv_pip_packages is defined and + - venv_facts_when_changed != [] + - (_install_venv_pip_packages is defined and _install_venv_pip_packages is mapping and _install_venv_pip_packages | changed) diff --git a/tasks/python_venv_wheel_build.yml b/tasks/python_venv_wheel_build.yml new file mode 100644 index 0000000..789d3eb --- /dev/null +++ b/tasks/python_venv_wheel_build.yml @@ -0,0 +1,85 @@ +--- +# Copyright 2018, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- name: Build the wheels on the build host + delegate_to: "{{ venv_build_host }}" + become: "{{ venv_build_host == 'localhost' }}" + when: + - venv_build_host != inventory_hostname + block: + - name: Install distro packages for wheel build + package: + name: "{{ venv_build_distro_package_list + venv_install_distro_package_list }}" + state: "{{ venv_distro_package_state }}" + update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" + cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(venv_distro_cache_valid_time, omit) }}" + when: + - (venv_build_distro_package_list | length > 0) or + (venv_install_distro_package_list | length > 0) + register: _install_build_distro_packages + until: _install_build_distro_packages is success + retries: 5 + delay: 2 + + - name: Ensure a fresh venv_build_host_venv_path if venv_rebuild is enabled + file: + path: "{{ venv_build_host_venv_path }}" + state: absent + when: + - venv_rebuild | bool + + - name: Create wheel directory on the build host + file: + path: "{{ venv_build_host_wheel_path }}" + state: directory + + # NOTE(odyssey4me): + # Not using --always-copy for CentOS/SuSE due to + # https://github.com/pypa/virtualenv/issues/565 + - name: Create the wheel build virtualenv (if it does not exist) + command: >- + virtualenv + {{ _venv_create_extra_options }} + --python={{ venv_python_executable }} + {{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }} + {{ venv_build_host_venv_path }} + args: + creates: "{{ venv_build_host_venv_path }}/bin/activate" + + - name: Upgrade the wheel build virtualenv pip/setuptools/wheel to the versions we want + pip: + name: + - pip + - setuptools + - wheel + state: "{{ venv_pip_package_state }}" + virtualenv: "{{ venv_build_host_venv_path }}" + extra_args: >- + --find-links {{ venv_build_host_wheel_path }}/ + --log /var/log/python_venv_build.log + {{ venv_pip_install_args }} + register: _update_virtualenv_packages + until: _update_virtualenv_packages is success + retries: 5 + delay: 2 + + - name: Build wheels for the packages to be installed into the venv + command: >- + {{ venv_build_host_venv_path }}/bin/pip wheel + --wheel-dir {{ venv_build_host_wheel_path }}/ + --find-links {{ venv_build_host_wheel_path }}/ + --log /var/log/python_wheel_build.log + {{ venv_pip_install_args }} + {{ venv_pip_packages | join(' ') }} diff --git a/tests/test-functional.yml b/tests/test-functional.yml deleted file mode 100644 index 2647fdf..0000000 --- a/tests/test-functional.yml +++ /dev/null @@ -1,170 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Clean up from previous tests - hosts: "{{ build_host }}:{{ install_hosts }}" - become: yes - any_errors_fatal: yes - tasks: - - name: Clean up files/folders from previous tests - file: - path: "{{ item.path }}" - state: absent - with_items: - - path: "{{ lookup('env', 'HOME') }}/archive" - - path: "{{ lookup('env', 'HOME') }}/venvs" - - - name: Clean up facts from previous tests - ini_file: - path: "/etc/ansible/facts.d/openstack_ansible.fact" - section: "{{ item }}" - state: absent - with_items: "{{ build_host.split(':') + install_hosts.split(':') }}" - - - name: Refresh the inventory to clear the added groups - meta: refresh_inventory - - - name: Set venv_build_archive_path and venv_install_source_path - set_fact: - venv_build_archive_path: >- - {%- if build_host == "container1" %} - {%- if ansible_distribution == "Ubuntu" %} - {%- set _path = "/var/www/html" %} - {%- elif ansible_distribution == "CentOS" %} - {%- set _path = "/usr/share/nginx/html" %} - {%- else %} - {%- set _path = "/srv/www/htdocs" %} - {%- endif %} - {%- else -%} - {%- set _path = lookup('env', 'HOME') ~ "/archive/venvs" -%} - {%- endif -%} - {{- _path }} - venv_install_source_path: >- - {%- if build_host == "container1" -%} - http://{{ hostvars['container1'].ansible_default_ipv4.address }} - {%- else -%} - {{- lookup('env', 'HOME') }}/archive/venvs - {%- endif -%} - - - name: Setup web server for url-based venv install - when: - - inventory_hostname == 'container1' - block: - - name: Install EPEL gpg keys - rpm_key: - key: "http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7" - state: present - when: - - ansible_pkg_mgr in ['yum', 'dnf'] - register: _add_yum_keys - until: _add_yum_keys is success - retries: 5 - delay: 2 - - - name: Install the EPEL repository - yum_repository: - name: epel-nginx - baseurl: "{{ (centos_epel_mirror | default ('http://download.fedoraproject.org/pub/epel')) ~ '/' ~ ansible_distribution_major_version ~ '/' ~ ansible_architecture }}" - description: 'Extra Packages for Enterprise Linux 7 - $basearch' - gpgcheck: yes - enabled: yes - state: present - includepkgs: 'nginx*' - when: - - ansible_pkg_mgr in ['yum', 'dnf'] - register: install_epel_repo - until: install_epel_repo is success - retries: 5 - delay: 2 - - - name: Install distro packages - package: - name: "nginx" - update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" - register: install - until: install is success - retries: 5 - delay: 2 - - - name: Enable and start nginx - service: - name: nginx - enabled: yes - daemon_reload: yes - state: restarted - -- name: Execute build - hosts: "{{ build_host }}" - become: yes - any_errors_fatal: yes - tasks: - - name: Execute venv build - include_role: - name: "python_venv_build" - private: yes - vars: - venv_build_only: yes - -- name: Execute install - hosts: "{{ install_hosts }}" - become: yes - any_errors_fatal: yes - tasks: - - - name: Execute venv install - include_role: - name: "python_venv_build" - private: yes - vars: - venv_facts_when_changed: - - section: "{{ inventory_hostname }}" - option: "test" - value: True - - - name: refresh local facts - setup: - filter: ansible_local - gather_subset: "!all" - - - name: Show the ansible_local facts - debug: - var: ansible_local - - - name: Verify that the facts were set - assert: - that: - - ansible_local['openstack_ansible'][inventory_hostname]['test'] | bool - - - name: Find files/folders on targets - find: - file_type: directory - get_checksum: no - recurse: no - paths: - - "{{ venv_install_destination_path | dirname }}" - register: _target_folders - - - name: Compile the folder list from the targets - set_fact: - _target_folder_list: "{{ _target_folders['files'] | map(attribute='path') | list }}" - - - name: Show the files/folder from the targets - debug: - var: _target_folder_list - - - name: Verify the folder list from the targets - assert: - that: - - "{{ venv_install_destination_path in _target_folder_list }}" diff --git a/tests/test.yml b/tests/test.yml index 1925c9f..e53ac96 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -16,38 +16,180 @@ - name: Prepare the host/containers import_playbook: common/test-setup-host.yml -- name: Verify building on a web server, and installing using a URL - import_playbook: test-functional.yml - vars: - build_host: container1 - install_hosts: "container2:container3" - venv_pip_packages: - - "Jinja2==2.10" - venv_install_destination_path: "{{ lookup('env', 'HOME') }}/venvs/test-venv" +- name: Prepare web server on localhost to serve python packages + hosts: localhost + connection: local + become: yes + any_errors_fatal: yes + tasks: + - name: Set venv_build_archive_path and venv_install_source_path + set_fact: + venv_build_host_wheel_path: >- + {%- if ansible_distribution == "Ubuntu" %} + {%- set _path = "/var/www/html" %} + {%- elif ansible_distribution == "CentOS" %} + {%- set _path = "/usr/share/nginx/html" %} + {%- else %} + {%- set _path = "/srv/www/htdocs" %} + {%- endif %} + {{- _path }} -- name: Verify building on localhost, and installing using a copy - import_playbook: test-functional.yml - vars: - build_host: localhost - install_hosts: "container1:container2:container3" - venv_pip_packages: - - "Jinja2==2.10" - venv_install_destination_path: "{{ lookup('env', 'HOME') }}/venvs/test-venv" + - name: Install EPEL gpg keys + rpm_key: + key: "http://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-7" + state: present + when: + - ansible_pkg_mgr in ['yum', 'dnf'] + register: _add_yum_keys + until: _add_yum_keys is success + retries: 5 + delay: 2 -- name: Verify building on a build host, and installing using a copy - import_playbook: test-functional.yml - vars: - build_host: container2 - install_hosts: "container1:container3" - venv_pip_packages: - - "Jinja2==2.10" - venv_install_destination_path: "{{ lookup('env', 'HOME') }}/venvs/test-venv" + - name: Install the EPEL repository + yum_repository: + name: epel-nginx + baseurl: "{{ (centos_epel_mirror | default ('http://download.fedoraproject.org/pub/epel')) ~ '/' ~ ansible_distribution_major_version ~ '/' ~ ansible_architecture }}" + description: 'Extra Packages for Enterprise Linux 7 - $basearch' + gpgcheck: yes + enabled: yes + state: present + includepkgs: 'nginx*' + when: + - ansible_pkg_mgr in ['yum', 'dnf'] + register: install_epel_repo + until: install_epel_repo is success + retries: 5 + delay: 2 -- name: Verify setting multiple build hosts behaves correctly - import_playbook: test-functional.yml + - name: Install distro packages + package: + name: "nginx" + update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" + register: install + until: install is success + retries: 5 + delay: 2 + + - name: Enable and start nginx + service: + name: nginx + enabled: yes + daemon_reload: yes + state: restarted + +- name: Verify not using a build host + hosts: "container1" + remote_user: root + any_errors_fatal: yes vars: - build_host: "localhost:container1" - install_hosts: "container2:container3" venv_pip_packages: - "Jinja2==2.10" - venv_install_destination_path: "{{ lookup('env', 'HOME') }}/venvs/test-venv" + venv_install_destination_path: "/openstack/venvs/test-venv" + tasks: + + - name: Execute venv install + include_role: + name: "python_venv_build" + private: yes + vars: + venv_facts_when_changed: + - section: "{{ inventory_hostname }}" + option: "test" + value: True + + - name: refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + + - name: Show the ansible_local facts + debug: + var: ansible_local + + - name: Verify that the facts were set + assert: + that: + - ansible_local['openstack_ansible'][inventory_hostname]['test'] | bool + + - name: Find files/folders on targets + find: + file_type: directory + get_checksum: no + recurse: no + paths: + - "{{ venv_install_destination_path | dirname }}" + register: _target_folders + + - name: Compile the folder list from the targets + set_fact: + _target_folder_list: "{{ _target_folders['files'] | map(attribute='path') | list }}" + + - name: Show the files/folder from the targets + debug: + var: _target_folder_list + + - name: Verify the folder list from the targets + assert: + that: + - "{{ venv_install_destination_path in _target_folder_list }}" + +- name: Verify using a build host + hosts: "container2:container3" + remote_user: root + any_errors_fatal: yes + vars: + venv_pip_packages: + - "Jinja2==2.10" + venv_install_destination_path: "/openstack/venvs/test-venv" + venv_pip_install_args: >- + --find-links http://{{ hostvars['localhost'].ansible_default_ipv4.address }} + --trusted-host {{ hostvars['localhost'].ansible_default_ipv4.address }} + venv_build_host: localhost + venv_build_host_wheel_path: "{{ hostvars['localhost']['venv_build_host_wheel_path'] }}" + tasks: + + - name: Execute venv install + include_role: + name: "python_venv_build" + private: yes + vars: + venv_facts_when_changed: + - section: "{{ inventory_hostname }}" + option: "test" + value: True + + - name: refresh local facts + setup: + filter: ansible_local + gather_subset: "!all" + + - name: Show the ansible_local facts + debug: + var: ansible_local + + - name: Verify that the facts were set + assert: + that: + - ansible_local['openstack_ansible'][inventory_hostname]['test'] | bool + + - name: Find files/folders on targets + find: + file_type: directory + get_checksum: no + recurse: no + paths: + - "{{ venv_install_destination_path | dirname }}" + register: _target_folders + + - name: Compile the folder list from the targets + set_fact: + _target_folder_list: "{{ _target_folders['files'] | map(attribute='path') | list }}" + + - name: Show the files/folder from the targets + debug: + var: _target_folder_list + + - name: Verify the folder list from the targets + assert: + that: + - "{{ venv_install_destination_path in _target_folder_list }}" diff --git a/vars/main.yml b/vars/main.yml deleted file mode 100644 index 02b28e6..0000000 --- a/vars/main.yml +++ /dev/null @@ -1,19 +0,0 @@ ---- -# Copyright 2018, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# NOTE(hwoarang): ansible_distribution may return a string with spaces -# such as "openSUSE Leap" so we need to replace the space with underscore -# in order to create a more sensible repo name for the distro. -venv_distro_arch_grouping: "{{ (ansible_distribution | lower) | replace(' ', '_') }}-{{ ansible_distribution_version.split('.')[:2] | join('.') }}-{{ ansible_architecture | lower }}"