From baab4e2e7c5d96a3cd52bcf2703b12f064f63b73 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Fri, 14 Oct 2016 15:23:07 -0500 Subject: [PATCH] Updates to support ignored packages and external indexes This change ensures that a venv create process can work when a dependency is crafted outside of the OpenStack global requirements system. The change keys off of the original package url setting the isolated perameter should "withindex=True" be found. The requirements.txt.j2 template file has been updated to allow packages to be ignored from wheel building should the `ignorerequirement=True` property be set within a given remote package. This ability is important as some packages, generally external testing repos, do not use requirement boundries and are known to break given enough time. Tagged tasks that MUST always be run have had tags set to "always". Change-Id: I5d11aca84dcb74c77f3d0e3f31ce6546458a3e84 Closes-Bug: #1631992 Signed-off-by: Kevin Carter --- defaults/main.yml | 1 - tasks/repo_post_build.yml | 7 +----- tasks/repo_venv_build.yml | 6 ++--- templates/op-venv-script.sh.j2 | 44 ++++++++++++++++++++++++++++++++-- templates/requirements.txt.j2 | 6 +++-- tests/test-vars.yml | 10 ++++---- 6 files changed, 54 insertions(+), 20 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 6d012d3..30437b0 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -81,7 +81,6 @@ repo_build_venv_dir: "/var/www/repo/venvs" repo_build_venv_pip_install_options: > --timeout 120 --find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }} - --no-index --log /var/log/repo/repo_venv_builder.log repo_build_venv_command_options: > {{ virtualenv_bin }} diff --git a/tasks/repo_post_build.yml b/tasks/repo_post_build.yml index 0c05b81..86c8356 100644 --- a/tasks/repo_post_build.yml +++ b/tasks/repo_post_build.yml @@ -17,12 +17,7 @@ command: "ls -1 {{ repo_build_output }}" register: built_wheels tags: - - repo-build-index-wheels - - repo-create-pool - - repo-copy-wheels-to-pool - - repo-create-release-links - - repo-create-links-index - - repo-create-release-manifest + - always - name: Create release process script template: diff --git a/tasks/repo_venv_build.yml b/tasks/repo_venv_build.yml index e0acd09..3dc125e 100644 --- a/tasks/repo_venv_build.yml +++ b/tasks/repo_venv_build.yml @@ -17,15 +17,13 @@ command: which virtualenv register: virtualenv_path tags: - - repo-command-bin - - repo-create-venv + - always - name: Set virtualenv command path set_fact: virtualenv_bin: "{{ virtualenv_path.stdout }}" tags: - - repo-command-bin - - repo-create-venv + - always - name: Check for created venvs command: > diff --git a/templates/op-venv-script.sh.j2 b/templates/op-venv-script.sh.j2 index 269328b..82c5c38 100644 --- a/templates/op-venv-script.sh.j2 +++ b/templates/op-venv-script.sh.j2 @@ -5,6 +5,8 @@ set -ev function venv_create { VENV_PATH="$1" VENV_FILE="$2" + ROLE_VENV_WITH_INDEX="$3" + VENV_VALUES="$4" # If the venv working directory already exists remove it [[ -d "/tmp/${VENV_PATH}" ]] && rm -rf "/tmp/${VENV_PATH}" @@ -23,7 +25,31 @@ function venv_create { # Run the pip install within the venv and specify a specific build directory which # resolves pip locking issues when run in parallel. - ${VENV_PATH}/bin/pip install --build "/tmp/${VENV_FILE}" {{ repo_build_venv_pip_install_options.split() | join(' ') }} $3 + {% set pip_command = [] %} + {% set _ = pip_command.append('${VENV_PATH}/bin/pip install') %} + {% set _ = pip_command.append('--build "/tmp/${VENV_FILE}"') %} + {% set _ = pip_command.append(repo_build_venv_pip_install_options.strip()) %} + + if [ "${ROLE_VENV_WITH_INDEX}" = false ]; then + {{ pip_command | join(' ') }} --no-index ${VENV_VALUES} + + {% if repo_build_pip_default_index is defined %} + {% set _ = pip_command.append('--index-url ' + repo_build_pip_default_index) %} + {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_default_index | netloc_no_port) %} + {% endif %} + {% if repo_build_pip_extra_index is defined %} + {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_index) %} + {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_index | netloc_no_port) %} + {% endif %} + {% if repo_build_pip_extra_indexes is defined %} + {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_indexes | join(' --extra-index-url ')) %} + {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ')) %} + {% endif %} + + else + {{ pip_command | join(' ') }} ${VENV_VALUES} + + fi # Deactivate the venv for good measure deactivate @@ -70,14 +96,27 @@ PID=() # If the venv archive already exists the creation process will be skipped pushd "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" {% for key, value in os_group.items() %} + +{% set venvwithindex = [] %} +{% for clone_item in local_packages.results.0.item.remote_package_parts %} +{% if clone_item['name'] == (key | replace('os_', '')) and 'venvwithindex=True' in clone_item['original'] %} +{% set _ = venvwithindex.append(true) %} +{% endif %} +{% endfor %} +{% if venvwithindex %} +ROLE_VENV_WITH_INDEX=true +{% else %} +ROLE_VENV_WITH_INDEX=false +{% endif %} ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ key | replace('os_', '') }}" ROLE_VENV_FILE="{{ key | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}" if [ ! -f "${ROLE_VENV_FILE}.tgz" ];then - venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "{{ value | join(' ') }}" & + venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "${ROLE_VENV_WITH_INDEX}" "{{ value | join(' ') }}" & pid[{{ loop.index }}]=$! fi unset ROLE_VENV_PATH unset ROLE_VENV_FILE +unset ROLE_VENV_WITH_INDEX {% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %} for job_pid in ${!pid[@]}; do wait ${pid[$job_pid]} || exit 99 @@ -86,3 +125,4 @@ done {% endif %} {% endfor %} popd + diff --git a/templates/requirements.txt.j2 b/templates/requirements.txt.j2 index 033ca1a..1651aab 100644 --- a/templates/requirements.txt.j2 +++ b/templates/requirements.txt.j2 @@ -58,8 +58,10 @@ {% for requirement_raw in requirement_data %} {% set name = requirement_raw | regex_replace('(\[|>=|<=|>|<|==|~=|!=).*$','') %} {% set name_normalized = name | replace('-', '_') | lower %} -{% set requirement_normalized = all_requirements[name_normalized] %} -{% set _ = selected_requirements.update({name_normalized: requirement_normalized}) %} +{% if name_normalized in all_requirements %} +{% set requirement_normalized = all_requirements[name_normalized] %} +{% set _ = selected_requirements.update({name_normalized: requirement_normalized}) %} +{% endif %} {% endfor %} {% endif %} {% endfor %} diff --git a/tests/test-vars.yml b/tests/test-vars.yml index 99c41d4..d0474ce 100644 --- a/tests/test-vars.yml +++ b/tests/test-vars.yml @@ -295,7 +295,7 @@ local_packages: - egg_name: rally fragment: null name: rally - original: git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&projectgroup=utility_all + original: git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&venvwithindex=True&projectgroup=utility_all project_group: utility_all url: https://git.openstack.org/openstack/rally version: e6f4dafc9a3a788c3388abad32269a38f4809123 @@ -337,7 +337,7 @@ local_packages: - egg_name: tempest fragment: null name: tempest - original: git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&projectgroup=utility_all + original: git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&venvwithindex=True&projectgroup=utility_all project_group: utility_all url: https://git.openstack.org/openstack/tempest version: 479bfaccf5b1db53afb9f0663de7544b25c4bca7 @@ -364,12 +364,12 @@ local_packages: - git+https://git.openstack.org/openstack/nova-lxd@f81277ab66bed71a9c37dc8df99eface79e6156f#egg=nova_lxd&gitname=nova-lxd&projectgroup=nova_compute - git+https://git.openstack.org/openstack/nova-powervm@80e413814424c679215585c7a8e5d33281a1dea0#egg=nova_powervm&gitname=nova-powervm&projectgroup=nova_compute - git+https://git.openstack.org/openstack/nova@110c12b4145a5a978e3dd9e99e65be1d4a86ee06#egg=nova&gitname=nova&projectgroup=nova_all - - git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&projectgroup=utility_all + - git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&venvwithindex=True&projectgroup=utility_all - git+https://git.openstack.org/openstack/requirements@4f211a5002af4b19a2f050e543b34e2212250fbf#egg=requirements&gitname=requirements&projectgroup=all - git+https://git.openstack.org/openstack/sahara-dashboard@2bb9843690e583bc7944a86fdb27058d3cb2e887#egg=sahara_dashboard&gitname=sahara-dashboard&projectgroup=horizon_all - git+https://git.openstack.org/openstack/sahara@5eeeab432b95f59b2a1c2db85f1f1a297bd22668#egg=sahara&gitname=sahara&projectgroup=sahara_all - git+https://git.openstack.org/openstack/swift@d5e484e692c05c5f04069b181a5e4037239bc57c#egg=swift&gitname=swift&projectgroup=swift_all - - git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&projectgroup=utility_all + - git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&venvwithindex=True&projectgroup=utility_all - git+https://github.com/SPICE/spice-html5@54cc41299bea8cd681ed0262735e0fd821cd774a#egg=spice_html5&gitname=spice-html5&projectgroup=nova_console - git+https://github.com/kanaka/novnc@7a16304e52b3fbd466337601f05a7def2493c563#egg=novnc&gitname=novnc&projectgroup=nova_console - git+https://github.com/projectcalico/felix@a2daf3725e33a76bdb01c0311d5827f159969a29#egg=calico&gitname=calico&projectgroup=neutron_all @@ -1567,4 +1567,4 @@ local_packages: - urllib3 - virtualenv - virtualenv-tools - - wheel \ No newline at end of file + - wheel