diff --git a/defaults/main.yml b/defaults/main.yml index 6d012d3..30437b0 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -81,7 +81,6 @@ repo_build_venv_dir: "/var/www/repo/venvs" repo_build_venv_pip_install_options: > --timeout 120 --find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }} - --no-index --log /var/log/repo/repo_venv_builder.log repo_build_venv_command_options: > {{ virtualenv_bin }} diff --git a/tasks/repo_post_build.yml b/tasks/repo_post_build.yml index 0c05b81..86c8356 100644 --- a/tasks/repo_post_build.yml +++ b/tasks/repo_post_build.yml @@ -17,12 +17,7 @@ command: "ls -1 {{ repo_build_output }}" register: built_wheels tags: - - repo-build-index-wheels - - repo-create-pool - - repo-copy-wheels-to-pool - - repo-create-release-links - - repo-create-links-index - - repo-create-release-manifest + - always - name: Create release process script template: diff --git a/tasks/repo_venv_build.yml b/tasks/repo_venv_build.yml index e0acd09..3dc125e 100644 --- a/tasks/repo_venv_build.yml +++ b/tasks/repo_venv_build.yml @@ -17,15 +17,13 @@ command: which virtualenv register: virtualenv_path tags: - - repo-command-bin - - repo-create-venv + - always - name: Set virtualenv command path set_fact: virtualenv_bin: "{{ virtualenv_path.stdout }}" tags: - - repo-command-bin - - repo-create-venv + - always - name: Check for created venvs command: > diff --git a/templates/op-venv-script.sh.j2 b/templates/op-venv-script.sh.j2 index 269328b..82c5c38 100644 --- a/templates/op-venv-script.sh.j2 +++ b/templates/op-venv-script.sh.j2 @@ -5,6 +5,8 @@ set -ev function venv_create { VENV_PATH="$1" VENV_FILE="$2" + ROLE_VENV_WITH_INDEX="$3" + VENV_VALUES="$4" # If the venv working directory already exists remove it [[ -d "/tmp/${VENV_PATH}" ]] && rm -rf "/tmp/${VENV_PATH}" @@ -23,7 +25,31 @@ function venv_create { # Run the pip install within the venv and specify a specific build directory which # resolves pip locking issues when run in parallel. - ${VENV_PATH}/bin/pip install --build "/tmp/${VENV_FILE}" {{ repo_build_venv_pip_install_options.split() | join(' ') }} $3 + {% set pip_command = [] %} + {% set _ = pip_command.append('${VENV_PATH}/bin/pip install') %} + {% set _ = pip_command.append('--build "/tmp/${VENV_FILE}"') %} + {% set _ = pip_command.append(repo_build_venv_pip_install_options.strip()) %} + + if [ "${ROLE_VENV_WITH_INDEX}" = false ]; then + {{ pip_command | join(' ') }} --no-index ${VENV_VALUES} + + {% if repo_build_pip_default_index is defined %} + {% set _ = pip_command.append('--index-url ' + repo_build_pip_default_index) %} + {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_default_index | netloc_no_port) %} + {% endif %} + {% if repo_build_pip_extra_index is defined %} + {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_index) %} + {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_index | netloc_no_port) %} + {% endif %} + {% if repo_build_pip_extra_indexes is defined %} + {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_indexes | join(' --extra-index-url ')) %} + {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ')) %} + {% endif %} + + else + {{ pip_command | join(' ') }} ${VENV_VALUES} + + fi # Deactivate the venv for good measure deactivate @@ -70,14 +96,27 @@ PID=() # If the venv archive already exists the creation process will be skipped pushd "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" {% for key, value in os_group.items() %} + +{% set venvwithindex = [] %} +{% for clone_item in local_packages.results.0.item.remote_package_parts %} +{% if clone_item['name'] == (key | replace('os_', '')) and 'venvwithindex=True' in clone_item['original'] %} +{% set _ = venvwithindex.append(true) %} +{% endif %} +{% endfor %} +{% if venvwithindex %} +ROLE_VENV_WITH_INDEX=true +{% else %} +ROLE_VENV_WITH_INDEX=false +{% endif %} ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ key | replace('os_', '') }}" ROLE_VENV_FILE="{{ key | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}" if [ ! -f "${ROLE_VENV_FILE}.tgz" ];then - venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "{{ value | join(' ') }}" & + venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "${ROLE_VENV_WITH_INDEX}" "{{ value | join(' ') }}" & pid[{{ loop.index }}]=$! fi unset ROLE_VENV_PATH unset ROLE_VENV_FILE +unset ROLE_VENV_WITH_INDEX {% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %} for job_pid in ${!pid[@]}; do wait ${pid[$job_pid]} || exit 99 @@ -86,3 +125,4 @@ done {% endif %} {% endfor %} popd + diff --git a/templates/requirements.txt.j2 b/templates/requirements.txt.j2 index 033ca1a..1651aab 100644 --- a/templates/requirements.txt.j2 +++ b/templates/requirements.txt.j2 @@ -58,8 +58,10 @@ {% for requirement_raw in requirement_data %} {% set name = requirement_raw | regex_replace('(\[|>=|<=|>|<|==|~=|!=).*$','') %} {% set name_normalized = name | replace('-', '_') | lower %} -{% set requirement_normalized = all_requirements[name_normalized] %} -{% set _ = selected_requirements.update({name_normalized: requirement_normalized}) %} +{% if name_normalized in all_requirements %} +{% set requirement_normalized = all_requirements[name_normalized] %} +{% set _ = selected_requirements.update({name_normalized: requirement_normalized}) %} +{% endif %} {% endfor %} {% endif %} {% endfor %} diff --git a/tests/test-vars.yml b/tests/test-vars.yml index 99c41d4..d0474ce 100644 --- a/tests/test-vars.yml +++ b/tests/test-vars.yml @@ -295,7 +295,7 @@ local_packages: - egg_name: rally fragment: null name: rally - original: git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&projectgroup=utility_all + original: git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&venvwithindex=True&projectgroup=utility_all project_group: utility_all url: https://git.openstack.org/openstack/rally version: e6f4dafc9a3a788c3388abad32269a38f4809123 @@ -337,7 +337,7 @@ local_packages: - egg_name: tempest fragment: null name: tempest - original: git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&projectgroup=utility_all + original: git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&venvwithindex=True&projectgroup=utility_all project_group: utility_all url: https://git.openstack.org/openstack/tempest version: 479bfaccf5b1db53afb9f0663de7544b25c4bca7 @@ -364,12 +364,12 @@ local_packages: - git+https://git.openstack.org/openstack/nova-lxd@f81277ab66bed71a9c37dc8df99eface79e6156f#egg=nova_lxd&gitname=nova-lxd&projectgroup=nova_compute - git+https://git.openstack.org/openstack/nova-powervm@80e413814424c679215585c7a8e5d33281a1dea0#egg=nova_powervm&gitname=nova-powervm&projectgroup=nova_compute - git+https://git.openstack.org/openstack/nova@110c12b4145a5a978e3dd9e99e65be1d4a86ee06#egg=nova&gitname=nova&projectgroup=nova_all - - git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&projectgroup=utility_all + - git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&venvwithindex=True&projectgroup=utility_all - git+https://git.openstack.org/openstack/requirements@4f211a5002af4b19a2f050e543b34e2212250fbf#egg=requirements&gitname=requirements&projectgroup=all - git+https://git.openstack.org/openstack/sahara-dashboard@2bb9843690e583bc7944a86fdb27058d3cb2e887#egg=sahara_dashboard&gitname=sahara-dashboard&projectgroup=horizon_all - git+https://git.openstack.org/openstack/sahara@5eeeab432b95f59b2a1c2db85f1f1a297bd22668#egg=sahara&gitname=sahara&projectgroup=sahara_all - git+https://git.openstack.org/openstack/swift@d5e484e692c05c5f04069b181a5e4037239bc57c#egg=swift&gitname=swift&projectgroup=swift_all - - git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&projectgroup=utility_all + - git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&venvwithindex=True&projectgroup=utility_all - git+https://github.com/SPICE/spice-html5@54cc41299bea8cd681ed0262735e0fd821cd774a#egg=spice_html5&gitname=spice-html5&projectgroup=nova_console - git+https://github.com/kanaka/novnc@7a16304e52b3fbd466337601f05a7def2493c563#egg=novnc&gitname=novnc&projectgroup=nova_console - git+https://github.com/projectcalico/felix@a2daf3725e33a76bdb01c0311d5827f159969a29#egg=calico&gitname=calico&projectgroup=neutron_all @@ -1567,4 +1567,4 @@ local_packages: - urllib3 - virtualenv - virtualenv-tools - - wheel \ No newline at end of file + - wheel