Updates to support ignored packages and external indexes

This change ensures that a venv create process can work when a dependency is crafted outside of the OpenStack global requirements system. The change keys off of the original package url setting the isolated perameter should "withindex=True" be found. The requirements.txt.j2 template file has been updated to allow packages to be ignored from wheel building should the `ignorerequirement=True` property be set within a given remote package. This ability is important as some packages, generally external testing repos, do not use requirement boundries and are known to break given enough time. Tagged tasks that MUST always be run have had tags set to "always". Change-Id: I5d11aca84dcb74c77f3d0e3f31ce6546458a3e84 Closes-Bug: #1631992 Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
2016-10-14 15:23:07 -05:00 · 2016-10-14 15:23:07 -05:00 · baab4e2e7c
parent 05026c23bd
commit baab4e2e7c
6 changed files with 54 additions and 20 deletions
--- a/defaults/main.yml
+++ b/defaults/main.yml
@ -81,7 +81,6 @@ repo_build_venv_dir: "/var/www/repo/venvs"
 repo_build_venv_pip_install_options: >
  --timeout 120
  --find-links {{ repo_build_release_path }}/{{ repo_build_release_tag }}
-  --no-index
  --log /var/log/repo/repo_venv_builder.log
 repo_build_venv_command_options: >
  {{ virtualenv_bin }}
--- a/tasks/repo_post_build.yml
+++ b/tasks/repo_post_build.yml
@ -17,12 +17,7 @@
  command: "ls -1 {{ repo_build_output }}"
  register: built_wheels
  tags:
-    - repo-build-index-wheels
-    - repo-create-pool
-    - repo-copy-wheels-to-pool
-    - repo-create-release-links
-    - repo-create-links-index
-    - repo-create-release-manifest
+    - always

 - name: Create release process script
  template:
--- a/tasks/repo_venv_build.yml
+++ b/tasks/repo_venv_build.yml
@ -17,15 +17,13 @@
  command: which virtualenv
  register: virtualenv_path
  tags:
-    - repo-command-bin
-    - repo-create-venv
+    - always

 - name: Set virtualenv command path
  set_fact:
    virtualenv_bin: "{{ virtualenv_path.stdout }}"
  tags:
-    - repo-command-bin
-    - repo-create-venv
+    - always

 - name: Check for created venvs
  command: >
--- a/templates/op-venv-script.sh.j2
+++ b/templates/op-venv-script.sh.j2
@ -5,6 +5,8 @@ set -ev
 function venv_create {
    VENV_PATH="$1"
    VENV_FILE="$2"
+    ROLE_VENV_WITH_INDEX="$3"
+    VENV_VALUES="$4"

    # If the venv working directory already exists remove it
    [[ -d "/tmp/${VENV_PATH}" ]] && rm -rf "/tmp/${VENV_PATH}"
@ -23,7 +25,31 @@ function venv_create {

    # Run the pip install within the venv and specify a specific build directory which
    #  resolves pip locking issues when run in parallel.
-    ${VENV_PATH}/bin/pip install --build "/tmp/${VENV_FILE}" {{ repo_build_venv_pip_install_options.split() | join(' ') }} $3
+    {% set pip_command = [] %}
+    {% set _ = pip_command.append('${VENV_PATH}/bin/pip install') %}
+    {% set _ = pip_command.append('--build "/tmp/${VENV_FILE}"') %}
+    {% set _ = pip_command.append(repo_build_venv_pip_install_options.strip()) %}
+
+    if [ "${ROLE_VENV_WITH_INDEX}" = false ]; then
+    {{ pip_command | join(' ') }} --no-index ${VENV_VALUES}
+
+    {% if repo_build_pip_default_index is defined %}
+    {% set _ = pip_command.append('--index-url ' + repo_build_pip_default_index) %}
+    {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_default_index | netloc_no_port) %}
+    {% endif %}
+    {% if repo_build_pip_extra_index is defined %}
+    {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_index)  %}
+    {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_index | netloc_no_port) %}
+    {% endif %}
+    {% if repo_build_pip_extra_indexes is defined %}
+    {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_indexes | join(' --extra-index-url ')) %}
+    {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ')) %}
+    {% endif %}
+
+    else
+    {{ pip_command | join(' ') }} ${VENV_VALUES}
+
+    fi

    # Deactivate the venv for good measure
    deactivate
@ -70,14 +96,27 @@ PID=()
 #  If the venv archive already exists the creation process will be skipped
 pushd "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}"
 {% for key, value in os_group.items() %}
+
+{% set venvwithindex = [] %}
+{% for clone_item in local_packages.results.0.item.remote_package_parts %}
+{%   if clone_item['name'] == (key | replace('os_', '')) and 'venvwithindex=True' in clone_item['original'] %}
+{%     set _ = venvwithindex.append(true) %}
+{%   endif %}
+{% endfor %}
+{% if venvwithindex %}
+ROLE_VENV_WITH_INDEX=true
+{% else %}
+ROLE_VENV_WITH_INDEX=false
+{% endif %}
 ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ key | replace('os_', '') }}"
 ROLE_VENV_FILE="{{ key | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}"
 if [ ! -f "${ROLE_VENV_FILE}.tgz" ];then
-    venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "{{ value | join(' ') }}" &
+    venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "${ROLE_VENV_WITH_INDEX}" "{{ value | join(' ') }}" &
    pid[{{ loop.index }}]=$!
 fi
 unset ROLE_VENV_PATH
 unset ROLE_VENV_FILE
+unset ROLE_VENV_WITH_INDEX
 {% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %}
 for job_pid in ${!pid[@]}; do
    wait ${pid[$job_pid]} || exit 99
@ -86,3 +125,4 @@ done
 {% endif %}
 {% endfor %}
 popd
+
--- a/templates/requirements.txt.j2
+++ b/templates/requirements.txt.j2
@ -58,8 +58,10 @@
 {%           for requirement_raw in requirement_data %}
 {%             set name = requirement_raw | regex_replace('(\[|>=|<=|>|<|==|~=|!=).*$','') %}
 {%             set name_normalized = name | replace('-', '_') | lower %}
-{%             set requirement_normalized = all_requirements[name_normalized] %}
-{%             set _ = selected_requirements.update({name_normalized: requirement_normalized}) %}
+{%             if name_normalized in all_requirements %}
+{%               set requirement_normalized = all_requirements[name_normalized] %}
+{%               set _ = selected_requirements.update({name_normalized: requirement_normalized}) %}
+{%             endif %}
 {%           endfor %}
 {%         endif %}
 {%       endfor %}
--- a/tests/test-vars.yml
+++ b/tests/test-vars.yml
@ -295,7 +295,7 @@ local_packages:
        - egg_name: rally
          fragment: null
          name: rally
-          original: git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&projectgroup=utility_all
+          original: git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&venvwithindex=True&projectgroup=utility_all
          project_group: utility_all
          url: https://git.openstack.org/openstack/rally
          version: e6f4dafc9a3a788c3388abad32269a38f4809123
@ -337,7 +337,7 @@ local_packages:
        - egg_name: tempest
          fragment: null
          name: tempest
-          original: git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&projectgroup=utility_all
+          original: git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&venvwithindex=True&projectgroup=utility_all
          project_group: utility_all
          url: https://git.openstack.org/openstack/tempest
          version: 479bfaccf5b1db53afb9f0663de7544b25c4bca7
@ -364,12 +364,12 @@ local_packages:
        - git+https://git.openstack.org/openstack/nova-lxd@f81277ab66bed71a9c37dc8df99eface79e6156f#egg=nova_lxd&gitname=nova-lxd&projectgroup=nova_compute
        - git+https://git.openstack.org/openstack/nova-powervm@80e413814424c679215585c7a8e5d33281a1dea0#egg=nova_powervm&gitname=nova-powervm&projectgroup=nova_compute
        - git+https://git.openstack.org/openstack/nova@110c12b4145a5a978e3dd9e99e65be1d4a86ee06#egg=nova&gitname=nova&projectgroup=nova_all
-        - git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&projectgroup=utility_all
+        - git+https://git.openstack.org/openstack/rally@e6f4dafc9a3a788c3388abad32269a38f4809123#egg=rally&gitname=rally&venvwithindex=True&projectgroup=utility_all
        - git+https://git.openstack.org/openstack/requirements@4f211a5002af4b19a2f050e543b34e2212250fbf#egg=requirements&gitname=requirements&projectgroup=all
        - git+https://git.openstack.org/openstack/sahara-dashboard@2bb9843690e583bc7944a86fdb27058d3cb2e887#egg=sahara_dashboard&gitname=sahara-dashboard&projectgroup=horizon_all
        - git+https://git.openstack.org/openstack/sahara@5eeeab432b95f59b2a1c2db85f1f1a297bd22668#egg=sahara&gitname=sahara&projectgroup=sahara_all
        - git+https://git.openstack.org/openstack/swift@d5e484e692c05c5f04069b181a5e4037239bc57c#egg=swift&gitname=swift&projectgroup=swift_all
-        - git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&projectgroup=utility_all
+        - git+https://git.openstack.org/openstack/tempest@479bfaccf5b1db53afb9f0663de7544b25c4bca7#egg=tempest&gitname=tempest&venvwithindex=True&projectgroup=utility_all
        - git+https://github.com/SPICE/spice-html5@54cc41299bea8cd681ed0262735e0fd821cd774a#egg=spice_html5&gitname=spice-html5&projectgroup=nova_console
        - git+https://github.com/kanaka/novnc@7a16304e52b3fbd466337601f05a7def2493c563#egg=novnc&gitname=novnc&projectgroup=nova_console
        - git+https://github.com/projectcalico/felix@a2daf3725e33a76bdb01c0311d5827f159969a29#egg=calico&gitname=calico&projectgroup=neutron_all
@ -1567,4 +1567,4 @@ local_packages:
            - urllib3
            - virtualenv
            - virtualenv-tools
-            - wheel
+            - wheel