From 7348e023aca51d1b914c2a8fdc0cf270c87be8e4 Mon Sep 17 00:00:00 2001 From: Kevin Carter Date: Wed, 23 Mar 2016 03:01:18 -0500 Subject: [PATCH] Updated repo_build process to be faster The repo_build process is slow and can do with some improvement. To achieve better speed of execution some several slow tasks, as profiled by the integrated gate have been moved into executable scripts using jobs. These scripts are generated from the playbooks and the Jinja2 variables created during the repo_build run time and make it possible to perform multiple actions in parallel. In test the results were significantly faster[0]. The test case has been updated to build all of the packages known to exist and be required for a real OpenStack environment. [0] - http://paste.openstack.org/show/491537/ Change-Id: I3a1872741a17dee25054a340859333b96203ca15 Signed-off-by: Kevin Carter --- defaults/main.yml | 2 +- tasks/repo_build.yml | 4 + tasks/repo_clone_git.yml | 24 +- tasks/repo_post_build.yml | 52 +--- tasks/repo_set_facts.yml | 36 ++- tasks/repo_venv_build.yml | 56 +--- templates/op-clone-script.sh.j2 | 49 +++ templates/op-release-script.sh.j2 | 60 ++++ templates/op-venv-script.sh.j2 | 79 +++++ tests/test.yml | 496 ++++++++++++++++++++++++++++-- 10 files changed, 730 insertions(+), 128 deletions(-) create mode 100644 templates/op-clone-script.sh.j2 create mode 100644 templates/op-release-script.sh.j2 create mode 100644 templates/op-venv-script.sh.j2 diff --git a/defaults/main.yml b/defaults/main.yml index f9d7a83..9a753ca 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -57,7 +57,7 @@ repo_build_pip_extra_indexes: # repo_build_pip_extra_index: "https://pypi.python.org/simple" repo_build_timeout: 120 - +repo_build_concurrency: "{{ ansible_processor_count | default(5) }}" repo_build_venv_build_dir: "/tmp/openstack-venv-builder" repo_build_venv_dir: "/var/www/repo/venvs" repo_build_venv_pip_install_options: > diff --git a/tasks/repo_build.yml b/tasks/repo_build.yml index 9543044..797b3c7 100644 --- a/tasks/repo_build.yml +++ b/tasks/repo_build.yml @@ -18,6 +18,7 @@ pip wheel --timeout {{ repo_build_timeout }} \ --wheel-dir {{ repo_build_output }} \ --find-links {{ repo_build_global_links_path }} \ + --find-links {{ repo_build_output }} \ --constraint {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt \ --index-url {{ repo_build_pip_default_index }} \ --trusted-host {{ repo_build_pip_default_index | netloc_no_port }} \ @@ -30,11 +31,13 @@ --requirement {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_global_requirements.txt tags: - repo-build-global-requirement-wheels + - repo-build-openstack-ansible-requirement-wheels - name: Create OpenStack-Ansible requirement wheels shell: | pip wheel --timeout {{ repo_build_timeout }} \ --wheel-dir {{ repo_build_output }} \ + --find-links {{ repo_build_global_links_path }} \ --find-links {{ repo_build_output }} \ --constraint {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt \ --index-url {{ repo_build_pip_default_index }} \ @@ -47,4 +50,5 @@ --log /var/log/repo/repo_builder.log \ --requirement {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_local_filtered.txt tags: + - repo-build-local-requirement-wheels - repo-build-openstack-ansible-requirement-wheels diff --git a/tasks/repo_clone_git.yml b/tasks/repo_clone_git.yml index eb5ab63..d8b56a1 100644 --- a/tasks/repo_clone_git.yml +++ b/tasks/repo_clone_git.yml @@ -13,13 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -- name: Clone all upstream git repositories - git: - repo: "{{ item['url'] }}" - dest: "{{ repo_build_git_dir }}/{{ item['name'] }}" - clone: yes - update: yes - version: "{{ item['version'] }}" - with_items: local_packages.results.0.item.remote_package_parts +- name: Create clone process script + template: + src: "op-clone-script.sh.j2" + dest: "/opt/op-clone-script.sh" + tags: + - repo-clone-repos + +- name: Run clone process script + shell: "bash /opt/op-clone-script.sh" + tags: + - repo-clone-repos + +- name: Remove clone process script + file: + path: "/opt/op-clone-script.sh" + state: absent tags: - repo-clone-repos diff --git a/tasks/repo_post_build.yml b/tasks/repo_post_build.yml index 7dc2cce..2160dc2 100644 --- a/tasks/repo_post_build.yml +++ b/tasks/repo_post_build.yml @@ -24,45 +24,25 @@ - repo-create-links-index - repo-create-release-manifest -- name: Create wheel pool structure - file: - path: "{{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}" - state: directory - owner: "{{ repo_build_service_user_name }}" - with_items: built_wheels.stdout_lines +- name: Create release process script + template: + src: "op-release-script.sh.j2" + dest: "/opt/op-release-script.sh" tags: - repo-create-pool -- name: Remove pool indexes if found +- name: Run release process script + shell: "bash /opt/op-release-script.sh" + tags: + - repo-create-pool + +- name: Remove release process script file: - path: "{{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}/index.html" + path: "/opt/op-clone-script.sh" state: absent - with_items: built_wheels.stdout_lines tags: - repo-create-pool -- name: Move wheels into place and ensure permissions - shell: | - if [ ! -f "{{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}/{{ item | lower }}" ];then - mv {{ repo_build_output }}/{{ item }} {{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}/{{ item | lower }} - elif ! diff {{ repo_build_output }}/{{ item }} {{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}/{{ item | lower }} > /dev/null;then - mv {{ repo_build_output }}/{{ item }} {{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}/{{ item | lower }} - fi - chown {{ repo_build_service_user_name }} {{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}/{{ item | lower }} - with_items: built_wheels.stdout_lines - tags: - - repo-copy-wheels-to-pool - -- name: Create release pool links - file: - dest: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/{{ item | lower }}" - src: "{{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}/{{ item | lower }}" - state: link - owner: "{{ repo_build_service_user_name }}" - with_items: built_wheels.stdout_lines - tags: - - repo-create-release-links - - name: Create release manifest template: src: "manifest.in.j2" @@ -91,16 +71,6 @@ tags: - repo-create-release-index -- name: Create general index links - file: - dest: "{{ repo_build_global_links_path }}/{{ item | lower }}" - src: "../pools/{{ item.split('-')[0] | lower }}/{{ item | lower }}" - state: link - owner: "{{ repo_build_service_user_name }}" - with_items: built_wheels.stdout_lines - tags: - - repo-create-links-index - - name: Index general links command: "ls -1 {{ repo_build_global_links_path }}" register: global_indexed_links diff --git a/tasks/repo_set_facts.yml b/tasks/repo_set_facts.yml index 6c523f2..a7dc4c1 100644 --- a/tasks/repo_set_facts.yml +++ b/tasks/repo_set_facts.yml @@ -39,9 +39,41 @@ - repo-set-requirement-names - repo-set-requirements +- name: Set stripped global requirement names + set_fact: + global_requirement_names_stripped: "{% for req in global_requirement_names %}{{ req.split('#')[0].strip().replace('-', '_') }}{% if not loop.last %}||{% endif %}{% endfor %}" + tags: + - repo-set-requirement-names-filtered + - repo-set-requirement-names + - repo-set-requirements + +- name: Set stripped local requirement names + set_fact: + local_requirement_names_stripped: "{% for req in local_packages.results.0.item.packages %}{{ req.split('#')[0].strip().replace('-', '_') }}{% if not loop.last %}||{% endif %}{% endfor %}" + tags: + - repo-set-requirement-names-filtered + - repo-set-requirement-names + - repo-set-requirements + +- name: Set global normalized requirement names + set_fact: + global_requirement_normalized: "{{ global_requirement_names_stripped.split('||') }}" + tags: + - repo-set-requirement-names-filtered + - repo-set-requirement-names + - repo-set-requirements + +- name: Set local normalized requirement names + set_fact: + local_requirement_normalized: "{{ local_requirement_names_stripped.split('||') }}" + tags: + - repo-set-requirement-names-filtered + - repo-set-requirement-names + - repo-set-requirements + - name: Set filtered requirement names set_fact: - global_requirement_names_filtered: "{{ local_packages.results.0.item.packages | filtered_list(global_requirement_names) }}" + global_requirement_names_filtered: "{{ local_requirement_normalized | filtered_list(global_requirement_normalized) }}" tags: - repo-set-requirement-names-filtered - repo-set-requirements @@ -65,7 +97,7 @@ - name: Set upper constraints set_fact: - upper_constraints: "{{ _upper_constraints | pip_constraint_update(local_packages.results.0.item.packages) }}" + upper_constraints: "{{ _upper_constraints | pip_constraint_update(local_requirement_normalized) }}" when: slurp_upper_constraints | success tags: - repo-set-constraints diff --git a/tasks/repo_venv_build.yml b/tasks/repo_venv_build.yml index 712da01..e0acd09 100644 --- a/tasks/repo_venv_build.yml +++ b/tasks/repo_venv_build.yml @@ -53,54 +53,24 @@ tags: - repo-create-venv -- name: Create role based venv - pip: - name: "{{ item.value | join(' ') }}" - state: present - virtualenv: "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" - virtualenv_site_packages: "no" - virtualenv_command: "{{ repo_build_venv_command_options }}" - extra_args: "{{ repo_build_venv_pip_install_options }}" - register: install_packages - until: install_packages|success - retries: 5 - delay: 2 - with_dict: local_packages.results.0.item.role_packages - when: - - '"os_" in item.key' - - "'{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz' not in existing_venvs" +- name: Create venv process script + template: + src: "op-venv-script.sh.j2" + dest: "/opt/op-venv-script.sh" tags: - repo-create-venv -- name: Create venv archive - shell: | - # This is to clean up pyc files which makes the archived venv smaller - # TODO(cloudnull) This should use the find module when we move to Ansible 2.0 - find "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" -name '*.pyc' -delete - # Create archive - tar czf "{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz" \ - -C "{{ repo_build_venv_build_dir }}/venvs/{{ item.key | replace('os_', '') }}" . - args: - chdir: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" - creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz" - with_dict: local_packages.results.0.item.role_packages - when: - - '"os_" in item.key' +- name: Run venv process script + shell: "bash /opt/op-venv-script.sh" tags: - - skip_ansible_lint + - repo-create-venv - repo-venv-compress-archive - repo-create-venv-archive - - repo-create-venv - -- name: Create venv archive checksum - shell: > - sha1sum "{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.tgz" | awk '{print $1}' > "{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.checksum" - args: - chdir: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" - creates: "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}/{{ item.key | replace('os_', '') }}-{{ repo_build_release_tag }}.checksum" - with_dict: local_packages.results.0.item.role_packages - when: - - '"os_" in item.key' - tags: - repo-create-venv-checksum + +- name: Remove venv process script + file: + path: "/opt/op-venv-script.sh" + state: absent + tags: - repo-create-venv diff --git a/templates/op-clone-script.sh.j2 b/templates/op-clone-script.sh.j2 new file mode 100644 index 0000000..ed08c0b --- /dev/null +++ b/templates/op-clone-script.sh.j2 @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +set -ve + +# Clone a git repository / Update git repository. +function git_clone { + repo="$1" + dest="$2" + version="$3" + + # If the git repository directory is locally found the function will ensure the remote + # is available within the repository. If the remote is not already in the local repository + # the repository URL will be hashed and the first 7 charactors of the hash will be used for + # remote name. Once the remotes are satisfied all refs will be fetched. + if [ -d "$dest/.git" ];then + pushd $dest + if ! git remote -v | grep -q "$repo"; then + remote_hash=$(echo "$repo" | sha1sum | awk '{print $1}') + git remote add ${remote_hash:0:7} "$repo" + fi + git fetch --all + popd + + # If the local target directory is not a valid git repository it will be removed and cloned. + elif [ -d "$dest" ];then + rm -rf "$dest" + git clone "$repo" "$dest" + else # If the local target directory does not exist it will be cloned. + git clone "$repo" "$dest" + fi + + # The version passed to the function will be checked out. + pushd "$dest" + git checkout $version + popd +} + +PID=() +# Run the git clone. This will loop over all of the package parts and clone all known repositories. +# Clone ops are done in parallel at a count of the known "ansible_processor_count" or using a default of 5. +{% for item in local_packages.results.0.item.remote_package_parts %} +git_clone "{{ item['url'] }}" "{{ repo_build_git_dir }}/{{ item['name'] }}" "{{ item['version'] }}" & +pid[{{ loop.index }}]=$! +{% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %} +for job_pid in ${!pid[@]}; do + wait ${pid[$job_pid]} || exit 99 +done + +{% endif %} +{% endfor %} diff --git a/templates/op-release-script.sh.j2 b/templates/op-release-script.sh.j2 new file mode 100644 index 0000000..b999bad --- /dev/null +++ b/templates/op-release-script.sh.j2 @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -ve + + +function build_repo { + PKG_DIRECTORY="$1" + PKG_LOWER="$2" + RAW_PKG="$3" + + # If the target PKG_DIRECTORY does not exist, create it + [[ ! -d "${PKG_DIRECTORY}" ]] && mkdir -p "${PKG_DIRECTORY}" + + # Make sure the repo service user owns the PKG_DIRECTORY + chown "{{ repo_build_service_user_name }}" "${PKG_DIRECTORY}" + + # If the target wheel PKG_DIRECTORY has an index.html file in it, remove it. This is + # in support of older archives that may have been created in the early liberty/kilo + # timeframe. + [[ -f "${PKG_DIRECTORY}/index.html" ]] && rm "${PKG_DIRECTORY}/index.html" + + # Move the built wheel into place if it does not exist or is different from one + # already found that is using the same name. + if [ ! -f "${PKG_DIRECTORY}/${PKG_LOWER}" ];then + mv "{{ repo_build_output }}/${RAW_PKG}" "${PKG_DIRECTORY}/${PKG_LOWER}" + elif ! diff "{{ repo_build_output }}/${RAW_PKG}" "${PKG_DIRECTORY}/${PKG_LOWER}" > /dev/null;then + mv "{{ repo_build_output }}/${RAW_PKG}" "${PKG_DIRECTORY}/${PKG_LOWER}" + fi + + # Make sure the repo service user owns the package + chown "{{ repo_build_service_user_name }}" "${PKG_DIRECTORY}/${PKG_LOWER}" + + # link the built package to the release reference + ln -sf "${PKG_DIRECTORY}/${PKG_LOWER}" "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/${PKG_LOWER}" + + # Make sure the repo service user owns the link + chown -h "{{ repo_build_service_user_name }}" "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/${PKG_LOWER}" + + # link the built package to the global links reference + ln -sf "${PKG_DIRECTORY}/${PKG_LOWER}" "{{ repo_build_global_links_path }}/${PKG_LOWER}" + + # Make sure the repo service user owns the link + chown -h "{{ repo_build_service_user_name }}" "{{ repo_build_global_links_path }}/${PKG_LOWER}" +} + + +# Loop through all built wheel. +{% for item in built_wheels.stdout_lines %} + +# Set the PKG_DIRECTORY variable normalizing the name +# Set the package name variable normalizing the name +DIRECTORY="{{ repo_build_pool_dir }}/{{ item.split('-')[0] | lower }}" +NAME_LOWER="{{ item | lower }}" +RAW_NAME="{{ item }}" + +build_repo "${DIRECTORY}" "${NAME_LOWER}" "${RAW_NAME}" + +unset PKG_DIRECTORY +unset PKG_LOWER + +{% endfor %} \ No newline at end of file diff --git a/templates/op-venv-script.sh.j2 b/templates/op-venv-script.sh.j2 new file mode 100644 index 0000000..000b3c3 --- /dev/null +++ b/templates/op-venv-script.sh.j2 @@ -0,0 +1,79 @@ +#!/usr/local/env bash + +set -ev + +function venv_create { + VENV_PATH="$1" + VENV_FILE="$2" + + # If the venv working directory already exists remove it + [[ -d "/tmp/${VENV_PATH}" ]] && rm -rf "/tmp/${VENV_PATH}" + + # If the pip build directory already exists remove it + [[ -d "/tmp/${VENV_FILE}" ]] && rm -rf "/tmp/${VENV_FILE}" + + # Create the virtualenv shell + {{ repo_build_venv_command_options }} "${VENV_PATH}" + + # Create the pip build directory + mkdir -p "/tmp/${VENV_FILE}" + + # Activate the python virtual environment for good measure + source "${VENV_PATH}/bin/activate" + + # Run the pip install within the venv and specify a specific build directory which + # resolves pip locking issues when run in parallel. + ${VENV_PATH}/bin/pip install --build "/tmp/${VENV_FILE}" {{ repo_build_venv_pip_install_options }} $3 + + # Deactivate the venv for good measure + deactivate + + # Find and remove all of the python pyc files + find "${VENV_PATH}" -type f -name '*.pyc' -delete + + # Create the archive + tar czf "${VENV_FILE}.tgz" -C "${VENV_PATH}" . + + # Create a checksum file for the archive + sha1sum "${VENV_FILE}.tgz" | awk '{print $1}' > "${VENV_FILE}.checksum" + + # Remove the working directories + rm -rf "${VENV_PATH}" + rm -rf "/tmp/${VENV_FILE}" +} + +# First operation is to sort and set the known os_* roles and create a unique dict. +# NOTE: this is a Jinja loop and will not be rendered within the script. For debugging +# purposes the group data will be rendered as a comment. +{% set os_group = {} %} +{% for key, value in local_packages.results.0.item.role_packages.items() %} +{% if "os_" in key %} +# venv to build {{ key }} +# * packages within the {{ key }} venv: {{ value }} +{% set _ = os_group.update({key: value}) %} +{% endif %} +{% endfor %} + +PID=() +# Run the venv create. This will loop over all of the os_group role packages and create a python virtual env. +# Venv creation is done parallel at a count of the known "ansible_processor_count" or using a default of 5. +# This loop will enter the venv build directory and craete tagged venvs in a distrobution directory +# If the venv archive already exists the creation process will be skipped +pushd "{{ repo_build_venv_dir }}/{{ repo_build_release_tag }}/{{ ansible_distribution | lower }}" +{% for key, value in os_group.items() %} +ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ key | replace('os_', '') }}" +ROLE_VENV_FILE="{{ key | replace('os_', '') }}-{{ repo_build_release_tag }}" +if [ ! -f "${ROLE_VENV_FILE}.tgz" ];then + venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "{{ value | join(' ') }}" & + pid[{{ loop.index }}]=$! +fi +unset ROLE_VENV_PATH +unset ROLE_VENV_FILE +{% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %} +for job_pid in ${!pid[@]}; do + wait ${pid[$job_pid]} || exit 99 +done + +{% endif %} +{% endfor %} +popd diff --git a/tests/test.yml b/tests/test.yml index 4aa186c..6df34e8 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -21,48 +21,478 @@ - name: Playbook for repo-build role testing hosts: repo_all - connection: local user: root pre_tasks: - name: Set local packages fact set_fact: local_packages: results: [ - { - "item": { - "packages": [ - "aodh", - ], - "remote_package_parts": [ - { - "url": "https://git.openstack.org/openstack/aodh", - "version": "239e1f629b26557ceadb92de3d62edcd87489b9d", - "name": "aodh", - "fragment": null, - "original": "git+https://git.openstack.org/openstack/aodh@239e1f629b26557ceadb92de3d62edcd87489b9d#egg=aodh&gitname=aodh", - "egg_name": "aodh" - }, - { - "url": "https://git.openstack.org/openstack/requirements", - "version": "332278d456e06870150835564342570ec9d5f5a0", - "name": "requirements", - "fragment": null, - "original": "git+https://git.openstack.org/openstack/requirements@332278d456e06870150835564342570ec9d5f5a0#egg=requirements&gitname=requirements", - "egg_name": "requirements" + { + "item": { + "packages": [ + "ansible-lint<=2.3.9", + "ansible>1.9,<2.0", + "aodh", + "argparse", + "bashate==0.5.0", + "ceilometer", + "ceilometermiddleware", + "cinder", + "cliff", + "configobj", + "django-appconf", + "django-openstack-auth", + "dnspython", + "ecdsa", + "fixtures", + "flake8==2.2.4", + "glance", + "greenlet", + "hacking>=0.10.0,<0.11", + "heat", + "horizon", + "httplib2", + "jinja2>=2.6", + "junitxml", + "keystone", + "keystonemiddleware", + "ldappool", + "libvirt-python", + "lxc-python2", + "lxml", + "mccabe==0.2.1", + "mysql-python", + "netaddr>=0.7.12", + "networking-plumgrid==2015.2.1.1", + "neutron", + "neutron_fwaas", + "neutron_lbaas", + "nose", + "nova", + "oslo.config", + "oslo.log", + "oslo.middleware", + "oslo.serialization", + "oslosphinx>=2.5.0", + "paramiko>=1.13.0", + "pbr", + "pep8==1.5.7", + "pip==8.1.1", + "ply", + "prettytable>=0.7,<0.8", + "pycrypto>=2.6", + "pyeclib", + "pyflakes==0.8.1", + "pymongo", + "pymysql", + "pysaml2", + "python-ceilometerclient", + "python-cephlibs", + "python-cinderclient", + "python-glanceclient", + "python-heatclient", + "python-keystoneclient", + "python-ldap", + "python-memcached", + "python-neutronclient", + "python-novaclient", + "python-openstackclient", + "python-subunit", + "python-swiftclient", + "python-troveclient", + "pyyaml>=3.1.0", + "reno>=0.1.1", + "repoze.lru", + "setuptools==20.3.1", + "sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2", + "swift", + "tempest-lib", + "testrepository", + "testscenarios", + "testtools", + "tooz", + "virtualenv", + "virtualenv-tools", + "warlock", + "websockify", + "wheel==0.29.0" + ], + "remote_package_parts": [ + { + "name": "aodh", + "fragment": null, + "url": "https://git.openstack.org/openstack/aodh", + "version": "9d71ae4f4f7a30beb001e867ad688bf60fc23ece", + "original": "git+https://git.openstack.org/openstack/aodh@9d71ae4f4f7a30beb001e867ad688bf60fc23ece#egg=aodh&gitname=aodh", + "egg_name": "aodh" + }, + { + "name": "ceilometer", + "fragment": null, + "url": "https://git.openstack.org/openstack/ceilometer", + "version": "04d91aad6315cfb1679b7a3d18474e9ebe033329", + "original": "git+https://git.openstack.org/openstack/ceilometer@04d91aad6315cfb1679b7a3d18474e9ebe033329#egg=ceilometer&gitname=ceilometer", + "egg_name": "ceilometer" + }, + { + "name": "cinder", + "fragment": null, + "url": "https://git.openstack.org/openstack/cinder", + "version": "545b8c4435469c72920d54e4a3ca3c74cbc66b95", + "original": "git+https://git.openstack.org/openstack/cinder@545b8c4435469c72920d54e4a3ca3c74cbc66b95#egg=cinder&gitname=cinder", + "egg_name": "cinder" + }, + { + "name": "glance", + "fragment": null, + "url": "https://git.openstack.org/openstack/glance", + "version": "e0ba0c4fd030d42d03ad1586c32b5925b0b6c6fe", + "original": "git+https://git.openstack.org/openstack/glance@e0ba0c4fd030d42d03ad1586c32b5925b0b6c6fe#egg=glance&gitname=glance", + "egg_name": "glance" + }, + { + "name": "heat", + "fragment": null, + "url": "https://git.openstack.org/openstack/heat", + "version": "e3f3f749a905aba302e9f0c457782d979440ee0c", + "original": "git+https://git.openstack.org/openstack/heat@e3f3f749a905aba302e9f0c457782d979440ee0c#egg=heat&gitname=heat", + "egg_name": "heat" + }, + { + "name": "horizon", + "fragment": null, + "url": "https://git.openstack.org/openstack/horizon", + "version": "4a5eb9644d36e3063b3ddcff03cdbdfadc0f1116", + "original": "git+https://git.openstack.org/openstack/horizon@4a5eb9644d36e3063b3ddcff03cdbdfadc0f1116#egg=horizon&gitname=horizon", + "egg_name": "horizon" + }, + { + "name": "keystone", + "fragment": null, + "url": "https://git.openstack.org/openstack/keystone", + "version": "8a56c161ee29e34e70c6334b048881e8fbbd7514", + "original": "git+https://git.openstack.org/openstack/keystone@8a56c161ee29e34e70c6334b048881e8fbbd7514#egg=keystone&gitname=keystone", + "egg_name": "keystone" + }, + { + "name": "neutron", + "fragment": null, + "url": "https://git.openstack.org/openstack/neutron", + "version": "5e48499a385b7c7e4a8384c02ae6aa0f856b24eb", + "original": "git+https://git.openstack.org/openstack/neutron@5e48499a385b7c7e4a8384c02ae6aa0f856b24eb#egg=neutron&gitname=neutron", + "egg_name": "neutron" + }, + { + "name": "neutron-fwaas", + "fragment": null, + "url": "https://git.openstack.org/openstack/neutron-fwaas", + "version": "3605570ecae8ef7f84a84627569386acc6a75e11", + "original": "git+https://git.openstack.org/openstack/neutron-fwaas@3605570ecae8ef7f84a84627569386acc6a75e11#egg=neutron_fwaas&gitname=neutron-fwaas", + "egg_name": "neutron_fwaas" + }, + { + "name": "neutron-lbaas", + "fragment": null, + "url": "https://git.openstack.org/openstack/neutron-lbaas", + "version": "e7ec75f61cce871c8bedff9c04345bbb14b9fda5", + "original": "git+https://git.openstack.org/openstack/neutron-lbaas@e7ec75f61cce871c8bedff9c04345bbb14b9fda5#egg=neutron_lbaas&gitname=neutron-lbaas", + "egg_name": "neutron_lbaas" + }, + { + "name": "neutron-vpnaas", + "fragment": null, + "url": "https://git.openstack.org/openstack/neutron-vpnaas", + "version": "0067e2671d1c33e630dff947ac7090c370152225", + "original": "git+https://git.openstack.org/openstack/neutron-vpnaas@0067e2671d1c33e630dff947ac7090c370152225#egg=neutron_vpnaas&gitname=neutron-vpnaas", + "egg_name": "neutron_vpnaas" + }, + { + "name": "nova", + "fragment": null, + "url": "https://git.openstack.org/openstack/nova", + "version": "2bc8128d7793cc72ca2e146de3a092e1fef5033b", + "original": "git+https://git.openstack.org/openstack/nova@2bc8128d7793cc72ca2e146de3a092e1fef5033b#egg=nova&gitname=nova", + "egg_name": "nova" + }, + { + "name": "novnc", + "fragment": null, + "url": "https://github.com/kanaka/novnc", + "version": "670dbddb54264fd0082d0aca1b3acb0f1814b1d2", + "original": "git+https://github.com/kanaka/novnc@670dbddb54264fd0082d0aca1b3acb0f1814b1d2#egg=novnc&gitname=novnc", + "egg_name": "novnc" + }, + { + "name": "requirements", + "fragment": null, + "url": "https://git.openstack.org/openstack/requirements", + "version": "f8cf7eba898a1424549c730d6692ec4e9573c0ed", + "original": "git+https://git.openstack.org/openstack/requirements@f8cf7eba898a1424549c730d6692ec4e9573c0ed#egg=requirements&gitname=requirements", + "egg_name": "requirements" + }, + { + "name": "spice-html5", + "fragment": null, + "url": "https://github.com/SPICE/spice-html5", + "version": "ab73d009487c8afd4def39b54a422499b4c13c40", + "original": "git+https://github.com/SPICE/spice-html5@ab73d009487c8afd4def39b54a422499b4c13c40#egg=spice_html5&gitname=spice-html5", + "egg_name": "spice_html5" + }, + { + "name": "swift", + "fragment": null, + "url": "https://git.openstack.org/openstack/swift", + "version": "62fb544224178377990752b16956ee2e13d9207c", + "original": "git+https://git.openstack.org/openstack/swift@62fb544224178377990752b16956ee2e13d9207c#egg=swift&gitname=swift", + "egg_name": "swift" + }, + { + "name": "tempest", + "fragment": null, + "url": "https://git.openstack.org/openstack/tempest", + "version": "5cc7ef78b4233444a4dcea1b1eb8f213c1548491", + "original": "git+https://git.openstack.org/openstack/tempest@5cc7ef78b4233444a4dcea1b1eb8f213c1548491#egg=tempest&gitname=tempest", + "egg_name": "tempest" + } + ], + "remote_packages": [ + "git+https://git.openstack.org/openstack/aodh@9d71ae4f4f7a30beb001e867ad688bf60fc23ece#egg=aodh&gitname=aodh", + "git+https://git.openstack.org/openstack/ceilometer@04d91aad6315cfb1679b7a3d18474e9ebe033329#egg=ceilometer&gitname=ceilometer", + "git+https://git.openstack.org/openstack/cinder@545b8c4435469c72920d54e4a3ca3c74cbc66b95#egg=cinder&gitname=cinder", + "git+https://git.openstack.org/openstack/glance@e0ba0c4fd030d42d03ad1586c32b5925b0b6c6fe#egg=glance&gitname=glance", + "git+https://git.openstack.org/openstack/heat@e3f3f749a905aba302e9f0c457782d979440ee0c#egg=heat&gitname=heat", + "git+https://git.openstack.org/openstack/horizon@4a5eb9644d36e3063b3ddcff03cdbdfadc0f1116#egg=horizon&gitname=horizon", + "git+https://git.openstack.org/openstack/keystone@8a56c161ee29e34e70c6334b048881e8fbbd7514#egg=keystone&gitname=keystone", + "git+https://git.openstack.org/openstack/neutron-fwaas@3605570ecae8ef7f84a84627569386acc6a75e11#egg=neutron_fwaas&gitname=neutron-fwaas", + "git+https://git.openstack.org/openstack/neutron-lbaas@e7ec75f61cce871c8bedff9c04345bbb14b9fda5#egg=neutron_lbaas&gitname=neutron-lbaas", + "git+https://git.openstack.org/openstack/neutron-vpnaas@0067e2671d1c33e630dff947ac7090c370152225#egg=neutron_vpnaas&gitname=neutron-vpnaas", + "git+https://git.openstack.org/openstack/neutron@5e48499a385b7c7e4a8384c02ae6aa0f856b24eb#egg=neutron&gitname=neutron", + "git+https://git.openstack.org/openstack/nova@2bc8128d7793cc72ca2e146de3a092e1fef5033b#egg=nova&gitname=nova", + "git+https://git.openstack.org/openstack/requirements@f8cf7eba898a1424549c730d6692ec4e9573c0ed#egg=requirements&gitname=requirements", + "git+https://git.openstack.org/openstack/swift@62fb544224178377990752b16956ee2e13d9207c#egg=swift&gitname=swift", + "git+https://git.openstack.org/openstack/tempest@5cc7ef78b4233444a4dcea1b1eb8f213c1548491#egg=tempest&gitname=tempest", + "git+https://github.com/SPICE/spice-html5@ab73d009487c8afd4def39b54a422499b4c13c40#egg=spice_html5&gitname=spice-html5", + "git+https://github.com/kanaka/novnc@670dbddb54264fd0082d0aca1b3acb0f1814b1d2#egg=novnc&gitname=novnc" + ], + "role_packages": { + "os_heat": [ + "PyMySQL", + "heat", + "httplib2", + "keystonemiddleware", + "pycrypto", + "python-ceilometerclient", + "python-cinderclient", + "python-glanceclient", + "python-heatclient", + "python-keystoneclient", + "python-memcached", + "python-neutronclient", + "python-novaclient", + "python-openstackclient", + "python-swiftclient", + "python-troveclient", + "virtualenv", + "virtualenv-tools" + ], + "os_nova": [ + "PyMySQL", + "httplib2", + "keystonemiddleware", + "libvirt-python", + "nova", + "pycrypto", + "python-cephlibs", + "python-keystoneclient", + "python-memcached", + "python-novaclient", + "virtualenv", + "virtualenv-tools", + "websockify" + ], + "os_keystone": [ + "PyMySQL", + "argparse", + "httplib2", + "keystone", + "keystonemiddleware", + "ldappool", + "lxml", + "oslo.log", + "oslo.middleware", + "pbr", + "pycrypto", + "pysaml2", + "python-keystoneclient", + "python-ldap", + "python-memcached", + "python-openstackclient", + "repoze.lru", + "virtualenv", + "virtualenv-tools" + ], + "galera_client": [ + "MySQL-python", + "pycrypto", + "python-memcached" + ], + "galera_server": [ + "MySQL-python", + "pycrypto" + ], + "rabbitmq_server": [ + "pycrypto" + ], + "os_aodh": [ + "aodh", + "ceilometermiddleware", + "httplib2", + "pycrypto", + "pymongo", + "python-ceilometerclient", + "python-keystoneclient", + "python-memcached", + "virtualenv", + "virtualenv-tools", + "warlock" + ], + "os_neutron": [ + "PyMySQL", + "cliff", + "configobj", + "httplib2", + "keystonemiddleware", + "neutron", + "neutron_fwaas", + "neutron_lbaas", + "pycrypto", + "python-glanceclient", + "python-keystoneclient", + "python-memcached", + "python-neutronclient", + "python-novaclient", + "repoze.lru", + "virtualenv", + "virtualenv-tools" + ], + "lxc_hosts": [ + "lxc-python2" + ], + "os_cinder": [ + "PyMySQL", + "cinder", + "ecdsa", + "httplib2", + "keystonemiddleware", + "pycrypto", + "python-cephlibs", + "python-cinderclient", + "python-keystoneclient", + "python-memcached", + "virtualenv", + "virtualenv-tools" + ], + "os_glance": [ + "PyMySQL", + "glance", + "httplib2", + "keystonemiddleware", + "pycrypto", + "python-cephlibs", + "python-cinderclient", + "python-glanceclient", + "python-keystoneclient", + "python-memcached", + "python-swiftclient", + "virtualenv", + "virtualenv-tools", + "warlock" + ], + "os_swift": [ + "ceilometermiddleware", + "dnspython", + "ecdsa", + "httplib2", + "keystonemiddleware", + "pycrypto", + "pyeclib", + "python-cinderclient", + "python-keystoneclient", + "python-memcached", + "python-swiftclient", + "swift", + "virtualenv", + "virtualenv-tools" + ], + "repo_build": [ + "virtualenv", + "virtualenv-tools", + "wheel" + ], + "pip_install": [], + "os_tempest": [ + "fixtures", + "junitxml", + "nose", + "oslo.serialization", + "python-ceilometerclient", + "python-cinderclient", + "python-glanceclient", + "python-heatclient", + "python-keystoneclient", + "python-memcached", + "python-neutronclient", + "python-novaclient", + "python-openstackclient", + "python-subunit", + "python-swiftclient", + "tempest-lib", + "testrepository", + "testscenarios", + "testtools", + "virtualenv", + "virtualenv-tools" + ], + "os_horizon": [ + "MySQL-python", + "PyMySQL", + "django-appconf", + "django-openstack-auth", + "greenlet", + "horizon", + "httplib2", + "keystonemiddleware", + "oslo.config", + "ply", + "pycrypto", + "python-keystoneclient", + "python-memcached", + "virtualenv", + "virtualenv-tools" + ], + "os_ceilometer": [ + "PyMySQL", + "ceilometer", + "ceilometermiddleware", + "httplib2", + "libvirt-python", + "pycrypto", + "pymongo", + "python-ceilometerclient", + "python-keystoneclient", + "python-memcached", + "tooz", + "virtualenv", + "virtualenv-tools", + "warlock" + ] + } } - ], - "remote_packages": [ - "git+https://git.openstack.org/openstack/requirements@332278d456e06870150835564342570ec9d5f5a0#egg=requirements&gitname=requirements", - "git+https://git.openstack.org/openstack/aodh@239e1f629b26557ceadb92de3d62edcd87489b9d#egg=aodh&gitname=aodh" - ], - "role_packages": { - } } - } ] - - roles: - role: "repo_server" - - role: "{{ rolename | basename }}" \ No newline at end of file + - role: "{{ rolename | basename }}" + vars: + openstack_release: testing