From 7e6e614ff035c428fa34384169da7dfc00ccb103 Mon Sep 17 00:00:00 2001 From: Chandan Kumar Date: Tue, 2 Oct 2018 13:29:51 +0530 Subject: [PATCH] Added support for installing tempest from distro In order to consume os_tempest role across different deployment tools, following changes are introduced: - tempest_install_method:distro|source through which user can install tempest from source as well as distro - tempest_workspace directory is added as when tempest is installed from distro, we need a place where tempest can be executed. It generalised the use of tempest_workspace at each place. - The default location of the default tempest configuration is now /etc/tempest/tempest.conf rather than the previous default of $HOME/.tempest/etc. Change-Id: Ie280cfa6b93fa65c56ed98066deb4f9c82a1a8db --- defaults/main.yml | 13 +- ...d-support-for-distro-89611067ce74fc2c.yaml | 11 ++ tasks/main.yml | 5 + tasks/stackviz.yml | 2 +- tasks/tempest_install.yml | 120 ++--------------- tasks/tempest_install_source.yml | 125 ++++++++++++++++++ tasks/tempest_post_install.yml | 17 ++- tasks/tempest_run.yml | 24 +++- tests/test-tempest-functional.yml | 2 + tox.ini | 8 ++ vars/distro_install.yml | 21 +++ vars/redhat-7.yml | 9 ++ vars/source_install.yml | 21 +++ vars/suse.yml | 6 + vars/ubuntu.yml | 4 + zuul.d/project.yaml | 1 + 16 files changed, 267 insertions(+), 122 deletions(-) create mode 100644 releasenotes/notes/add-support-for-distro-89611067ce74fc2c.yaml create mode 100644 tasks/tempest_install_source.yml create mode 100644 vars/distro_install.yml create mode 100644 vars/source_install.yml diff --git a/defaults/main.yml b/defaults/main.yml index 12abdada..7142ba40 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -16,6 +16,12 @@ ## Verbosity Options debug: False +# Install openstack tempest +# set the tempest_install_method to source or distro +# on choosing source it will install from git or venv +# on choosing distro it will install based on distribution +tempest_install_method: "source" + # Set the package install state for distribution and pip packages # # Options are 'present' and 'latest' tempest_package_state: "latest" @@ -75,9 +81,12 @@ tempest_log_dir: "/var/log/tempest" # package: ironic #for installing the plugin from packages tempest_plugins: "{{ _tempest_plugins['keystone'] + _tempest_plugins['cinder'] + _tempest_plugins['heat'] + _tempest_plugins['neutron'] }}" +# tempest_workspace where tempest can be runned +tempest_workspace: "{{ ansible_env.HOME }}/workspace" + # The location where the test whitelist/blacklist will be placed -tempest_test_whitelist_file_path: "{{ tempest_venv_bin | dirname }}/workspace/etc/tempest_whitelist.txt" -tempest_test_blacklist_file_path: "{{ tempest_venv_bin | dirname }}/workspace/etc/tempest_blacklist.txt" +tempest_test_whitelist_file_path: "{{ tempest_workspace }}/etc/tempest_whitelist.txt" +tempest_test_blacklist_file_path: "{{ tempest_workspace }}/etc/tempest_blacklist.txt" # Tests to execute: # This sets up a list of tests to execute based on what's deployed in the environment. diff --git a/releasenotes/notes/add-support-for-distro-89611067ce74fc2c.yaml b/releasenotes/notes/add-support-for-distro-89611067ce74fc2c.yaml new file mode 100644 index 00000000..9936187a --- /dev/null +++ b/releasenotes/notes/add-support-for-distro-89611067ce74fc2c.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + The ``os_tempest`` role now has the ability to install from distribution packages by setting + ``tempest_install_method`` to ``distro``. + - | + The new variable ``tempest_workspace`` has been introduced to set the location of the tempest + workspace. + - | + The default location of the default tempest configuration is now ``/etc/tempest/tempest.conf`` + rather than the previous default of ``$HOME/.tempest/etc``. diff --git a/tasks/main.yml b/tasks/main.yml index 28250f29..8f5323a9 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -25,6 +25,11 @@ tags: - always +- name: Gather variables for installation method + include_vars: "{{ tempest_install_method }}_install.yml" + tags: + - always + - include_tasks: tempest_install.yml tags: - tempest-install diff --git a/tasks/stackviz.yml b/tasks/stackviz.yml index 9bbd3985..373a869f 100644 --- a/tasks/stackviz.yml +++ b/tasks/stackviz.yml @@ -36,7 +36,7 @@ . {{ stackviz_venv_bin }}/activate stackviz-export -f {{ tempest_log_dir }}/testrepository.subunit --env {{ tempest_log_dir }}/stackviz/data args: - chdir: "{{ tempest_venv_bin | dirname }}/workspace" + chdir: "{{ tempest_workspace }}" executable: /bin/bash tags: # don't trigger ANSIBLE0013 diff --git a/tasks/tempest_install.yml b/tasks/tempest_install.yml index ebd41fa5..e03e0a94 100644 --- a/tasks/tempest_install.yml +++ b/tasks/tempest_install.yml @@ -1,5 +1,6 @@ --- # Copyright 2014, Rackspace US, Inc. +# Copyright 2018, Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,114 +14,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -# TODO(odyssey4me): -# This can be simplified once all the roles are using -# python_venv_build. We can then switch to using a -# set of constraints in pip.conf inside the venv, -# perhaps prepared by giving a giving a list of -# constraints to the role. -- name: Create developer mode constraint file - copy: - dest: "/opt/developer-pip-constraints.txt" - content: | - {% for item in tempest_developer_constraints %} - {{ item }} - {% endfor %} - when: tempest_developer_mode | bool - -- name: Ensure remote wheel building is disabled in developer mode - set_fact: - venv_build_host: "{{ ansible_hostname }}" - when: - - tempest_developer_mode | bool - -- name: Install the python venv - include_role: - name: "python_venv_build" - private: yes - vars: - venv_install_destination_path: "{{ tempest_venv_bin | dirname }}" - venv_install_distro_package_list: "{{ tempest_distro_packages }}" - venv_pip_install_args: "{{ tempest_pip_install_args }}" - venv_pip_packages: "{{ tempest_pip_packages }}" - venv_facts_when_changed: - - section: "tempest" - option: "venv_tag" - value: "{{ tempest_venv_tag }}" - -- name: Get tempest plugins from git - git: - repo: "{{ item.repo }}" - dest: "/opt/{{ item.name|replace('-', '_') }}_{{ item.branch|replace('/', '_') }}" - version: "{{ item.branch }}" - force: yes - with_items: "{{ tempest_plugins }}" - when: item.repo is defined - register: git_clone - until: git_clone is success +- name: Install distro packages + package: + name: "{{ tempest_package_list }}" + state: "{{ tempest_package_state }}" + update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}" + cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(cache_timeout, omit) }}" + register: install_packages + until: install_packages is success retries: 5 delay: 2 -- name: Check for the existance of the test-requirements.txt file - stat: - path: "/opt/{{ item.name|replace('-', '_') }}_{{ item.branch|replace('/', '_') }}/test-requirements.txt" - with_items: "{{ tempest_plugins }}" - when: item.repo is defined - register: _test_requirements_stat - -- name: Install tempest plugin requirements - pip: - requirements: "{{ item.stat.path }}" - state: "{{ tempest_pip_package_state }}" - virtualenv: "{{ tempest_venv_bin | dirname }}" - virtualenv_site_packages: "no" - extra_args: >- - {{ tempest_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} - {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} - {{ pip_install_options | default('') }} - --isolated - with_items: "{{ _test_requirements_stat.results }}" - when: - - "item.item.repo is defined" - - "item.stat.exists" - - "(item.item.install_test_requirements | default(True)) | bool" - register: install_tempest_plugin_requirements - until: install_tempest_plugin_requirements is success - retries: 5 - delay: 2 - -- name: Install tempest plugins from cloned repo - pip: - name: "/opt/{{ item.name|replace('-', '_') }}_{{ item.branch|replace('/', '_') }}" - state: "{{ tempest_pip_package_state }}" - virtualenv: "{{ tempest_venv_bin | dirname }}" - virtualenv_site_packages: "no" - extra_args: >- - {{ tempest_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} - {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} - {{ pip_install_options | default('') }} - --isolated - with_items: "{{ tempest_plugins }}" - when: item.repo is defined - register: install_tempest_plugins - until: install_tempest_plugins is success - retries: 5 - delay: 2 - -- name: Install tempest plugins from packages - pip: - name: "{{ item.package }}" - state: "{{ tempest_pip_package_state }}" - virtualenv: "{{ tempest_venv_bin | dirname }}" - virtualenv_site_packages: "no" - extra_args: >- - {{ tempest_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} - {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} - {{ pip_install_options | default('') }} - --isolated - with_items: "{{ tempest_plugins }}" - when: item.package is defined - register: install_tempest_plugins_packages - until: install_tempest_plugins_packages is success - retries: 5 - delay: 2 +- name: Install Tempest packages from PIP + include_tasks: tempest_install_source.yml + when: tempest_install_method == 'source' diff --git a/tasks/tempest_install_source.yml b/tasks/tempest_install_source.yml new file mode 100644 index 00000000..513888aa --- /dev/null +++ b/tasks/tempest_install_source.yml @@ -0,0 +1,125 @@ +--- +# Copyright 2014, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO(odyssey4me): +# This can be simplified once all the roles are using +# python_venv_build. We can then switch to using a +# set of constraints in pip.conf inside the venv, +# perhaps prepared by giving a giving a list of +# constraints to the role. +- name: Create developer mode constraint file + copy: + dest: "/opt/developer-pip-constraints.txt" + content: | + {% for item in tempest_developer_constraints %} + {{ item }} + {% endfor %} + when: tempest_developer_mode | bool + +- name: Ensure remote wheel building is disabled in developer mode + set_fact: + venv_build_host: "{{ ansible_hostname }}" + when: + - tempest_developer_mode | bool + +- name: Install the python venv + include_role: + name: "python_venv_build" + private: yes + vars: + venv_install_destination_path: "{{ tempest_venv_bin | dirname }}" + venv_pip_install_args: "{{ tempest_pip_install_args }}" + venv_pip_packages: "{{ tempest_pip_packages }}" + venv_facts_when_changed: + - section: "tempest" + option: "venv_tag" + value: "{{ tempest_venv_tag }}" + +- name: Get tempest plugins from git + git: + repo: "{{ item.repo }}" + dest: "/opt/{{ item.name|replace('-', '_') }}_{{ item.branch|replace('/', '_') }}" + version: "{{ item.branch }}" + force: yes + with_items: "{{ tempest_plugins }}" + when: item.repo is defined + register: git_clone + until: git_clone is success + retries: 5 + delay: 2 + +- name: Check for the existance of the test-requirements.txt file + stat: + path: "/opt/{{ item.name|replace('-', '_') }}_{{ item.branch|replace('/', '_') }}/test-requirements.txt" + with_items: "{{ tempest_plugins }}" + when: item.repo is defined + register: _test_requirements_stat + +- name: Install tempest plugin requirements + pip: + requirements: "{{ item.stat.path }}" + state: "{{ tempest_pip_package_state }}" + virtualenv: "{{ tempest_venv_bin | dirname }}" + virtualenv_site_packages: "no" + extra_args: >- + {{ tempest_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + --isolated + with_items: "{{ _test_requirements_stat.results }}" + when: + - "item.item.repo is defined" + - "item.stat.exists" + - "(item.item.install_test_requirements | default(True)) | bool" + register: install_tempest_plugin_requirements + until: install_tempest_plugin_requirements is success + retries: 5 + delay: 2 + +- name: Install tempest plugins from cloned repo + pip: + name: "/opt/{{ item.name|replace('-', '_') }}_{{ item.branch|replace('/', '_') }}" + state: "{{ tempest_pip_package_state }}" + virtualenv: "{{ tempest_venv_bin | dirname }}" + virtualenv_site_packages: "no" + extra_args: >- + {{ tempest_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + --isolated + with_items: "{{ tempest_plugins }}" + when: item.repo is defined + register: install_tempest_plugins + until: install_tempest_plugins is success + retries: 5 + delay: 2 + +- name: Install tempest plugins from packages + pip: + name: "{{ item.package }}" + state: "{{ tempest_pip_package_state }}" + virtualenv: "{{ tempest_venv_bin | dirname }}" + virtualenv_site_packages: "no" + extra_args: >- + {{ tempest_developer_mode | ternary(pip_install_developer_constraints | default('--constraint /opt/developer-pip-constraints.txt'), '') }} + {{ (pip_install_upper_constraints is defined) | ternary('--constraint ' + pip_install_upper_constraints | default(''),'') }} + {{ pip_install_options | default('') }} + --isolated + with_items: "{{ tempest_plugins }}" + when: item.package is defined + register: install_tempest_plugins_packages + until: install_tempest_plugins_packages is success + retries: 5 + delay: 2 diff --git a/tasks/tempest_post_install.yml b/tasks/tempest_post_install.yml index 60028a5e..0b4fba1a 100644 --- a/tasks/tempest_post_install.yml +++ b/tasks/tempest_post_install.yml @@ -1,5 +1,6 @@ --- # Copyright 2014, Rackspace US, Inc. +# Copyright 2018, Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,13 +23,14 @@ mode: "{{ item.mode|default('0755') }}" with_items: - path: "{{ tempest_log_dir | realpath }}" - - path: "{{ ansible_env.HOME }}/.tempest/etc" - path: "{{ tempest_image_dir }}" + - path: "{{ tempest_workspace }}" + - path: "/etc/tempest" - name: Copy tempest config config_template: src: "tempest.conf.j2" - dest: "{{ ansible_env.HOME }}/.tempest/etc/tempest.conf" + dest: "/etc/tempest/tempest.conf" owner: "root" group: "root" mode: "0644" @@ -37,10 +39,12 @@ - name: Initialise tempest workspace shell: | - if [ ! -d {{ tempest_venv_bin | dirname }}/workspace ] - then - . {{ tempest_venv_bin }}/activate - tempest init {{ tempest_venv_bin | dirname }}/workspace + set -e + if [ ! -d {{ tempest_workspace }}/etc ]; then + if [ -d {{ tempest_venv_bin }} ]; then + . {{ tempest_venv_bin }}/activate + fi + tempest init {{ tempest_workspace }} exit 3 fi args: @@ -75,4 +79,3 @@ dest: "{{ tempest_test_blacklist_file_path }}" when: - tempest_test_blacklist | length > 0 - diff --git a/tasks/tempest_run.yml b/tasks/tempest_run.yml index f43560f0..238bac33 100644 --- a/tasks/tempest_run.yml +++ b/tasks/tempest_run.yml @@ -16,10 +16,14 @@ - block: - name: Execute tempest tests shell: | + set -e + if [ -d {{ tempest_venv_bin }} ] + then . {{ tempest_venv_bin }}/activate + fi tempest run {% if tempest_run_concurrency is defined %}{% if tempest_run_concurrency | int > 0 %}--concurrency {{ tempest_run_concurrency | int }}{% else %}--serial{% endif %}{% endif %} --whitelist-file {{ tempest_test_whitelist_file_path }}{% if tempest_test_blacklist | length > 0 %} --blacklist-file {{ tempest_test_blacklist_file_path }}{% endif %} args: - chdir: "{{ tempest_venv_bin | dirname }}/workspace" + chdir: "{{ tempest_workspace }}" executable: /bin/bash changed_when: false register: tempest_tests_run @@ -35,10 +39,14 @@ # NOTE(mattt): At present we're not placing this file in a valid location # to be used by the OpenStack Health dashboard. shell: | + set -e + if [ -d {{ tempest_venv_bin }} ] + then . {{ tempest_venv_bin }}/activate + fi stestr last --subunit > {{ tempest_log_dir }}/testrepository.subunit args: - chdir: "{{ tempest_venv_bin | dirname }}/workspace" + chdir: "{{ tempest_workspace }}" executable: /bin/bash changed_when: false failed_when: false @@ -48,10 +56,14 @@ - name: Generate xml subunit results shell: | + set -e + if [ -d {{ tempest_venv_bin }} ] + then . {{ tempest_venv_bin }}/activate + fi subunit2junitxml {{ tempest_log_dir }}/testrepository.subunit > {{ tempest_log_dir }}/tempest_results.xml args: - chdir: "{{ tempest_venv_bin | dirname }}/workspace" + chdir: "{{ tempest_workspace }}" executable: /bin/bash changed_when: false failed_when: false @@ -61,10 +73,14 @@ - name: Generate html subunit results shell: | + set -e + if [ -d {{ tempest_venv_bin }} ] + then . {{ tempest_venv_bin }}/activate + fi subunit2html {{ tempest_log_dir }}/testrepository.subunit {{ tempest_log_dir }}/stestr_results.html args: - chdir: "{{ tempest_venv_bin | dirname }}/workspace" + chdir: "{{ tempest_workspace }}" executable: /bin/bash changed_when: false failed_when: false diff --git a/tests/test-tempest-functional.yml b/tests/test-tempest-functional.yml index 417dcb2f..fa4457f1 100644 --- a/tests/test-tempest-functional.yml +++ b/tests/test-tempest-functional.yml @@ -28,6 +28,8 @@ exit 1 fi with_items: "{{ tempest_plugins }}" + when: + - tempest_install_method | default('source') == 'source' tags: - skip_ansible_lint diff --git a/tox.ini b/tox.ini index 71a77a27..8ca8bac7 100644 --- a/tox.ini +++ b/tox.ini @@ -93,6 +93,14 @@ commands = bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" +[testenv:distro_install] +setenv = + {[testenv]setenv} + ANSIBLE_PARAMETERS=-e @{toxinidir}/tests/common/test-distro_install-vars.yml +commands = + bash -c "{toxinidir}/tests/common/test-ansible-functional.sh" + + [testenv:linters] basepython = python3 commands = diff --git a/vars/distro_install.yml b/vars/distro_install.yml new file mode 100644 index 00000000..4d03dfa4 --- /dev/null +++ b/vars/distro_install.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2018, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Compile a list of the distro packages to install based on +# whether the host is in the host group and the service is +# enabled. +# +tempest_package_list: "{{ tempest_distro_packages + tempest_service_distro_packages }}" diff --git a/vars/redhat-7.yml b/vars/redhat-7.yml index 33b3b7c8..88ebed8d 100644 --- a/vars/redhat-7.yml +++ b/vars/redhat-7.yml @@ -1,5 +1,6 @@ --- # Copyright 2016, Rackspace US, Inc. +# Copyright 2018, Red Hat, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -15,3 +16,11 @@ tempest_distro_packages: - git + +tempest_service_distro_packages: + - openstack-tempest + - python-tempestconf + - python-subunit + - python-keystone-tests-tempest + - python-sahara-tests-tempest + - subunit-filters diff --git a/vars/source_install.yml b/vars/source_install.yml new file mode 100644 index 00000000..2fda1038 --- /dev/null +++ b/vars/source_install.yml @@ -0,0 +1,21 @@ +--- +# Copyright 2018, Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# +# Compile a list of the distro packages to install based on +# whether the host is in the host group and the service is +# enabled. +# +tempest_package_list: "{{ tempest_distro_packages }}" diff --git a/vars/suse.yml b/vars/suse.yml index 49d5230d..189eaa18 100644 --- a/vars/suse.yml +++ b/vars/suse.yml @@ -16,3 +16,9 @@ tempest_distro_packages: - git-core + +tempest_service_distro_packages: + - openstack-tempest + - "{{ (ansible_distribution_version is version ('42', '>=')) | ternary('python-python-subunit', 'python2-python-subunit') }}" + - "{{ (ansible_distribution_version is version ('42', '>=')) | ternary('python-os-testr', 'python2-os-testr') }}" + - python-junitxml diff --git a/vars/ubuntu.yml b/vars/ubuntu.yml index 470d2fdf..3361588e 100644 --- a/vars/ubuntu.yml +++ b/vars/ubuntu.yml @@ -18,3 +18,7 @@ cache_timeout: 600 tempest_distro_packages: - git-core + +tempest_service_distro_packages: + - openstack-tempest + - python-subunit diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index e4b83578..12a420c8 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -18,3 +18,4 @@ - check-requirements - publish-openstack-docs-pti - release-notes-jobs-python3 + - openstack-ansible-role-distro_install-jobs