Standardise tempest role

All of our roles have a standard pattern for installing services,
however tempest does things slightly differently.  I think most of this
is legacy and this commit brings tempest in line (for the most part)
with how other roles are written.

Note that tempest now has a ``tempest`` cli, and in a future commit we
will update this role to use that instead.  This will allow us to skip
the additional step of cloning the tempest git repo to act as a working
directory when running tests.

For some background as to why this work is being done, newer versions
of tempest are failing when run in an IRR because we are not installing
urllib3 into tempest's venv.  This is not an issue when we do an
integrated build because the repo server actually installs tempest into
the tempest venv, and all requirements get satisfied.  However, when we
install tempest without a repo server we simply install
tempest_pip_packages (which previously did not include tempest itself)
into the tempest venv.  We could update update tempest_pip_packages to
include urllib3, but doing so creates duplicate work as this
requirement is already captured in tempest's requirements.txt.

Change-Id: I6f3c36a8150b83afabae8d397d5fc7340dbc93fd
Depends-On: Ie700c061cf0efa3d1fcda9e74618bb8c6e9ae371

defaults/main.yml

@@ -22,7 +22,22 @@ verbose: True
 
 tempest_git_repo: https://git.openstack.org/openstack/tempest
 tempest_git_install_branch: master
+tempest_requirements_git_repo: https://git.openstack.org/openstack/requirements
+tempest_requirements_git_install_branch: master
 tempest_developer_mode: false
+tempest_developer_constraints:
+  - "git+{{ tempest_git_repo }}@{{ tempest_git_install_branch }}#egg=tempest"
+
+# Name of the virtual env to deploy into
+tempest_venv_tag: untagged
+# NOTE(mattt): The eventual goal is to get the tempest installed into
+#              /openstack/venvs like all other venvs, but for now we point
+#              ``tempest_venv_bin`` where we currently install tempest.
+#tempest_venv_bin: "/openstack/venvs/tempest-{{ tempest_venv_tag }}/bin"
+tempest_venv_bin: "/opt/tempest_{{ tempest_venv_tag }}/bin"
+
+# Set this to enable or disable installing in a venv
+tempest_venv_enabled: true
 
 ## Tempest Plugins
 # By default, no tempest plugins are installed.  Override ``tempest_plugins``
@@ -103,12 +118,9 @@ tempest_requires_pip_packages:
   - virtualenv-tools
 
 tempest_pip_packages:
-  - fixtures
+  - tempest
   - junitxml
   - nose
-  - oslo.concurrency
-  - oslo.serialization
-  - pyOpenSSL
   - python-ceilometerclient
   - python-cinderclient
   - python-glanceclient
@@ -120,10 +132,6 @@ tempest_pip_packages:
   - python-openstackclient
   - python-subunit
   - python-swiftclient
-  - tempest-lib
-  - testrepository
-  - testscenarios
-  - testtools
 
 # Please update SHA in tempest_images below when changing the cirros version.
 cirros_version: 0.3.4

tasks/tempest_install.yml

@@ -13,41 +13,105 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Install required pip packages
+- name: Create developer mode constraint file
+  copy:
+    dest: "/opt/developer-pip-constraints.txt"
+    content: |
+      {% for item in tempest_developer_constraints %}
+      {{ item }}
+      {% endfor %}
+  when:
+    - tempest_developer_mode | bool
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Clone requirements git repository
+  git:
+    repo: "{{ tempest_requirements_git_repo }}"
+    dest: "/opt/requirements"
+    clone: yes
+    update: yes
+    version: "{{ tempest_requirements_git_install_branch }}"
+  when:
+    - tempest_developer_mode | bool
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Add constraints to pip_install_options fact for developer mode
+  set_fact:
+    pip_install_options_fact: "{{ pip_install_options|default('') }} --constraint /opt/developer-pip-constraints.txt --constraint /opt/requirements/upper-constraints.txt"
+  when:
+    - tempest_developer_mode | bool
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Set pip_install_options_fact when not in developer mode
+  set_fact:
+    pip_install_options_fact: "{{ pip_install_options|default('') }}"
+  when:
+    - not tempest_developer_mode | bool
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Install requires pip packages
   pip:
     name: "{{ item }}"
     state: latest
-    extra_args: "{{ pip_install_options|default('') }}"
+    extra_args: "{{ pip_install_options_fact }}"
   register: install_packages
   until: install_packages|success
   retries: 5
   delay: 2
-  with_items: tempest_requires_pip_packages
+  with_items: "{{ tempest_requires_pip_packages }}"
   tags:
-    - tempest-pip-requires-packages
     - tempest-install
     - tempest-pip-packages
 
-- name: Get tempest from git
-  git:
-    repo: "{{ tempest_git_repo }}"
-    dest: "{{ tempest_git_dest }}"
-    version: "{{ tempest_git_install_branch }}"
-    force: yes
-  register: git_clone
-  until: git_clone|success
-  retries: 5
-  delay: 2
+- name: Get local venv checksum
+  stat:
+    path: "/var/cache/{{ tempest_venv_download_url | basename }}"
+    get_md5: False
+  when:
+    - not tempest_developer_mode | bool
+    - tempest_venv_enabled | bool
+  register: local_venv_stat
   tags:
-    - tempest-git-clone
-    - tempest-pip-install
+    - tempest-install
+    - tempest-pip-packages
 
+- name: Get remote venv checksum
+  uri:
+    url: "{{ tempest_venv_download_url | replace('tgz', 'checksum') }}"
+    return_content: True
+  when:
+    - not tempest_developer_mode | bool
+    - tempest_venv_enabled | bool
+  register: remote_venv_checksum
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+# TODO: When project moves to ansible 2 we can pass this a sha256sum which will:
+#       a) allow us to remove force: yes
+#       b) allow the module to calculate the checksum of dest file which would
+#          result in file being downloaded only if provided and dest sha256sum
+#          checksums differ
 - name: Attempt venv download
   get_url:
     url: "{{ tempest_venv_download_url }}"
     dest: "/var/cache/{{ tempest_venv_download_url | basename }}"
+    force: yes
   ignore_errors: true
   register: get_venv
+  when:
+    - not tempest_developer_mode | bool
+    - tempest_venv_enabled | bool
+    - (local_venv_stat.stat.exists == False or
+        {{ local_venv_stat.stat.checksum is defined and local_venv_stat.stat.checksum != remote_venv_checksum.content | trim }})
   tags:
     - tempest-install
     - tempest-pip-packages
@@ -55,16 +119,48 @@
 - name: Set tempest get_venv fact
   set_fact:
     tempest_get_venv: "{{ get_venv }}"
+  when: tempest_venv_enabled | bool
   tags:
     - tempest-install
     - tempest-pip-packages
 
+- name: Remove existing venv
+  file:
+    path: "{{ tempest_venv_bin | dirname }}"
+    state: absent
+  when:
+    - tempest_venv_enabled | bool
+    - tempest_get_venv | changed
+  tags:
+    - tempest-install
+    - tempest-pip-packages
+
+# NOTE(mattt): The goal is to get tempest installed into /openstack/venvs like
+#              all other services, and then use the tempest cli to create our
+#              working dir.  This will mean we no longer need to git clone
+#              tempest.
+- name: Get tempest from git
+  git:
+    repo: "{{ tempest_git_repo }}"
+    dest: "{{ tempest_venv_bin | dirname }}"
+    version: "{{ tempest_git_install_branch }}"
+    force: yes
+  register: git_clone
+  until: git_clone|success
+  retries: 5
+  delay: 2
+  tags:
+    - tempest-install
+    - tempest-git-clone
+
 - name: Create tempest venv dir
   file:
-    path: "{{ tempest_git_dest }}"
+    path: "{{ tempest_venv_bin | dirname }}"
     state: directory
   when:
-    - tempest_get_venv | success
+    - not tempest_developer_mode | bool
+    - tempest_venv_enabled | bool
+    - tempest_get_venv | changed
   tags:
     - tempest-install
     - tempest-pip-packages
@@ -72,57 +168,62 @@
 - name: Unarchive pre-built venv
   unarchive:
     src: "/var/cache/{{ tempest_venv_download_url | basename }}"
-    dest: "{{ tempest_git_dest }}"
+    dest: "{{ tempest_venv_bin | dirname }}"
     copy: "no"
   when:
-    - tempest_get_venv | success
+    - not tempest_developer_mode | bool
+    - tempest_venv_enabled | bool
+    - tempest_get_venv | changed
   tags:
     - tempest-install
     - tempest-pip-packages
 
 - name: Update virtualenv path
   command: >
-    virtualenv-tools --update-path=auto {{ tempest_git_dest }}
+    virtualenv-tools --update-path=auto {{ tempest_venv_bin | dirname }}
   when:
+    - not tempest_developer_mode | bool
+    - tempest_venv_enabled | bool
     - tempest_get_venv | success
   tags:
     - tempest-install
     - tempest-pip-packages
 
-- name: Install pip packages for tempest (prebuilt venv)
-  pip:
-    name: "{{ tempest_git_dest }}"
-    state: latest
-    virtualenv: "{{ tempest_git_dest }}"
-    virtualenv_site_packages: "no"
-    extra_args: "{{ pip_install_options|default('') }}"
-  register: install_packages
-  until: install_packages|success
-  retries: 5
-  delay: 2
-  when:
-    - tempest_get_venv | success
-  tags:
-    - tempest-pip-packages
-    - tempest-pip-install
-
-- name: Install pip packages for tempest (no prebuilt venv)
+- name: Install pip packages (venv)
   pip:
     name: "{{ item }}"
     state: latest
-    virtualenv: "{{ tempest_git_dest }}"
+    virtualenv: "{{ tempest_venv_bin | dirname }}"
     virtualenv_site_packages: "no"
-    extra_args: "{{ pip_install_options|default('') }}"
+    extra_args: "{{ pip_install_options_fact }}"
   register: install_packages
   until: install_packages|success
   retries: 5
   delay: 2
+  with_items: "{{ tempest_pip_packages }}"
   when:
-    - tempest_get_venv | failed
-  with_items: tempest_pip_packages
+    - tempest_venv_enabled | bool
+    - tempest_get_venv | failed or tempest_developer_mode | bool
   tags:
+    - tempest-install
+    - tempest-pip-packages
+
+- name: Install pip packages (no venv)
+  pip:
+    name: "{{ item }}"
+    state: latest
+    extra_args: "{{ pip_install_options_fact }}"
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 2
+  with_items: "{{ tempest_pip_packages }}"
+  when:
+    - not tempest_venv_enabled | bool
+    - not tempest_developer_mode | bool
+  tags:
+    - tempest-install
     - tempest-pip-packages
-    - tempest-pip-install
 
 - name: Get tempest plugins from git
   git:
@@ -136,19 +237,39 @@
   retries: 5
   delay: 2
   tags:
-    - git-clone
+    - tempest-install
+    - tempest-plugins-git-clone
 
-- name: Install tempest plugins
+- name: Install tempest plugins (venv)
   pip:
     name: "/opt/{{ item.name }}_{{ item.branch|replace('/', '_') }}"
     state: latest
     virtualenv: "{{ tempest_git_dest }}"
     virtualenv_site_packages: "no"
-    extra_args: "{{ pip_install_options|default('') }}"
+    extra_args: "{{ pip_install_options_fact }}"
   with_items: "{{ tempest_plugins }}"
   register: install_packages
   until: install_packages|success
   retries: 5
   delay: 2
+  when:
+    - tempest_venv_enabled | bool
   tags:
-    - pip-install
+    - tempest-install
+    - tempest-plugins-pip-install
+
+- name: Install tempest plugins (no venv)
+  pip:
+    name: "/opt/{{ item.name }}_{{ item.branch|replace('/', '_') }}"
+    state: present
+    extra_args: "{{ pip_install_options_fact }}"
+  with_items: "{{ tempest_plugins }}"
+  register: install_packages
+  until: install_packages|success
+  retries: 5
+  delay: 2
+  when:
+    - not tempest_venv_enabled | bool
+  tags:
+    - tempest-install
+    - tempest-plugins-pip-install

tasks/tempest_post_install.yml

@@ -43,8 +43,8 @@
     group: "root"
     mode: "{{ item.mode|default('0755') }}"
   with_items:
-    - { path: "{{ tempest_git_dest }}/locks", mode: "0777" }
-    - { path: "{{ tempest_git_dest }}/etc" }
+    - { path: "{{ tempest_venv_bin | dirname }}/locks", mode: "0777" }
+    - { path: "{{ tempest_venv_bin | dirname }}/etc" }
     - { path: "{{ tempest_image_dir }}" }
   tags:
     - tempest-dirs
@@ -87,7 +87,7 @@
 - name: Copy tempest config
   config_template:
     src: "tempest.conf.j2"
-    dest: "{{ tempest_git_dest }}/etc/tempest.conf"
+    dest: "{{ tempest_venv_bin | dirname }}/etc/tempest.conf"
     owner: "root"
     group: "root"
     mode: "0644"

templates/openstack_tempest_gate.sh.j2

@@ -149,10 +149,10 @@ for test_list_name in $test_lists; do
 done
 
 # work in tempest directory
-pushd {{ tempest_git_dest }}
+pushd {{ tempest_venv_bin | dirname }}
 
 # Load the tempest venv for a tempest run
-source {{ tempest_git_dest }}/bin/activate
+source {{ tempest_venv_bin | dirname }}/bin/activate
 
 # read creds into environment
 source /root/openrc

templates/tempest.conf.j2

@@ -160,7 +160,7 @@ instance_type = tempest1
 
 
 [oslo_concurrency]
-lock_path = {{ tempest_git_dest }}/locks
+lock_path = {{ tempest_venv_bin | dirname }}/locks
 
 
 [scenario]

tests/test-vars.yml

@@ -38,9 +38,8 @@ openrc_os_auth_url: "http://127.0.0.1:5000/v3"
 openrc_os_domain_name: "Default"
 openrc_os_password: "{{ keystone_auth_admin_password }}"
 tempest_developer_mode: True
-tempest_git_repo: https://git.openstack.org/openstack/tempest
-tempest_git_install_branch: 534a8dc60dfef116156b8f9ee60071a9bf4e4f90
-tempest_git_dest: "/opt/tempest_{{ tempest_git_install_branch | replace('/', '_') }}"
+tempest_git_install_branch: master
+tempest_requirements_git_install_branch: master
 tempest_log_dir: "/var/log/"
 tempest_main_group: keystone_all
 tempest_service_available_aodh: False
@@ -52,3 +51,4 @@ tempest_service_available_horizon: False
 tempest_service_available_neutron: False
 tempest_service_available_nova: False
 tempest_service_available_swift: False
+tempest_venv_tag: "{{ tempest_git_install_branch }}"