Refactored validate-tempest role for undercloud and containers

* Introduced new workflow for validate-tempest role
- Remove previously created tempest artifacts
- Install tempest
- Create tempest workspace
- Create tempest resources
- Generate tempest configurations
- Run tempest
- Removed unused ironic undercloud fake services
- Removing public network from undercloud as there is no public network
- Added tempest_dir var to give path to tempest workspace
- use {{ working_dir }}/tempest_git for clonning stuff
- Added test_black_regex to skip a series of tests
- Fixed permission issue for tempest_dir to generate subunit files
- Install stestr subunit-filters and subunit for generating subunit files
- Added tempest_skip_master containing skip list for undercloud
- Introduced tempest_conf_version for installing particular version of
  tempestconf from git
- Enable set -e to find the exit status of command running inside
  container

Change-Id: I021432e222fae0e57a13ca4859ba77e990ac6c67

roles/validate-tempest/README.md

@@ -17,7 +17,7 @@ Role Variables
 * `tempest_format`: venv/packages - Which tempest installation to use - either install python virtual environment
                     with installed there python modules from requirements file, or to use installed with RDO RPM packages
 * `tempest_log_file` - name of log file for tempest run
-* `test_regex` - tests regular expression for testr run, i.e. smoke or tempest.api.object_storage|keystone_tempest_plugin.
+* `test_regex` - tests regular expression for tempest run, i.e. smoke or tempest.api.object_storage|keystone_tempest_plugin.
 * `run_tempest`: false/true - to run tempest or not
 * `tempest_config`: false/true - whether to prepare the script which configures and runs tempest or not
 * `tempest_whitelist`: list - list of tests you want to be executed. set `skip_file_src`
@@ -39,6 +39,11 @@ Role Variables
                           removed from tempest.conf file.
                           Format: section.key: value
 * `public_physical_network`: <string> The name of the border physical network (default: datacentre).
+* `tempest_container_registry`: <string> The name of the container registry to use (default: docker.io/tripleomaster)
+* `tempest_container_namespace`: <string> The name of tempest container image to use (default: centos-binary-tempest)
+* `tempest_container_tag`: <string> The tag of the tempest container image to use (default: current-tripleo)
+* `tempest_dir`: <string> The path to tempest workspace directory (default: /home/stack/tempest)
+* `test_black_regex`: <list> A set of tempest tests to skip (default: [])
 
 Skip tests file
 ---------------

roles/validate-tempest/defaults/main.yml

@@ -6,6 +6,7 @@ public_net_pool_end: "{{ floating_ip_cidr|nthhost(120) }}"
 public_net_gateway: "{{ floating_ip_cidr|nthhost(1) }}"
 tempest_log_file: 'tempest_output.log'
 test_regex: smoke
+test_black_regex: []
 check_tempest_bugs: false
 
 public_net_name: public
@@ -22,7 +23,11 @@ tempest_config: true
 tempest_overcloud: true
 run_tempest: false
 post_tempest: true
-tempest_format: packages # venv or packages
+tempest_format: packages # venv or packages or container
+tempest_container_registry: "{{ docker_registry_host }}/{{ docker_registry_namespace }}"
+tempest_container_namespace: centos-binary-tempest
+tempest_container_tag: "{{ docker_image_tag }}"
+tempest_dir: "{{ working_dir }}/tempest"
 tempest_whitelist_file_src: "whitelist_file.j2"
 tempest_whitelist_file: "whitelist_file.conf"
 tempest_whitelist: []
@@ -42,6 +47,8 @@ tempest_exit_on_failure: true
 # For forked-tempest, we have branched name in synced with releases.
 tempest_version_dict: { 'newton': 'newton', 'ocata': '16.1.0', 'pike': '17.1.0', 'queens': '17.2.0', 'master': 'master'}
 tempest_version: "{{ tempest_version_dict[release] }}"
+tempest_conf_version_dict: {'ocata': '1.1.3', 'pike': '1.1.3', 'queens': '1.1.4', 'master': 'master'}
+tempest_conf_version: "{{ tempest_conf_version_dict[release] }}"
 tempestmail_config: config.yaml
 tempestmail_log_server: http://logs.openstack.org
 tempest_track_resources: true

roles/validate-tempest/tasks/main.yml

@@ -1,8 +1,18 @@
 ---
-- include: undercloud-config.yml
-  when: tempest_undercloud|bool
+- include: tempest-venv.yml
+  when: tempest_config|bool and tempest_format == 'venv'
   tags:
-    - tempest-undercloud-config
+    - pre-tempest-config
+
+- include: tempest-rpm.yml
+  when: tempest_config|bool and tempest_format == 'packages'
+  tags:
+    - pre-tempest-config
+
+- include: tempest-containers.yml
+  when: tempest_config|bool and tempest_format == 'container'
+  tags:
+    - pre-tempest-config
 
 - include: pre-tempest.yml
   when: tempest_config|bool or run_tempest|bool

roles/validate-tempest/tasks/pre-tempest.yml

@@ -4,13 +4,17 @@
       tempest_workers: "{{ ansible_processor_vcpus|int // 2 }}"
   when: tempest_workers is not defined
 
+- name: Set rc file to be sourced to run tempest
+  set_fact:
+      rc_file: "{{ working_dir }}/{% if tempest_undercloud %}stackrc{% else %}overcloudrc{% endif %}"
+
 - name: Create overcloud tempest setup script
   template:
     src: tempest-setup.j2
     dest: "{{ working_dir }}/tempest-setup.sh"
     mode: 0744
 
-- name: Load skip list variables
+- name: Load skip list variables (undercloud or overcloud)
   include_vars:
     file: "tempest_skip_{{ release }}.yml"
   when: skip_file_src != ''

roles/validate-tempest/tasks/tempest-containers.yml

@@ -0,0 +1,8 @@
+---
+- name: Set tempest init command
+  set_fact:
+      tempest_init: "{% if release == 'newton' %}/usr/share/openstack-tempest-*/tools/configure-tempest-directory{% else %}tempest init {{ tempest_dir }}{% endif %}"
+
+- name: Set tempestconf call
+  set_fact:
+      tempestconf: "{% if release == 'newton' %}{{ working_dir }}/tools/config_tempest.py{% else %}/usr/bin/discover-tempest-config{% endif %}"

roles/validate-tempest/tasks/tempest-results.yml

@@ -1,6 +1,20 @@
 ---
 - ignore_errors: true
   block:
+    - name: Install packages to generate subunit results
+      become: yes
+      package: name={{ item }} state=present
+      with_items:
+        - python-stestr
+        - python-subunit
+        - subunit-filters
+      when: tempest_format == 'container'
+
+    - name: Change permission of tempest container directory
+      shell: |
+        sudo chmod -R 777 {{ tempest_dir }}
+      when: tempest_format == 'container'
+
     - name: Generate testrepository.subunit results file
       shell: >
         {% if tempest_format == 'venv' %}source {{ working_dir }}/tempest_git/.venv/bin/activate; {% endif %}

roles/validate-tempest/tasks/tempest-rpm.yml

@@ -0,0 +1,39 @@
+---
+- name: Install openstack services tempest plugin
+  package: name={{ item }} state=present
+  become: true
+  with_items:
+    - python-ceilometer-tests
+    - python-zaqar-tests
+    - python-ironic-inspector-tests
+    - python-gnocchi-tests
+    - python-aodh-tests
+    - python-mistral-tests
+    - python-heat-tests
+    - python-keystone-tests
+    - python-ironic-tests
+    - python-neutron-tests
+    - python-cinder-tests
+  when: release == 'newton'
+
+- name: Install openstack tempest
+  package: name={{ item }} state=present
+  become: true
+  with_items:
+    - openstack-tempest
+    - python-junitxml
+
+- name: Install python-tempestconf
+  package:
+    name: python-tempestconf
+    state: present
+  become: true
+  when: release != 'newton'
+
+- name: Set tempest init command
+  set_fact:
+      tempest_init: "{% if release == 'newton' %}/usr/share/openstack-tempest-*/tools/configure-tempest-directory{% else %}tempest init {{ tempest_dir }}{% endif %}"
+
+- name: Set tempestconf call
+  set_fact:
+      tempestconf: "{% if release == 'newton' %}{{ working_dir }}/tools/config_tempest.py{% else %}/usr/bin/discover-tempest-config{% endif %}"

roles/validate-tempest/tasks/tempest-venv.yml

@@ -0,0 +1,37 @@
+---
+- name: Cloning tempest from redhat-openstack repository
+  git:
+    repo: 'https://github.com/redhat-openstack/tempest.git'
+    dest: '{{ working_dir }}/tempest_git'
+    version: '{{ tempest_version }}'
+  when: release == 'newton'
+
+- name: Cloning tempest from openstack repository
+  git:
+    repo: 'https://github.com/openstack/tempest.git'
+    dest: '{{ working_dir }}/tempest_git'
+    version: '{{ tempest_version }}'
+  when: release != 'newton'
+
+- name: Cloning python-tempestconf
+  git:
+    repo: 'https://github.com/openstack/python-tempestconf.git'
+    dest: '{{ working_dir }}/python-tempestconf'
+    version: '{{ tempest_conf_version }}'
+  when: release != 'newton'
+
+- name: Install packages required for create venv
+  package: name={{ item }} state=present
+  with_items:
+    - libffi-devel
+    - openssl-devel
+    - python-virtualenv
+    - gcc
+
+- name: Set tempest init command
+  set_fact:
+      tempest_init: "{{ working_dir }}/tempest_git/tools/{% if release == 'newton' %}configure-tempest-directory{% else %}with_env.sh tempest init{% endif %}"
+
+- name: Set tempestconf call
+  set_fact:
+      tempestconf: "{% if release == 'newton' %}{{ working_dir }}/tools/config_tempest.py{% else %}{{ working_dir }}/tempest_git/tools/with_venv.sh discover-tempest-config{% endif %}"

roles/validate-tempest/tasks/undercloud-config.yml

@@ -1,25 +0,0 @@
----
-- name: Add fake driver to Ironic config
-  become: yes
-  lineinfile:
-    backup: true
-    backrefs: true
-    dest: '/etc/ironic/ironic.conf'
-    regexp: '^(enabled_drivers=(?:(?!fake).)*)$'
-    line: '\1,fake'
-
-- name: Restart the Ironic conductor service
-  become: yes
-  service: name=openstack-ironic-conductor state=restarted
-
-- name: Ensure the fake driver is enabled
-  shell: >
-    source {{ working_dir }}/stackrc;
-    ironic driver-list;
-  register: driver_list
-  failed_when: "'fake' not in driver_list.stdout"
-
-- name: Copy tempest input file to undercloud
-  copy:
-    src: "tempest-undercloud-config.conf"
-    dest: "{{ working_dir }}/{{ tempest_deployer_input_file }}"

roles/validate-tempest/templates/cleanup-network.sh.j2

@@ -0,0 +1,72 @@
+{% if release == 'newton' %}
+
+for i in $(neutron floatingip-list -c id -f value)
+do
+    neutron floatingip-disassociate $i
+    neutron floatingip-delete $i
+done
+for i in $(neutron router-list -c id -f value); do neutron router-gateway-clear $i; done
+for r in $(neutron router-list -c id -f value); do
+    for p in $(neutron router-port-list $r -c id -f value); do
+        neutron router-interface-delete $r port=$p || true
+    done
+done
+for i in $(neutron router-list -c id -f value); do neutron router-delete $i; done
+for i in $(neutron port-list -c id -f value); do neutron port-delete $i; done
+for i in $(neutron net-list -c id -f value); do neutron net-delete $i; done
+
+neutron net-create {{ public_net_name }} --router:external=True \
+{% if public_physical_network != '' %}
+    --provider:network_type {{ public_network_type }} \
+    {% if public_segmentation_id != '' %}
+    --provider:segmentation_id {{ public_segmentation_id }} \
+{% endif %}
+    --provider:physical_network {{ public_physical_network }}
+{% endif %}
+
+public_net_id=$(neutron net-show {{ public_net_name }} -f value -c id)
+
+neutron subnet-create --name ext-subnet \
+    --allocation-pool \
+    start={{ public_net_pool_start }},end={{ public_net_pool_end }} \
+    --disable-dhcp \
+    --gateway {{ public_net_gateway }} \
+{{ public_net_name }} {{ floating_ip_cidr }}
+
+{% else %}
+
+for i in $(openstack floating ip list -c ID -f value)
+do
+    openstack floating ip unset --port $i
+    openstack floating ip delete $i
+done
+for i in $(openstack router list -c ID -f value); do openstack router unset --external-gateway $i; done
+for r in $(openstack router list -c ID -f value); do
+    for p in $(openstack port list --router $r -c ID -f value); do
+        openstack router remove subnet $r $p || true
+    done
+done
+for i in $(openstack router list -c ID -f value); do openstack router delete $i; done
+for i in $(openstack port list -c ID -f value); do openstack port delete $i; done
+for i in $(openstack network list -c ID -f value); do openstack network delete $i; done
+
+openstack network create {{ public_net_name }} --external \
+{% if public_physical_network != '' %}
+    --provider-network-type {{ public_network_type }} \
+    {% if public_segmentation_id != '' %}
+    --provider-segment {{ public_segmentation_id }} \
+{% endif %}
+    --provider-physical-network {{ public_physical_network }}
+{% endif %}
+
+
+public_net_id=$(openstack network show {{ public_net_name }} -f value -c id)
+
+openstack subnet create ext-subnet \
+    --allocation-pool \
+    start={{ public_net_pool_start }},end={{ public_net_pool_end }} \
+    --no-dhcp \
+    --gateway {{ public_net_gateway }} \
+    --network {{ public_net_name }} \
+    --subnet-range {{ floating_ip_cidr }}
+{% endif %}

roles/validate-tempest/templates/configure-tempest.sh.j2

@@ -3,158 +3,112 @@
 ## Configure tempest
 ## -----------------
 
-{% if tempest_overcloud|bool %}
-## ::
-source {{ working_dir }}/overcloudrc
-
 ## * Clean up from any previous tempest run
 ## ::
 
-rm -rf {{ working_dir }}/tempest
 # On doing tempest init workspace, it will create workspace directory
 # as well as .workspace directory to store workspace information
 # We need to delete .workspace directory otherwise tempest init failed
 # to create tempest directory.
 rm -rf {{ working_dir }}/.tempest
-rm -rf {{ working_dir }}/tempest_git
+rm -rf {{ tempest_dir }}
 rm -rf {{ working_dir }}/python-tempestconf
 
+# Source rc file
+source {{ rc_file }}
+
+## Create Tempest resources
+## ------------------------
+## * For overcloud
+
+{% if tempest_overcloud|bool %}
+
 ## * Clean up network if it exists from previous run
 ## ::
 
-for i in $(neutron floatingip-list -c id -f value)
-do
-    neutron floatingip-disassociate $i
-    neutron floatingip-delete $i
-done
-for i in $(neutron router-list -c id -f value); do neutron router-gateway-clear $i; done
-for r in $(neutron router-list -c id -f value); do
-    for p in $(neutron router-port-list $r -c id -f value); do
-        neutron router-interface-delete $r port=$p || true
-    done
-done
-for i in $(neutron router-list -c id -f value); do neutron router-delete $i; done
-for i in $(neutron port-list -c id -f value); do neutron port-delete $i; done
-for i in $(neutron net-list -c id -f value); do neutron net-delete $i; done
+{% include 'cleanup-network.sh.j2' %}
 
-neutron net-create {{ public_net_name }} --router:external=True \
-{% if public_physical_network != '' %}
-    --provider:network_type {{ public_network_type }} \
-    {% if public_segmentation_id != '' %}
-    --provider:segmentation_id {{ public_segmentation_id }} \
-{% endif %}
-    --provider:physical_network {{ public_physical_network }}
-{% endif %}
-
-
-public_net_id=$(neutron net-show {{ public_net_name }} -f value -c id)
-
-neutron subnet-create --name ext-subnet \
-    --allocation-pool \
-    start={{ public_net_pool_start }},end={{ public_net_pool_end }} \
-    --disable-dhcp \
-    --gateway {{ public_net_gateway }} \
-    {{ public_net_name }} {{ floating_ip_cidr }}
-{% else %}
-source {{ working_dir }}/stackrc
-public_net_id=$(neutron net-show {{ undercloud_public_net_name }} -f value -c id)
 {% endif %}
 
 ## * Ensure creator and Member role is present
 ## * Member role is needed for Heat tests.
 ## * creator role is needed for Barbican for running volume encryption tests.
 ## ::
+
 openstack role show Member > /dev/null || openstack role create Member
 
 openstack role show creator > /dev/null || openstack role create creator
 
-## * Generate a tempest configuration
+# Create Tempest directory
+
+mkdir {{ tempest_dir }}
+
+## Install openstack-tempest
+## -------------------------
+## * Using git
 ## ::
 
-mkdir {{ working_dir }}/tempest
 {% if tempest_format == "venv" %}
-{% if release == 'newton' %}
-# Clone all the git related stuff in a seperate directory
-git clone https://github.com/redhat-openstack/tempest {{ working_dir }}/tempest_git
-{% else %}
-git clone https://github.com/openstack/tempest {{ working_dir }}/tempest_git
-{% endif %}
-# set tempest_version
-pushd {{ working_dir }}/tempest_git
-git fetch origin {{ tempest_version }}
-git checkout {{ tempest_version }}
-popd
-# Install required dependency for creating venv
-sudo yum install -y libffi-devel openssl-devel python-virtualenv gcc
 # Create .venv in tempest_git directory with --system-site-packages to access tempest plugins
 virtualenv --system-site-packages {{ working_dir }}/tempest_git/.venv
 {{ working_dir }}/tempest_git/tools/with_venv.sh pip install -U pip
 {{ working_dir }}/tempest_git/tools/with_venv.sh pip install -U setuptools
 {{ working_dir }}/tempest_git/tools/with_venv.sh pip install {{ working_dir }}/tempest_git junitxml
-# Create Tempest Workspace using tempest git
-cd {{ working_dir }}/tempest
-{% if release == 'newton' %}
-{{ working_dir }}/tempest_git/tools/configure-tempest-directory
-{% else %}
-{{ working_dir }}/tempest_git/tools/with_venv.sh tempest init {{ working_dir }}/tempest
-{% endif %}
-{% elif tempest_format == "packages" %}
-# Install OpenStack Tempest, python-junitxml for Newton
-# From Ocata, config_tempest is moved to python-tempestconf. So for
-# Ocata onwards, Install python-tempestconf
-sudo yum -y install openstack-tempest python-junitxml {% if release != 'newton' %}python-tempestconf{% endif %}
 
-# Create Tempest Workspace from tempest rdo package
-{% if release == 'newton' %}
-cd {{ working_dir }}/tempest
-/usr/share/openstack-tempest-*/tools/configure-tempest-directory
-{% else %}
-tempest init {{ working_dir }}/tempest
-{% endif %}
-{% endif %}
-
-# Install OpenStack Services Tempest plugin
-{% if release == "newton" %}
-# FIXME(chkumar246): Install tempest plugin from package currently then switch to install_test_packages script
-sudo yum -y install python-ceilometer-tests python-zaqar-tests python-ironic-inspector-tests \
-    python-gnocchi-tests python-aodh-tests python-mistral-tests python-heat-tests python-keystone-tests \
-    python-ironic-tests python-neutron-tests python-cinder-tests
-{% endif %}
-
-# Generate tempest configuration files
-{% if release == 'newton' %}
-export TEMPESTCONF="{{ working_dir }}/tempest/tools/config_tempest.py"
-{% else %}
-{% if tempest_format == "venv" %}
 # Install python-tempestconf
-git clone https://git.openstack.org/openstack/python-tempestconf {{ working_dir }}/python-tempestconf
+{% if release != 'newton'%}
 {{ working_dir }}/tempest_git/tools/with_venv.sh pip install {{ working_dir }}/python-tempestconf
-export TEMPESTCONF="{{ working_dir }}/tempest_git/tools/with_venv.sh discover-tempest-config"
-{% elif tempest_format == "packages" %}
-export TEMPESTCONF="/usr/bin/discover-tempest-config"
 {% endif %}
 {% endif %}
 
-# Go to Tempest Workspace
-cd {{ working_dir }}/tempest
+{% if tempest_format == "container" %}
+export RCFILE="{{ rc_file }}"
+cat <<'EOF' >> {{ working_dir }}/tempest_container.sh
+# Set the exit status for the command
+set -e
+# Install OpenStack client  to create tempest resources with in containers
+{% endif %}
 
+## Create Tempest Workspace
+## ------------------------
+## ::
+
+# Create Tempest workspace
+pushd {{ tempest_dir }}
+{{ tempest_init }}
+popd
+
+## Generate tempest configuration using python-tempestconf
+## -------------------------------------------------------
+## ::
+export TEMPESTCONF="{{ tempestconf }}"
+
+# Source rc file for tempestconf generation
+source {{ rc_file }}
+
+{% if not tempest_overcloud|bool %}
+export OS_AUTH_URL="$OS_AUTH_URL/v$OS_IDENTITY_API_VERSION"
+{% endif %}
 # Generate Tempest Config file using python-tempestconf
 # Notice aodh_plugin will be set to False if telemetry service is disabled
 # TODO(arxcruz) In the future the
 # compute_feature_enabled.attach_encrypted_volume should be handled by
 # python-tempestconf tool
-${TEMPESTCONF} --out etc/tempest.conf \
-  --network-id $public_net_id \
+cd {{ tempest_dir }}
+$TEMPESTCONF --out etc/tempest.conf \
+  {% if tempest_overcloud|bool %}
   --deployer-input ~/{{ tempest_deployer_input_file }} \
+  --network-id $public_net_id \
+  {% endif %}
   --image {{ tempest_test_image_path }} \
   --debug \
-  {% if tempest_conf_removal %}
+  {% if tempest_conf_removal and tempest_overcloud|bool %}
   {% for key, value in tempest_conf_removal.iteritems() %}
   --remove {{ key }}={{ value }} \
   {% endfor %}
   {% endif %}
   --create \
-  {% if release in ['newton','ocata', 'pike'] %}
+  {% if release in ['ocata', 'pike'] %}
   identity.uri $OS_AUTH_URL \
   identity.admin_password $OS_PASSWORD \
   identity.admin_username $OS_USERNAME \
@@ -185,4 +139,3 @@ ${TEMPESTCONF} --out etc/tempest.conf \
   orchestration.stack_owner_role Member
 
 ### --stop_docs
-

roles/validate-tempest/templates/run-tempest.sh.j2

@@ -8,7 +8,7 @@
 {% if tempest_format == "venv" %}
 export TEMPESTCLI='{{ working_dir }}/tempest_git/tools/with_venv.sh tempest'
 export OSTESTR='{{ working_dir }}/tempest_git/tools/with_venv.sh ostestr'
-{% elif tempest_format == "packages" %}
+{% else %}
 export OSTESTR='ostestr'
 export TEMPESTCLI='/usr/bin/tempest'
 {% endif %}
@@ -24,6 +24,7 @@ $TEMPESTCLI cleanup --init-saved-state
 
 {% if release not in ["master", "queens"] %} $OSTESTR {% else %} $TEMPESTCLI run {% endif %}{% if test_regex != '' %} --regex '({{ test_regex }})' {% endif %}
 {% if tempest_whitelist|length > 0 %} --whitelist_file={{ working_dir }}/{{ tempest_whitelist_file }} {% endif %}
+{% if release not in ["newton", "ocata", "pike"] %} {% if test_black_regex|length > 0 %} --black-regex='{{ test_black_regex|join('|') }}' {% endif %} {% endif %}
 {% if skip_file_src != '' %} --blacklist_file={{ working_dir }}/{{ skip_file }} {% endif %}
 {% if tempest_workers is defined %} --concurrency {{ tempest_workers }} {% endif %}
 {% if tempest_until_failure|bool %} --until-failure {% endif %}
@@ -34,4 +35,26 @@ $TEMPESTCLI cleanup --init-saved-state
 $TEMPESTCLI cleanup --dry-run
 {% endif %}
 
+{% if tempest_format == "container" %}
+EOF
+
+chmod +x {{ working_dir }}/tempest_container.sh
+# Run tempest container using docker mouting required files
+sudo docker run -i -v $RCFILE:$RCFILE \
+    {% if skip_file_src != '' %}
+    -v {{ working_dir }}/{{ skip_file }}:{{ working_dir }}/{{ skip_file }} \
+    {% endif %}
+    {% if tempest_whitelist|length > 0 %}
+    -v {{ working_dir }}/{{ tempest_whitelist_file }}:{{ working_dir }}/{{ tempest_whitelist_file }} \
+    -v {{ working_dir }}/{{ tempest_deployer_input_file }}:{{ working_dir }}/{{ tempest_deployer_input_file }} \
+    {% endif %}
+    -v {{ tempest_dir }}:{{ tempest_dir }} \
+    -v {{ working_dir }}/tempest_container.sh:{{ working_dir }}/tempest_container.sh \
+    {% if not 'http' in tempest_test_image_path %}
+    -v {{ tempest_test_image_path }}:{{ tempest_test_image_path }} \
+    {% endif %}
+    {{ tempest_container_registry }}/{{ tempest_container_namespace }}:{{ tempest_container_tag }} \
+    /usr/bin/bash -c 'set -e; {{ working_dir }}/tempest_container.sh'
+
+{% endif %}
 ### --stop_docs

roles/validate-tempest/templates/skip_file.j2

@@ -1,4 +1,5 @@
 {% for skip_test in known_failures %}
+{% if (tempest_undercloud and skip_test.undercloud is defined and skip_test.undercloud) or (not tempest_undercloud and skip_test.undercloud is not defined) %}
 # {{ skip_test.reason }}
 {% if skip_test.bz is defined %}
 # {{ skip_test.bz }}
@@ -7,4 +8,5 @@
 # {{ skip_test.lp }}
 {% endif %}
 {{ skip_test.test }}
+{% endif %}
 {% endfor %}

roles/validate-tempest/vars/tempest_skip_master.yml

@@ -32,3 +32,91 @@ known_failures:
   - test: 'tempest.api.compute.servers.test_attach_interfaces.AttachInterfacesTestJSON.est_reassign_port_between_servers'
     reason: 'performance regressions & concurrency issues under investigation'
     lp: 'https://bugs.launchpad.net/tripleo/+bug/1759583'
+  # skip list for undercloud
+  - test: 'tempest.api.network.test_networks.NetworksTest.test_external_network_visibility'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.network.test_floating_ips.FloatingIPTestJSON'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.servers'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_existent_host'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_add_host_as_user'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_aggregates_negative.AggregatesAdminNegativeTestJSON.test_aggregate_remove_host_as_user'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_create_server.ServersWithSpecificFlavorTestJSON'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_aggregates.AggregatesAdminTestJSON'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_create_server.ServersWithSpecificFlavorTestJSON'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_hypervisor'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_hypervisor_negative'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_delete_server'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_hosts.HostsAdminTestJSON.test_show_host_detail'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_servers'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_simple_tenant_usage'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_servers_negative'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.floating_ips'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_server_diagnostics'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.admin.test_server_diagnostics_negative'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.images'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.network.admin.test_external_networks_negative'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.network.admin.test_floating_ips_admin_actions'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.network.admin.test_routers'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.network.admin.test_routers_negative'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.network.test_floating_ips_negative'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.network.test_networks.NetworksIpV6Test.test_external_network_visibility'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.network.admin.test_ports'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.compute.security_groups.test_security_groups.SecurityGroupsTestJSON'
+    reason: 'Running on undercloud'
+    undercloud: true
+  - test: 'tempest.api.identity.admin.v3.test_roles.RolesV3TestJSON'
+    reason: 'Running on undercloud'
+    undercloud: true