From 33110fb7e87a57cda947da20926d8dd9912c19d4 Mon Sep 17 00:00:00 2001 From: Slawek Kaplonski Date: Fri, 23 Nov 2018 15:53:10 +0100 Subject: [PATCH] Migrate neutron-functional job to zuul v3 syntax This patch migrates also neutron-functional-python27 job to zuul v3 syntax. Neutron's functional tests have to be run on host with deployed minimal devstack but without running any e.g. neutron services. Because of that new job's template inherits from devstack-minimal job instead of devstack-tox-functional and we need to have own run_functional_job and post_functional_job playbooks. It also adds ansible roles: - configure_functional_tests - setup_logdir - fetch_journal_log which are used in new neutron-functional job definition. Those roles can be also used later e.g. for fullstack job. Change-Id: I80bc17c8c9f43050ac0c21176fbc4be46c11ce35 Related-bug: #1804844 --- .zuul.yaml | 34 +++++-- neutron/tests/contrib/gate_hook.sh | 4 +- playbooks/configure_functional_job.yaml | 4 + .../neutron-functional-python27/post.yaml | 80 ---------------- .../neutron-functional-python27/run.yaml | 54 ----------- playbooks/legacy/neutron-functional/post.yaml | 96 ------------------- playbooks/legacy/neutron-functional/run.yaml | 55 ----------- playbooks/post_functional_job.yaml | 5 + playbooks/run_functional_job.yaml | 10 ++ roles/configure_functional_tests/README.rst | 18 ++++ .../defaults/main.yaml | 5 + .../tasks/main.yaml | 19 ++++ roles/fetch_journal_log/README.rst | 16 ++++ roles/fetch_journal_log/defaults/main.yaml | 2 + roles/fetch_journal_log/tasks/main.yaml | 23 +++++ roles/setup_logdir/README.rst | 8 ++ roles/setup_logdir/defaults/main.yaml | 1 + roles/setup_logdir/tasks/main.yaml | 9 ++ 18 files changed, 150 insertions(+), 293 deletions(-) create mode 100644 playbooks/configure_functional_job.yaml delete mode 100644 playbooks/legacy/neutron-functional-python27/post.yaml delete mode 100644 playbooks/legacy/neutron-functional-python27/run.yaml delete mode 100644 playbooks/legacy/neutron-functional/post.yaml delete mode 100644 playbooks/legacy/neutron-functional/run.yaml create mode 100644 playbooks/post_functional_job.yaml create mode 100644 playbooks/run_functional_job.yaml create mode 100644 roles/configure_functional_tests/README.rst create mode 100644 roles/configure_functional_tests/defaults/main.yaml create mode 100644 roles/configure_functional_tests/tasks/main.yaml create mode 100644 roles/fetch_journal_log/README.rst create mode 100644 roles/fetch_journal_log/defaults/main.yaml create mode 100644 roles/fetch_journal_log/tasks/main.yaml create mode 100644 roles/setup_logdir/README.rst create mode 100644 roles/setup_logdir/defaults/main.yaml create mode 100644 roles/setup_logdir/tasks/main.yaml diff --git a/.zuul.yaml b/.zuul.yaml index a2e9396d083..3e7c6355c70 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -102,24 +102,46 @@ - job: name: neutron-functional - parent: legacy-dsvm-base - run: playbooks/legacy/neutron-functional/run.yaml - post-run: playbooks/legacy/neutron-functional/post.yaml + parent: devstack-minimal + description: Run neutron functional tests timeout: 7800 required-projects: - - openstack-infra/devstack-gate + - git.openstack.org/openstack-dev/devstack - openstack/neutron + roles: + - zuul: openstack-dev/devstack + pre-run: playbooks/configure_functional_job.yaml + run: playbooks/run_functional_job.yaml + post-run: playbooks/post_functional_job.yaml irrelevant-files: - ^.*\.rst$ - ^doc/.*$ - ^neutron/locale/.*$ - ^releasenotes/.*$ + vars: + devstack_services: + # Ignore any default set by devstack. Emit a "disable_all_services". + base: false + devstack_localrc: + USE_PYTHON3: true + INSTALL_TESTONLY_PACKAGES: true + DATABASE_PASSWORD: stackdb + tox_envlist: dsvm-functional + tox_install_siblings: false + zuul_copy_output: + # We need to copy directory with logs to have it in job artifacts also, + # /opt/stack/logs is default logs directory defined in neutron's + # tox.ini file + '{{ devstack_base_dir }}/logs/dsvm-functional-logs': logs - job: name: neutron-functional-python27 parent: neutron-functional - run: playbooks/legacy/neutron-functional-python27/run.yaml - post-run: playbooks/legacy/neutron-functional-python27/post.yaml + description: Run neutron functional tests with Python 2.7 + vars: + devstack_localrc: + USE_PYTHON3: false + tox_envlist: dsvm-functional-python27 - job: name: neutron-fullstack diff --git a/neutron/tests/contrib/gate_hook.sh b/neutron/tests/contrib/gate_hook.sh index 6ec291d8392..86673734e00 100644 --- a/neutron/tests/contrib/gate_hook.sh +++ b/neutron/tests/contrib/gate_hook.sh @@ -2,7 +2,7 @@ set -ex -VENV=${1:-"dsvm-functional"} +VENV=${1:-"dsvm-fullstack"} FLAVOR=${2:-"all"} GATE_DEST=$BASE/new @@ -55,7 +55,7 @@ function load_rc_for_rally { case $VENV in -"dsvm-functional"|"dsvm-functional-python27"|"dsvm-fullstack") +"dsvm-fullstack") # The following need to be set before sourcing # configure_for_func_testing. GATE_STACK_USER=stack diff --git a/playbooks/configure_functional_job.yaml b/playbooks/configure_functional_job.yaml new file mode 100644 index 00000000000..01dccba3866 --- /dev/null +++ b/playbooks/configure_functional_job.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - setup_logdir + - configure_functional_tests diff --git a/playbooks/legacy/neutron-functional-python27/post.yaml b/playbooks/legacy/neutron-functional-python27/post.yaml deleted file mode 100644 index dac875340aa..00000000000 --- a/playbooks/legacy/neutron-functional-python27/post.yaml +++ /dev/null @@ -1,80 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*nose_results.html - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testr_results.html.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.testrepository/tmp* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testrepository.subunit.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}/tox' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.tox/*/log/* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/neutron-functional-python27/run.yaml b/playbooks/legacy/neutron-functional-python27/run.yaml deleted file mode 100644 index 538a66a8909..00000000000 --- a/playbooks/legacy/neutron-functional-python27/run.yaml +++ /dev/null @@ -1,54 +0,0 @@ -- hosts: all - name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack-infra/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - git://git.openstack.org \ - openstack-infra/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export PYTHONUNBUFFERED=true - export DEVSTACK_GATE_TEMPEST=0 - export DEVSTACK_GATE_EXERCISES=0 - export DEVSTACK_GATE_NEUTRON=1 - export DEVSTACK_GATE_INSTALL_TESTONLY=1 - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - - function gate_hook { - bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional-python27 - } - export -f gate_hook - - function post_test_hook { - bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional-python27 - } - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/legacy/neutron-functional/post.yaml b/playbooks/legacy/neutron-functional/post.yaml deleted file mode 100644 index 3b3df752f8f..00000000000 --- a/playbooks/legacy/neutron-functional/post.yaml +++ /dev/null @@ -1,96 +0,0 @@ -- hosts: primary - tasks: - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*nose_results.html - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testr_results.html.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.testrepository/tmp* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=**/*testrepository.subunit.gz - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}/tox' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/.tox/*/log/* - - --include=*/ - - --exclude=* - - --prune-empty-dirs - - # TODO(slaweq): this should be moved to separate role and used in zuulv3 - # jobs definitions also - - name: Store journal logs in {{ ansible_user_dir }}/workspace/logs/journal.log - become: yes - shell: - cmd: | - /bin/journalctl -a > {{ ansible_user_dir }}/workspace/logs/journal.log - - - name: Set journal.log file permissions - become: yes - file: - path: '{{ ansible_user_dir }}/workspace/logs/journal.log' - owner: '{{ ansible_user }}' - group: '{{ ansible_user }}' - mode: 0644 - - - name: Copy files from {{ ansible_user_dir }}/workspace/ on node - synchronize: - src: '{{ ansible_user_dir }}/workspace/' - dest: '{{ zuul.executor.log_root }}' - mode: pull - copy_links: true - verify_host: true - rsync_opts: - - --include=/logs/** - - --include=*/ - - --exclude=* - - --prune-empty-dirs diff --git a/playbooks/legacy/neutron-functional/run.yaml b/playbooks/legacy/neutron-functional/run.yaml deleted file mode 100644 index 397178ca01d..00000000000 --- a/playbooks/legacy/neutron-functional/run.yaml +++ /dev/null @@ -1,55 +0,0 @@ -- hosts: all - name: Autoconverted job legacy-neutron-dsvm-functional from old job gate-neutron-dsvm-functional-ubuntu-xenial-nv - tasks: - - - name: Ensure legacy workspace directory - file: - path: '{{ ansible_user_dir }}/workspace' - state: directory - - - shell: - cmd: | - set -e - set -x - cat > clonemap.yaml << EOF - clonemap: - - name: openstack-infra/devstack-gate - dest: devstack-gate - EOF - /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ - git://git.openstack.org \ - openstack-infra/devstack-gate - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' - - - shell: - cmd: | - set -e - set -x - export USE_PYTHON3=true - export PYTHONUNBUFFERED=true - export DEVSTACK_GATE_TEMPEST=0 - export DEVSTACK_GATE_EXERCISES=0 - export DEVSTACK_GATE_NEUTRON=1 - export DEVSTACK_GATE_INSTALL_TESTONLY=1 - export BRANCH_OVERRIDE=default - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - - function gate_hook { - bash -xe $BASE/new/neutron/neutron/tests/contrib/gate_hook.sh dsvm-functional - } - export -f gate_hook - - function post_test_hook { - bash -xe $BASE/new/neutron/neutron/tests/contrib/post_test_hook.sh dsvm-functional - } - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - executable: /bin/bash - chdir: '{{ ansible_user_dir }}/workspace' - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/post_functional_job.yaml b/playbooks/post_functional_job.yaml new file mode 100644 index 00000000000..2537f1cbd9a --- /dev/null +++ b/playbooks/post_functional_job.yaml @@ -0,0 +1,5 @@ +- hosts: all + roles: + - fetch_journal_log + - fetch-tox-output + - fetch-subunit-output diff --git a/playbooks/run_functional_job.yaml b/playbooks/run_functional_job.yaml new file mode 100644 index 00000000000..e85c2eee965 --- /dev/null +++ b/playbooks/run_functional_job.yaml @@ -0,0 +1,10 @@ +- hosts: all + roles: + - run-devstack + # Run bindep and test-setup after devstack so that they won't interfere + - role: bindep + bindep_profile: test + bindep_dir: "{{ zuul_work_dir }}" + - test-setup + - ensure-tox + - tox diff --git a/roles/configure_functional_tests/README.rst b/roles/configure_functional_tests/README.rst new file mode 100644 index 00000000000..0fb2f575289 --- /dev/null +++ b/roles/configure_functional_tests/README.rst @@ -0,0 +1,18 @@ +Configure host to run on it Neutron functional/fullstack tests + +**Role Variables** + +.. zuul:rolevar:: tests_venv + :default: {{ tox_envlist }} + +.. zuul:rolevar:: base_dir + :default: {{ ansible_user_dir }}/src/git.openstack.org + +.. zuul:rolevar:: gate_dest_dir + :default: {{ base_dir }}/openstack + +.. zuul:rolevar:: devstack_dir + :default: {{ base_dir }}/openstack-dev/devstack + +.. zuul:rolevar:: neutron_dir + :default: {{ gate_dest_dir }}/neutron diff --git a/roles/configure_functional_tests/defaults/main.yaml b/roles/configure_functional_tests/defaults/main.yaml new file mode 100644 index 00000000000..e849e80429b --- /dev/null +++ b/roles/configure_functional_tests/defaults/main.yaml @@ -0,0 +1,5 @@ +tests_venv: "{{ tox_envlist }}" +base_dir: "{{ ansible_user_dir }}/src/git.openstack.org" +gate_dest_dir: "{{ base_dir }}/openstack" +devstack_dir: "{{ base_dir }}/openstack-dev/devstack" +neutron_dir: "{{ gate_dest_dir }}/neutron" diff --git a/roles/configure_functional_tests/tasks/main.yaml b/roles/configure_functional_tests/tasks/main.yaml new file mode 100644 index 00000000000..8e4f791de4c --- /dev/null +++ b/roles/configure_functional_tests/tasks/main.yaml @@ -0,0 +1,19 @@ +- shell: + cmd: | + set -e + set -x + GATE_STACK_USER={{ ansible_user }} + IS_GATE=True + + BASE_DIR={{ base_dir }} + GATE_DEST={{ gate_dest_dir }} + NEUTRON_PATH={{ neutron_dir }} + DEVSTACK_PATH={{ devstack_dir }} + VENV={{ tests_venv }} + + source $DEVSTACK_PATH/functions + source $NEUTRON_PATH/devstack/lib/ovs + source $NEUTRON_PATH/tools/configure_for_func_testing.sh + + configure_host_for_func_testing + executable: /bin/bash diff --git a/roles/fetch_journal_log/README.rst b/roles/fetch_journal_log/README.rst new file mode 100644 index 00000000000..71276c3f6a4 --- /dev/null +++ b/roles/fetch_journal_log/README.rst @@ -0,0 +1,16 @@ +Collect journal log from test run + +By default, this stores journal log into log file and store it in +"journal_log_file_path" + +**Role Variables** + +.. zuul:rolevar:: journal_log_path + :default: {{ ansible_user_dir }}/workspace/logs + + Path where journal log file will be stored on job's node. + +.. zuul:rolevar:: journal_log_file_name + :default: {{ journal_log_path }}/journal.log + + Name of journal log file. diff --git a/roles/fetch_journal_log/defaults/main.yaml b/roles/fetch_journal_log/defaults/main.yaml new file mode 100644 index 00000000000..dfc540ba9d1 --- /dev/null +++ b/roles/fetch_journal_log/defaults/main.yaml @@ -0,0 +1,2 @@ +journal_log_path: "{{ ansible_user_dir }}/logs" +journal_log_file_name: "{{ journal_log_path }}/journal.log" diff --git a/roles/fetch_journal_log/tasks/main.yaml b/roles/fetch_journal_log/tasks/main.yaml new file mode 100644 index 00000000000..c67ecd075f1 --- /dev/null +++ b/roles/fetch_journal_log/tasks/main.yaml @@ -0,0 +1,23 @@ +- name: Ensure {{ journal_log_path }} exists + become: yes + msg: "Ensure logdir {{ journal_log_path }} exists and have correct permission" + file: + path: "{{ journal_log_path }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: 0775 + +- name: Store journal logs in {{ journal_log_file_name }} + become: yes + shell: + cmd: | + /bin/journalctl -a > {{ journal_log_file_name }} + +- name: Set journal.log file permissions + become: yes + file: + path: '{{ journal_log_file_name }}' + owner: '{{ ansible_user }}' + group: '{{ ansible_user }}' + mode: 0644 diff --git a/roles/setup_logdir/README.rst b/roles/setup_logdir/README.rst new file mode 100644 index 00000000000..159cc8c7ec9 --- /dev/null +++ b/roles/setup_logdir/README.rst @@ -0,0 +1,8 @@ +Configure logs dir to be accessible for ``stack`` user. + +**Role Variables** + +.. zuul:rolevar:: logdir + :default: /opt/stack/logs + + Name of the directory where logs will be stored. diff --git a/roles/setup_logdir/defaults/main.yaml b/roles/setup_logdir/defaults/main.yaml new file mode 100644 index 00000000000..f056848826e --- /dev/null +++ b/roles/setup_logdir/defaults/main.yaml @@ -0,0 +1 @@ +logdir: /opt/stack/logs diff --git a/roles/setup_logdir/tasks/main.yaml b/roles/setup_logdir/tasks/main.yaml new file mode 100644 index 00000000000..1476ae2a5b5 --- /dev/null +++ b/roles/setup_logdir/tasks/main.yaml @@ -0,0 +1,9 @@ +- name: Ensure logdir exists + become: yes + msg: "Ensure logdir {{ logdir }} exists and have correct permission" + file: + path: "{{ logdir }}" + state: directory + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + mode: 0775