diff --git a/playbooks/tripleo-ci/pre.yaml b/playbooks/tripleo-ci/pre.yaml new file mode 100644 index 000000000..059921221 --- /dev/null +++ b/playbooks/tripleo-ci/pre.yaml @@ -0,0 +1,5 @@ +- hosts: all + name: Common tasks for nodes preparation + roles: + - role: common + - role: prepare-node diff --git a/playbooks/tripleo-ci/run-v3.yaml b/playbooks/tripleo-ci/run-v3.yaml index e5909437e..6af4fdd7e 100644 --- a/playbooks/tripleo-ci/run-v3.yaml +++ b/playbooks/tripleo-ci/run-v3.yaml @@ -1,178 +1,5 @@ -- hosts: all - name: Common tasks for nodes preparation - tasks: - - - name: Include common vars - include_vars: - file: "common.yaml" - - - name: Swap is essential as we are not meeting memory requirements - include_role: - name: configure-swap - - - name: Set legacy log path - include_role: - name: set-zuul-log-path-fact - - - name: Ensure legacy workspace directory - file: - path: '{{ workspace }}' - state: directory - - - name: Ensure legacy logs directory - file: - path: '{{ workspace }}/logs' - state: directory - - - name: Assure src folder has safe permissions - # Workaround for https://bugs.launchpad.net/tripleo/+bug/1807703 - file: - path: /home/zuul/src/ - state: directory - mode: u=rwX,g=rX,o=rX - recurse: yes - - - name: Copy tripleo-ci to openstack namespace dir - shell: - cmd: | - set -e - set -x - # we still need all the repos under the same root directory, but instead of using - # zuul-cloner, use the wrapped commands directly - # reflink is an safer alternative to hardlinking which could fail - cp -dR --reflink=auto /home/zuul/src/git.openstack.org/openstack-infra/tripleo-ci . - executable: /bin/bash - chdir: '{{ tripleo_root }}' - environment: '{{ zuul | zuul_legacy_vars }}' - - - set_fact: - zuul_dump: "{{ {'zuul': zuul} }}" - - - name: Dump zuul variables on the host to pass it to inner ansible - copy: - content: "{{ zuul_dump|to_nice_yaml }}" - dest: "{{ workspace }}/logs//zuul-variables.yaml" - - - name: Remove package exclusions from package manager - become: true - ini_file: - path: "{{ item }}" - section: main - option: exclude - state: absent - failed_when: false - with_items: - - /etc/yum.conf - - /etc/dnf/dnf.conf - - hosts: - primary - - centos-7 - name: tripleo CI jobs runner - tasks: - - block: - - name: Check overridable settings - vars: - allowed_overrides: - - 'run_tempest' - - 'tempest_whitelist' - - 'test_black_regex' - - 'tempest_format' - - 'tempest_extra_config' - - 'tempest_plugins' - - 'standalone_environment_files' - - 'standalone_container_cli' - - 'test_white_regex' - - 'tempest_workers' - fail: - msg: "ERROR: {{ item }} is not overridable." - when: item not in "{{ allowed_overrides }}" - with_items: "{{ featureset_override }}" - - name: Generate standalone_custom_env_files from standalone_environment_files - set_fact: - featureset_override: "{{ featureset_override | combine(custom_env, recursive=true) }}" - vars: - tht_dir: /usr/share/openstack-tripleo-heat-templates/ - custom_env: - standalone_custom_env_files: | - {{ featureset_override.standalone_environment_files | - default([]) | - map('regex_replace', '^', tht_dir ) | list }} - when: featureset_override['standalone_environment_files'] is defined - - include_role: - name: ceph-loop-device - when: "{{ standalone_ceph|default(false) }}" - - name: Generate featureset overriding file - template: - src: templates/featureset-override.j2 - dest: "{{ tripleo_root }}/tripleo-ci/featureset-override.yaml" - mode: 0755 - force: yes - - name: Include some extra for generating toci scripts - set_fact: - extra_vars: "--extra-vars @{{ tripleo_root }}/tripleo-ci/featureset-override.yaml" - when: featureset_override is defined - - - name: render toci_gate_test script - template: - src: templates/toci_gate_test.sh.j2 - dest: "{{ tripleo_root }}/tripleo-ci/toci_gate_test.sh" - mode: 0755 - force: yes - - - name: render toci_quickstart script - template: - src: templates/toci_quickstart.sh.j2 - dest: "{{ tripleo_root }}/tripleo-ci/toci_quickstart.sh" - mode: 0755 - force: yes - - - name: render common_vars script - template: - src: templates/common_vars.bash.j2 - dest: "{{ tripleo_root }}/tripleo-ci/scripts/common_vars.bash" - force: yes - - - name: render oooq_common_functions script - template: - src: templates/oooq_common_functions.sh.j2 - dest: "{{ tripleo_root }}/tripleo-ci/scripts/oooq_common_functions.sh" - force: yes - - - shell: - cmd: | - set -e - set -x - {% if nodes is defined %} - export TOCI_JOBTYPE={{ environment_type }}-{{ nodes }}-featureset{{ featureset }} - {% else %} - export TOCI_JOBTYPE={{ environment_type }}-featureset{{ featureset }} - {% endif %} - export DISTRIBUTION={{ ansible_distribution }} - export DISTRIBUTION_MAJOR_VERSION={{ ansible_distribution_major_version }} - - {% if nodes is defined %} - export NODES_FILE={{ nodes_file }} - {% endif %} - - # ZUUL does not set these for periodic jobs - export ZUUL_BRANCH=${ZUUL_BRANCH:-master} - export ZUUL_REF=${ZUUL_REF:-None} - - # Compatibility with zuul v3 variables - export JOB_NAME={{ zuul.job }} - export LOG_PATH={{ zuul_log_path }} - - export BRANCH_OVERRIDE="{{ branch_override | default('default') }}" - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - - export PYTHONUNBUFFERED=true - export TRIPLEO_ROOT={{ tripleo_root }} - # NOTE(trown) We can move this into the multinode environment file once we - # migrate the 2-node jobs over to zuulv3 - bash -xe $TRIPLEO_ROOT/tripleo-ci/toci_gate_test.sh - executable: /bin/bash - chdir: '{{ tripleo_root }}' - environment: '{{ zuul | zuul_legacy_vars }}' + roles: + - role: common + - role: run-test diff --git a/roles/common/tasks/main.yml b/roles/common/tasks/main.yml new file mode 100644 index 000000000..5dbe502f0 --- /dev/null +++ b/roles/common/tasks/main.yml @@ -0,0 +1,5 @@ +# In ansible 2.7 should be enough to add the option public: true to the calling include_role +# for this role. THis is the best workaround +- name: Include the vars to be available for the playbook + include_vars: + dir: vars diff --git a/playbooks/tripleo-ci/vars/common.yaml b/roles/common/vars/main.yaml similarity index 100% rename from playbooks/tripleo-ci/vars/common.yaml rename to roles/common/vars/main.yaml diff --git a/roles/prepare-node/tasks/main.yaml b/roles/prepare-node/tasks/main.yaml new file mode 100644 index 000000000..c8ccc78d6 --- /dev/null +++ b/roles/prepare-node/tasks/main.yaml @@ -0,0 +1,59 @@ +- name: Swap is essential as we are not meeting memory requirements + include_role: + name: configure-swap + +- name: Set legacy log path + include_role: + name: set-zuul-log-path-fact + +- name: Ensure legacy workspace directory + file: + path: '{{ workspace }}' + state: directory + +- name: Ensure legacy logs directory + file: + path: '{{ workspace }}/logs' + state: directory + +- name: Assure src folder has safe permissions + # Workaround for https://bugs.launchpad.net/tripleo/+bug/1807703 + file: + path: /home/zuul/src/ + state: directory + mode: u=rwX,g=rX,o=rX + recurse: yes + +- name: Copy tripleo-ci to openstack namespace dir + shell: + cmd: | + set -e + set -x + # we still need all the repos under the same root directory, but instead of using + # zuul-cloner, use the wrapped commands directly + # reflink is an safer alternative to hardlinking which could fail + cp -dR --reflink=auto /home/zuul/src/git.openstack.org/openstack-infra/tripleo-ci . + executable: /bin/bash + chdir: '{{ tripleo_root }}' + environment: '{{ zuul | zuul_legacy_vars }}' + +- name: "set zull_dump" + set_fact: + zuul_dump: "{{ {'zuul': zuul} }}" + +- name: Dump zuul variables on the host to pass it to inner ansible + copy: + content: "{{ zuul_dump|to_nice_yaml }}" + dest: "{{ workspace }}/logs/zuul-variables.yaml" + +- name: Remove package exclusions from package manager + become: true + ini_file: + path: "{{ item }}" + section: main + option: exclude + state: absent + failed_when: false + with_items: + - /etc/yum.conf + - /etc/dnf/dnf.conf diff --git a/roles/run-test/tasks/main.yaml b/roles/run-test/tasks/main.yaml new file mode 100644 index 000000000..398d33158 --- /dev/null +++ b/roles/run-test/tasks/main.yaml @@ -0,0 +1,110 @@ +- name: Set legacy log path + include_role: + name: set-zuul-log-path-fact + +- block: + - name: Check overridable settings + vars: + allowed_overrides: + - 'run_tempest' + - 'tempest_whitelist' + - 'test_black_regex' + - 'tempest_format' + - 'tempest_extra_config' + - 'tempest_plugins' + - 'standalone_environment_files' + - 'standalone_container_cli' + - 'test_white_regex' + - 'tempest_workers' + fail: + msg: "ERROR: {{ item }} is not overridable." + when: item not in "{{ allowed_overrides }}" + with_items: "{{ featureset_override }}" + - name: Generate standalone_custom_env_files from standalone_environment_files + set_fact: + featureset_override: "{{ featureset_override | combine(custom_env, recursive=true) }}" + vars: + tht_dir: /usr/share/openstack-tripleo-heat-templates/ + custom_env: + standalone_custom_env_files: | + {{ featureset_override.standalone_environment_files | + default([]) | + map('regex_replace', '^', tht_dir ) | list }} + when: featureset_override['standalone_environment_files'] is defined + - include_role: + name: ceph-loop-device + when: "{{ standalone_ceph|default(false) }}" + - name: Generate featureset overriding file + template: + src: templates/featureset-override.j2 + dest: "{{ tripleo_root }}/tripleo-ci/featureset-override.yaml" + mode: 0755 + force: yes + - name: Include some extra for generating toci scripts + set_fact: + extra_vars: "--extra-vars @{{ tripleo_root }}/tripleo-ci/featureset-override.yaml" + when: featureset_override is defined + +- name: render toci_gate_test script + template: + src: templates/toci_gate_test.sh.j2 + dest: "{{ tripleo_root }}/tripleo-ci/toci_gate_test.sh" + mode: 0755 + force: yes + +- name: render toci_quickstart script + template: + src: templates/toci_quickstart.sh.j2 + dest: "{{ tripleo_root }}/tripleo-ci/toci_quickstart.sh" + mode: 0755 + force: yes + +- name: render common_vars script + template: + src: templates/common_vars.bash.j2 + dest: "{{ tripleo_root }}/tripleo-ci/scripts/common_vars.bash" + force: yes + +- name: render oooq_common_functions script + template: + src: templates/oooq_common_functions.sh.j2 + dest: "{{ tripleo_root }}/tripleo-ci/scripts/oooq_common_functions.sh" + force: yes + +- shell: + cmd: | + set -e + set -x + {% if nodes is defined %} + export TOCI_JOBTYPE={{ environment_type }}-{{ nodes }}-featureset{{ featureset }} + {% else %} + export TOCI_JOBTYPE={{ environment_type }}-featureset{{ featureset }} + {% endif %} + export DISTRIBUTION={{ ansible_distribution }} + export DISTRIBUTION_MAJOR_VERSION={{ ansible_distribution_major_version }} + + {% if nodes is defined %} + export NODES_FILE={{ nodes_file }} + {% endif %} + + # ZUUL does not set these for periodic jobs + export ZUUL_BRANCH=${ZUUL_BRANCH:-master} + export ZUUL_REF=${ZUUL_REF:-None} + + # Compatibility with zuul v3 variables + export JOB_NAME={{ zuul.job }} + export LOG_PATH={{ zuul_log_path }} + + export BRANCH_OVERRIDE="{{ branch_override | default('default') }}" + if [ "$BRANCH_OVERRIDE" != "default" ] ; then + export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE + fi + + export PYTHONUNBUFFERED=true + export TRIPLEO_ROOT={{ tripleo_root }} + # NOTE(trown) We can move this into the multinode environment file once we + # migrate the 2-node jobs over to zuulv3 + bash -xe $TRIPLEO_ROOT/tripleo-ci/toci_gate_test.sh + executable: /bin/bash + chdir: '{{ tripleo_root }}' + environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/playbooks/tripleo-ci/templates/common_vars.bash.j2 b/roles/run-test/templates/common_vars.bash.j2 similarity index 100% rename from playbooks/tripleo-ci/templates/common_vars.bash.j2 rename to roles/run-test/templates/common_vars.bash.j2 diff --git a/playbooks/tripleo-ci/templates/featureset-override.j2 b/roles/run-test/templates/featureset-override.j2 similarity index 100% rename from playbooks/tripleo-ci/templates/featureset-override.j2 rename to roles/run-test/templates/featureset-override.j2 diff --git a/playbooks/tripleo-ci/templates/oooq_common_functions.sh.j2 b/roles/run-test/templates/oooq_common_functions.sh.j2 similarity index 100% rename from playbooks/tripleo-ci/templates/oooq_common_functions.sh.j2 rename to roles/run-test/templates/oooq_common_functions.sh.j2 diff --git a/playbooks/tripleo-ci/templates/toci_gate_test.sh.j2 b/roles/run-test/templates/toci_gate_test.sh.j2 similarity index 100% rename from playbooks/tripleo-ci/templates/toci_gate_test.sh.j2 rename to roles/run-test/templates/toci_gate_test.sh.j2 diff --git a/playbooks/tripleo-ci/templates/toci_quickstart.sh.j2 b/roles/run-test/templates/toci_quickstart.sh.j2 similarity index 100% rename from playbooks/tripleo-ci/templates/toci_quickstart.sh.j2 rename to roles/run-test/templates/toci_quickstart.sh.j2 diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 46ab86842..b3cf16496 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -34,6 +34,7 @@ - playbooks/nodepool-provider/pre.yaml - playbooks/openstack-zuul-jobs/legacy/pre.yaml - playbooks/tripleo-ci/ceph.yaml + - playbooks/tripleo-ci/pre.yaml run: playbooks/tripleo-ci/run-v3.yaml post-run: playbooks/tripleo-ci/post.yaml irrelevant-files: