Add py3/f28 support to tripleo-container-build-base

Enable Fedora 28 and Python3 support on tripleo-buildcontainers
playbook and on tripleo-repos role.

Centos-7 job would run upstream and its fedora-28 counterpart would
be run on RDO and be non voting

Uses: https://review.openstack.org/637212
User-Story: https://tree.taiga.io/project/tripleo-ci-board/us/650
Change-Id: Ie46d0d7fa7c41427bb44bcfad5cf59af745ac627
Co-Authored-By: Sorin Sbarnea <ssbarnea@redhat.com>

playbooks/tripleo-buildcontainers/pre.yaml

@@ -2,8 +2,10 @@
   name: TripleO Setup Container Registry and repos mirror
   roles:
     - role: tripleo-repos
-      tripleo_repos_repository: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tripleo-repos'].src_dir }}"
       override_repos: "{{ buildcontainers_override_repos | default('') }}"
+      tripleo_repos_repository: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/tripleo-repos'].src_dir }}"
+    - role: bindep
+      bindep_dir: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/python-tripleoclient'].src_dir }}"
   tasks:
     - name: Include common vars
       include_vars:
@@ -72,7 +74,7 @@
         - name: Start apache
           service:
             name: httpd
-            state: running
+            state: started
 
         - name: Fetch delorean repos
           shell: |

playbooks/tripleo-buildcontainers/run.yaml

@@ -1,8 +1,5 @@
 - hosts: all
   name: TripleO container image building job
-  roles:
-    - role: bindep
-      bindep_dir: "{{ ansible_user_dir }}/{{ zuul.projects['git.openstack.org/openstack/python-tripleoclient'].src_dir }}"
   tasks:
     - name: Include common vars
       include_vars:
@@ -15,28 +12,37 @@
         push_tag: "{{ push_tag | default('latest') }}"
         push_containers: "{{ push_containers | default(false) | bool }}"
 
-    - name: Install pip
-      become: true
-      package:
-        name: python-pip
-        state: present
+    - name: grab kolla patch if needed
+      when: ansible_pkg_mgr == "dnf"
+      shell: |
+        set -euxo pipefail
+        git config --global user.email "zuul@openstack.org"
+        git config --global user.name "Zuul"
+        git remote add upstream https://git.openstack.org/openstack/kolla
+        git ls-remote https://git.openstack.org/openstack/kolla | \
+          grep -E refs/changes/[[:digit:]]+/{{ kolla_f28_patch }}/ | \
+          awk '{print $2}' | \
+          sort -t / -k 5 -g -r | \
+          head -1 | \
+          xargs -I{} git fetch https://git.openstack.org/openstack/kolla {} && \
+          git checkout -b f28 FETCH_HEAD && \
+          git pull --rebase upstream master
 
-    - name: Install virtualenv
-      become: true
-      package:
-        name: python-virtualenv
-        state: present
+      args:
+        chdir: "{{ openstack_git_root }}/kolla"
+        warn: false
+      register: result
+      changed_when: "'nothing to commit, working directory clean' not in result.stdout_lines"
 
-    - name: pip install required items
-      become: true
+    - name: pip install kolla
       pip:
-        name: "{{ item }}"
-        state: present
+        name:
+          - "file://{{ openstack_git_root }}/kolla"
+          - "file://{{ openstack_git_root }}/tripleo-common"
+          - "file://{{ openstack_git_root }}/python-tripleoclient"
         virtualenv: "{{ workspace }}/venv"
-      with_items:
-        - "file://{{ openstack_git_root }}/kolla"
-        - "file://{{ openstack_git_root }}/tripleo-common"
-        - "file://{{ openstack_git_root }}/python-tripleoclient"
+        virtualenv_python: "{{ virtualenv_python[ansible_pkg_mgr] }}"
+        virtualenv_site_packages: true
 
     # TODO(aschultz): make the kolla-build branch aware
     - name: Generate kolla-build.conf
@@ -51,13 +57,18 @@
         chdir: '{{ workspace }}'
       shell: |
         set -x
-        BUILDAH=""
         source {{ workspace }}/venv/bin/activate
         TRIPLEO_COMMON_PATH="{{ openstack_git_root }}/tripleo-common"
 
-        if [[ "{{ use_buildah | default(False) }}" == "True" ]]; then BUILDAH="--use-buildah"; fi
         openstack overcloud container image build \
             --config-file $TRIPLEO_COMMON_PATH/container-images/overcloud_containers.yaml \
-            --kolla-config-file {{ workspace }}/kolla-build.conf $BUILDAH > {{ workspace }}/build.log 2> {{ workspace }}/build-err.log
+            --kolla-config-file {{ workspace }}/kolla-build.conf \
+            {% for item in exclude_containers[ansible_pkg_mgr] %}
+             --exclude {{ item }} \
+            {% endfor %}
+            {% if use_buildah is defined and use_buildah %}
+             --use-buildah \
+            {% endif %}
+            > {{ workspace }}/build.log 2> {{ workspace }}/build-err.log
         RESULT=$?
         exit $RESULT

playbooks/tripleo-buildcontainers/templates/kolla-build.conf.j2

@@ -1,5 +1,5 @@
 [DEFAULT]
-base=centos
+base={{ ansible_distribution|lower }}
 type=binary
 registry={{ push_registry }}
 tag={{ push_tag }}

playbooks/tripleo-buildcontainers/vars/common.yaml

@@ -1,2 +1,20 @@
 workspace: "{{ ansible_user_dir }}/workspace"
 openstack_git_root: "{{ ansible_user_dir }}/src/git.openstack.org/openstack"
+
+# https://review.openstack.org/#/c/624838/
+kolla_f28_patch: 624838
+
+virtualenv_python:
+  yum: python
+  dnf: python3
+
+exclude_containers:
+  dnf:
+    - fluentd
+    - opendaylight
+    - skydive-agent
+    - skydive-analyzer
+    - sensu-client
+    - sensu-base
+  yum:
+    - opendaylight

roles/run-test/templates/toci_gate_test.sh.j2

@@ -126,7 +126,7 @@ EMIT_RELEASES_EXTRA_ARGS="$EMIT_RELEASES_EXTRA_ARGS --is-periodic"
 if [[ -f "$RELEASES_SCRIPT" ]] && [[ {{ featureset }} =~ 010|011|037|047|050|056 ]]; then
 
     if [[  -f "$RELEASES_FILE_OUTPUT" ]]; then
-       echo "Importing ready releases.sh file - skipping releases.sh file generation."
+        echo "Importing ready releases.sh file - skipping releases.sh file generation."
     else
         python $RELEASES_SCRIPT \
             --stable-release ${STABLE_RELEASE:-"master"} \

roles/tripleo-repos/defaults/main.yml

@@ -3,6 +3,25 @@
 tripleo_repos_repository: "git+https://git.openstack.org/openstack/tripleo-repos"
 workspace: "{{ ansible_user_dir }}"
 centos_mirror_host: "http://mirror.centos.org"
+fedora_mirror_host: "https://mirrors.fedoraproject.org"
 rdo_mirror_host: "https://trunk.rdoproject.org"
 override_repos: ""
+virtualenv_python:
+  yum: python
+  dnf: python3
 
+python_pip:
+  yum: python-pip
+  dnf: python3-pip
+
+python_setuptools:
+  yum: python-setuptools
+  dnf: python3-setuptools
+
+python_virtualenv:
+  yum: python-virtualenv
+  dnf: python3-virtualenv
+
+python_libselinux:
+  yum: libselinux-python
+  dnf: python3-libselinux

roles/tripleo-repos/tasks/main.yml

@@ -1,14 +1,13 @@
 ---
-- name: Install pip
+- name: install system packages
   become: true
   package:
-    name: python-pip
-    state: present
-
-- name: Install virtualenv
-  become: true
-  package:
-    name: python-virtualenv
+    name:
+      - "git"
+      - "{{ python_libselinux[ansible_pkg_mgr] }}"
+      - "{{ python_setuptools[ansible_pkg_mgr] }}"
+      - "{{ python_virtualenv[ansible_pkg_mgr] }}"
+      - "{{ python_pip[ansible_pkg_mgr] }}"
     state: present
 
 - name: pip install tripleo-repos
@@ -16,6 +15,8 @@
   pip:
     name: "{{ tripleo_repos_repository }}"
     virtualenv: "{{ workspace }}/venv"
+    virtualenv_python: "{{ virtualenv_python[ansible_pkg_mgr] }}"
+    virtualenv_site_packages: true
 
 - name: Get branch
   set_fact:
@@ -29,9 +30,13 @@
     source {{ workspace }}/venv/bin/activate
     if [ -f /etc/ci/mirror_info.sh ]; then
       source /etc/ci/mirror_info.sh
-      CENTOS_MIRROR_HOST="http://${NODEPOOL_MIRROR_HOST}"
+      DISTRO_MIRROR_HOST="http://${NODEPOOL_MIRROR_HOST}"
     fi
-    CENTOS_MIRROR_HOST=${CENTOS_MIRROR_HOST:-"{{ centos_mirror_host }}"}
+    {% if ansible_distribution | lower == 'fedora' %}
+    DISTRO_MIRROR_HOST=${DISTRO_MIRROR_HOST:-"{{ fedora_mirror_host }}"}
+    {% else %}
+    DISTRO_MIRROR_HOST=${DISTRO_MIRROR_HOST:-"{{ centos_mirror_host }}"}
+    {% endif %}
     RDO_MIRROR_HOST=${NODEPOOL_RDO_PROXY:-"{{ rdo_mirror_host }}"}
 
     if [ "{{ override_repos }}" != "" ]; then
@@ -42,6 +47,19 @@
         REPO=current
     fi
 
-    tripleo-repos -b "{{ ci_branch | default('master') }}" \
-                  --centos-mirror $CENTOS_MIRROR_HOST \
-                  --rdo-mirror $RDO_MIRROR_HOST $REPO
+    tripleo-repos \
+      -b "{{ ci_branch | default('master') }}" \
+      --mirror $DISTRO_MIRROR_HOST \
+      --rdo-mirror $RDO_MIRROR_HOST $REPO
+  register: result
+  changed_when: "'Installed:' in result.stdout_lines"
+
+# TODO(rfolco): normalize centos/fedora repo names, fedora sets delorean.repo and fedora-stable.repo,
+# while centos sets delorean.repo and delorean-master-testing.repo.
+- name: rename fedora-stable.repo
+  become: true
+  shell: |
+    set -ex
+    # Rename fedora-stable.repo as kola.conf.j2 template expects delorean*
+    mv /etc/yum.repos.d/fedora-stable.repo /etc/yum.repos.d/delorean-master-testing.repo
+  when: ansible_distribution|lower  == "fedora"

zuul.d/layout.yaml

@@ -6,13 +6,13 @@
 
 - project:
     templates:
-      - tripleo-standalone-scenarios-full
+      - tripleo-build-containers-jobs
       - tripleo-multinode-baremetal-full
+      - tripleo-multinode-branchful
       - tripleo-multinode-container-full
       - tripleo-multinode-experimental
+      - tripleo-standalone-scenarios-full
       - tripleo-undercloud-jobs
-      - tripleo-multinode-branchful
-      - tripleo-build-containers-jobs
     check:
       jobs:
         - openstack-tox-linters