From baf6a5b5245d43d38474ecfeb75e1b30111fd797 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Evrard Date: Thu, 11 Oct 2018 11:13:51 +0200 Subject: [PATCH] Always build requirements image in the pipeline Without this patch, the loci builder automatically consumes a previously published requirements image when the project is not "requirements". This also means the first step in checks/gates for non requirement images would be to pull a requirement. Currently there is neither a "pipeline" nor consumption of artifacts between jobs. Which means the "requirements" build and the other image build have to be considered as independant. However, by default, the "non-requirements" image built is consuming previously published "requirements" image, but not immediately previous built requirements image. This is a problem, as it leaves us open to a race condition where a new build of "requirements" and "non-requirements" can succeed in a commit, but would cause all the next "non-requirements" builds to fail, as it is possible the newly published "requirements" to be incompatible with the next "non-requirements", as they are not tested together. This should fix the problem by ensuring that "requirements" are always built and consumed in the "non-requirement" building process, instead of re-using the previously built requirements. For this, project needs to be overriden to build the requirement image, and therefore a new profile 'requirements' was added to the buildargs of the distros. Requirements repo also needs to be cloned in the gating environment, and therefore was added to all the required projects. This will also allow new branches to be created, as there would be no need of a previous build and publish of requirements in the gating. Change-Id: I093e4dfc0eef03c47d2d029011fdb5429242ae79 --- .zuul.d/cinder.yaml | 1 + .zuul.d/glance.yaml | 1 + .zuul.d/heat.yaml | 1 + .zuul.d/horizon.yaml | 1 + .zuul.d/ironic.yaml | 1 + .zuul.d/keystone.yaml | 1 + .zuul.d/neutron.yaml | 1 + .zuul.d/nova.yaml | 1 + .zuul.d/octavia.yaml | 1 + playbooks/loci-builder.yaml | 56 ++++++++++++++++++++++++++++++++----- playbooks/vars.yaml | 14 ++++++++++ 11 files changed, 72 insertions(+), 7 deletions(-) diff --git a/.zuul.d/cinder.yaml b/.zuul.d/cinder.yaml index 0676853..9673071 100644 --- a/.zuul.d/cinder.yaml +++ b/.zuul.d/cinder.yaml @@ -16,6 +16,7 @@ project: cinder required-projects: - openstack/loci + - openstack/requirements - openstack/cinder - job: diff --git a/.zuul.d/glance.yaml b/.zuul.d/glance.yaml index 2ee11ef..159e07c 100644 --- a/.zuul.d/glance.yaml +++ b/.zuul.d/glance.yaml @@ -16,6 +16,7 @@ project: glance required-projects: - openstack/loci + - openstack/requirements - openstack/glance - job: diff --git a/.zuul.d/heat.yaml b/.zuul.d/heat.yaml index 4c4dbac..c05834f 100644 --- a/.zuul.d/heat.yaml +++ b/.zuul.d/heat.yaml @@ -16,6 +16,7 @@ project: heat required-projects: - openstack/loci + - openstack/requirements - openstack/heat - job: diff --git a/.zuul.d/horizon.yaml b/.zuul.d/horizon.yaml index a142d71..9c3e706 100644 --- a/.zuul.d/horizon.yaml +++ b/.zuul.d/horizon.yaml @@ -16,6 +16,7 @@ project: horizon required-projects: - openstack/loci + - openstack/requirements - openstack/horizon - job: diff --git a/.zuul.d/ironic.yaml b/.zuul.d/ironic.yaml index 11ca146..14ffa50 100644 --- a/.zuul.d/ironic.yaml +++ b/.zuul.d/ironic.yaml @@ -16,6 +16,7 @@ project: ironic required-projects: - openstack/loci + - openstack/requirements - openstack/ironic - job: diff --git a/.zuul.d/keystone.yaml b/.zuul.d/keystone.yaml index da7db6f..8f09f42 100644 --- a/.zuul.d/keystone.yaml +++ b/.zuul.d/keystone.yaml @@ -16,6 +16,7 @@ project: keystone required-projects: - openstack/loci + - openstack/requirements - openstack/keystone - job: diff --git a/.zuul.d/neutron.yaml b/.zuul.d/neutron.yaml index 3f9d0ab..f91d47a 100644 --- a/.zuul.d/neutron.yaml +++ b/.zuul.d/neutron.yaml @@ -16,6 +16,7 @@ project: neutron required-projects: - openstack/loci + - openstack/requirements - openstack/neutron - job: diff --git a/.zuul.d/nova.yaml b/.zuul.d/nova.yaml index 78e8ec4..e683d3a 100644 --- a/.zuul.d/nova.yaml +++ b/.zuul.d/nova.yaml @@ -16,6 +16,7 @@ project: nova required-projects: - openstack/loci + - openstack/requirements - openstack/nova - job: diff --git a/.zuul.d/octavia.yaml b/.zuul.d/octavia.yaml index e4965a4..d64542b 100644 --- a/.zuul.d/octavia.yaml +++ b/.zuul.d/octavia.yaml @@ -16,6 +16,7 @@ project: octavia required-projects: - openstack/loci + - openstack/requirements - openstack/octavia - job: diff --git a/playbooks/loci-builder.yaml b/playbooks/loci-builder.yaml index ca179c4..a57d58c 100644 --- a/playbooks/loci-builder.yaml +++ b/playbooks/loci-builder.yaml @@ -9,7 +9,13 @@ - "pydep.txt" environment: LC_ALL: C - + # NOTE(evrardjp): While reuse_requirements is very nice and optimises + # checks and gating, there is a race condition here, because + # we are consuming prebuild wheels (see vars.yaml) by default: + # The jobs in zuul can be building a new "requirements" image, working + # And the new "nova" image would still build on previous "requirements" + # image that was last published. This could cause an issue. Instead + # in gating we should build directly what will be consumed. # NOTE(SamYaple): This process is so we can take advantage of the infra # DockerHub mirroring as configured through the Docker daemon. We do this # instead of calling fetch_wheels initially. All-in-all this saves @@ -25,6 +31,7 @@ async: 1000 poll: 0 register: pull + - async_status: jid: "{{ item.ansible_job_id }}" with_items: "{{ pull.results }}" @@ -34,11 +41,14 @@ - pull_result.finished retries: 1000 delay: 5 - when: project != 'requirements' + when: + - reuse_requirements | bool + - project != 'requirements' - name: Build base images block: - - docker_image: + - name: "Build base image for {{ item.name }} asynchronously" + docker_image: path: "{{ loci_src_dir }}/dockerfiles/{{ item.name }}" name: base tag: "{{ item.name }}" @@ -47,6 +57,7 @@ async: 1000 poll: 0 register: base + - async_status: jid: "{{ item.ansible_job_id }}" with_items: "{{ base.results }}" @@ -57,18 +68,49 @@ retries: 1000 delay: 5 - - name: Build project images + - name: Build requirements image block: - - docker_image: + - name: "Build requirements image for {{ item.name }}" + docker_image: path: "{{ loci_src_dir }}" - name: loci/{{ project }} + name: loci/requirements tag: "{{ branch }}-{{ item.name }}" + repository: 172.17.0.1:5000/loci/requirements + push: yes pull: False - buildargs: "{{ item.buildargs.project }}" + buildargs: "{{ item.buildargs.requirements }}" with_items: "{{ distros }}" async: 1000 poll: 0 register: build + + - async_status: + jid: "{{ item.ansible_job_id }}" + with_items: "{{ build.results }}" + register: build_result + until: + - build_result.finished is defined + - build_result.finished + retries: 1000 + delay: 5 + when: + - (not reuse_requirements) | bool + - project != 'requirements' + + - name: Build project images + block: + - name: "Build {{ project }} image for {{ item.name }}" + docker_image: + path: "{{ loci_src_dir }}" + name: loci/{{ project }} + tag: "{{ branch }}-{{ item.name }}" + pull: False + buildargs: "{{ item.buildargs.project }}" + with_items: "{{ distros }}" + async: 1000 + poll: 0 + register: build + - async_status: jid: "{{ item.ansible_job_id }}" with_items: "{{ build.results }}" diff --git a/playbooks/vars.yaml b/playbooks/vars.yaml index bf5dc80..735d60b 100644 --- a/playbooks/vars.yaml +++ b/playbooks/vars.yaml @@ -7,6 +7,8 @@ docker_daemon: insecure-registries: - 172.17.0.1:5000 +reuse_requirements: False + # NOTE(SamYaple): When running in the loci repo, the project is "loci", but # when running as a post job for cinder, the project is "cinder". We statically # declare the path rather than using zuul variables so thats not an issue @@ -26,6 +28,10 @@ distros: PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }} WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-centos FROM: base:centos + requirements: + PROJECT: requirements + PROJECT_REPO: http://172.17.0.1/git/openstack/requirements + FROM: base:centos - name: ubuntu image: ubuntu:xenial buildargs: @@ -41,6 +47,10 @@ distros: PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }} WHEELS: 172.17.0.1:5000/loci/requirements:{{ branch }}-ubuntu FROM: base:ubuntu + requirements: + PROJECT: requirements + PROJECT_REPO: http://172.17.0.1/git/openstack/requirements + FROM: base:ubuntu # - name: debian # image: debian:stretch # buildargs: @@ -57,3 +67,7 @@ distros: # PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }} # WHEELS: 172.17.0.1:5000/loci/requirements:master-debian # FROM: base:debian +# requirements: +# PROJECT: requirements +# PROJECT_REPO: http://172.17.0.1/git/openstack/requirements +# FROM: base:debian