From 3c0baa2c57f6907819a56486e8d3aadb72bed98a Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Fri, 22 Feb 2019 08:28:27 -0800 Subject: [PATCH] Add docker image jobs Change-Id: I6d5621686c80ca9a328feb84906bb1cccc31c980 Depends-On: https://review.openstack.org/638706 --- playbooks/buildset-registry/post.yaml | 3 + playbooks/buildset-registry/pre.yaml | 9 +++ playbooks/buildset-registry/run.yaml | 7 +++ playbooks/docker-image/README.rst | 80 ++++++++++++++++++++++++++ playbooks/docker-image/credentials.rst | 22 +++++++ playbooks/docker-image/pre.yaml | 3 + playbooks/docker-image/promote.yaml | 3 + playbooks/docker-image/run.yaml | 3 + playbooks/docker-image/upload.yaml | 3 + zuul.yaml | 71 +++++++++++++++++++++++ 10 files changed, 204 insertions(+) create mode 100644 playbooks/buildset-registry/post.yaml create mode 100644 playbooks/buildset-registry/pre.yaml create mode 100644 playbooks/buildset-registry/run.yaml create mode 100644 playbooks/docker-image/README.rst create mode 100644 playbooks/docker-image/credentials.rst create mode 100644 playbooks/docker-image/pre.yaml create mode 100644 playbooks/docker-image/promote.yaml create mode 100644 playbooks/docker-image/run.yaml create mode 100644 playbooks/docker-image/upload.yaml diff --git a/playbooks/buildset-registry/post.yaml b/playbooks/buildset-registry/post.yaml new file mode 100644 index 0000000..c64e8ef --- /dev/null +++ b/playbooks/buildset-registry/post.yaml @@ -0,0 +1,3 @@ +- hosts: localhost + roles: + - push-to-intermediate-registry diff --git a/playbooks/buildset-registry/pre.yaml b/playbooks/buildset-registry/pre.yaml new file mode 100644 index 0000000..bbf1de5 --- /dev/null +++ b/playbooks/buildset-registry/pre.yaml @@ -0,0 +1,9 @@ +- hosts: all + roles: + - install-docker + - run-buildset-registry + - use-buildset-registry + +- hosts: localhost + roles: + - pull-from-intermediate-registry diff --git a/playbooks/buildset-registry/run.yaml b/playbooks/buildset-registry/run.yaml new file mode 100644 index 0000000..f94afa5 --- /dev/null +++ b/playbooks/buildset-registry/run.yaml @@ -0,0 +1,7 @@ +- hosts: localhost + tasks: + - name: Pause the job + zuul_return: + data: + zuul: + pause: true diff --git a/playbooks/docker-image/README.rst b/playbooks/docker-image/README.rst new file mode 100644 index 0000000..0f54e68 --- /dev/null +++ b/playbooks/docker-image/README.rst @@ -0,0 +1,80 @@ +This is one of a collection of jobs which are designed to work +together to build, upload, and promote docker images in a gating +context: + + * :zuul:job:`opendev-build-docker-image`: Build the images. + * :zuul:job:`opendev-upload-docker-image`: Build and stage the images on dockerhub. + * :zuul:job:`opendev-promote-docker-image`: Promote previously uploaded images. + +The :zuul:job:`opendev-build-docker-image` job is designed to be used in +a `check` pipeline and simply builds the images to verify that +the build functions. + +The :zuul:job:`opendev-upload-docker-image` job builds and uploads the +images to Docker Hub, but only with a single tag corresponding to the +change ID. This job is designed in a `gate` pipeline so that the +build produced by the gate is staged and can later be promoted to +production if the change is successful. + +The :zuul:job:`opendev-promote-docker-image` job is designed to be +used in a `promote` pipeline. It requires no nodes and runs very +quickly on the Zuul executor. It simply re-tags a previously uploaded +image for a change with whatever tags are supplied by +:zuul:jobvar:`opendev-build-docker-image.docker_images.tags`. It also +removes the change ID tag from the repository in Docker Hub, and +removes any similar change ID tags more than 24 hours old. This keeps +the repository tidy in the case that gated changes fail to merge after +uploading their staged images. + +They all accept the same input data, principally a list of +dictionaries representing the images to build. YAML anchors_ can be +used to supply the same data to all three jobs. + +**Job Variables** + +.. zuul:jobvar:: zuul_work_dir + :default: {{ zuul.project.src_dir }} + + The project directory. Serves as the base for + :zuul:jobvar:`opendev-build-docker-image.docker_images.context`. + +.. zuul:jobvar:: docker_images + :type: list + + A list of images to build. Each item in the list should have: + + .. zuul:jobvar:: context + + The docker build context; this should be a directory underneath + :zuul:jobvar:`opendev-build-docker-image.zuul_work_dir`. + + .. zuul:jobvar:: repository + + The name of the target repository in dockerhub for the + image. Supply this even if the image is not going to be + uploaded (it will be tagged with this in the local + registry). + + .. zuul:jobvar:: path + + Optional: the directory that should be passed to docker build. + Useful for building images with a Dockerfile in the context + directory but a source repository elsewhere. + + .. zuul:jobvar:: build_args + :type: list + + Optional: a list of values to pass to the docker ``--build-arg`` + parameter. + + .. zuul:jobvar:: target + + Optional: the target for a multi-stage build. + + .. zuul:jobvar:: tags + :type: list + :default: ['latest'] + + A list of tags to be added to the image when promoted. + +.. _anchors: https://yaml.org/spec/1.2/spec.html#&%20anchor// diff --git a/playbooks/docker-image/credentials.rst b/playbooks/docker-image/credentials.rst new file mode 100644 index 0000000..4587dbf --- /dev/null +++ b/playbooks/docker-image/credentials.rst @@ -0,0 +1,22 @@ +.. zuul:jobvar:: docker_credentials + :type: dict + + This is expected to be a Zuul Secret with these keys: + + .. zuul:jobvar:: username + + The Docker Hub username. + + .. zuul:jobvar:: password + + The Docker Hub password. + + .. zuul:jobvar:: repository + + Optional; if supplied this is a regular expression which + restricts to what repositories the image may be uploaded. The + following example allows projects to upload images to + repositories within an organization based on their own names:: + + repository: "^myorgname/{{ zuul.project.short_name }}.*" + diff --git a/playbooks/docker-image/pre.yaml b/playbooks/docker-image/pre.yaml new file mode 100644 index 0000000..52f5b5b --- /dev/null +++ b/playbooks/docker-image/pre.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - install-docker diff --git a/playbooks/docker-image/promote.yaml b/playbooks/docker-image/promote.yaml new file mode 100644 index 0000000..8160bf4 --- /dev/null +++ b/playbooks/docker-image/promote.yaml @@ -0,0 +1,3 @@ +- hosts: localhost + roles: + - promote-docker-image diff --git a/playbooks/docker-image/run.yaml b/playbooks/docker-image/run.yaml new file mode 100644 index 0000000..d3525ad --- /dev/null +++ b/playbooks/docker-image/run.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - build-docker-image diff --git a/playbooks/docker-image/upload.yaml b/playbooks/docker-image/upload.yaml new file mode 100644 index 0000000..712f726 --- /dev/null +++ b/playbooks/docker-image/upload.yaml @@ -0,0 +1,3 @@ +- hosts: all + roles: + - upload-docker-image diff --git a/zuul.yaml b/zuul.yaml index d72fff3..fc2433e 100644 --- a/zuul.yaml +++ b/zuul.yaml @@ -143,6 +143,77 @@ secrets: - site_logs +- secret: + name: opendev-intermediate-registry + data: + host: insecure-ci-registry.opendev.org + port: 5000 + username: zuul + password: !encrypted/pkcs1-oaep + - AT70uhYcK9nhoiysHdUjr2sBiKK8T0LVNTQ8uwCd0UtXQll+sUtopXlr/P8IqfJ6kDoIq + ZAkptew+p2GB2a6OAfJE5LVCD420XBp4oePDrV6kmxkZTX9Wd8V5/zh/wLi2n5PmI4s0c + DfexUX+WYj5S90It41IyOwGVOBOfEqS4heh892q6Qj66+u1/i90JcXeSnPiMaqCr0UIl5 + hs26L3J6VM4ikO88XvmYwSKaHFWJfHM1G5BgD0DjnRdpXX1aVNxTeJXQvfOJnK2da0tzI + nRHRCxhRPvSWmvjoGSZDJxY6v1DfjuT4AsLTZ/xnYUBNBrwbppjlqzLJ871Vw3GQDdfDG + qcQ+jCwhp6BSoA7EKXaWhGIqDHzsSxeBbIqs2afu0YqEOQthxUzrsPHuE1y7qijS3DNN8 + scOGG5x3nisd++sesSOO+rs0DbZHJ1TNAdpd/rmrUtbPi++AkDbx+fEriGQdhX111IwFY + V2femI+Y/SJepu9yf1ps9hnj1jqoBF9ano6fRBJAXSg58fBu17YOxFWAllTFcHzFjXQCz + 38gEa9baO7nxIQ5tK/0dPFvNwoTpPZD4M3Z7kSFO3jWZ7Ur+5rUGnt3R8jp38UyVC5g0J + o/0JvF1nS8CyID5c3oyVBMSx39xWhgYzNtKZRAdB5qEBK4Lbjy7TnDS4FE4WKA= + +- job: + name: opendev-buildset-registry + description: | + Inherit from this job to get a buildset registry which interacts + with the intermediate CI registry to share speculative container + images between projects. + pre-run: playbooks/buildset-registry/pre.yaml + post-run: playbooks/buildset-registry/post.yaml + secrets: + - secret: opendev-intermediate-registry + name: intermediate_registry + +- job: + name: opendev-build-docker-image + parent: opendev-buildset-registry + description: | + Starts a buildset registry and builds one or more docker images. + + Analog of build-docker-image job, but with a buildset registry. + + .. include:: ../../playbooks/docker-image/README.rst + pre-run: playbooks/docker-image/pre.yaml + run: playbooks/docker-image/run.yaml + provides: docker-image + requires: docker-image + +- job: + name: opendev-upload-docker-image + parent: opendev-build-docker-image + description: | + Starts a buildset registry and builds and uploads one or more + docker images to docker.io. + + Analog of upload-docker-image job, but with a buildset registry. + + .. include:: ../../playbooks/docker-image/README.rst + .. include:: ../../playbooks/docker-image/credentials.rst + post-run: playbooks/docker-image/upload.yaml + +- job: + name: opendev-promote-docker-image + parent: promote-docker-image + description: | + Retag a previously-uploaded docker image. + + Analog of promote-docker-image job. + + .. include:: ../../playbooks/docker-image/README.rst + .. include:: ../../playbooks/docker-image/credentials.rst + post-run: playbooks/docker-image/promote.yaml + nodeset: + nodes: [] + - project: templates: - build-tox-docs