From e3d41e874b8d8c6031bbb2c4db57f2a796cb58ef Mon Sep 17 00:00:00 2001 From: Sam Yaple Date: Wed, 4 Oct 2017 02:27:34 -0400 Subject: [PATCH] Use infra mirrors We have to stop pinning Docker because upstream only mirrors the latest version. Hopefully this won't turn into an issue. Change-Id: I33bb9527cf3d8718361d84b1efff62426d7b711b --- dockerfiles/centos/CentOS-OpenStack.repo | 6 --- dockerfiles/centos/CentOS.repo | 23 +++++++++ dockerfiles/centos/Dockerfile | 10 +++- dockerfiles/ubuntu/Dockerfile | 19 ++++++- dockerfiles/ubuntu/sources.list | 6 +++ playbooks/files/apache.conf | 10 ++-- playbooks/files/apache2-systemd.conf | 2 +- playbooks/loci-builder.yaml | 63 ++++++++++++++++-------- playbooks/post.yaml | 8 +-- playbooks/setup-gate.yaml | 14 ++++-- playbooks/vars.yaml | 31 +++++++++++- scripts/install.sh | 26 +++------- scripts/pip_install.sh | 2 +- scripts/requirements.sh | 7 +-- scripts/setup_pip.sh | 16 ++++++ 15 files changed, 177 insertions(+), 66 deletions(-) delete mode 100644 dockerfiles/centos/CentOS-OpenStack.repo create mode 100644 dockerfiles/centos/CentOS.repo create mode 100644 dockerfiles/ubuntu/sources.list create mode 100755 scripts/setup_pip.sh diff --git a/dockerfiles/centos/CentOS-OpenStack.repo b/dockerfiles/centos/CentOS-OpenStack.repo deleted file mode 100644 index 95fc4f7..0000000 --- a/dockerfiles/centos/CentOS-OpenStack.repo +++ /dev/null @@ -1,6 +0,0 @@ -[centos-openstack] -name=CentOS-7 - OpenStack -baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-pike/ -gpgcheck=1 -enabled=1 -gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud diff --git a/dockerfiles/centos/CentOS.repo b/dockerfiles/centos/CentOS.repo new file mode 100644 index 0000000..6e948f6 --- /dev/null +++ b/dockerfiles/centos/CentOS.repo @@ -0,0 +1,23 @@ +[base] +name=CentOS-$releasever - Base +baseurl=http://%%PACKAGE_MIRROR%%/centos/$releasever/os/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +[updates] +name=CentOS-$releasever - Updates +baseurl=http://%%PACKAGE_MIRROR%%/centos/$releasever/updates/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +[extras] +name=CentOS-$releasever - Extras +baseurl=http://%%PACKAGE_MIRROR%%/centos/$releasever/extras/$basearch/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +[openstack] +name=CentOS-$releasever - OpenStack +baseurl=http://%%PACKAGE_MIRROR%%/centos/\$releasever/cloud/$basearch/openstack-pike/ +gpgcheck=1 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud diff --git a/dockerfiles/centos/Dockerfile b/dockerfiles/centos/Dockerfile index fe4fbf3..166edd5 100644 --- a/dockerfiles/centos/Dockerfile +++ b/dockerfiles/centos/Dockerfile @@ -1,5 +1,13 @@ ARG FROM=centos:7 FROM ${FROM} -COPY CentOS-OpenStack.repo /etc/yum.repos.d/ +ARG PACKAGE_MIRROR=mirror.centos.org +ARG PIP_INDEX_URL=https://pypi.python.org/simple/ +ARG PIP_TRUSTED_HOST=pypi.python.org +ENV PIP_INDEX_URL=${PIP_INDEX_URL} +ENV PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST} + +RUN rm -rf /etc/yum.repos.d/* +COPY CentOS.repo /etc/yum.repos.d/ COPY RPM-GPG-KEY-CentOS-SIG-Cloud /etc/pki/rpm-gpg/ +RUN sed -i "s|%%PACKAGE_MIRROR%%|${PACKAGE_MIRROR}|g" /etc/yum.repos.d/CentOS.repo diff --git a/dockerfiles/ubuntu/Dockerfile b/dockerfiles/ubuntu/Dockerfile index 78efda3..87f855c 100644 --- a/dockerfiles/ubuntu/Dockerfile +++ b/dockerfiles/ubuntu/Dockerfile @@ -1,5 +1,22 @@ ARG FROM=ubuntu:xenial FROM ${FROM} -COPY cloud-archive.list ceph.list /etc/apt/sources.list.d/ +ARG UBUNTU_URL=http://archive.ubuntu.com/ubuntu/ +ARG CLOUD_ARCHIVE_URL=http://ubuntu-cloud.archive.canonical.com/ubuntu/ +ARG CEPH_URL=http://download.ceph.com/debian-luminous/ +ARG ALLOW_UNAUTHENTICATED=false +ARG PIP_INDEX_URL=https://pypi.python.org/simple/ +ARG PIP_TRUSTED_HOST=pypi.python.org +ENV PIP_INDEX_URL=${PIP_INDEX_URL} +ENV PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST} + +COPY sources.list /etc/apt/ COPY cloud-archive.gpg ceph.gpg /etc/apt/trusted.gpg.d/ +RUN sed -i \ + -e "s|%%UBUNTU_URL%%|${UBUNTU_URL}|g" \ + -e "s|%%CLOUD_ARCHIVE_URL%%|${CLOUD_ARCHIVE_URL}|g" \ + -e "s|%%CEPH_URL%%|${CEPH_URL}|g" \ + /etc/apt/sources.list + +# NOTE(SamYaple): Remove this when infra starts signing thier mirrors +RUN echo "APT::Get::AllowUnauthenticated \"${ALLOW_UNAUTHENTICATED}\";" > /etc/apt/apt.conf.d/allow-unathenticated diff --git a/dockerfiles/ubuntu/sources.list b/dockerfiles/ubuntu/sources.list new file mode 100644 index 0000000..d3c7bf3 --- /dev/null +++ b/dockerfiles/ubuntu/sources.list @@ -0,0 +1,6 @@ +deb %%UBUNTU_URL%% xenial main universe +deb %%UBUNTU_URL%% xenial-updates main universe +deb %%UBUNTU_URL%% xenial-backports main universe +deb %%UBUNTU_URL%% xenial-security main universe +deb %%CEPH_URL%% xenial main +deb %%CLOUD_ARCHIVE_URL%% xenial-updates/pike main diff --git a/playbooks/files/apache.conf b/playbooks/files/apache.conf index c8bb905..cb3f573 100644 --- a/playbooks/files/apache.conf +++ b/playbooks/files/apache.conf @@ -10,10 +10,12 @@ LoadModule authn_core_module /usr/lib/apache2/modules/mod_authn_core.so LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so LoadModule cgi_module /usr/lib/apache2/modules/mod_cgi.so -Listen 80 - - +# NOTE(SamYaple): 172.17.0.1 is the network we use for Docker so it will be in +# the same subnet as the internal addesses in the build containers +Listen 172.17.0.1:80 + SetEnv GIT_PROJECT_ROOT /home/zuul/src/git.openstack.org/ SetEnv GIT_HTTP_EXPORT_ALL - ScriptAlias / /usr/lib/git-core/git-http-backend/ + ScriptAlias /git/ /usr/lib/git-core/git-http-backend/ + DocumentRoot /webroot diff --git a/playbooks/files/apache2-systemd.conf b/playbooks/files/apache2-systemd.conf index 575a40a..ed8a880 100644 --- a/playbooks/files/apache2-systemd.conf +++ b/playbooks/files/apache2-systemd.conf @@ -1,3 +1,3 @@ [Service] ExecStart= -ExecStart=/usr/sbin/apache2 -k start -f /webroot/apache.conf +ExecStart=/usr/sbin/apache2 -f /webroot/apache.conf diff --git a/playbooks/loci-builder.yaml b/playbooks/loci-builder.yaml index 72413fb..6e2a5a1 100644 --- a/playbooks/loci-builder.yaml +++ b/playbooks/loci-builder.yaml @@ -2,37 +2,60 @@ tasks: - include_vars: vars.yaml - # NOTE(SamYaple): Unused currently - - name: Gather wheels + # NOTE(SamYaple): This process is so we can take advantage of the infra + # DockerHub mirroring as configured through the Docker daemon. We do this + # instead of calling fetch_wheels initially. All-in-all this saves + # bandwidth and time. + - name: Gather wheels to local registry block: - docker_image: - name: openstackloci/requirements:{{ item }} + name: openstackloci/requirements + tag: "{{ item.name }}" + repository: 172.17.0.1:5000/openstackloci/requirements + push: yes with_items: "{{ distros }}" - - command: "docker save -o /tmp/wheels-{{ item }}.img openstackloci/requirements:{{ item }}" - with_items: "{{ distros }}" - - command: "{{ zuul.project.src_dir }}/scripts/fetch_wheels.py" - environment: - WHEELS: /tmp/wheels-{{ item }}.img - WHEELS_DEST: "/webroot/{{ item }}.tar.gz" - with_items: "{{ distros }}" - when: False + async: 1000 + poll: 0 + register: pull + - async_status: + jid: "{{ item.ansible_job_id }}" + with_items: "{{ pull.results }}" + register: pull_result + until: + - pull_result.finished is defined + - pull_result.finished + retries: 60 + delay: 5 + when: project != 'requirements' - - name: Build images + - name: Build base images block: - docker_image: - path: "{{ zuul.project.src_dir }}/dockerfiles/{{ item }}" + path: "{{ zuul.project.src_dir }}/dockerfiles/{{ item.name }}" name: base - tag: "{{ item }}" + tag: "{{ item.name }}" + buildargs: "{{ item.buildargs.base }}" with_items: "{{ distros }}" + async: 1000 + poll: 0 + register: base + - async_status: + jid: "{{ item.ansible_job_id }}" + with_items: "{{ base.results }}" + register: base_result + until: + - base_result.finished is defined + - base_result.finished + retries: 30 + delay: 5 + + - name: Build project images + block: - docker_image: path: "{{ zuul.project.src_dir }}" - name: openstackloci/{{ project }}:master-{{ item }} + name: openstackloci/{{ project }}:master-{{ item.name }} pull: False - buildargs: - PROJECT: "{{ project }}" - #PROJECT_REPO: http://172.17.0.1/openstack/{{ project }} - #WHEELS: http://172.17.0.1/{{ item }}.tar.gz - FROM: base:{{ item }} + buildargs: "{{ item.buildargs.project }}" with_items: "{{ distros }}" async: 1000 poll: 0 diff --git a/playbooks/post.yaml b/playbooks/post.yaml index 5ae2488..279d1ad 100644 --- a/playbooks/post.yaml +++ b/playbooks/post.yaml @@ -18,12 +18,12 @@ # NOTE(SamYaple): Unused right now - name: Extract wheels for tarball.o.o block: - - command: "docker save -o /tmp/wheels-{{ item }}.img openstackloci/{{ project }}:master-{{ item }}" + - command: "docker save -o /tmp/wheels-{{ item.name }}.img openstackloci/{{ project }}:master-{{ item.name }}" with_items: "{{ distros }}" - command: "{{ zuul.project.src_dir }}/scripts/fetch_wheels.py" environment: - WHEELS: /tmp/wheels-{{ item }}.img - WHEELS_DEST: "{{ zuul.executor.work_root }}/artifacts/{{ item }}.tar.gz" + WHEELS: /tmp/wheels-{{ item.name }}.img + WHEELS_DEST: "{{ zuul.executor.work_root }}/artifacts/{{ item.name }}.tar.gz" with_items: "{{ distros }}" become: True when: False @@ -32,7 +32,7 @@ block: - command: docker login -u {{ loci_docker_login.user }} -p {{ loci_docker_login.password }} no_log: True - - command: docker push openstackloci/{{ project }}:master-{{ item }} + - command: docker push openstackloci/{{ project }}:master-{{ item.name }} with_items: "{{ distros }}" become: True when: loci_docker_login is defined diff --git a/playbooks/setup-gate.yaml b/playbooks/setup-gate.yaml index 73cd2a2..ca2b9d1 100644 --- a/playbooks/setup-gate.yaml +++ b/playbooks/setup-gate.yaml @@ -36,20 +36,28 @@ - apt_key: url: https://download.docker.com/linux/ubuntu/gpg - apt_repository: - repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable + repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker xenial stable - apt: name: "{{ item }}" + allow_unauthenticated: True with_items: - - docker-ce=17.06* + - docker-ce - python-pip - pip: name: docker + - docker_container: + name: registry + image: registry:2 + state: started + published_ports: + - 172.17.0.1:5000:5000 # NOTE(SamYaple): Allow all connections from containers to host so the # containers can access the http server for git and wheels - iptables: + action: insert chain: INPUT in_interface: docker0 - policy: ACCEPT + jump: ACCEPT become: True - name: Setup http server for git and wheels diff --git a/playbooks/vars.yaml b/playbooks/vars.yaml index ebd97d1..8466eb9 100644 --- a/playbooks/vars.yaml +++ b/playbooks/vars.yaml @@ -4,7 +4,34 @@ docker_daemon: - "http://{{ zuul_site_mirror_fqdn }}:8081/registry-1.docker/" storage-driver: overlay2 debug: True + insecure-registries: + - 172.17.0.1:5000 distros: - - centos - - ubuntu + - name: centos + image: centos:7 + buildargs: + base: + PACKAGE_MIRROR: "{{ zuul_site_mirror_fqdn }}" + PIP_INDEX_URL: http://{{ zuul_site_mirror_fqdn }}/pypi/simple + PIP_TRUSTED_HOST: "{{ zuul_site_mirror_fqdn }}" + project: + PROJECT: "{{ project }}" + PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }} + WHEELS: 172.17.0.1:5000/openstackloci/requirements:centos + FROM: base:centos + - name: ubuntu + image: ubuntu:xenial + buildargs: + base: + UBUNTU_URL: http://{{ zuul_site_mirror_fqdn }}/ubuntu/ + CLOUD_ARCHIVE_URL: http://{{ zuul_site_mirror_fqdn }}/ubuntu-cloud-archive/ + CEPH_URL: http://{{ zuul_site_mirror_fqdn }}/ceph-deb-luminous/ + ALLOW_UNAUTHENTICATED: "true" + PIP_INDEX_URL: http://{{ zuul_site_mirror_fqdn }}/pypi/simple + PIP_TRUSTED_HOST: "{{ zuul_site_mirror_fqdn }}" + project: + PROJECT: "{{ project }}" + PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }} + WHEELS: 172.17.0.1:5000/openstackloci/requirements:ubuntu + FROM: base:ubuntu diff --git a/scripts/install.sh b/scripts/install.sh index 0ddc4a3..40da577 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -14,7 +14,7 @@ case ${distro} in ca-certificates \ netbase \ python \ - virtualenv \ + python-pip \ lsb-release \ sudo ;; @@ -22,7 +22,7 @@ case ${distro} in yum upgrade -y yum install -y --setopt=skip_missing_names_on_install=False \ git \ - python-virtualenv \ + python-pip \ redhat-lsb-core \ sudo ;; @@ -33,24 +33,12 @@ case ${distro} in esac if [[ "${PROJECT}" == 'requirements' ]]; then - /opt/loci/scripts/requirements.sh + $(dirname $0)/requirements.sh exit 0 fi -mkdir -p /opt/loci/ -cp $(dirname $0)/{clone_project.sh,pip_install.sh,fetch_wheels.py} /opt/loci/ - -# NOTE(SamYaple): --system-site-packages flag allows python to use libraries -# outside of the virtualenv if they do not exist inside the venv. This is a -# requirement for using python-rbd which is not pip installable and is only -# available in packaged form. -virtualenv --system-site-packages /var/lib/openstack/ -source /var/lib/openstack/bin/activate -pip install -U pip -pip install -U setuptools wheel - +$(dirname $0)/setup_pip.sh $(dirname $0)/clone_project.sh - $(dirname $0)/pip_install.sh \ /tmp/${PROJECT} \ pycrypto \ @@ -72,18 +60,20 @@ case ${distro} in if [[ ! -z ${PACKAGES} ]]; then apt-get install -y --no-install-recommends ${PACKAGES[@]} fi + pip uninstall -y virtualenv apt-get purge -y --auto-remove \ git \ - virtualenv + python-pip rm -rf /var/lib/apt/lists/* ;; centos) if [[ ! -z ${PACKAGES} ]]; then yum -y --setopt=skip_missing_names_on_install=False install ${PACKAGES[@]} fi + pip uninstall -y virtualenv yum -y autoremove \ git \ - python-virtualenv + python-pip yum clean all ;; *) diff --git a/scripts/pip_install.sh b/scripts/pip_install.sh index 5d7fe60..8c83fed 100755 --- a/scripts/pip_install.sh +++ b/scripts/pip_install.sh @@ -4,7 +4,7 @@ set -ex packages=$@ -/opt/loci/fetch_wheels.py +$(dirname $0)/fetch_wheels.py mkdir -p /tmp/wheels/ # NOTE(SamYaple): We exclude all files starting with '.' as these can be diff --git a/scripts/requirements.sh b/scripts/requirements.sh index 64b72f0..9710edc 100755 --- a/scripts/requirements.sh +++ b/scripts/requirements.sh @@ -98,14 +98,11 @@ case ${distro} in ;; esac -/opt/loci/scripts/clone_project.sh +$(dirname $0)/setup_pip.sh +$(dirname $0)/clone_project.sh mv /tmp/requirements/{global-requirements.txt,upper-constraints.txt} / -python -m virtualenv /builder -pip install -U pip -pip install -U wheel setuptools - # NOTE(SamYaple): Build all deps in parallel. This is safe because we are # constrained on the version and we are building with --no-deps pushd $(mktemp -d) diff --git a/scripts/setup_pip.sh b/scripts/setup_pip.sh new file mode 100755 index 0000000..5653e81 --- /dev/null +++ b/scripts/setup_pip.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -ex + +pip install -U virtualenv + +# NOTE(SamYaple): --system-site-packages flag allows python to use libraries +# outside of the virtualenv if they do not exist inside the venv. This is a +# requirement for using python-rbd which is not pip installable and is only +# available in packaged form. +# --no-pip --no-setuptools --no-wheel is declared because it was breaking pypi +# mirrors until setuptools is setup properly +virtualenv --no-pip --no-setuptools --no-wheel --system-site-packages /var/lib/openstack/ +source /var/lib/openstack/bin/activate +pip install -U pip +pip install -U setuptools wheel