Use infra mirrors
We have to stop pinning Docker because upstream only mirrors the latest version. Hopefully this won't turn into an issue. Change-Id: I33bb9527cf3d8718361d84b1efff62426d7b711b
This commit is contained in:
Sam Yaple
parent
335506aaa2
commit
e3d41e874b
|
|
@ -1,6 +0,0 @@
|
|||
[centos-openstack]
|
||||
name=CentOS-7 - OpenStack
|
||||
baseurl=http://mirror.centos.org/centos/7/cloud/$basearch/openstack-pike/
|
||||
gpgcheck=1
|
||||
enabled=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
[base]
|
||||
name=CentOS-$releasever - Base
|
||||
baseurl=http://%%PACKAGE_MIRROR%%/centos/$releasever/os/$basearch/
|
||||
gpgcheck=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
|
||||
|
||||
[updates]
|
||||
name=CentOS-$releasever - Updates
|
||||
baseurl=http://%%PACKAGE_MIRROR%%/centos/$releasever/updates/$basearch/
|
||||
gpgcheck=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
|
||||
|
||||
[extras]
|
||||
name=CentOS-$releasever - Extras
|
||||
baseurl=http://%%PACKAGE_MIRROR%%/centos/$releasever/extras/$basearch/
|
||||
gpgcheck=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
|
||||
|
||||
[openstack]
|
||||
name=CentOS-$releasever - OpenStack
|
||||
baseurl=http://%%PACKAGE_MIRROR%%/centos/\$releasever/cloud/$basearch/openstack-pike/
|
||||
gpgcheck=1
|
||||
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Cloud
|
||||
|
|
@ -1,5 +1,13 @@
|
|||
ARG FROM=centos:7
|
||||
FROM ${FROM}
|
||||
|
||||
COPY CentOS-OpenStack.repo /etc/yum.repos.d/
|
||||
ARG PACKAGE_MIRROR=mirror.centos.org
|
||||
ARG PIP_INDEX_URL=https://pypi.python.org/simple/
|
||||
ARG PIP_TRUSTED_HOST=pypi.python.org
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX_URL}
|
||||
ENV PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST}
|
||||
|
||||
RUN rm -rf /etc/yum.repos.d/*
|
||||
COPY CentOS.repo /etc/yum.repos.d/
|
||||
COPY RPM-GPG-KEY-CentOS-SIG-Cloud /etc/pki/rpm-gpg/
|
||||
RUN sed -i "s|%%PACKAGE_MIRROR%%|${PACKAGE_MIRROR}|g" /etc/yum.repos.d/CentOS.repo
|
||||
|
|
|
|||
|
|
@ -1,5 +1,22 @@
|
|||
ARG FROM=ubuntu:xenial
|
||||
FROM ${FROM}
|
||||
|
||||
COPY cloud-archive.list ceph.list /etc/apt/sources.list.d/
|
||||
ARG UBUNTU_URL=http://archive.ubuntu.com/ubuntu/
|
||||
ARG CLOUD_ARCHIVE_URL=http://ubuntu-cloud.archive.canonical.com/ubuntu/
|
||||
ARG CEPH_URL=http://download.ceph.com/debian-luminous/
|
||||
ARG ALLOW_UNAUTHENTICATED=false
|
||||
ARG PIP_INDEX_URL=https://pypi.python.org/simple/
|
||||
ARG PIP_TRUSTED_HOST=pypi.python.org
|
||||
ENV PIP_INDEX_URL=${PIP_INDEX_URL}
|
||||
ENV PIP_TRUSTED_HOST=${PIP_TRUSTED_HOST}
|
||||
|
||||
COPY sources.list /etc/apt/
|
||||
COPY cloud-archive.gpg ceph.gpg /etc/apt/trusted.gpg.d/
|
||||
RUN sed -i \
|
||||
-e "s|%%UBUNTU_URL%%|${UBUNTU_URL}|g" \
|
||||
-e "s|%%CLOUD_ARCHIVE_URL%%|${CLOUD_ARCHIVE_URL}|g" \
|
||||
-e "s|%%CEPH_URL%%|${CEPH_URL}|g" \
|
||||
/etc/apt/sources.list
|
||||
|
||||
# NOTE(SamYaple): Remove this when infra starts signing thier mirrors
|
||||
RUN echo "APT::Get::AllowUnauthenticated \"${ALLOW_UNAUTHENTICATED}\";" > /etc/apt/apt.conf.d/allow-unathenticated
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
deb %%UBUNTU_URL%% xenial main universe
|
||||
deb %%UBUNTU_URL%% xenial-updates main universe
|
||||
deb %%UBUNTU_URL%% xenial-backports main universe
|
||||
deb %%UBUNTU_URL%% xenial-security main universe
|
||||
deb %%CEPH_URL%% xenial main
|
||||
deb %%CLOUD_ARCHIVE_URL%% xenial-updates/pike main
|
||||
|
|
@ -10,10 +10,12 @@ LoadModule authn_core_module /usr/lib/apache2/modules/mod_authn_core.so
|
|||
LoadModule authz_core_module /usr/lib/apache2/modules/mod_authz_core.so
|
||||
LoadModule cgi_module /usr/lib/apache2/modules/mod_cgi.so
|
||||
|
||||
Listen 80
|
||||
|
||||
<VirtualHost *:80>
|
||||
# NOTE(SamYaple): 172.17.0.1 is the network we use for Docker so it will be in
|
||||
# the same subnet as the internal addesses in the build containers
|
||||
Listen 172.17.0.1:80
|
||||
<VirtualHost 172.17.0.1:80>
|
||||
SetEnv GIT_PROJECT_ROOT /home/zuul/src/git.openstack.org/
|
||||
SetEnv GIT_HTTP_EXPORT_ALL
|
||||
ScriptAlias / /usr/lib/git-core/git-http-backend/
|
||||
ScriptAlias /git/ /usr/lib/git-core/git-http-backend/
|
||||
DocumentRoot /webroot
|
||||
</VirtualHost>
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
[Service]
|
||||
ExecStart=
|
||||
ExecStart=/usr/sbin/apache2 -k start -f /webroot/apache.conf
|
||||
ExecStart=/usr/sbin/apache2 -f /webroot/apache.conf
|
||||
|
|
|
|||
|
|
@ -2,37 +2,60 @@
|
|||
tasks:
|
||||
- include_vars: vars.yaml
|
||||
|
||||
# NOTE(SamYaple): Unused currently
|
||||
- name: Gather wheels
|
||||
# NOTE(SamYaple): This process is so we can take advantage of the infra
|
||||
# DockerHub mirroring as configured through the Docker daemon. We do this
|
||||
# instead of calling fetch_wheels initially. All-in-all this saves
|
||||
# bandwidth and time.
|
||||
- name: Gather wheels to local registry
|
||||
block:
|
||||
- docker_image:
|
||||
name: openstackloci/requirements:{{ item }}
|
||||
name: openstackloci/requirements
|
||||
tag: "{{ item.name }}"
|
||||
repository: 172.17.0.1:5000/openstackloci/requirements
|
||||
push: yes
|
||||
with_items: "{{ distros }}"
|
||||
- command: "docker save -o /tmp/wheels-{{ item }}.img openstackloci/requirements:{{ item }}"
|
||||
with_items: "{{ distros }}"
|
||||
- command: "{{ zuul.project.src_dir }}/scripts/fetch_wheels.py"
|
||||
environment:
|
||||
WHEELS: /tmp/wheels-{{ item }}.img
|
||||
WHEELS_DEST: "/webroot/{{ item }}.tar.gz"
|
||||
with_items: "{{ distros }}"
|
||||
when: False
|
||||
async: 1000
|
||||
poll: 0
|
||||
register: pull
|
||||
- async_status:
|
||||
jid: "{{ item.ansible_job_id }}"
|
||||
with_items: "{{ pull.results }}"
|
||||
register: pull_result
|
||||
until:
|
||||
- pull_result.finished is defined
|
||||
- pull_result.finished
|
||||
retries: 60
|
||||
delay: 5
|
||||
when: project != 'requirements'
|
||||
|
||||
- name: Build images
|
||||
- name: Build base images
|
||||
block:
|
||||
- docker_image:
|
||||
path: "{{ zuul.project.src_dir }}/dockerfiles/{{ item }}"
|
||||
path: "{{ zuul.project.src_dir }}/dockerfiles/{{ item.name }}"
|
||||
name: base
|
||||
tag: "{{ item }}"
|
||||
tag: "{{ item.name }}"
|
||||
buildargs: "{{ item.buildargs.base }}"
|
||||
with_items: "{{ distros }}"
|
||||
async: 1000
|
||||
poll: 0
|
||||
register: base
|
||||
- async_status:
|
||||
jid: "{{ item.ansible_job_id }}"
|
||||
with_items: "{{ base.results }}"
|
||||
register: base_result
|
||||
until:
|
||||
- base_result.finished is defined
|
||||
- base_result.finished
|
||||
retries: 30
|
||||
delay: 5
|
||||
|
||||
- name: Build project images
|
||||
block:
|
||||
- docker_image:
|
||||
path: "{{ zuul.project.src_dir }}"
|
||||
name: openstackloci/{{ project }}:master-{{ item }}
|
||||
name: openstackloci/{{ project }}:master-{{ item.name }}
|
||||
pull: False
|
||||
buildargs:
|
||||
PROJECT: "{{ project }}"
|
||||
#PROJECT_REPO: http://172.17.0.1/openstack/{{ project }}
|
||||
#WHEELS: http://172.17.0.1/{{ item }}.tar.gz
|
||||
FROM: base:{{ item }}
|
||||
buildargs: "{{ item.buildargs.project }}"
|
||||
with_items: "{{ distros }}"
|
||||
async: 1000
|
||||
poll: 0
|
||||
|
|
|
|||
|
|
@ -18,12 +18,12 @@
|
|||
# NOTE(SamYaple): Unused right now
|
||||
- name: Extract wheels for tarball.o.o
|
||||
block:
|
||||
- command: "docker save -o /tmp/wheels-{{ item }}.img openstackloci/{{ project }}:master-{{ item }}"
|
||||
- command: "docker save -o /tmp/wheels-{{ item.name }}.img openstackloci/{{ project }}:master-{{ item.name }}"
|
||||
with_items: "{{ distros }}"
|
||||
- command: "{{ zuul.project.src_dir }}/scripts/fetch_wheels.py"
|
||||
environment:
|
||||
WHEELS: /tmp/wheels-{{ item }}.img
|
||||
WHEELS_DEST: "{{ zuul.executor.work_root }}/artifacts/{{ item }}.tar.gz"
|
||||
WHEELS: /tmp/wheels-{{ item.name }}.img
|
||||
WHEELS_DEST: "{{ zuul.executor.work_root }}/artifacts/{{ item.name }}.tar.gz"
|
||||
with_items: "{{ distros }}"
|
||||
become: True
|
||||
when: False
|
||||
|
|
@ -32,7 +32,7 @@
|
|||
block:
|
||||
- command: docker login -u {{ loci_docker_login.user }} -p {{ loci_docker_login.password }}
|
||||
no_log: True
|
||||
- command: docker push openstackloci/{{ project }}:master-{{ item }}
|
||||
- command: docker push openstackloci/{{ project }}:master-{{ item.name }}
|
||||
with_items: "{{ distros }}"
|
||||
become: True
|
||||
when: loci_docker_login is defined
|
||||
|
|
|
|||
|
|
@ -36,20 +36,28 @@
|
|||
- apt_key:
|
||||
url: https://download.docker.com/linux/ubuntu/gpg
|
||||
- apt_repository:
|
||||
repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu xenial stable
|
||||
repo: deb http://{{ zuul_site_mirror_fqdn }}/deb-docker xenial stable
|
||||
- apt:
|
||||
name: "{{ item }}"
|
||||
allow_unauthenticated: True
|
||||
with_items:
|
||||
- docker-ce=17.06*
|
||||
- docker-ce
|
||||
- python-pip
|
||||
- pip:
|
||||
name: docker
|
||||
- docker_container:
|
||||
name: registry
|
||||
image: registry:2
|
||||
state: started
|
||||
published_ports:
|
||||
- 172.17.0.1:5000:5000
|
||||
# NOTE(SamYaple): Allow all connections from containers to host so the
|
||||
# containers can access the http server for git and wheels
|
||||
- iptables:
|
||||
action: insert
|
||||
chain: INPUT
|
||||
in_interface: docker0
|
||||
policy: ACCEPT
|
||||
jump: ACCEPT
|
||||
become: True
|
||||
|
||||
- name: Setup http server for git and wheels
|
||||
|
|
|
|||
|
|
@ -4,7 +4,34 @@ docker_daemon:
|
|||
- "http://{{ zuul_site_mirror_fqdn }}:8081/registry-1.docker/"
|
||||
storage-driver: overlay2
|
||||
debug: True
|
||||
insecure-registries:
|
||||
- 172.17.0.1:5000
|
||||
|
||||
distros:
|
||||
- centos
|
||||
- ubuntu
|
||||
- name: centos
|
||||
image: centos:7
|
||||
buildargs:
|
||||
base:
|
||||
PACKAGE_MIRROR: "{{ zuul_site_mirror_fqdn }}"
|
||||
PIP_INDEX_URL: http://{{ zuul_site_mirror_fqdn }}/pypi/simple
|
||||
PIP_TRUSTED_HOST: "{{ zuul_site_mirror_fqdn }}"
|
||||
project:
|
||||
PROJECT: "{{ project }}"
|
||||
PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
|
||||
WHEELS: 172.17.0.1:5000/openstackloci/requirements:centos
|
||||
FROM: base:centos
|
||||
- name: ubuntu
|
||||
image: ubuntu:xenial
|
||||
buildargs:
|
||||
base:
|
||||
UBUNTU_URL: http://{{ zuul_site_mirror_fqdn }}/ubuntu/
|
||||
CLOUD_ARCHIVE_URL: http://{{ zuul_site_mirror_fqdn }}/ubuntu-cloud-archive/
|
||||
CEPH_URL: http://{{ zuul_site_mirror_fqdn }}/ceph-deb-luminous/
|
||||
ALLOW_UNAUTHENTICATED: "true"
|
||||
PIP_INDEX_URL: http://{{ zuul_site_mirror_fqdn }}/pypi/simple
|
||||
PIP_TRUSTED_HOST: "{{ zuul_site_mirror_fqdn }}"
|
||||
project:
|
||||
PROJECT: "{{ project }}"
|
||||
PROJECT_REPO: http://172.17.0.1/git/openstack/{{ project }}
|
||||
WHEELS: 172.17.0.1:5000/openstackloci/requirements:ubuntu
|
||||
FROM: base:ubuntu
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ case ${distro} in
|
|||
ca-certificates \
|
||||
netbase \
|
||||
python \
|
||||
virtualenv \
|
||||
python-pip \
|
||||
lsb-release \
|
||||
sudo
|
||||
;;
|
||||
|
|
@ -22,7 +22,7 @@ case ${distro} in
|
|||
yum upgrade -y
|
||||
yum install -y --setopt=skip_missing_names_on_install=False \
|
||||
git \
|
||||
python-virtualenv \
|
||||
python-pip \
|
||||
redhat-lsb-core \
|
||||
sudo
|
||||
;;
|
||||
|
|
@ -33,24 +33,12 @@ case ${distro} in
|
|||
esac
|
||||
|
||||
if [[ "${PROJECT}" == 'requirements' ]]; then
|
||||
/opt/loci/scripts/requirements.sh
|
||||
$(dirname $0)/requirements.sh
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p /opt/loci/
|
||||
cp $(dirname $0)/{clone_project.sh,pip_install.sh,fetch_wheels.py} /opt/loci/
|
||||
|
||||
# NOTE(SamYaple): --system-site-packages flag allows python to use libraries
|
||||
# outside of the virtualenv if they do not exist inside the venv. This is a
|
||||
# requirement for using python-rbd which is not pip installable and is only
|
||||
# available in packaged form.
|
||||
virtualenv --system-site-packages /var/lib/openstack/
|
||||
source /var/lib/openstack/bin/activate
|
||||
pip install -U pip
|
||||
pip install -U setuptools wheel
|
||||
|
||||
$(dirname $0)/setup_pip.sh
|
||||
$(dirname $0)/clone_project.sh
|
||||
|
||||
$(dirname $0)/pip_install.sh \
|
||||
/tmp/${PROJECT} \
|
||||
pycrypto \
|
||||
|
|
@ -72,18 +60,20 @@ case ${distro} in
|
|||
if [[ ! -z ${PACKAGES} ]]; then
|
||||
apt-get install -y --no-install-recommends ${PACKAGES[@]}
|
||||
fi
|
||||
pip uninstall -y virtualenv
|
||||
apt-get purge -y --auto-remove \
|
||||
git \
|
||||
virtualenv
|
||||
python-pip
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
;;
|
||||
centos)
|
||||
if [[ ! -z ${PACKAGES} ]]; then
|
||||
yum -y --setopt=skip_missing_names_on_install=False install ${PACKAGES[@]}
|
||||
fi
|
||||
pip uninstall -y virtualenv
|
||||
yum -y autoremove \
|
||||
git \
|
||||
python-virtualenv
|
||||
python-pip
|
||||
yum clean all
|
||||
;;
|
||||
*)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ set -ex
|
|||
|
||||
packages=$@
|
||||
|
||||
/opt/loci/fetch_wheels.py
|
||||
$(dirname $0)/fetch_wheels.py
|
||||
|
||||
mkdir -p /tmp/wheels/
|
||||
# NOTE(SamYaple): We exclude all files starting with '.' as these can be
|
||||
|
|
|
|||
|
|
@ -98,14 +98,11 @@ case ${distro} in
|
|||
;;
|
||||
esac
|
||||
|
||||
/opt/loci/scripts/clone_project.sh
|
||||
$(dirname $0)/setup_pip.sh
|
||||
$(dirname $0)/clone_project.sh
|
||||
|
||||
mv /tmp/requirements/{global-requirements.txt,upper-constraints.txt} /
|
||||
|
||||
python -m virtualenv /builder
|
||||
pip install -U pip
|
||||
pip install -U wheel setuptools
|
||||
|
||||
# NOTE(SamYaple): Build all deps in parallel. This is safe because we are
|
||||
# constrained on the version and we are building with --no-deps
|
||||
pushd $(mktemp -d)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -ex
|
||||
|
||||
pip install -U virtualenv
|
||||
|
||||
# NOTE(SamYaple): --system-site-packages flag allows python to use libraries
|
||||
# outside of the virtualenv if they do not exist inside the venv. This is a
|
||||
# requirement for using python-rbd which is not pip installable and is only
|
||||
# available in packaged form.
|
||||
# --no-pip --no-setuptools --no-wheel is declared because it was breaking pypi
|
||||
# mirrors until setuptools is setup properly
|
||||
virtualenv --no-pip --no-setuptools --no-wheel --system-site-packages /var/lib/openstack/
|
||||
source /var/lib/openstack/bin/activate
|
||||
pip install -U pip
|
||||
pip install -U setuptools wheel
|
||||
Loading…
Reference in New Issue