Add mini-mirror image

This commit adds a mini-mirror image to OpenStack-Helm images for
the mini-mirror chart [0].

[0] https://review.openstack.org/623314

Change-Id: I55d90560ef7c07e22d2a997d2e32a0ad95edb301
This commit is contained in:
Drew Walters 2018-11-22 02:58:49 -06:00
parent 5a5fc44981
commit c08c1f9f66
8 changed files with 347 additions and 0 deletions

View File

@ -0,0 +1,39 @@
# Copyright 2019, AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
FROM ubuntu:16.04 as aptly
ARG APTLY_CONFIG_PATH=etc/aptly.conf
ARG MIRROR_SOURCE_DIR=sources
ARG RELEASE_SIGN_KEY_PATH=etc
ARG RELEASE_SIGN_KEY_PASSPHRASE
COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf
COPY "${MIRROR_SOURCE_DIR}" /opt/sources
COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg
COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh
RUN apt-get update
RUN apt-get install -y aptly wget
RUN /opt/publish_snapshots.sh "${RELEASE_SIGN_KEY_PASSPHRASE}"
FROM nginx
ARG APTLY_SNAPSHOT_DIR=/srv
# NOTE(drewwalters96): This must match the location provided in the NGINX
# config file.
COPY --from=aptly /opt/.aptly/public "${APTLY_SNAPSHOT_DIR}"

103
mini-mirror/README.rst Normal file
View File

@ -0,0 +1,103 @@
Mini-mirror Image Build
=======================
Mini-mirror is a service that mirrors existing Debian/Ubuntu repositories and
can be used as an APT source for OpenStack-Helm deployments with no internet
connectivity.
Build Requirements
------------------
Add mirror sources
~~~~~~~~~~~~~~~~~~
Mini-mirror requires a directory at build-time that contains the repositories
and packages that will be mirrored.
.. code::
sources/
| -- source1/
|-- source.txt
|-- packages.txt
| -- source2/
|-- source.txt
|-- packages.txt
Sources are defined as directories containing the files:
* source.txt - contains location and metadata information for a source.
* packages.txt - contains a list of packages, formatted as `package queries <https://www.aptly.info/doc/feature/query/>`_
for a source.
Example ``source.txt`` format:
.. code::
source_url source_key_url dist components
Example ``packages.txt`` format:
.. code::
package1
package2
package3 (>=3.6)
To specify the location of your sources directory, export the following
environment variable:
.. code:: bash
export MIRROR_SOURCE_DIR=/path/to/sources
Generate a signing key
~~~~~~~~~~~~~~~~~~~~~~
.. WARNING::
The demo image published in the ``OpenStack-Helm-Addons`` repository is not
signed. It should NOT be used in production and signing should be enabled
in the Aptly config file.
Mini-mirror signs the release file during the image build process. Supply a
path to a valid GPG key using the ``RELEASE_SIGN_KEY`` environment variable.
.. code:: bash
export RELEASE_SIGN_KEY_PATH=key.gpg
Additionally, supply your GPG key passphrase with the
``RELEASE_SIGN_KEY_PASSPHRASE`` environment variable:
.. code:: bash
export RELEASE_SIGN_KEY_PASSPHRASE=passphrase
Create an Aptly config file (optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Specify the location of your Aptly config file using the ``APTLY_CONFIG_PATH``
environment variable:
.. code:: bash
export APTLY_CONFIG_PATH=aptly.conf
Proxy
~~~~~
If building the mini-mirror image behind a proxy server, define the standard
``HTTP_PROXY``, ``HTTPS_PROXY``, and ``NO_PROXY`` environment variables. They
will be passed as build-args.
Build
-----
To build the mini-mirror image, execute the following:
.. code:: bash
export DISTRO=ubuntu
./build.sh

52
mini-mirror/build.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/bash
#
# Copyright 2019 The Openstack-Helm Authors.
# Copyright 2019, AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
SCRIPT=$(realpath "$0")
SCRIPT_DIR=$(dirname "${SCRIPT}")
## Only build from main folder
cd "${SCRIPT_DIR}"/.. || exit
IMAGE="mini-mirror"
VERSION=${VERSION:-latest}
DISTRO=${DISTRO:-ubuntu}
REGISTRY_URI=${REGISTRY_URI:-"openstackhelm/"}
EXTRA_TAG_INFO=${EXTRA_TAG_INFO:-""}
HTTP_PROXY=${HTTP_PROXY:-""}
HTTPS_PROXY=${HTTPS_PROXY:-""}
NO_PROXY=${NO_PROXY:-"127.0.0.1,localhost"}
APTLY_CONFIG_PATH=${APTLY_CONFIG_PATH:-"etc/aptly.conf"}
MIRROR_SOURCE_DIR=${MIRROR_SOURCE_DIR:-"sources"}
RELEASE_SIGN_KEY_PATH=${RELEASE_SIGN_KEY_PATH:-"etc"}
RELEASE_SIGN_KEY_PASSPHRASE=${RELEASE_SIGN_KEY_PASSPHRASE:-""}
docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \
-t "${REGISTRY_URI}""${IMAGE}":"${VERSION}"-"${DISTRO}""${EXTRA_TAG_INFO}" \
--build-arg http_proxy="${HTTP_PROXY}" \
--build-arg https_proxy="${HTTPS_PROXY}" \
--build-arg HTTP_PROXY="${HTTP_PROXY}" \
--build-arg HTTPS_PROXY="${HTTPS_PROXY}" \
--build-arg no_proxy="${HTTP_PROXY}" \
--build-arg NO_PROXY="${HTTP_PROXY}" \
--build-arg APTLY_CONFIG_PATH="${APTLY_CONFIG_PATH}" \
--build-arg MIRROR_SOURCE_DIR="${MIRROR_SOURCE_DIR}" \
--build-arg RELEASE_SIGN_KEY_PATH="${RELEASE_SIGN_KEY_PATH}" \
--build-arg RELEASE_SIGN_KEY_PASSPHRASE="${RELEASE_SIGN_KEY_PASSPHRASE}" \
${extra_build_args} "${IMAGE}"
cd - || exit

View File

@ -0,0 +1,57 @@
{
"rootDir": "/opt/.aptly",
"downloadConcurrency": 4,
"downloadSpeedLimit": 0,
"architectures": [],
"dependencyFollowSuggests": false,
"dependencyFollowRecommends": false,
"dependencyFollowAllVariants": false,
"dependencyFollowSource": false,
"dependencyVerboseResolve": false,
"gpgDisableSign": true,
"gpgDisableVerify": true,
"gpgProvider": "gpg",
"downloadSourcePackages": false,
"skipLegacyPool": true,
"ppaDistributorID": "ubuntu",
"ppaCodename": "",
"FileSystemPublishEndpoints": {
"test": {
"rootDir": "/opt/aptly-publish",
"linkMethod": "copy",
"verifyMethod": "md5"
}
},
"S3PublishEndpoints": {
"test": {
"region": "us-east-1",
"bucket": "repo",
"endpoint": "",
"awsAccessKeyID": "",
"awsSecretAccessKey": "",
"prefix": "",
"acl": "public-read",
"storageClass": "",
"encryptionMethod": "",
"plusWorkaround": false,
"disableMultiDel": false,
"forceSigV2": false,
"debug": false
}
},
"SwiftPublishEndpoints": {
"test": {
"container": "repo",
"osname": "",
"password": "",
"prefix": "",
"authurl": "",
"tenant": "",
"tenantid": "",
"domain": "",
"domainid": "",
"tenantdomain": "",
"tenantdomainid": ""
}
}
}

View File

@ -0,0 +1 @@
aptly

View File

@ -0,0 +1 @@
http://repo.aptly.info/ https://www.aptly.info/pubkey.txt squeeze main

View File

@ -0,0 +1,56 @@
#!/bin/bash
#
# Copyright 2019, AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -e
for source in /opt/sources/*; do
read -r -a info < "${source}"/source.txt
repo=${info[0]}
key=${info[1]}
distro=${info[2]}
components=${info[*]:3}
# Import source key
wget --no-check-certificate -O - "${key}" | gpg --no-default-keyring \
--keyring trustedkeys.gpg --import
snapshots=()
while read -r package; do
snapshots+=("$package")
# NOTE(drewwalters96): Separate snapshots by package until aptly supports
# multiple package queries for mirrors/snapshots.
aptly mirror create -filter="${package}" -filter-with-deps "${package}" \
"${repo}" "${distro}" "${components}"
aptly mirror update "${package}"
aptly snapshot create "${package}" from mirror "${package}"
done < "${source}"/packages.txt
# Combine package snapshots into single source snapshot
aptly snapshot merge "${source}" "${snapshots[@]}"
done
# Combine source snapshots
read -r -a snapshots <<< "$(ls -d /opt/sources/*)"
aptly snapshot merge minimirror "${snapshots[@]}"
# Publish snapshot
if [ ! -z "$1" ]; then
gpg --import /opt/release.gpg
aptly publish snapshot -batch=true -passphrase="${1}" minimirror
else
aptly publish snapshot minimirror
fi

38
zuul.d/mini-mirror.yaml Normal file
View File

@ -0,0 +1,38 @@
---
# Copyright 2018, SUSE LINUX GmbH.
# Copyright 2019, AT&T Intellectual Property
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- project:
check:
jobs:
- openstack-helm-images-mini-mirror-ubuntu_xenial
gate:
jobs:
- openstack-helm-images-mini-mirror-ubuntu_xenial
periodic:
jobs:
- openstack-helm-images-mini-mirror-ubuntu_xenial
- job:
name: openstack-helm-images-mini-mirror
parent: openstack-helm-images-base
abstract: true
vars:
image_path: mini-mirror
- job:
name: openstack-helm-images-mini-mirror-ubuntu_xenial
parent: openstack-helm-images-mini-mirror
vars:
distro: "ubuntu"