diff --git a/mini-mirror/Dockerfile.ubuntu b/mini-mirror/Dockerfile.ubuntu new file mode 100644 index 00000000..cf6b478b --- /dev/null +++ b/mini-mirror/Dockerfile.ubuntu @@ -0,0 +1,39 @@ +# Copyright 2019, AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:16.04 as aptly + +ARG APTLY_CONFIG_PATH=etc/aptly.conf +ARG MIRROR_SOURCE_DIR=sources +ARG RELEASE_SIGN_KEY_PATH=etc +ARG RELEASE_SIGN_KEY_PASSPHRASE + +COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf +COPY "${MIRROR_SOURCE_DIR}" /opt/sources +COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg + +COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh + +RUN apt-get update +RUN apt-get install -y aptly wget + +RUN /opt/publish_snapshots.sh "${RELEASE_SIGN_KEY_PASSPHRASE}" + +FROM nginx + +ARG APTLY_SNAPSHOT_DIR=/srv + +# NOTE(drewwalters96): This must match the location provided in the NGINX +# config file. +COPY --from=aptly /opt/.aptly/public "${APTLY_SNAPSHOT_DIR}" diff --git a/mini-mirror/README.rst b/mini-mirror/README.rst new file mode 100644 index 00000000..a5693ffc --- /dev/null +++ b/mini-mirror/README.rst @@ -0,0 +1,103 @@ +Mini-mirror Image Build +======================= + +Mini-mirror is a service that mirrors existing Debian/Ubuntu repositories and +can be used as an APT source for OpenStack-Helm deployments with no internet +connectivity. + +Build Requirements +------------------ + +Add mirror sources +~~~~~~~~~~~~~~~~~~ + +Mini-mirror requires a directory at build-time that contains the repositories +and packages that will be mirrored. + +.. code:: + + sources/ + | -- source1/ + |-- source.txt + |-- packages.txt + | -- source2/ + |-- source.txt + |-- packages.txt + +Sources are defined as directories containing the files: + +* source.txt - contains location and metadata information for a source. +* packages.txt - contains a list of packages, formatted as `package queries `_ + for a source. + +Example ``source.txt`` format: + + .. code:: + + source_url source_key_url dist components + +Example ``packages.txt`` format: + +.. code:: + + package1 + package2 + package3 (>=3.6) + +To specify the location of your sources directory, export the following +environment variable: + +.. code:: bash + + export MIRROR_SOURCE_DIR=/path/to/sources + +Generate a signing key +~~~~~~~~~~~~~~~~~~~~~~ + +.. WARNING:: + + The demo image published in the ``OpenStack-Helm-Addons`` repository is not + signed. It should NOT be used in production and signing should be enabled + in the Aptly config file. + +Mini-mirror signs the release file during the image build process. Supply a +path to a valid GPG key using the ``RELEASE_SIGN_KEY`` environment variable. + +.. code:: bash + + export RELEASE_SIGN_KEY_PATH=key.gpg + +Additionally, supply your GPG key passphrase with the +``RELEASE_SIGN_KEY_PASSPHRASE`` environment variable: + +.. code:: bash + + export RELEASE_SIGN_KEY_PASSPHRASE=passphrase + +Create an Aptly config file (optional) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Specify the location of your Aptly config file using the ``APTLY_CONFIG_PATH`` +environment variable: + +.. code:: bash + + export APTLY_CONFIG_PATH=aptly.conf + +Proxy +~~~~~ + +If building the mini-mirror image behind a proxy server, define the standard +``HTTP_PROXY``, ``HTTPS_PROXY``, and ``NO_PROXY`` environment variables. They +will be passed as build-args. + +Build +----- + +To build the mini-mirror image, execute the following: + +.. code:: bash + + export DISTRO=ubuntu + ./build.sh + diff --git a/mini-mirror/build.sh b/mini-mirror/build.sh new file mode 100755 index 00000000..8c44a663 --- /dev/null +++ b/mini-mirror/build.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# +# Copyright 2019 The Openstack-Helm Authors. +# Copyright 2019, AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT=$(realpath "$0") +SCRIPT_DIR=$(dirname "${SCRIPT}") +## Only build from main folder +cd "${SCRIPT_DIR}"/.. || exit + +IMAGE="mini-mirror" +VERSION=${VERSION:-latest} +DISTRO=${DISTRO:-ubuntu} +REGISTRY_URI=${REGISTRY_URI:-"openstackhelm/"} +EXTRA_TAG_INFO=${EXTRA_TAG_INFO:-""} + +HTTP_PROXY=${HTTP_PROXY:-""} +HTTPS_PROXY=${HTTPS_PROXY:-""} +NO_PROXY=${NO_PROXY:-"127.0.0.1,localhost"} + +APTLY_CONFIG_PATH=${APTLY_CONFIG_PATH:-"etc/aptly.conf"} +MIRROR_SOURCE_DIR=${MIRROR_SOURCE_DIR:-"sources"} +RELEASE_SIGN_KEY_PATH=${RELEASE_SIGN_KEY_PATH:-"etc"} +RELEASE_SIGN_KEY_PASSPHRASE=${RELEASE_SIGN_KEY_PASSPHRASE:-""} + +docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \ + -t "${REGISTRY_URI}""${IMAGE}":"${VERSION}"-"${DISTRO}""${EXTRA_TAG_INFO}" \ + --build-arg http_proxy="${HTTP_PROXY}" \ + --build-arg https_proxy="${HTTPS_PROXY}" \ + --build-arg HTTP_PROXY="${HTTP_PROXY}" \ + --build-arg HTTPS_PROXY="${HTTPS_PROXY}" \ + --build-arg no_proxy="${HTTP_PROXY}" \ + --build-arg NO_PROXY="${HTTP_PROXY}" \ + --build-arg APTLY_CONFIG_PATH="${APTLY_CONFIG_PATH}" \ + --build-arg MIRROR_SOURCE_DIR="${MIRROR_SOURCE_DIR}" \ + --build-arg RELEASE_SIGN_KEY_PATH="${RELEASE_SIGN_KEY_PATH}" \ + --build-arg RELEASE_SIGN_KEY_PASSPHRASE="${RELEASE_SIGN_KEY_PASSPHRASE}" \ + ${extra_build_args} "${IMAGE}" + +cd - || exit diff --git a/mini-mirror/etc/aptly.conf b/mini-mirror/etc/aptly.conf new file mode 100644 index 00000000..78fc7f1f --- /dev/null +++ b/mini-mirror/etc/aptly.conf @@ -0,0 +1,57 @@ +{ + "rootDir": "/opt/.aptly", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": [], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": true, + "gpgDisableVerify": true, + "gpgProvider": "gpg", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "FileSystemPublishEndpoints": { + "test": { + "rootDir": "/opt/aptly-publish", + "linkMethod": "copy", + "verifyMethod": "md5" + } + }, + "S3PublishEndpoints": { + "test": { + "region": "us-east-1", + "bucket": "repo", + "endpoint": "", + "awsAccessKeyID": "", + "awsSecretAccessKey": "", + "prefix": "", + "acl": "public-read", + "storageClass": "", + "encryptionMethod": "", + "plusWorkaround": false, + "disableMultiDel": false, + "forceSigV2": false, + "debug": false + } + }, + "SwiftPublishEndpoints": { + "test": { + "container": "repo", + "osname": "", + "password": "", + "prefix": "", + "authurl": "", + "tenant": "", + "tenantid": "", + "domain": "", + "domainid": "", + "tenantdomain": "", + "tenantdomainid": "" + } + } +} diff --git a/mini-mirror/sources/aptly-example/packages.txt b/mini-mirror/sources/aptly-example/packages.txt new file mode 100644 index 00000000..a7620ba6 --- /dev/null +++ b/mini-mirror/sources/aptly-example/packages.txt @@ -0,0 +1 @@ +aptly diff --git a/mini-mirror/sources/aptly-example/source.txt b/mini-mirror/sources/aptly-example/source.txt new file mode 100644 index 00000000..6eabe16a --- /dev/null +++ b/mini-mirror/sources/aptly-example/source.txt @@ -0,0 +1 @@ +http://repo.aptly.info/ https://www.aptly.info/pubkey.txt squeeze main diff --git a/mini-mirror/tools/publish_snapshots.sh b/mini-mirror/tools/publish_snapshots.sh new file mode 100755 index 00000000..e5a9882b --- /dev/null +++ b/mini-mirror/tools/publish_snapshots.sh @@ -0,0 +1,56 @@ +#!/bin/bash +# +# Copyright 2019, AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +for source in /opt/sources/*; do + read -r -a info < "${source}"/source.txt + repo=${info[0]} + key=${info[1]} + distro=${info[2]} + components=${info[*]:3} + + # Import source key + wget --no-check-certificate -O - "${key}" | gpg --no-default-keyring \ + --keyring trustedkeys.gpg --import + + snapshots=() + while read -r package; do + snapshots+=("$package") + + # NOTE(drewwalters96): Separate snapshots by package until aptly supports + # multiple package queries for mirrors/snapshots. + aptly mirror create -filter="${package}" -filter-with-deps "${package}" \ + "${repo}" "${distro}" "${components}" + aptly mirror update "${package}" + aptly snapshot create "${package}" from mirror "${package}" + done < "${source}"/packages.txt + + # Combine package snapshots into single source snapshot + aptly snapshot merge "${source}" "${snapshots[@]}" +done + +# Combine source snapshots +read -r -a snapshots <<< "$(ls -d /opt/sources/*)" +aptly snapshot merge minimirror "${snapshots[@]}" + +# Publish snapshot +if [ ! -z "$1" ]; then + gpg --import /opt/release.gpg + aptly publish snapshot -batch=true -passphrase="${1}" minimirror +else + aptly publish snapshot minimirror +fi diff --git a/zuul.d/mini-mirror.yaml b/zuul.d/mini-mirror.yaml new file mode 100644 index 00000000..8f63e834 --- /dev/null +++ b/zuul.d/mini-mirror.yaml @@ -0,0 +1,38 @@ +--- +# Copyright 2018, SUSE LINUX GmbH. +# Copyright 2019, AT&T Intellectual Property +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- project: + check: + jobs: + - openstack-helm-images-mini-mirror-ubuntu_xenial + gate: + jobs: + - openstack-helm-images-mini-mirror-ubuntu_xenial + periodic: + jobs: + - openstack-helm-images-mini-mirror-ubuntu_xenial +- job: + name: openstack-helm-images-mini-mirror + parent: openstack-helm-images-base + abstract: true + vars: + image_path: mini-mirror + +- job: + name: openstack-helm-images-mini-mirror-ubuntu_xenial + parent: openstack-helm-images-mini-mirror + vars: + distro: "ubuntu"