Revert "mirror/ubuntu: speed up downloading the packages"

It brokes master

This reverts commit 4b21d952c0.

Change-Id: I3973aad0754642ea702f704428f442c4a527645b
This commit is contained in:
Dmitry Pyzhov 2015-02-18 15:40:52 +00:00
parent 4b21d952c0
commit 2f0d6a3b82
9 changed files with 368 additions and 375 deletions

View File

@ -5,7 +5,7 @@ make-changelog: $(BUILD_DIR)/mirror/make-changelog.done
clean: clean-mirror
clean-mirror:
clean-mirror: clean-ubuntu
sudo rm -rf $(BUILD_DIR)/mirror
include $(SOURCE_DIR)/mirror/centos/module.mk

View File

@ -0,0 +1,86 @@
APT_CONF_TEMPLATES := apt-ftparchive-deb.conf apt-ftparchive-udeb.conf apt-ftparchive-release.conf Release-amd64
empty_line:=
define insert_ubuntu_version
sed -i $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo/$(1) \
-e 's|@@UBUNTU_RELEASE@@|$(UBUNTU_RELEASE)|g' \
-e 's|@@UBUNTU_RELEASE_NUMBER@@|$(UBUNTU_RELEASE_NUMBER)|g'
$(empty_line)
endef
ifeq (,$(findstring clean,$(MAKECMDGOALS)))
include $(BUILD_DIR)/ubuntu_installer_kernel_version.mk
endif
define newline
endef
# there are two blank lines here, this is not an error
define apt_sources_list
$(if $(subst none,,$(USE_MIRROR)),
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main,
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main universe multiverse restricted
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-updates main universe multiverse restricted
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-security main universe multiverse restricted
deb $(MIRROR_UBUNTU_SECURITY) $(UBUNTU_RELEASE)-security main universe multiverse restricted
deb $(MIRROR_FUEL_UBUNTU) $(UBUNTU_RELEASE) main
)
$(if $(EXTRA_DEB_REPOS),$(subst |,$(newline)deb ,deb $(EXTRA_DEB_REPOS)))
endef
define policy_rc_d
#!/bin/sh
# suppress starting services in the staging chroot
exit 101
endef
$(BUILD_DIR)/mirror/ubuntu/createchroot.done: export APT_SOURCES_LIST:=$(apt_sources_list)
$(BUILD_DIR)/mirror/ubuntu/createchroot.done: export POLICY_RC_D:=$(policy_rc_d)
$(BUILD_DIR)/mirror/ubuntu/createchroot.done:
mkdir -p $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot
# Prevent services from being started inside the staging chroot
policy_rc_d="$(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/usr/sbin/policy-rc.d"; \
mkdir -p "$${policy_rc_d%/*}"; \
echo "$${POLICY_RC_D}" > "$${policy_rc_d}"; \
chmod 755 "$${policy_rc_d}"
mkdir -p $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/init.d
# Avoid base packages' configuration errors by preventing the postinst
# scripts from fiddling with the services' start order.
# Suppresses the scary error messages
# "Errors were while processing: <long-list-of-base-packages>'
# and makes it possible to install non-trivial packages in the chroot
touch $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/init.d/.legacy-bootordering
# copy the mirroring script and the list of the packages
mkdir -p $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo
rsync -a $(SOURCE_DIR)/mirror/ubuntu/files/ $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo/
cp -a $(SOURCE_DIR)/requirements-deb.txt $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot
$(foreach f,$(APT_CONF_TEMPLATES),$(call insert_ubuntu_version,$(f)))
chmod +x $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo/mkrepo.sh
# bootstrap Ubuntu
sudo debootstrap --no-check-gpg --arch=$(UBUNTU_ARCH) \
--variant=minbase --include=apt-utils,wget,bzip2 \
$(UBUNTU_RELEASE) $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot $(MIRROR_UBUNTU)
sudo rm -f $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt/sources.list.d/*.list
echo "$${APT_SOURCES_LIST}" | sudo tee $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt/sources.list
echo 'APT::Get::AllowUnauthenticated 1;' | sudo tee $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt/apt.conf.d/02mirantis-unauthenticated
sudo cp -a $(SOURCE_DIR)/mirror/ubuntu/files/preferences $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt
sudo rm -f $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/resolv.conf
sudo cp /etc/resolv.conf $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/resolv.conf
extra_env=""; \
if [ -n "$$HTTP_PROXY" ] || [ -n "$$http_proxy" ]; then \
HTTP_PROXY="$${HTTP_PROXY:-$${http_proxy}}"; \
echo "Acquire::http { Proxy \"$$HTTP_PROXY\"; };" | sudo tee $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt/apt.conf.d/03-use-proxy; \
extra_env="HTTP_PROXY=$${HTTP_PROXY} http_proxy=$${HTTP_PROXY}"; \
fi; \
sudo chroot $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot env \
UBUNTU_RELEASE='$(UBUNTU_RELEASE)' \
UBUNTU_INSTALLER_KERNEL_VERSION='$(UBUNTU_INSTALLER_KERNEL_VERSION)' \
UBUNTU_KERNEL_FLAVOR='$(UBUNTU_KERNEL_FLAVOR)' \
$$extra_env \
/repo/mkrepo.sh
sudo rsync -a --delete $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo/* $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/
sudo rm -rf $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot
$(ACTION.TOUCH)

View File

@ -0,0 +1,6 @@
Archive: @@UBUNTU_RELEASE@@
Version: @@UBUNTU_RELEASE_NUMBER@@
Component: main
Origin: Ubuntu
Label: Ubuntu
Architecture: amd64

View File

@ -0,0 +1,25 @@
Dir {
ArchiveDir "/repo/";
};
TreeDefault {
Directory "pool";
};
BinDirectory "pool/main" {
Packages "dists/@@UBUNTU_RELEASE@@/main/binary-amd64/Packages";
BinOverride "/repo/indices/override.@@UBUNTU_RELEASE@@.main";
ExtraOverride "/repo/indices/override.@@UBUNTU_RELEASE@@.extra.main";
};
Default {
Packages {
Extensions ".deb";
Compress ". gzip";
};
};
Contents {
Compress "gzip";
};

View File

@ -0,0 +1,8 @@
APT::FTPArchive::Release::Origin "Ubuntu";
APT::FTPArchive::Release::Label "Ubuntu";
APT::FTPArchive::Release::Suite "@@UBUNTU_RELEASE@@";
APT::FTPArchive::Release::Version "@@UBUNTU_RELEASE_NUMBER@@";
APT::FTPArchive::Release::Codename "@@UBUNTU_RELEASE@@";
APT::FTPArchive::Release::Architectures "amd64";
APT::FTPArchive::Release::Components "main";
APT::FTPArchive::Release::Description "Ubuntu @@UBUNTU_RELEASE@@ @@UBUNTU_RELEASE_NUMBER@@ LTS";

View File

@ -0,0 +1,24 @@
Dir {
ArchiveDir "/repo/";
};
TreeDefault {
Directory "pool";
};
BinDirectory "pool/debian-installer" {
Packages "dists/@@UBUNTU_RELEASE@@/main/debian-installer/binary-amd64/Packages";
BinOverride "/repo/indices/override.@@UBUNTU_RELEASE@@.main.debian-installer";
};
Default {
Packages {
Extensions ".udeb";
Compress ". gzip";
};
};
Contents {
Compress "gzip";
};

View File

@ -0,0 +1,212 @@
#!/bin/bash
if [ -z "$UBUNTU_RELEASE" ]; then
echo 'mkrepo.sh: UBUNTU_RELEASE is not defined'
exit 1
fi
if [ -z "$UBUNTU_INSTALLER_KERNEL_VERSION" ]; then
echo 'mkrepo.sh: UBUNTU_INSTALLER_KERNEL_VERSION is not defined'
exit 1
fi
if [ -z "$UBUNTU_KERNEL_FLAVOR" ]; then
echo 'mkrepo.sh UBUNTU_KERNEL_FLAVOR is not defined'
exit 1
fi
mkdir -p /repo/download/
cat >> /requirements-deb.txt << EOF
linux-image-${UBUNTU_INSTALLER_KERNEL_VERSION}
linux-headers-${UBUNTU_INSTALLER_KERNEL_VERSION}
linux-image-generic-${UBUNTU_KERNEL_FLAVOR}
linux-headers-generic-${UBUNTU_KERNEL_FLAVOR}
EOF
requirements_add_essential_pkgs () {
# All essential packages are already installed, so ask dpkg for a list
dpkg-query -W -f='${Package} ${Essential}\n' > /tmp/essential.pkgs
sed -i /tmp/essential.pkgs -n -e 's/\([^ ]\+\).*yes$/\1/p'
cat /tmp/essential.pkgs >> /requirements-deb.txt
}
# Note: apt-get install --print-uris package
# is not going to print anything if the package is already installed. Thus
# the base packages will be omitted if we use the main APT/dpkg settings.
# Pretend that no package has been installed by creating an alternative APT
# state and configuration directories.
# Previously we used to copy all debs from the APT cache which is unreliable:
# - a wrong version of the package might be included
# - multiple revisions of the same package might be included
apt_altstate="/apt-altstate"
rm -rf "$apt_altstate"
apt_lists_dir="$apt_altstate/var/lib/apt/lists"
apt_cache_dir="$apt_altstate/var/cache/apt"
null_dpkg_status="$apt_altstate/var/lib/dpkg/status"
mkdir -p "$apt_lists_dir"
mkdir -p "$apt_cache_dir"
mkdir -p "${null_dpkg_status%/*}"
touch "${null_dpkg_status}"
apt_altstate_opts="-o APT::Get::AllowUnauthenticated=1"
apt_altstate_opts="${apt_altstate_opts} -o Dir::State::Lists=${apt_lists_dir}"
apt_altstate_opts="${apt_altstate_opts} -o Dir::State::status=${null_dpkg_status}"
apt_altstate_opts="${apt_altstate_opts} -o Dir::Cache=${apt_cache_dir}"
if ! apt-get $apt_altstate_opts update; then
echo "mkrepo.sh: failed to populate alt apt state"
exit 1
fi
requirements_add_essential_pkgs
has_apt_errors=''
rm -f /apt-errors.log
while read pkg; do
downloads_list="/downloads_${pkg}.list"
if ! apt-get $apt_altstate_opts --print-uris --yes -qq install $pkg >"${downloads_list}" 2>>"/apt-errors.log"; then
echo "package $pkg can not be installed" >>/apt-errors.log
# run apt-get once more to get a verbose error message
apt-get $apt_altstate_opts --print-uris --yes install $pkg >>/apt-errors.log 2>&1 || true
has_apt_errors='yes'
fi
sed -i "${downloads_list}" -n -e "s/^'\([^']\+\)['].*$/\1/p"
done < /requirements-deb.txt
if [ -n "$has_apt_errors" ]; then
echo 'some packages are not installable' >&2
cat < /apt-errors.log >&2
exit 1
fi
rm -rf "$apt_altstate"
cat /downloads_*.list | sort | uniq > /repo/download/download_urls.list
rm /downloads_*.list /apt-errors.log
(cat /repo/download/download_urls.list | xargs -n1 -P4 wget -nv -P /repo/download/) || exit 1
# Make structure and mocks for multiarch
for dir in binary-i386 binary-amd64; do
mkdir -p /repo/dists/${UBUNTU_RELEASE}/main/$dir /repo/dists/${UBUNTU_RELEASE}/main/debian-installer/$dir
touch /repo/dists/${UBUNTU_RELEASE}/main/$dir/Packages /repo/dists/${UBUNTU_RELEASE}/main/debian-installer/$dir/Packages
done
mkdir -p /repo/pool/debian-installer /repo/pool/main
cd /repo/pool/debian-installer
###################
# Grab every udeb
###################
wrkdir=`dirname $(pwd)`/`basename $(pwd)`
rm -f ${wrkdir}/UPackages.tmp ${wrkdir}/override.${UBUNTU_RELEASE}.main ${wrkdir}/override.${UBUNTU_RELEASE}.extra.main
touch ${wrkdir}/UPackages.tmp ${wrkdir}/override.${UBUNTU_RELEASE}.main ${wrkdir}/override.${UBUNTU_RELEASE}.extra.main
# Prepare temp apt dirs
[ -d ${wrkdir}/apt.tmp ] && rm -rf ${wrkdir}/apt.tmp
mkdir -p ${wrkdir}/apt.tmp/lists
mkdir -p ${wrkdir}/apt.tmp/sources
mkdir -p ${wrkdir}/apt.tmp/cache
# Extract all specified repos (except backports repo)
for list in /etc/apt/sources.list; do
for repo in `cat $list| grep -v backports | sed 's| \+|\||g' | grep "^deb|"`; do
repourl=`echo $repo | awk -F '|' '{print $2}'`
repodist=`echo $repo | awk -F '|' '{print $3}'`
repos=`echo $repo | awk -F '|' '{for(i=4; i<=NF; ++i) {print $i}}'`
for repo in $repos; do
echo "deb ${repourl} ${repodist} ${repo}/debian-installer" >> ${wrkdir}/apt.tmp/sources/sources.list
packagesfile=`wget -nv -qO - ${repourl}/dists/${repodist}/Release | \
egrep '[0-9a-f]{64}' | \
grep ${repo}/debian-installer/binary-amd64/Packages.bz2 | \
awk '{print $3}'`
if [ -n "$packagesfile" ]; then
bz=${repourl}/dists/${repodist}/$packagesfile
wget -nv -qO - $bz | bzip2 -cdq | sed -ne 's/^Package: //p' >> ${wrkdir}/UPackages.tmp
else
bz=${repourl}/dists/${repodist}/${repo}/debian-installer/binary-amd64/Packages
wget -nv -qO - $bz | sed -ne 's/^Package: //p' >> ${wrkdir}/UPackages.tmp
fi
# Getting indices
wget -nv -O - ${repourl}/indices/override.${repodist}.${repo} >> ${wrkdir}/override.${UBUNTU_RELEASE}.main
wget -nv -O - ${repourl}/indices/override.${repodist}.extra.${repo} >> ${wrkdir}/override.${UBUNTU_RELEASE}.extra.main
wget -nv -O - ${repourl}/indices/override.${repodist}.${repo}.debian-installer >> ${wrkdir}/override.${UBUNTU_RELEASE}.main.debian-installer
done
done
done
# Collect all udebs except packages with suffux generic or virtual
packages=`cat ${wrkdir}/UPackages.tmp | sort -u | egrep -v "generic|virtual"`
# Find modules for ${UBUNTU_INSTALLER_KERNEL_VERSION} kernel (installer runs with this version)
for package in `cat ${wrkdir}/UPackages.tmp | egrep "generic|virtual" | grep ${UBUNTU_INSTALLER_KERNEL_VERSION}`; do
packages="$packages $package"
done
# Update apt temp cache
apt-get -o Dir::Etc::SourceParts="${wrkdir}/apt.tmp/sources/parts" \
-o Dir::Etc::SourceList="${wrkdir}/apt.tmp/sources/sources.list" \
-o Dir::State::Lists="${wrkdir}/apt.tmp/lists" \
-o Dir::Cache="${wrkdir}/apt.tmp/cache" \
update
# Download udebs
apt-get -o Dir::Etc::SourceParts="${wrkdir}/apt.tmp/sources/parts" \
-o Dir::Etc::SourceList="${wrkdir}/apt.tmp/sources/sources.list" \
-o Dir::State::Lists="${wrkdir}/apt.tmp/lists" \
-o Dir::Cache="${wrkdir}/apt.tmp/cache" \
download $packages
rm -f ${wrkdir}/UPackages.tmp
rm -rf ${wrkdir}/apt.tmp
# Get rid of urlencoded names
for i in $(ls | grep %) ; do mv $i $(echo $i | echo -e $(sed 's/%/\\x/g')) ; done
sort_packages_file () {
local pkg_file="$1"
local pkg_file_gz="${pkg_file}.gz"
local pkg_file_bz2="${pkg_file}.bz2"
apt-sortpkgs "$pkg_file" > "${pkg_file}.new" || exit 1
if [ -e "$pkg_file_gz" ]; then
gzip -c "${pkg_file}.new" > "$pkg_file_gz"
fi
if [ -e "$pkg_file_bz2" ]; then
bzip2 -k "${pkg_file}.new" > "$pkg_file_bz2"
fi
mv "${pkg_file}.new" "${pkg_file}"
}
##########################################
# Move all stuff to the our package pool
##########################################
mv /repo/download/*deb /repo/pool/main
cd /repo/pool/main
# urlencode again
for i in $(ls | grep %) ; do mv $i $(echo $i | echo -e $(sed 's/%/\\x/g')) ; done
mkdir -p /repo/indices
cd /repo/indices
for idx in override.${UBUNTU_RELEASE}.main override.${UBUNTU_RELEASE}.extra.main override.${UBUNTU_RELEASE}.main.debian-installer; do
cat ${wrkdir}/$idx | sort -u > /repo/indices/$idx
done
rm -f ${wrkdir}/override.${UBUNTU_RELEASE}.main ${wrkdir}/override.${UBUNTU_RELEASE}.extra.main ${wrkdir}/override.${UBUNTU_RELEASE}.main.debian-installer
cd /repo
# Just because apt scan will produce crap
cp -a Release-amd64 Release-i386
sed -i 's/amd64/i386/g' Release-i386
for amd64dir in $(find . -name binary-amd64) ; do
cp -a Release-amd64 $amd64dir/Release
done
for i386dir in $(find . -name binary-i386) ; do
cp -a Release-i386 $i386dir/Release
done
apt-ftparchive -c apt-ftparchive-release.conf generate apt-ftparchive-deb.conf
apt-ftparchive -c apt-ftparchive-release.conf generate apt-ftparchive-udeb.conf
# Work around the base system installation failure.
# XXX: someone should rewrite this script to use debmirror and reprepro
for pkg_file in `find dists -type f -name Packages`; do
sort_packages_file $pkg_file
done
apt-ftparchive -c apt-ftparchive-release.conf release dists/${UBUNTU_RELEASE}/ > dists/${UBUNTU_RELEASE}/Release
# some cleanup...
rm -rf apt-ftparchive*conf Release-amd64 Release-i386 mkrepo.sh preferences

View File

@ -1,24 +1,15 @@
.PHONY: mirror-ubuntu
.PHONY: clean-ubuntu mirror-ubuntu
# This module downloads ubuntu installation images.
include $(SOURCE_DIR)/mirror/ubuntu/boot.mk
include $(SOURCE_DIR)/mirror/ubuntu/createchroot.mk
clean: clean-ubuntu
mirror-ubuntu: $(BUILD_DIR)/mirror/ubuntu/build.done
$(BUILD_DIR)/mirror/ubuntu/repo.done: $(BUILD_DIR)/ubuntu_installer_kernel_version.mk
$(MAKE) -f $(SOURCE_DIR)/mirror/ubuntu/repo.mk \
MIRROR_UBUNTU=$(MIRROR_UBUNTU) \
MIRROR_FUEL_UBUNTU=$(MIRROR_FUEL_UBUNTU) \
EXTRA_DEB_REPOS='$(EXTRA_DEB_REPOS)' \
UBUNTU_RELEASE=$(UBUNTU_RELEASE) \
UBUNTU_RELEASE_NUMBER=$(UBUNTU_RELEASE_NUMBER) \
UBUNTU_ARCH=$(UBUNTU_ARCH) \
UBUNTU_KERNEL_FLAVOR=$(UBUNTU_KERNEL_FLAVOR) \
LOCAL_MIRROR_UBUNTU=$(LOCAL_MIRROR_UBUNTU) \
SOURCE_DIR=$(SOURCE_DIR) \
BUILD_DIR=$(BUILD_DIR)
$(ACTION.TOUCH)
clean-ubuntu:
(mount -l | grep -q $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/proc && sudo umount $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/proc) || exit 0
$(BUILD_DIR)/mirror/ubuntu/build.done: \
$(BUILD_DIR)/mirror/ubuntu/boot.done \
$(BUILD_DIR)/mirror/ubuntu/repo.done
$(BUILD_DIR)/mirror/ubuntu/createchroot.done
$(ACTION.TOUCH)

View File

@ -1,359 +0,0 @@
#!/usr/bin/make -f
# the root APT repo metadata:
repo_release_file:=$(LOCAL_MIRROR_UBUNTU)/dists/$(UBUNTU_RELEASE)/Release
all: $(repo_release_file)
workdir?=$(BUILD_DIR)/mirror/ubuntu
target_dir?=$(LOCAL_MIRROR_UBUNTU)/pool/main
udeb_target_dir?=$(LOCAL_MIRROR_UBUNTU)/pool/debian-installer
indices_dir?=$(LOCAL_MIRROR_UBUNTU)/indices
deb_download_urls:=$(workdir)/deb_download_urls.list
udeb_download_urls:=$(workdir)/udeb_download_urls.list
udebs_list:=$(workdir)/udebs.list
deb_download_done:=$(workdir)/deb_download.done
udeb_download_done:=$(workdir)/udeb_download.done
deb_download: $(deb_download_done)
deb_download_urls: $(deb_download_urls)
# The version of some packages might contain an epoch, which is separated by
# a semicolon, which is a bit special for make. Therefore replace semicolons
# with a special sequence
decode_versions=$(subst ___,:,$(1))
# Replace a special sequence denoting the semicolon with the semicolon character
encode_versions=$(subst :,___,$(1))
requirements_deb:=$(call encode_versions,$(shell cat $(SOURCE_DIR)/requirements-deb.txt))
include $(BUILD_DIR)/ubuntu_installer_kernel_version.mk
requirements_deb+=\
linux-image-generic-$(UBUNTU_KERNEL_FLAVOR) \
linux-headers-generic-$(UBUNTU_KERNEL_FLAVOR) \
linux-image-$(UBUNTU_INSTALLER_KERNEL_VERSION) \
linux-headers-$(UBUNTU_INSTALLER_KERNEL_VERSION)
per_pkg_downloads_urls:=$(requirements_deb:%=$(workdir)/urllists/%.list)
$(info per_pkg_downloads_urls=$(per_pkg_downloads_urls))
# Setup a separate APT/dpkg databases to not interfere with the system one.
# Note: apt-get install --print-uris package
# is not going to print anything if the package is already installed. Thus
# the base packages will be omitted if we use the main APT/dpkg settings.
# Pretend that no package has been installed by creating an alternative APT
# state and configuration directories.
define newline
endef
define apt_sources_list
$(if $(subst none,,$(USE_MIRROR)),
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main main/debian-installer,
# USE_MIRROR=none
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main universe multiverse restricted
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-updates main universe multiverse restricted
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-security main universe multiverse restricted
deb $(MIRROR_FUEL_UBUNTU) $(UBUNTU_RELEASE) main
deb $(MIRROR_UBUNTU_SECURITY) $(UBUNTU_RELEASE)-security main universe multiverse restricted
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main/debian-installer universe/debian-installer multiverse/debian-installer restricted/debian-installer
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-updates main/debian-installer universe/debian-installer multiverse/debian-installer restricted/debian-installer
deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-security main/debian-installer universe/debian-installer multiverse/debian-installer restricted/debian-installer
deb $(MIRROR_FUEL_UBUNTU) $(UBUNTU_RELEASE) main/debian-installer
deb $(MIRROR_UBUNTU_SECURITY) $(UBUNTU_RELEASE)-security main/debian-installer universe/debian-installer multiverse/debian-installer restricted/debian-installer
)
$(if $(EXTRA_DEB_REPOS),$(subst |,$(newline)deb ,deb $(EXTRA_DEB_REPOS)))
endef
apt_altstate:=$(workdir)/apt-altstate
apt_alt_conf:=$(apt_altstate)/etc/apt/apt.conf
define apt_config
Dir::Etc::SourceList "$(apt_altstate)/etc/apt/sources.list";
Dir::Etc::SourceParts "$(apt_altstate)/etc/apt/sources.list.d";
Dir::Etc::Preferences "$(apt_altstate)/etc/apt/preferences";
Dir::Etc::PreferencesParts "$(apt_altstate)/etc/apt/preferences.d";
Dir::State::Lists "$(apt_altstate)/var/lib/apt/lists";
Dir::State::Status "$(apt_altstate)/var/lib/dpkg/status";
Dir::Cache "$(apt_altstate)/var/cache/apt";
APT::Install-Recommends "1";
APT::Get::AllowUnauthenticated "true";
$(if $(strip $(HTTP_PROXY)),
Acquire::http::Proxy "$(strip $(HTTP_PROXY))";)
endef
apt_setup_done:=$(workdir)/apt_setup.done
$(apt_setup_done): export apt_conf:=$(apt_config)
$(apt_setup_done): export sources_lists:=$(apt_sources_list)
$(apt_setup_done):
rm -rf "$(apt_altstate)" && \
mkdir -p "$(apt_altstate)/etc/apt/sources.list.d" && \
mkdir -p "$(apt_altstate)/etc/apt/preferences.d" && \
mkdir -p "$(apt_altstate)/var/lib/apt/lists" && \
mkdir -p "$(apt_altstate)/var/cache/apt/archives/partial" && \
mkdir -p "$(apt_altstate)/var/lib/dpkg" && \
touch "$(apt_altstate)/var/lib/dpkg/status" && \
echo "$$apt_conf" > "$(apt_altstate)/etc/apt/apt.conf" && \
echo "$$sources_lists" > "$(apt_altstate)/etc/apt/sources.list" && \
cp $(SOURCE_DIR)/mirror/ubuntu/files/preferences "$(apt_altstate)/etc/apt" && \
mkdir -p "$(workdir)/urllists" "$(workdir)/log" && \
apt-get -c "$(apt_alt_conf)" update && \
touch $@
$(apt_altstate)/var/lib/dpkg/available: $(apt_setup_done)
apt-cache -c "$(apt_alt_conf)" dumpavail > "$@.tmp" && \
mv "$@.tmp" "$@"
# Get the download URLs for each package in requirements-deb.txt.
# This takes into account package dependencies. Note that downloading
# all packages at once is not possible due to the conflicts.
$(per_pkg_downloads_urls): $(workdir)/urllists/%.list: $(apt_setup_done)
apt_log="$(workdir)/log/$*.log"; \
pkg="$(call decode_versions,$*)"; \
if ! apt-get -c "$(apt_alt_conf)" --print-uris --yes -qq install "$$pkg" >"$@.tmp" 2>"$${apt_log}"; then \
echo "package $* can not be installed" >> "$${apt_log}"; \
apt-get -c "$(apt_alt_conf)" -o Debug::pkgProblemResolver=true --print-uris --yes install "$$pkg" >>"$${apt_log}" 2>&1; \
exit 1; \
fi; \
mv $@.tmp $@
# Declaring dependency on an Essential package is not mandatory,
# hence we need to make sure the essential packages are mirrored too.
# XXX: dpkg-query works with dpkg database (and we have it empty on purpose)
# Also dpkg-query --load-avail produces wrong result, hence sed hackery
$(workdir)/essential_pkgs.list: $(apt_altstate)/var/lib/dpkg/available
sed -rne '/^Package/ h; /^Essential:\s+yes$$/ { g; s/^Package:\s+(.+)$$/\1/p; d }' < $< > $@.unsorted
sort -u < $@.unsorted > $@.tmp && \
mv $@.tmp $@
essential_pkgs_urls:=$(workdir)/urllists/essential_pkgs.list
# XXX: take a shortcut for essential packages: just ask apt to download them
# (or rather print their URL). An essential package can only depend on other
# essential packages, therefore dependency calculation can be skipped
$(essential_pkgs_urls): $(workdir)/essential_pkgs.list
apt_log="$(workdir)/log/essential_pkgs.log"; \
if ! apt-get -c "$(apt_alt_conf)" download -qq --print-uris `cat $<` > $@.tmp 2>"$$apt_log"; then \
echo 'Essential packages can not be installed' >>"$$apt_log"; \
exit 1; \
fi; \
mv $@.tmp $@
# udebs are required for the provisioning with cobbler + Debian installer
# Download almost all udebs, skip the modules for the kernels different
# from that used by the installer (to save *a lot* of disk space).
$(udeb_download_urls): $(workdir)/udeb_pkgs_filtered.list
apt_log="$(workdir)/log/udeb_pkgs.log"; \
if ! apt-get -c "$(apt_alt_conf)" download -qq --print-uris `cat $<` > $@.unsorted 2>"$$apt_log"; then \
echo 'Udebs can not be downloaded' >>"$$apt_log"; \
apt-get -c "$(apt_alt_conf)" download -o Debug::pkgProblemResolver=yes `cat $<` >>"$$apt_log" 2>&1; \
exit 1; \
fi
sed -rne "s/^'([^']+)['].*$$/\1/p" -i $@.unsorted && \
sort -u < $@.unsorted > $@.tmp && \
mv $@.tmp $@
# udebs are in Section: debian-installer or {universe,multiverse}/debian-installer
$(workdir)/udeb_pkgs.list: $(apt_altstate)/var/lib/dpkg/available
sed -rne '/^Package/h; /^Section:\s+.*debian-installer/ { g; s/Package:\s+(.+)$$/\1/p }' < $< > $@.unsorted
sort -u < $@.unsorted > $@.tmp && \
mv $@.tmp $@
# Skip kernel modules udebs for kernels other than the one used by the installer
$(workdir)/udeb_pkgs_filtered.list: $(workdir)/udeb_pkgs.list
sed -re '/^(.+-modules-.*|kernel-image|kernel-signed-image).*-generic-di$$/ { s/$(UBUNTU_INSTALLER_KERNEL_VERSION)/&/p; d; }' < $< > $@.unsorted && \
sort -u < $@.unsorted > $@.tmp && \
mv $@.tmp $@
$(deb_download_urls): $(per_pkg_downloads_urls) $(essential_pkgs_urls)
mkdir -p "$(dir $@)" && \
cat $^ > $@.tmp.all && \
sed -rne "s/^'([^']+)['].*$$/\1/p" -i $@.tmp.all && \
sort -u < $@.tmp.all > $@.tmp && \
mv $@.tmp $@
$(deb_download_done): $(deb_download_urls)
mkdir -p "$(target_dir)" && \
xargs -n1 -P4 wget -nv -P "$(target_dir)" < $< || exit 1
touch $@
$(udeb_download_done): $(udeb_download_urls)
mkdir -p "$(udeb_target_dir)" && \
xargs -n1 -P4 wget -nv -P "$(udeb_target_dir)" < $< || exit 1
touch $@
combined_overrides:=$(LOCAL_MIRROR_UBUNTU)/indices/override.$(UBUNTU_RELEASE).main
combined_extra_overrides:=$(LOCAL_MIRROR_UBUNTU)/indices/override.$(UBUNTU_RELEASE).extra.main
combined_di_overrides:=$(LOCAL_MIRROR_UBUNTU)/indices/main.debian-installer
define apt_ftparchive_conf
Dir {
ArchiveDir "$(LOCAL_MIRROR_UBUNTU)";
};
TreeDefault {
Directory "pool";
}
BinDirectory "pool/main" {
Packages "dists/$(UBUNTU_RELEASE)/main/binary-$(UBUNTU_ARCH)/Packages";
BinOverride "$(combined_overrides)";
ExtraOverride "$(combined_extra_overrides)";
};
BinDirectory "pool/debian-installer" {
Packages "dists/$(UBUNTU_RELEASE)/main/debian-installer/binary-$(UBUNTU_ARCH)/Packages";
BinOverride "$(combined_di_overrides)";
};
Default {
Packages {
Extensions ".deb .udeb";
Compress ". gzip";
};
};
Contents {
Compress "gzip";
};
endef
apt_repo_components:=main main/debian-installer
define apt_ftparchive_release_conf
APT::FTPArchive::Release::Origin "Ubuntu";
APT::FTPArchive::Release::Label "Ubuntu";
APT::FTPArchive::Release::Suite "$(UBUNTU_RELEASE)";
APT::FTPArchive::Release::Version "$(UBUNTU_RELEASE_NUMBER)";
APT::FTPArchive::Release::Codename "$(UBUNTU_RELEASE)";
APT::FTPArchive::Release::Architectures "$(UBUNTU_ARCH)";
APT::FTPArchive::Release::Components "main";
APT::FTPArchive::Release::Description "Ubuntu $(UBUNTU_RELEASE) $(UBUNTU_RELEASE_NUMBER) LTS";
endef
define component_release
Archive: $(UBUNTU_RELEASE)
Version: $(UBUNTU_RELEASE_NUMBER)
Component: main
Origin: Ubuntu
Label: Ubuntu
Architecture: $(UBUNTU_ARCH)
endef
$(repo_release_file): export APT_FTPARCHIVE_CONF:=$(apt_ftparchive_conf)
$(repo_release_file): export APT_FTPARCHIVE_RELEASE_CONF:=$(apt_ftparchive_release_conf)
$(repo_release_file): export COMPONENT_RELEASE:=$(component_release)
$(repo_release_file): $(deb_download_done) $(udeb_download_done)
mkdir -p "$(workdir)/conf" && \
echo "$$APT_FTPARCHIVE_CONF" > "$(workdir)/conf/apt-ftparchive.conf" && \
echo "$$APT_FTPARCHIVE_RELEASE_CONF" > "$(workdir)/conf/apt-ftparchive-release.conf"
for component in $(apt_repo_components); do \
dists_dir="$(dir $@)$$component/binary-$(UBUNTU_ARCH)"; \
rm -rf "$$dists_dir"; \
mkdir -p "$$dists_dir" && \
echo "$$COMPONENT_RELEASE" > "$$dists_dir/Release"; \
done
apt-ftparchive -c "$(workdir)/conf/apt-ftparchive-release.conf" generate "$(workdir)/conf/apt-ftparchive.conf"
### XXX: The Packages file produced by apt-ftparchive generate is not sorted properly.
### XXX: Sort them manually to prevent the provisioning error.
for component in $(apt_repo_components); do \
pkgs_file="$(dir $@)$$component/binary-$(UBUNTU_ARCH)/Packages"; \
rm -f "$${pkgs_file}.gz" && \
apt-sortpkgs "$$pkgs_file" > "$${pkgs_file}.new" && \
gzip -c < "$${pkgs_file}.new" > "$${pkgs_file}.gz.new" && \
mv "$${pkgs_file}.new" "$${pkgs_file}" && \
mv "$${pkgs_file}.gz.new" "$${pkgs_file}.gz"; \
done
apt-ftparchive -c "$(workdir)/conf/apt-ftparchive-release.conf" release "$(dir $@)" > "$@.tmp"
mv "$@.tmp" "$@"
$(repo_release_file): $(combined_overrides) \
$(combined_extra_overrides) \
$(combined_di_overrides)
ifeq (none,$(strip $(USE_MIRROR)))
overrides:=\
override.$(UBUNTU_RELEASE).main \
override.$(UBUNTU_RELEASE).restricted \
override.$(UBUNTU_RELEASE).universe \
override.$(UBUNTU_RELEASE).multiverse \
override.$(UBUNTU_RELEASE)-updates.main \
override.$(UBUNTU_RELEASE)-updates.restricted \
override.$(UBUNTU_RELEASE)-updates.universe \
override.$(UBUNTU_RELEASE)-updates.multiverse \
override.$(UBUNTU_RELEASE)-security.main \
override.$(UBUNTU_RELEASE)-security.restricted \
override.$(UBUNTU_RELEASE)-security.universe \
override.$(UBUNTU_RELEASE)-security.multiverse \
$(empty)
extra_overrides:=\
override.$(UBUNTU_RELEASE).extra.main \
override.$(UBUNTU_RELEASE).extra.restricted \
override.$(UBUNTU_RELEASE).extra.universe \
override.$(UBUNTU_RELEASE).extra.multiverse \
override.$(UBUNTU_RELEASE)-updates.extra.main \
override.$(UBUNTU_RELEASE)-updates.extra.restricted \
override.$(UBUNTU_RELEASE)-updates.extra.universe \
override.$(UBUNTU_RELEASE)-updates.extra.multiverse \
override.$(UBUNTU_RELEASE)-security.extra.main \
override.$(UBUNTU_RELEASE)-security.extra.restricted \
override.$(UBUNTU_RELEASE)-security.extra.universe \
override.$(UBUNTU_RELEASE)-security.extra.multiverse \
$(empty)
di_overrides:=\
override.$(UBUNTU_RELEASE).main.debian-installer \
override.$(UBUNTU_RELEASE).restricted.debian-installer \
override.$(UBUNTU_RELEASE).universe.debian-installer \
override.$(UBUNTU_RELEASE).multiverse.debian-installer \
override.$(UBUNTU_RELEASE)-updates.main.debian-installer \
override.$(UBUNTU_RELEASE)-updates.restricted.debian-installer \
override.$(UBUNTU_RELEASE)-updates.universe.debian-installer \
override.$(UBUNTU_RELEASE)-updates.multiverse.debian-installer \
override.$(UBUNTU_RELEASE)-security.main.debian-installer \
override.$(UBUNTU_RELEASE)-security.restricted.debian-installer \
override.$(UBUNTU_RELEASE)-security.universe.debian-installer \
override.$(UBUNTU_RELEASE)-security.multiverse.debian-installer \
$(empty)
overrides_dl:=$(foreach idx,$(overrides) $(extra_overrides) $(di_overrides),$(workdir)/indices/$(idx))
$(overrides_dl): $(workdir)/indices/%:
mkdir -p "$(dir $@)" && \
rm -f "$@.tmp" && \
wget -nv -O "$@.tmp" "$(MIRROR_UBUNTU)/indices/$*" && \
mv "$@.tmp" "$@"
$(combined_overrides): $(overrides:%=$(workdir)/indices/%)
mkdir -p "$(dir $@)" && \
cat $^ > "$@.unsorted" && \
sed -re '/Supported/d' -i "$@.unsorted" && \
sort -u < "$@.unsorted" > "$@.tmp" && \
mv "$@.tmp" "$@"
$(combined_extra_overrides): $(extra_overrides:%=$(workdir)/indices/%)
mkdir -p "$(dir $@)" && \
cat $^ > "$@.unsorted" && \
sed -re '/Supported/d' -i "$@.unsorted" && \
sort -u < "$@.unsorted" > "$@.tmp" && \
mv "$@.tmp" "$@"
$(combined_di_overrides): $(di_overrides:%=$(workdir)/indices/%)
mkdir -p "$(dir $@)" && \
cat $^ > "$@.unsorted" && \
sed -re '/Supported/d' -i "$@.unsorted" && \
sort -u < "$@.unsorted" > "$@.tmp" && \
mv "$@.tmp" "$@"
else
overrides_dl:=$(combined_overrides) $(combined_extra_overrides) $(combined_di_overrides)
$(overrides_dl): $(LOCAL_MIRROR_UBUNTU)/indices/%:
mkdir -p "$(dir $@)" && \
rm -f "$@.tmp" && \
wget -nv -O "$@.tmp" "$(MIRROR_UBUNTU)/indices/$*" && \
mv $@.tmp
endif