diff --git a/mirror/module.mk b/mirror/module.mk index fceffa90f..14249f362 100644 --- a/mirror/module.mk +++ b/mirror/module.mk @@ -5,7 +5,7 @@ make-changelog: $(BUILD_DIR)/mirror/make-changelog.done clean: clean-mirror -clean-mirror: +clean-mirror: clean-ubuntu sudo rm -rf $(BUILD_DIR)/mirror include $(SOURCE_DIR)/mirror/centos/module.mk diff --git a/mirror/ubuntu/createchroot.mk b/mirror/ubuntu/createchroot.mk new file mode 100644 index 000000000..5a2fb3309 --- /dev/null +++ b/mirror/ubuntu/createchroot.mk @@ -0,0 +1,86 @@ + +APT_CONF_TEMPLATES := apt-ftparchive-deb.conf apt-ftparchive-udeb.conf apt-ftparchive-release.conf Release-amd64 +empty_line:= +define insert_ubuntu_version + sed -i $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo/$(1) \ + -e 's|@@UBUNTU_RELEASE@@|$(UBUNTU_RELEASE)|g' \ + -e 's|@@UBUNTU_RELEASE_NUMBER@@|$(UBUNTU_RELEASE_NUMBER)|g' + $(empty_line) +endef + +ifeq (,$(findstring clean,$(MAKECMDGOALS))) +include $(BUILD_DIR)/ubuntu_installer_kernel_version.mk +endif + +define newline + + +endef +# there are two blank lines here, this is not an error + +define apt_sources_list +$(if $(subst none,,$(USE_MIRROR)), +deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main, +deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main universe multiverse restricted +deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-updates main universe multiverse restricted +deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-security main universe multiverse restricted +deb $(MIRROR_UBUNTU_SECURITY) $(UBUNTU_RELEASE)-security main universe multiverse restricted +deb $(MIRROR_FUEL_UBUNTU) $(UBUNTU_RELEASE) main +) +$(if $(EXTRA_DEB_REPOS),$(subst |,$(newline)deb ,deb $(EXTRA_DEB_REPOS))) +endef + +define policy_rc_d +#!/bin/sh +# suppress starting services in the staging chroot +exit 101 +endef + +$(BUILD_DIR)/mirror/ubuntu/createchroot.done: export APT_SOURCES_LIST:=$(apt_sources_list) +$(BUILD_DIR)/mirror/ubuntu/createchroot.done: export POLICY_RC_D:=$(policy_rc_d) + +$(BUILD_DIR)/mirror/ubuntu/createchroot.done: + mkdir -p $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot + # Prevent services from being started inside the staging chroot + policy_rc_d="$(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/usr/sbin/policy-rc.d"; \ + mkdir -p "$${policy_rc_d%/*}"; \ + echo "$${POLICY_RC_D}" > "$${policy_rc_d}"; \ + chmod 755 "$${policy_rc_d}" + mkdir -p $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/init.d + # Avoid base packages' configuration errors by preventing the postinst + # scripts from fiddling with the services' start order. + # Suppresses the scary error messages + # "Errors were while processing: ' + # and makes it possible to install non-trivial packages in the chroot + touch $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/init.d/.legacy-bootordering + # copy the mirroring script and the list of the packages + mkdir -p $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo + rsync -a $(SOURCE_DIR)/mirror/ubuntu/files/ $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo/ + cp -a $(SOURCE_DIR)/requirements-deb.txt $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot + $(foreach f,$(APT_CONF_TEMPLATES),$(call insert_ubuntu_version,$(f))) + chmod +x $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo/mkrepo.sh + # bootstrap Ubuntu + sudo debootstrap --no-check-gpg --arch=$(UBUNTU_ARCH) \ + --variant=minbase --include=apt-utils,wget,bzip2 \ + $(UBUNTU_RELEASE) $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot $(MIRROR_UBUNTU) + sudo rm -f $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt/sources.list.d/*.list + echo "$${APT_SOURCES_LIST}" | sudo tee $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt/sources.list + echo 'APT::Get::AllowUnauthenticated 1;' | sudo tee $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt/apt.conf.d/02mirantis-unauthenticated + sudo cp -a $(SOURCE_DIR)/mirror/ubuntu/files/preferences $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt + sudo rm -f $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/resolv.conf + sudo cp /etc/resolv.conf $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/resolv.conf + extra_env=""; \ + if [ -n "$$HTTP_PROXY" ] || [ -n "$$http_proxy" ]; then \ + HTTP_PROXY="$${HTTP_PROXY:-$${http_proxy}}"; \ + echo "Acquire::http { Proxy \"$$HTTP_PROXY\"; };" | sudo tee $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/etc/apt/apt.conf.d/03-use-proxy; \ + extra_env="HTTP_PROXY=$${HTTP_PROXY} http_proxy=$${HTTP_PROXY}"; \ + fi; \ + sudo chroot $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot env \ + UBUNTU_RELEASE='$(UBUNTU_RELEASE)' \ + UBUNTU_INSTALLER_KERNEL_VERSION='$(UBUNTU_INSTALLER_KERNEL_VERSION)' \ + UBUNTU_KERNEL_FLAVOR='$(UBUNTU_KERNEL_FLAVOR)' \ + $$extra_env \ + /repo/mkrepo.sh + sudo rsync -a --delete $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/repo/* $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/ + sudo rm -rf $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot + $(ACTION.TOUCH) diff --git a/mirror/ubuntu/files/Release-amd64 b/mirror/ubuntu/files/Release-amd64 new file mode 100644 index 000000000..8db3bf88b --- /dev/null +++ b/mirror/ubuntu/files/Release-amd64 @@ -0,0 +1,6 @@ +Archive: @@UBUNTU_RELEASE@@ +Version: @@UBUNTU_RELEASE_NUMBER@@ +Component: main +Origin: Ubuntu +Label: Ubuntu +Architecture: amd64 diff --git a/mirror/ubuntu/files/apt-ftparchive-deb.conf b/mirror/ubuntu/files/apt-ftparchive-deb.conf new file mode 100644 index 000000000..311d7fe5f --- /dev/null +++ b/mirror/ubuntu/files/apt-ftparchive-deb.conf @@ -0,0 +1,25 @@ +Dir { + ArchiveDir "/repo/"; +}; + +TreeDefault { + Directory "pool"; +}; + +BinDirectory "pool/main" { + Packages "dists/@@UBUNTU_RELEASE@@/main/binary-amd64/Packages"; + BinOverride "/repo/indices/override.@@UBUNTU_RELEASE@@.main"; + ExtraOverride "/repo/indices/override.@@UBUNTU_RELEASE@@.extra.main"; +}; + +Default { + Packages { + Extensions ".deb"; + Compress ". gzip"; + }; +}; + +Contents { + Compress "gzip"; +}; + diff --git a/mirror/ubuntu/files/apt-ftparchive-release.conf b/mirror/ubuntu/files/apt-ftparchive-release.conf new file mode 100644 index 000000000..4f12f9739 --- /dev/null +++ b/mirror/ubuntu/files/apt-ftparchive-release.conf @@ -0,0 +1,8 @@ +APT::FTPArchive::Release::Origin "Ubuntu"; +APT::FTPArchive::Release::Label "Ubuntu"; +APT::FTPArchive::Release::Suite "@@UBUNTU_RELEASE@@"; +APT::FTPArchive::Release::Version "@@UBUNTU_RELEASE_NUMBER@@"; +APT::FTPArchive::Release::Codename "@@UBUNTU_RELEASE@@"; +APT::FTPArchive::Release::Architectures "amd64"; +APT::FTPArchive::Release::Components "main"; +APT::FTPArchive::Release::Description "Ubuntu @@UBUNTU_RELEASE@@ @@UBUNTU_RELEASE_NUMBER@@ LTS"; diff --git a/mirror/ubuntu/files/apt-ftparchive-udeb.conf b/mirror/ubuntu/files/apt-ftparchive-udeb.conf new file mode 100644 index 000000000..ceaf48b54 --- /dev/null +++ b/mirror/ubuntu/files/apt-ftparchive-udeb.conf @@ -0,0 +1,24 @@ +Dir { + ArchiveDir "/repo/"; +}; + +TreeDefault { + Directory "pool"; +}; + +BinDirectory "pool/debian-installer" { + Packages "dists/@@UBUNTU_RELEASE@@/main/debian-installer/binary-amd64/Packages"; + BinOverride "/repo/indices/override.@@UBUNTU_RELEASE@@.main.debian-installer"; +}; + +Default { + Packages { + Extensions ".udeb"; + Compress ". gzip"; + }; +}; + +Contents { + Compress "gzip"; +}; + diff --git a/mirror/ubuntu/files/mkrepo.sh b/mirror/ubuntu/files/mkrepo.sh new file mode 100644 index 000000000..055e274c6 --- /dev/null +++ b/mirror/ubuntu/files/mkrepo.sh @@ -0,0 +1,212 @@ +#!/bin/bash + +if [ -z "$UBUNTU_RELEASE" ]; then + echo 'mkrepo.sh: UBUNTU_RELEASE is not defined' + exit 1 +fi + +if [ -z "$UBUNTU_INSTALLER_KERNEL_VERSION" ]; then + echo 'mkrepo.sh: UBUNTU_INSTALLER_KERNEL_VERSION is not defined' + exit 1 +fi + +if [ -z "$UBUNTU_KERNEL_FLAVOR" ]; then + echo 'mkrepo.sh UBUNTU_KERNEL_FLAVOR is not defined' + exit 1 +fi + +mkdir -p /repo/download/ + +cat >> /requirements-deb.txt << EOF +linux-image-${UBUNTU_INSTALLER_KERNEL_VERSION} +linux-headers-${UBUNTU_INSTALLER_KERNEL_VERSION} +linux-image-generic-${UBUNTU_KERNEL_FLAVOR} +linux-headers-generic-${UBUNTU_KERNEL_FLAVOR} +EOF + +requirements_add_essential_pkgs () { + # All essential packages are already installed, so ask dpkg for a list + dpkg-query -W -f='${Package} ${Essential}\n' > /tmp/essential.pkgs + sed -i /tmp/essential.pkgs -n -e 's/\([^ ]\+\).*yes$/\1/p' + cat /tmp/essential.pkgs >> /requirements-deb.txt +} + +# Note: apt-get install --print-uris package +# is not going to print anything if the package is already installed. Thus +# the base packages will be omitted if we use the main APT/dpkg settings. +# Pretend that no package has been installed by creating an alternative APT +# state and configuration directories. +# Previously we used to copy all debs from the APT cache which is unreliable: +# - a wrong version of the package might be included +# - multiple revisions of the same package might be included +apt_altstate="/apt-altstate" +rm -rf "$apt_altstate" +apt_lists_dir="$apt_altstate/var/lib/apt/lists" +apt_cache_dir="$apt_altstate/var/cache/apt" +null_dpkg_status="$apt_altstate/var/lib/dpkg/status" + +mkdir -p "$apt_lists_dir" +mkdir -p "$apt_cache_dir" +mkdir -p "${null_dpkg_status%/*}" +touch "${null_dpkg_status}" + +apt_altstate_opts="-o APT::Get::AllowUnauthenticated=1" +apt_altstate_opts="${apt_altstate_opts} -o Dir::State::Lists=${apt_lists_dir}" +apt_altstate_opts="${apt_altstate_opts} -o Dir::State::status=${null_dpkg_status}" +apt_altstate_opts="${apt_altstate_opts} -o Dir::Cache=${apt_cache_dir}" + +if ! apt-get $apt_altstate_opts update; then + echo "mkrepo.sh: failed to populate alt apt state" + exit 1 +fi + +requirements_add_essential_pkgs +has_apt_errors='' +rm -f /apt-errors.log +while read pkg; do + downloads_list="/downloads_${pkg}.list" + if ! apt-get $apt_altstate_opts --print-uris --yes -qq install $pkg >"${downloads_list}" 2>>"/apt-errors.log"; then + echo "package $pkg can not be installed" >>/apt-errors.log + # run apt-get once more to get a verbose error message + apt-get $apt_altstate_opts --print-uris --yes install $pkg >>/apt-errors.log 2>&1 || true + has_apt_errors='yes' + fi + sed -i "${downloads_list}" -n -e "s/^'\([^']\+\)['].*$/\1/p" +done < /requirements-deb.txt + +if [ -n "$has_apt_errors" ]; then + echo 'some packages are not installable' >&2 + cat < /apt-errors.log >&2 + exit 1 +fi + +rm -rf "$apt_altstate" + +cat /downloads_*.list | sort | uniq > /repo/download/download_urls.list +rm /downloads_*.list /apt-errors.log +(cat /repo/download/download_urls.list | xargs -n1 -P4 wget -nv -P /repo/download/) || exit 1 +# Make structure and mocks for multiarch +for dir in binary-i386 binary-amd64; do + mkdir -p /repo/dists/${UBUNTU_RELEASE}/main/$dir /repo/dists/${UBUNTU_RELEASE}/main/debian-installer/$dir + touch /repo/dists/${UBUNTU_RELEASE}/main/$dir/Packages /repo/dists/${UBUNTU_RELEASE}/main/debian-installer/$dir/Packages +done +mkdir -p /repo/pool/debian-installer /repo/pool/main +cd /repo/pool/debian-installer + +################### +# Grab every udeb +################### + +wrkdir=`dirname $(pwd)`/`basename $(pwd)` + +rm -f ${wrkdir}/UPackages.tmp ${wrkdir}/override.${UBUNTU_RELEASE}.main ${wrkdir}/override.${UBUNTU_RELEASE}.extra.main +touch ${wrkdir}/UPackages.tmp ${wrkdir}/override.${UBUNTU_RELEASE}.main ${wrkdir}/override.${UBUNTU_RELEASE}.extra.main + +# Prepare temp apt dirs +[ -d ${wrkdir}/apt.tmp ] && rm -rf ${wrkdir}/apt.tmp +mkdir -p ${wrkdir}/apt.tmp/lists +mkdir -p ${wrkdir}/apt.tmp/sources +mkdir -p ${wrkdir}/apt.tmp/cache + +# Extract all specified repos (except backports repo) +for list in /etc/apt/sources.list; do + for repo in `cat $list| grep -v backports | sed 's| \+|\||g' | grep "^deb|"`; do + repourl=`echo $repo | awk -F '|' '{print $2}'` + repodist=`echo $repo | awk -F '|' '{print $3}'` + repos=`echo $repo | awk -F '|' '{for(i=4; i<=NF; ++i) {print $i}}'` + for repo in $repos; do + echo "deb ${repourl} ${repodist} ${repo}/debian-installer" >> ${wrkdir}/apt.tmp/sources/sources.list + packagesfile=`wget -nv -qO - ${repourl}/dists/${repodist}/Release | \ + egrep '[0-9a-f]{64}' | \ + grep ${repo}/debian-installer/binary-amd64/Packages.bz2 | \ + awk '{print $3}'` + if [ -n "$packagesfile" ]; then + bz=${repourl}/dists/${repodist}/$packagesfile + wget -nv -qO - $bz | bzip2 -cdq | sed -ne 's/^Package: //p' >> ${wrkdir}/UPackages.tmp + else + bz=${repourl}/dists/${repodist}/${repo}/debian-installer/binary-amd64/Packages + wget -nv -qO - $bz | sed -ne 's/^Package: //p' >> ${wrkdir}/UPackages.tmp + fi + # Getting indices + wget -nv -O - ${repourl}/indices/override.${repodist}.${repo} >> ${wrkdir}/override.${UBUNTU_RELEASE}.main + wget -nv -O - ${repourl}/indices/override.${repodist}.extra.${repo} >> ${wrkdir}/override.${UBUNTU_RELEASE}.extra.main + wget -nv -O - ${repourl}/indices/override.${repodist}.${repo}.debian-installer >> ${wrkdir}/override.${UBUNTU_RELEASE}.main.debian-installer + done + done +done + +# Collect all udebs except packages with suffux generic or virtual +packages=`cat ${wrkdir}/UPackages.tmp | sort -u | egrep -v "generic|virtual"` + +# Find modules for ${UBUNTU_INSTALLER_KERNEL_VERSION} kernel (installer runs with this version) +for package in `cat ${wrkdir}/UPackages.tmp | egrep "generic|virtual" | grep ${UBUNTU_INSTALLER_KERNEL_VERSION}`; do + packages="$packages $package" +done + +# Update apt temp cache +apt-get -o Dir::Etc::SourceParts="${wrkdir}/apt.tmp/sources/parts" \ + -o Dir::Etc::SourceList="${wrkdir}/apt.tmp/sources/sources.list" \ + -o Dir::State::Lists="${wrkdir}/apt.tmp/lists" \ + -o Dir::Cache="${wrkdir}/apt.tmp/cache" \ + update + +# Download udebs +apt-get -o Dir::Etc::SourceParts="${wrkdir}/apt.tmp/sources/parts" \ + -o Dir::Etc::SourceList="${wrkdir}/apt.tmp/sources/sources.list" \ + -o Dir::State::Lists="${wrkdir}/apt.tmp/lists" \ + -o Dir::Cache="${wrkdir}/apt.tmp/cache" \ + download $packages + +rm -f ${wrkdir}/UPackages.tmp +rm -rf ${wrkdir}/apt.tmp + +# Get rid of urlencoded names +for i in $(ls | grep %) ; do mv $i $(echo $i | echo -e $(sed 's/%/\\x/g')) ; done + +sort_packages_file () { + local pkg_file="$1" + local pkg_file_gz="${pkg_file}.gz" + local pkg_file_bz2="${pkg_file}.bz2" + apt-sortpkgs "$pkg_file" > "${pkg_file}.new" || exit 1 + if [ -e "$pkg_file_gz" ]; then + gzip -c "${pkg_file}.new" > "$pkg_file_gz" + fi + if [ -e "$pkg_file_bz2" ]; then + bzip2 -k "${pkg_file}.new" > "$pkg_file_bz2" + fi + mv "${pkg_file}.new" "${pkg_file}" +} + +########################################## +# Move all stuff to the our package pool +########################################## +mv /repo/download/*deb /repo/pool/main +cd /repo/pool/main +# urlencode again +for i in $(ls | grep %) ; do mv $i $(echo $i | echo -e $(sed 's/%/\\x/g')) ; done +mkdir -p /repo/indices +cd /repo/indices +for idx in override.${UBUNTU_RELEASE}.main override.${UBUNTU_RELEASE}.extra.main override.${UBUNTU_RELEASE}.main.debian-installer; do + cat ${wrkdir}/$idx | sort -u > /repo/indices/$idx +done +rm -f ${wrkdir}/override.${UBUNTU_RELEASE}.main ${wrkdir}/override.${UBUNTU_RELEASE}.extra.main ${wrkdir}/override.${UBUNTU_RELEASE}.main.debian-installer +cd /repo +# Just because apt scan will produce crap +cp -a Release-amd64 Release-i386 +sed -i 's/amd64/i386/g' Release-i386 +for amd64dir in $(find . -name binary-amd64) ; do + cp -a Release-amd64 $amd64dir/Release +done +for i386dir in $(find . -name binary-i386) ; do + cp -a Release-i386 $i386dir/Release +done +apt-ftparchive -c apt-ftparchive-release.conf generate apt-ftparchive-deb.conf +apt-ftparchive -c apt-ftparchive-release.conf generate apt-ftparchive-udeb.conf +# Work around the base system installation failure. +# XXX: someone should rewrite this script to use debmirror and reprepro +for pkg_file in `find dists -type f -name Packages`; do + sort_packages_file $pkg_file +done +apt-ftparchive -c apt-ftparchive-release.conf release dists/${UBUNTU_RELEASE}/ > dists/${UBUNTU_RELEASE}/Release +# some cleanup... +rm -rf apt-ftparchive*conf Release-amd64 Release-i386 mkrepo.sh preferences diff --git a/mirror/ubuntu/module.mk b/mirror/ubuntu/module.mk index 7a4a0b240..f64469c32 100644 --- a/mirror/ubuntu/module.mk +++ b/mirror/ubuntu/module.mk @@ -1,24 +1,15 @@ -.PHONY: mirror-ubuntu +.PHONY: clean-ubuntu mirror-ubuntu # This module downloads ubuntu installation images. include $(SOURCE_DIR)/mirror/ubuntu/boot.mk +include $(SOURCE_DIR)/mirror/ubuntu/createchroot.mk +clean: clean-ubuntu mirror-ubuntu: $(BUILD_DIR)/mirror/ubuntu/build.done -$(BUILD_DIR)/mirror/ubuntu/repo.done: $(BUILD_DIR)/ubuntu_installer_kernel_version.mk - $(MAKE) -f $(SOURCE_DIR)/mirror/ubuntu/repo.mk \ - MIRROR_UBUNTU=$(MIRROR_UBUNTU) \ - MIRROR_FUEL_UBUNTU=$(MIRROR_FUEL_UBUNTU) \ - EXTRA_DEB_REPOS='$(EXTRA_DEB_REPOS)' \ - UBUNTU_RELEASE=$(UBUNTU_RELEASE) \ - UBUNTU_RELEASE_NUMBER=$(UBUNTU_RELEASE_NUMBER) \ - UBUNTU_ARCH=$(UBUNTU_ARCH) \ - UBUNTU_KERNEL_FLAVOR=$(UBUNTU_KERNEL_FLAVOR) \ - LOCAL_MIRROR_UBUNTU=$(LOCAL_MIRROR_UBUNTU) \ - SOURCE_DIR=$(SOURCE_DIR) \ - BUILD_DIR=$(BUILD_DIR) - $(ACTION.TOUCH) +clean-ubuntu: + (mount -l | grep -q $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/proc && sudo umount $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/chroot/proc) || exit 0 $(BUILD_DIR)/mirror/ubuntu/build.done: \ $(BUILD_DIR)/mirror/ubuntu/boot.done \ - $(BUILD_DIR)/mirror/ubuntu/repo.done + $(BUILD_DIR)/mirror/ubuntu/createchroot.done $(ACTION.TOUCH) diff --git a/mirror/ubuntu/repo.mk b/mirror/ubuntu/repo.mk deleted file mode 100644 index 2bf8aa663..000000000 --- a/mirror/ubuntu/repo.mk +++ /dev/null @@ -1,359 +0,0 @@ -#!/usr/bin/make -f - -# the root APT repo metadata: -repo_release_file:=$(LOCAL_MIRROR_UBUNTU)/dists/$(UBUNTU_RELEASE)/Release - -all: $(repo_release_file) - -workdir?=$(BUILD_DIR)/mirror/ubuntu -target_dir?=$(LOCAL_MIRROR_UBUNTU)/pool/main -udeb_target_dir?=$(LOCAL_MIRROR_UBUNTU)/pool/debian-installer -indices_dir?=$(LOCAL_MIRROR_UBUNTU)/indices - -deb_download_urls:=$(workdir)/deb_download_urls.list -udeb_download_urls:=$(workdir)/udeb_download_urls.list -udebs_list:=$(workdir)/udebs.list -deb_download_done:=$(workdir)/deb_download.done -udeb_download_done:=$(workdir)/udeb_download.done - -deb_download: $(deb_download_done) -deb_download_urls: $(deb_download_urls) - -# The version of some packages might contain an epoch, which is separated by -# a semicolon, which is a bit special for make. Therefore replace semicolons -# with a special sequence -decode_versions=$(subst ___,:,$(1)) -# Replace a special sequence denoting the semicolon with the semicolon character -encode_versions=$(subst :,___,$(1)) - -requirements_deb:=$(call encode_versions,$(shell cat $(SOURCE_DIR)/requirements-deb.txt)) -include $(BUILD_DIR)/ubuntu_installer_kernel_version.mk -requirements_deb+=\ -linux-image-generic-$(UBUNTU_KERNEL_FLAVOR) \ -linux-headers-generic-$(UBUNTU_KERNEL_FLAVOR) \ -linux-image-$(UBUNTU_INSTALLER_KERNEL_VERSION) \ -linux-headers-$(UBUNTU_INSTALLER_KERNEL_VERSION) - -per_pkg_downloads_urls:=$(requirements_deb:%=$(workdir)/urllists/%.list) -$(info per_pkg_downloads_urls=$(per_pkg_downloads_urls)) - -# Setup a separate APT/dpkg databases to not interfere with the system one. -# Note: apt-get install --print-uris package -# is not going to print anything if the package is already installed. Thus -# the base packages will be omitted if we use the main APT/dpkg settings. -# Pretend that no package has been installed by creating an alternative APT -# state and configuration directories. - -define newline - - -endef - -define apt_sources_list -$(if $(subst none,,$(USE_MIRROR)), -deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main main/debian-installer, - -# USE_MIRROR=none -deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main universe multiverse restricted -deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-updates main universe multiverse restricted -deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-security main universe multiverse restricted -deb $(MIRROR_FUEL_UBUNTU) $(UBUNTU_RELEASE) main -deb $(MIRROR_UBUNTU_SECURITY) $(UBUNTU_RELEASE)-security main universe multiverse restricted -deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE) main/debian-installer universe/debian-installer multiverse/debian-installer restricted/debian-installer -deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-updates main/debian-installer universe/debian-installer multiverse/debian-installer restricted/debian-installer -deb $(MIRROR_UBUNTU) $(UBUNTU_RELEASE)-security main/debian-installer universe/debian-installer multiverse/debian-installer restricted/debian-installer -deb $(MIRROR_FUEL_UBUNTU) $(UBUNTU_RELEASE) main/debian-installer -deb $(MIRROR_UBUNTU_SECURITY) $(UBUNTU_RELEASE)-security main/debian-installer universe/debian-installer multiverse/debian-installer restricted/debian-installer -) -$(if $(EXTRA_DEB_REPOS),$(subst |,$(newline)deb ,deb $(EXTRA_DEB_REPOS))) -endef - -apt_altstate:=$(workdir)/apt-altstate -apt_alt_conf:=$(apt_altstate)/etc/apt/apt.conf - -define apt_config -Dir::Etc::SourceList "$(apt_altstate)/etc/apt/sources.list"; -Dir::Etc::SourceParts "$(apt_altstate)/etc/apt/sources.list.d"; -Dir::Etc::Preferences "$(apt_altstate)/etc/apt/preferences"; -Dir::Etc::PreferencesParts "$(apt_altstate)/etc/apt/preferences.d"; -Dir::State::Lists "$(apt_altstate)/var/lib/apt/lists"; -Dir::State::Status "$(apt_altstate)/var/lib/dpkg/status"; -Dir::Cache "$(apt_altstate)/var/cache/apt"; -APT::Install-Recommends "1"; -APT::Get::AllowUnauthenticated "true"; -$(if $(strip $(HTTP_PROXY)), -Acquire::http::Proxy "$(strip $(HTTP_PROXY))";) -endef - -apt_setup_done:=$(workdir)/apt_setup.done - -$(apt_setup_done): export apt_conf:=$(apt_config) -$(apt_setup_done): export sources_lists:=$(apt_sources_list) -$(apt_setup_done): - rm -rf "$(apt_altstate)" && \ - mkdir -p "$(apt_altstate)/etc/apt/sources.list.d" && \ - mkdir -p "$(apt_altstate)/etc/apt/preferences.d" && \ - mkdir -p "$(apt_altstate)/var/lib/apt/lists" && \ - mkdir -p "$(apt_altstate)/var/cache/apt/archives/partial" && \ - mkdir -p "$(apt_altstate)/var/lib/dpkg" && \ - touch "$(apt_altstate)/var/lib/dpkg/status" && \ - echo "$$apt_conf" > "$(apt_altstate)/etc/apt/apt.conf" && \ - echo "$$sources_lists" > "$(apt_altstate)/etc/apt/sources.list" && \ - cp $(SOURCE_DIR)/mirror/ubuntu/files/preferences "$(apt_altstate)/etc/apt" && \ - mkdir -p "$(workdir)/urllists" "$(workdir)/log" && \ - apt-get -c "$(apt_alt_conf)" update && \ - touch $@ - -$(apt_altstate)/var/lib/dpkg/available: $(apt_setup_done) - apt-cache -c "$(apt_alt_conf)" dumpavail > "$@.tmp" && \ - mv "$@.tmp" "$@" - - -# Get the download URLs for each package in requirements-deb.txt. -# This takes into account package dependencies. Note that downloading -# all packages at once is not possible due to the conflicts. -$(per_pkg_downloads_urls): $(workdir)/urllists/%.list: $(apt_setup_done) - apt_log="$(workdir)/log/$*.log"; \ - pkg="$(call decode_versions,$*)"; \ - if ! apt-get -c "$(apt_alt_conf)" --print-uris --yes -qq install "$$pkg" >"$@.tmp" 2>"$${apt_log}"; then \ - echo "package $* can not be installed" >> "$${apt_log}"; \ - apt-get -c "$(apt_alt_conf)" -o Debug::pkgProblemResolver=true --print-uris --yes install "$$pkg" >>"$${apt_log}" 2>&1; \ - exit 1; \ - fi; \ - mv $@.tmp $@ - -# Declaring dependency on an Essential package is not mandatory, -# hence we need to make sure the essential packages are mirrored too. -# XXX: dpkg-query works with dpkg database (and we have it empty on purpose) -# Also dpkg-query --load-avail produces wrong result, hence sed hackery -$(workdir)/essential_pkgs.list: $(apt_altstate)/var/lib/dpkg/available - sed -rne '/^Package/ h; /^Essential:\s+yes$$/ { g; s/^Package:\s+(.+)$$/\1/p; d }' < $< > $@.unsorted - sort -u < $@.unsorted > $@.tmp && \ - mv $@.tmp $@ - -essential_pkgs_urls:=$(workdir)/urllists/essential_pkgs.list -# XXX: take a shortcut for essential packages: just ask apt to download them -# (or rather print their URL). An essential package can only depend on other -# essential packages, therefore dependency calculation can be skipped -$(essential_pkgs_urls): $(workdir)/essential_pkgs.list - apt_log="$(workdir)/log/essential_pkgs.log"; \ - if ! apt-get -c "$(apt_alt_conf)" download -qq --print-uris `cat $<` > $@.tmp 2>"$$apt_log"; then \ - echo 'Essential packages can not be installed' >>"$$apt_log"; \ - exit 1; \ - fi; \ - mv $@.tmp $@ - - -# udebs are required for the provisioning with cobbler + Debian installer -# Download almost all udebs, skip the modules for the kernels different -# from that used by the installer (to save *a lot* of disk space). -$(udeb_download_urls): $(workdir)/udeb_pkgs_filtered.list - apt_log="$(workdir)/log/udeb_pkgs.log"; \ - if ! apt-get -c "$(apt_alt_conf)" download -qq --print-uris `cat $<` > $@.unsorted 2>"$$apt_log"; then \ - echo 'Udebs can not be downloaded' >>"$$apt_log"; \ - apt-get -c "$(apt_alt_conf)" download -o Debug::pkgProblemResolver=yes `cat $<` >>"$$apt_log" 2>&1; \ - exit 1; \ - fi - sed -rne "s/^'([^']+)['].*$$/\1/p" -i $@.unsorted && \ - sort -u < $@.unsorted > $@.tmp && \ - mv $@.tmp $@ - -# udebs are in Section: debian-installer or {universe,multiverse}/debian-installer -$(workdir)/udeb_pkgs.list: $(apt_altstate)/var/lib/dpkg/available - sed -rne '/^Package/h; /^Section:\s+.*debian-installer/ { g; s/Package:\s+(.+)$$/\1/p }' < $< > $@.unsorted - sort -u < $@.unsorted > $@.tmp && \ - mv $@.tmp $@ - -# Skip kernel modules udebs for kernels other than the one used by the installer -$(workdir)/udeb_pkgs_filtered.list: $(workdir)/udeb_pkgs.list - sed -re '/^(.+-modules-.*|kernel-image|kernel-signed-image).*-generic-di$$/ { s/$(UBUNTU_INSTALLER_KERNEL_VERSION)/&/p; d; }' < $< > $@.unsorted && \ - sort -u < $@.unsorted > $@.tmp && \ - mv $@.tmp $@ - - -$(deb_download_urls): $(per_pkg_downloads_urls) $(essential_pkgs_urls) - mkdir -p "$(dir $@)" && \ - cat $^ > $@.tmp.all && \ - sed -rne "s/^'([^']+)['].*$$/\1/p" -i $@.tmp.all && \ - sort -u < $@.tmp.all > $@.tmp && \ - mv $@.tmp $@ - -$(deb_download_done): $(deb_download_urls) - mkdir -p "$(target_dir)" && \ - xargs -n1 -P4 wget -nv -P "$(target_dir)" < $< || exit 1 - touch $@ - -$(udeb_download_done): $(udeb_download_urls) - mkdir -p "$(udeb_target_dir)" && \ - xargs -n1 -P4 wget -nv -P "$(udeb_target_dir)" < $< || exit 1 - touch $@ - - -combined_overrides:=$(LOCAL_MIRROR_UBUNTU)/indices/override.$(UBUNTU_RELEASE).main -combined_extra_overrides:=$(LOCAL_MIRROR_UBUNTU)/indices/override.$(UBUNTU_RELEASE).extra.main -combined_di_overrides:=$(LOCAL_MIRROR_UBUNTU)/indices/main.debian-installer - -define apt_ftparchive_conf -Dir { - ArchiveDir "$(LOCAL_MIRROR_UBUNTU)"; -}; -TreeDefault { - Directory "pool"; -} -BinDirectory "pool/main" { - Packages "dists/$(UBUNTU_RELEASE)/main/binary-$(UBUNTU_ARCH)/Packages"; - BinOverride "$(combined_overrides)"; - ExtraOverride "$(combined_extra_overrides)"; -}; -BinDirectory "pool/debian-installer" { - Packages "dists/$(UBUNTU_RELEASE)/main/debian-installer/binary-$(UBUNTU_ARCH)/Packages"; - BinOverride "$(combined_di_overrides)"; -}; -Default { - Packages { - Extensions ".deb .udeb"; - Compress ". gzip"; - }; -}; -Contents { - Compress "gzip"; -}; -endef - -apt_repo_components:=main main/debian-installer - -define apt_ftparchive_release_conf -APT::FTPArchive::Release::Origin "Ubuntu"; -APT::FTPArchive::Release::Label "Ubuntu"; -APT::FTPArchive::Release::Suite "$(UBUNTU_RELEASE)"; -APT::FTPArchive::Release::Version "$(UBUNTU_RELEASE_NUMBER)"; -APT::FTPArchive::Release::Codename "$(UBUNTU_RELEASE)"; -APT::FTPArchive::Release::Architectures "$(UBUNTU_ARCH)"; -APT::FTPArchive::Release::Components "main"; -APT::FTPArchive::Release::Description "Ubuntu $(UBUNTU_RELEASE) $(UBUNTU_RELEASE_NUMBER) LTS"; -endef - -define component_release -Archive: $(UBUNTU_RELEASE) -Version: $(UBUNTU_RELEASE_NUMBER) -Component: main -Origin: Ubuntu -Label: Ubuntu -Architecture: $(UBUNTU_ARCH) -endef - -$(repo_release_file): export APT_FTPARCHIVE_CONF:=$(apt_ftparchive_conf) -$(repo_release_file): export APT_FTPARCHIVE_RELEASE_CONF:=$(apt_ftparchive_release_conf) -$(repo_release_file): export COMPONENT_RELEASE:=$(component_release) -$(repo_release_file): $(deb_download_done) $(udeb_download_done) - mkdir -p "$(workdir)/conf" && \ - echo "$$APT_FTPARCHIVE_CONF" > "$(workdir)/conf/apt-ftparchive.conf" && \ - echo "$$APT_FTPARCHIVE_RELEASE_CONF" > "$(workdir)/conf/apt-ftparchive-release.conf" - for component in $(apt_repo_components); do \ - dists_dir="$(dir $@)$$component/binary-$(UBUNTU_ARCH)"; \ - rm -rf "$$dists_dir"; \ - mkdir -p "$$dists_dir" && \ - echo "$$COMPONENT_RELEASE" > "$$dists_dir/Release"; \ - done - apt-ftparchive -c "$(workdir)/conf/apt-ftparchive-release.conf" generate "$(workdir)/conf/apt-ftparchive.conf" -### XXX: The Packages file produced by apt-ftparchive generate is not sorted properly. -### XXX: Sort them manually to prevent the provisioning error. - for component in $(apt_repo_components); do \ - pkgs_file="$(dir $@)$$component/binary-$(UBUNTU_ARCH)/Packages"; \ - rm -f "$${pkgs_file}.gz" && \ - apt-sortpkgs "$$pkgs_file" > "$${pkgs_file}.new" && \ - gzip -c < "$${pkgs_file}.new" > "$${pkgs_file}.gz.new" && \ - mv "$${pkgs_file}.new" "$${pkgs_file}" && \ - mv "$${pkgs_file}.gz.new" "$${pkgs_file}.gz"; \ - done - apt-ftparchive -c "$(workdir)/conf/apt-ftparchive-release.conf" release "$(dir $@)" > "$@.tmp" - mv "$@.tmp" "$@" - -$(repo_release_file): $(combined_overrides) \ - $(combined_extra_overrides) \ - $(combined_di_overrides) - -ifeq (none,$(strip $(USE_MIRROR))) -overrides:=\ -override.$(UBUNTU_RELEASE).main \ -override.$(UBUNTU_RELEASE).restricted \ -override.$(UBUNTU_RELEASE).universe \ -override.$(UBUNTU_RELEASE).multiverse \ -override.$(UBUNTU_RELEASE)-updates.main \ -override.$(UBUNTU_RELEASE)-updates.restricted \ -override.$(UBUNTU_RELEASE)-updates.universe \ -override.$(UBUNTU_RELEASE)-updates.multiverse \ -override.$(UBUNTU_RELEASE)-security.main \ -override.$(UBUNTU_RELEASE)-security.restricted \ -override.$(UBUNTU_RELEASE)-security.universe \ -override.$(UBUNTU_RELEASE)-security.multiverse \ -$(empty) - -extra_overrides:=\ -override.$(UBUNTU_RELEASE).extra.main \ -override.$(UBUNTU_RELEASE).extra.restricted \ -override.$(UBUNTU_RELEASE).extra.universe \ -override.$(UBUNTU_RELEASE).extra.multiverse \ -override.$(UBUNTU_RELEASE)-updates.extra.main \ -override.$(UBUNTU_RELEASE)-updates.extra.restricted \ -override.$(UBUNTU_RELEASE)-updates.extra.universe \ -override.$(UBUNTU_RELEASE)-updates.extra.multiverse \ -override.$(UBUNTU_RELEASE)-security.extra.main \ -override.$(UBUNTU_RELEASE)-security.extra.restricted \ -override.$(UBUNTU_RELEASE)-security.extra.universe \ -override.$(UBUNTU_RELEASE)-security.extra.multiverse \ -$(empty) - -di_overrides:=\ -override.$(UBUNTU_RELEASE).main.debian-installer \ -override.$(UBUNTU_RELEASE).restricted.debian-installer \ -override.$(UBUNTU_RELEASE).universe.debian-installer \ -override.$(UBUNTU_RELEASE).multiverse.debian-installer \ -override.$(UBUNTU_RELEASE)-updates.main.debian-installer \ -override.$(UBUNTU_RELEASE)-updates.restricted.debian-installer \ -override.$(UBUNTU_RELEASE)-updates.universe.debian-installer \ -override.$(UBUNTU_RELEASE)-updates.multiverse.debian-installer \ -override.$(UBUNTU_RELEASE)-security.main.debian-installer \ -override.$(UBUNTU_RELEASE)-security.restricted.debian-installer \ -override.$(UBUNTU_RELEASE)-security.universe.debian-installer \ -override.$(UBUNTU_RELEASE)-security.multiverse.debian-installer \ -$(empty) - -overrides_dl:=$(foreach idx,$(overrides) $(extra_overrides) $(di_overrides),$(workdir)/indices/$(idx)) - -$(overrides_dl): $(workdir)/indices/%: - mkdir -p "$(dir $@)" && \ - rm -f "$@.tmp" && \ - wget -nv -O "$@.tmp" "$(MIRROR_UBUNTU)/indices/$*" && \ - mv "$@.tmp" "$@" - -$(combined_overrides): $(overrides:%=$(workdir)/indices/%) - mkdir -p "$(dir $@)" && \ - cat $^ > "$@.unsorted" && \ - sed -re '/Supported/d' -i "$@.unsorted" && \ - sort -u < "$@.unsorted" > "$@.tmp" && \ - mv "$@.tmp" "$@" - -$(combined_extra_overrides): $(extra_overrides:%=$(workdir)/indices/%) - mkdir -p "$(dir $@)" && \ - cat $^ > "$@.unsorted" && \ - sed -re '/Supported/d' -i "$@.unsorted" && \ - sort -u < "$@.unsorted" > "$@.tmp" && \ - mv "$@.tmp" "$@" - -$(combined_di_overrides): $(di_overrides:%=$(workdir)/indices/%) - mkdir -p "$(dir $@)" && \ - cat $^ > "$@.unsorted" && \ - sed -re '/Supported/d' -i "$@.unsorted" && \ - sort -u < "$@.unsorted" > "$@.tmp" && \ - mv "$@.tmp" "$@" - -else -overrides_dl:=$(combined_overrides) $(combined_extra_overrides) $(combined_di_overrides) - -$(overrides_dl): $(LOCAL_MIRROR_UBUNTU)/indices/%: - mkdir -p "$(dir $@)" && \ - rm -f "$@.tmp" && \ - wget -nv -O "$@.tmp" "$(MIRROR_UBUNTU)/indices/$*" && \ - mv $@.tmp -endif