Add building of Fuel deb packages during ISO build.

1) Add routines for building deb packages and adding them to local mirror.
2) Add commands for creating Ubuntu-based build sandbox.
3) Update deb specs for Fuel components.

Closes-Bug: 1373400
Change-Id: If098654d1158aca6676f9cbb3fc3100fc2e8674d
This commit is contained in:
Vitaly Parakhin 2014-09-08 14:15:49 +03:00
parent 863cc95244
commit 967ec61eb6
13 changed files with 269 additions and 24 deletions

View File

@ -1,5 +1,64 @@
.PHONY: clean clean-deb
include $(SOURCE_DIR)/packages/deb/debian-boot/module.mk
$(BUILD_DIR)/packages/deb/build.done: $(BUILD_DIR)/packages/deb/debian-boot/build.done
clean: clean-deb
clean-deb:
-sudo umount $(BUILD_DIR)/packages/deb/SANDBOX/proc
-sudo umount $(BUILD_DIR)/packages/deb/SANDBOX/dev
sudo rm -rf $(BUILD_DIR)/packages/deb
# Usage:
# (eval (call build_deb,package_name))
define build_deb
$(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1.done
$(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1-repocleanup.done
$(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/mirror/build.done
$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX
$(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_UP:=$$(SANDBOX_UBUNTU_UP)
$(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_DOWN:=$$(SANDBOX_UBUNTU_DOWN)
$(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/repos/repos.done
mkdir -p $(BUILD_DIR)/packages/deb/packages $(BUILD_DIR)/packages/deb/sources
mkdir -p $$(SANDBOX_UBUNTU)
mkdir -p $$(SANDBOX_UBUNTU)/proc
sed -e "s/@@UBUNTU_RELEASE@@/$(UBUNTU_RELEASE)/g" $$(SOURCE_DIR)/packages/multistrap.conf | sudo tee $$(SANDBOX_UBUNTU)/multistrap.conf
sudo sh -c "$$$${SANDBOX_UBUNTU_UP}"
sudo chroot $$(SANDBOX_UBUNTU) /bin/bash -c "apt-get update"
sudo mkdir -p $$(SANDBOX_UBUNTU)/tmp/$1
ifeq ($1,nailgun-net-check)
tar zxf $(BUILD_DIR)/packages/sources/$1/$1*.tar.gz -C $(BUILD_DIR)/packages/deb/sources
sudo cp -r $(BUILD_DIR)/packages/deb/sources/$1*/* $$(SANDBOX_UBUNTU)/tmp/$1/
endif
sudo cp -r $(BUILD_DIR)/packages/sources/$1/* $$(SANDBOX_UBUNTU)/tmp/$1/
sudo cp -r $(SOURCE_DIR)/packages/deb/specs/$1/* $$(SANDBOX_UBUNTU)/tmp/$1/
dpkg-checkbuilddeps $(SOURCE_DIR)/packages/deb/specs/$1/debian/control 2>&1 | sed 's/^dpkg-checkbuilddeps: Unmet build dependencies: //g' | sed 's/([^()]*)//g;s/|//g' | sudo tee $$(SANDBOX_UBUNTU)/tmp/$1.installdeps
sudo chroot $$(SANDBOX_UBUNTU) /bin/bash -c "cat /tmp/$1.installdeps | xargs apt-get -y install"
sudo chroot $$(SANDBOX_UBUNTU) /bin/bash -c "cd /tmp/$1 ; DEB_BUILD_OPTIONS=nocheck debuild -us -uc -b -d"
cp $$(SANDBOX_UBUNTU)/tmp/*$1*.deb $(BUILD_DIR)/packages/deb/packages
sudo rm -rf $$(SANDBOX_UBUNTU)/tmp/*
sudo sh -c "$$$${SANDBOX_UBUNTU_DOWN}"
$$(ACTION.TOUCH)
$(BUILD_DIR)/packages/deb/$1-repocleanup.done: $(BUILD_DIR)/mirror/build.done
sudo find $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/pool/main -regex '.*$1_[^-]+-[^-]+.*' -delete
$$(ACTION.TOUCH)
endef
$(eval $(call build_deb,fencing-agent))
$(eval $(call build_deb,nailgun-mcagents))
$(eval $(call build_deb,nailgun-net-check))
$(eval $(call build_deb,nailgun-agent))
$(BUILD_DIR)/packages/deb/repo.done:
sudo find $(BUILD_DIR)/packages/deb/packages -name '*.deb' -exec cp -u {} $(LOCAL_MIRROR_UBUNTU_OS_BASEURL)/pool/main \;
echo "Applying fix for upstream bug in dpkg..."
-sudo patch -N /usr/bin/dpkg-scanpackages < $(SOURCE_DIR)/packages/dpkg.patch
sudo $(SOURCE_DIR)/packages/regenerate_ubuntu_repo $(LOCAL_MIRROR_UBUNTU_OS_BASEURL) $(UBUNTU_RELEASE)
$(ACTION.TOUCH)
$(BUILD_DIR)/packages/deb/build.done: $(BUILD_DIR)/packages/deb/repo.done \
$(BUILD_DIR)/packages/deb/debian-boot/build.done
$(ACTION.TOUCH)

View File

@ -1,3 +1,9 @@
fencing-agent (0.1.0-ubuntu1) precise; urgency=low
* Update code from upstream
-- OSCI Jenkins <dburmistrov@mirantis.com> Wed, 03 Sep 2014 15:17:07 +0400
fencing-agent (0.1.0-1ubuntu0) precise; urgency=low
* Add fencing agent script as a separated task

View File

@ -1,3 +1,9 @@
nailgun-agent (3.9.2-ubuntu6) precise; urgency=low
* Update code from upstream
-- OSCI Jenkins <dburmistrov@mirantis.com> Wed, 03 Sep 2014 15:16:22 +0400
nailgun-agent (3.9.2-1ubuntu10) precise; urgency=low
* Add new version code

View File

@ -1 +1 @@
* * * * * root flock -w 0 -o /var/lock/agent.lock -c "/opt/nailgun/bin/agent >> /var/log/nailgun-agent.log 2>&1"
* * * * * root flock -w 0 -o /var/lock/agent.lock -c "/opt/nailgun/bin/agent 2>&1 | tee -a /var/log/nailgun-agent.log | /usr/bin/logger -t nailgun-agent"

View File

@ -1,3 +1,9 @@
nailgun-mcagents (0.0.1-ubuntu1) precise; urgency=low
* Update code from upstream
-- OSCI Jenkins <dburmistrov@mirantis.com> Wed, 03 Sep 2014 15:20:13 +0400
nailgun-mcagents (0.0.1) unstable; urgency=low
* Initial release.

View File

@ -11,7 +11,7 @@ clean:
build-arch:
dh_testdir
mkdir -p debian/nailgun-mcagents/usr/share/mcollective/plugins/mcollective/agent/
tar -xzf nailgun-mcagents.tar.gz -C debian/nailgun-mcagents/usr/share/mcollective/plugins/mcollective/agent/
tar -xzf mcagents.tar.gz -C debian/nailgun-mcagents/usr/share/mcollective/plugins/mcollective/agent/
build: build-arch build-indep

View File

@ -1,3 +1,9 @@
nailgun-net-check (0.2-ubuntu1) precise; urgency=low
* Update code from upstream
-- OSCI Jenkins <dburmistrov@mirantis.com> Wed, 03 Sep 2014 15:18:56 +0400
nailgun-net-check (0.2-0) unstable; urgency=low
* Initial release.

49
packages/dpkg.patch Normal file
View File

@ -0,0 +1,49 @@
From 09efec567e6b0baa835c1921c222a5a967cf193a Mon Sep 17 00:00:00 2001
From: Dennis Dmitriev <ddmitriev@mirantis.com>
Date: Tue, 19 Aug 2014 18:31:33 +0300
Subject: [PATCH] Allow correctly parcing indices/override.* files
Allow parcing indices/override.* files when architecture
prefixes are used in package names there. It allows to use
dpkg-scanpackages with Ubuntu repositories.
Originally, dpkg-scanpackages just skip the following lines because it
compares package names without any modification, and cannot find the
apropriate package because 'a2jmidid' != 'a2jmidid/amd64'
For example, several lines from override.precise.extra.main:
...
a2jmidid Bugs https://bugs.launchpad.net/ubuntu/+filebug
a2jmidid Origin Ubuntu
a2jmidid/amd64 Task ubuntustudio-generation, ubuntustudio-recording
a2jmidid/armel Task ubuntustudio-generation, ubuntustudio-recording
a2jmidid/armhf Task ubuntustudio-generation, ubuntustudio-recording
a2jmidid/i386 Task ubuntustudio-generation, ubuntustudio-recording
a2jmidid/powerpc Task ubuntustudio-generation, ubuntustudio-recording
...
Related-Bug: 1358785
---
scripts/dpkg-scanpackages.pl | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/scripts/dpkg-scanpackages.pl b/scripts/dpkg-scanpackages.pl
index 9a3d59c..3d6724b 100755
--- a/scripts/dpkg-scanpackages.pl
+++ b/scripts/dpkg-scanpackages.pl
@@ -143,7 +143,12 @@ sub load_override_extra
s/\s+$//;
next unless $_;
- my ($p, $field, $value) = split(/\s+/, $_, 3);
+ my ($pr, $field, $value) = split(/\s+/, $_, 3);
+
+ my ($p, $parch) = split(/\//, $pr, 2);
+ if (defined($options{arch}) and defined($parch)) {
+ next unless ($options{arch} eq $parch);
+ }
next unless defined($packages{$p});
--
1.8.5.5

View File

@ -5,12 +5,10 @@ include $(SOURCE_DIR)/packages/deb/module.mk
ifneq ($(BUILD_PACKAGES),0)
$(BUILD_DIR)/packages/build.done: \
$(BUILD_DIR)/packages/deb/build.done \
$(BUILD_DIR)/packages/rpm/build.done
$(BUILD_DIR)/packages/rpm/build.done $(BUILD_DIR)/packages/deb/build.done
endif
$(BUILD_DIR)/packages/build.done: \
$(BUILD_DIR)/packages/deb/build.done
$(BUILD_DIR)/packages/build.done:
$(ACTION.TOUCH)
packages: $(BUILD_DIR)/packages/build.done

40
packages/multistrap.conf Normal file
View File

@ -0,0 +1,40 @@
[General]
# arch and directory can be specified on the command line.
#arch=armel
#directory=/opt/multistrap/
# same as --tidy-up option if set to true
cleanup=false
# retain the sources outside the rootfs for distribution
# specify a directory to which all the .debs can be moved.
# or override with the --source-dir option.
# same as --no-auth option if set to true
# keyring packages listed in each debootstrap will
# still be installed.
noauth=true
# retries not needed.
#retries=5
# extract all downloaded archives
unpack=true
bootstrap=Ubuntu Updates Security Mirantis
# the order of sections is no longer important.
# debootstrap determines which repository is used to
# calculate the list of Priority: required packages
# the order of sections is no longer important.
# aptsources is a list of sections to be listed
# in the /etc/apt/sources.list.d/multistrap.sources.list
# of the target.
aptsources=Ubuntu Updates Security Mirantis
[Ubuntu]
packages=apt wget bzip2 apt-utils build-essential python-setuptools devscripts debhelper fakeroot
source=http://mirror.yandex.ru/ubuntu
suite=@@UBUNTU_RELEASE@@
omitdebsrc=true
[Security]
source=http://mirror.yandex.ru/ubuntu/
suite=@@UBUNTU_RELEASE@@-security
omitdebsrc=true
[Updates]
source=http://mirror.yandex.ru/ubuntu/
suite=@@UBUNTU_RELEASE@@-updates
omitdebsrc=true

57
packages/regenerate_ubuntu_repo Executable file
View File

@ -0,0 +1,57 @@
#!/bin/bash
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# This script will rebuild local ubuntu mirror.
# Based on the method described here:
# http://troubleshootingrange.blogspot.com/2012/09/hosting-simple-apt-repository-on-centos.html
# Example:
# regenerate_ubuntu_repo /path/to/ubuntu/repo precise
REPO_PATH=$1
REPONAME=$2
BINDIR=${REPO_PATH}/dists/${REPONAME}/main
binoverride=indices/override.${REPONAME}.main
extraoverride=indices/override.${REPONAME}.extra.main
package_deb=${BINDIR}/binary-amd64/Packages
release_header=`head -8 ${REPO_PATH}/dists/${REPONAME}/Release`
cd ${REPO_PATH}
echo "Regenerating Ubuntu local mirror..."
# Scan *.deb packages
dpkg-scanpackages -m --extra-override ${extraoverride} -a amd64 pool/main ${binoverride} > $package_deb 2>/dev/null
gzip -9c $package_deb > ${package_deb}.gz
# Generate release file
cd ${REPO_PATH}/dists/${REPONAME}
echo "$release_header" > Release
# Generate hashes
c1=(MD5Sum: SHA1: SHA256: SHA512:)
c2=(md5 sha1 sha256 sha512)
i=0
while [ $i -lt ${#c1[*]} ]; do
echo ${c1[i]} >> Release
for hashme in `find main -type f \( -name "Package*" -o -name "Release*" \)`; do
chash=`openssl dgst -${c2[$i]} ${hashme}|cut -d" " -f 2`
size=`stat -c %s ${hashme}`
echo " ${chash} ${size} ${hashme}" >> Release
done
i=$(( $i + 1));
done

View File

@ -12,42 +12,42 @@ RPM_SOURCES:=$(BUILD_DIR)/packages/rpm/SOURCES
# Usage:
# (eval (call prepare_file_source,package_name,file_name,source_path))
define prepare_file_source
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/rpm/sources/$1/$2
$(BUILD_DIR)/packages/rpm/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/rpm/sources/$1
cp $3 $(BUILD_DIR)/packages/rpm/sources/$1/$2
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/sources/$1/$2
$(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/sources/$1
cp $3 $(BUILD_DIR)/packages/sources/$1/$2
endef
# Usage:
# (eval (call prepare_python_source,package_name,file_name,source_path))
define prepare_python_source
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/rpm/sources/$1/$2
$(BUILD_DIR)/packages/rpm/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/rpm/sources/$1
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/sources/$1/$2
$(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/sources/$1
ifeq ($1,nailgun)
cd $3 && npm install && grunt build --static-dir=compressed_static
rm -rf $3/static
mv $3/compressed_static $3/static
endif
cd $3 && python setup.py sdist -d $(BUILD_DIR)/packages/rpm/sources/$1
cd $3 && python setup.py sdist -d $(BUILD_DIR)/packages/sources/$1
endef
# Usage:
# (eval (call prepare_tgz_source,package_name,file_name,source_path))
define prepare_tgz_source
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/rpm/sources/$1/$2
$(BUILD_DIR)/packages/rpm/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/rpm/sources/$1
cd $3 && tar zcf $(BUILD_DIR)/packages/rpm/sources/$1/$2 *
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/sources/$1/$2
$(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/sources/$1
cd $3 && tar zcf $(BUILD_DIR)/packages/sources/$1/$2 *
endef
# Usage:
# (eval (call prepare_ruby21_source,package_name,file_name,source_path))
define prepare_ruby21_source
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/rpm/sources/$1/$2
$(BUILD_DIR)/packages/rpm/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/rpm/sources/$1
cd $3 && gem build *.gemspec && cp $2 $(BUILD_DIR)/packages/rpm/sources/$1/$2
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/sources/$1/$2
$(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/sources/$1
cd $3 && gem build *.gemspec && cp $2 $(BUILD_DIR)/packages/sources/$1/$2
endef
# Usage:
@ -72,7 +72,7 @@ $(BUILD_DIR)/packages/rpm/$1.done: \
sudo sh -c "$$$${SANDBOX_UP}"
sudo yum -c $$(SANDBOX)/etc/yum.conf --installroot=$$(SANDBOX) -y --nogpgcheck install ruby rpm-build tar python-setuptools python-pbr
sudo mkdir -p $$(SANDBOX)/tmp/SOURCES
sudo cp -r $(BUILD_DIR)/packages/rpm/sources/$1/* $$(SANDBOX)/tmp/SOURCES
sudo cp -r $(BUILD_DIR)/packages/sources/$1/* $$(SANDBOX)/tmp/SOURCES
sudo cp $(SOURCE_DIR)/packages/rpm/specs/$1.spec $$(SANDBOX)/tmp
sudo chroot $$(SANDBOX) rpmbuild --nodeps -vv --define "_topdir /tmp" -ba /tmp/$1.spec
cp $$(SANDBOX)/tmp/RPMS/*/$1-*.rpm $(BUILD_DIR)/packages/rpm/RPMS/x86_64

View File

@ -50,3 +50,21 @@ sync
umount $(SANDBOX)/proc
umount $(SANDBOX)/dev
endef
define SANDBOX_UBUNTU_UP
mount | grep -q $(SANDBOX_UBUNTU)/proc || sudo mount -t proc none $(SANDBOX_UBUNTU)/proc
[ -f $(SANDBOX_UBUNTU)/etc/debian_version ] || sudo multistrap -a amd64 -f $(SANDBOX_UBUNTU)/multistrap.conf -d $(SANDBOX_UBUNTU)/
sudo chroot $(SANDBOX_UBUNTU) /bin/bash -c "locale-gen en_US.UTF-8 ; dpkg-reconfigure locales"
sudo chroot $(SANDBOX_UBUNTU) /bin/bash -c "dpkg --configure -a || exit 0"
sudo chroot $(SANDBOX_UBUNTU) /bin/bash -c "rm -rf /var/run/*"
sudo chroot $(SANDBOX_UBUNTU) /bin/bash -c "dpkg --configure -a || exit 0"
echo 'APT::Get::AllowUnauthenticated 1;' | sudo tee $(SANDBOX_UBUNTU)/etc/apt/apt.conf.d/02mirantis-unauthenticated
[ -n "$(EXTRA_DEB_REPOS)" ] && echo "$(EXTRA_DEB_REPOS)" | tr '|' '\n' | while read repo; do echo deb $$repo; done | sudo tee $(SANDBOX_UBUNTU)/etc/apt/sources.list.d/extra.list || exit 0
sudo cp /etc/resolv.conf $(SANDBOX_UBUNTU)/etc/resolv.conf
mount | grep -q $(SANDBOX_UBUNTU)/proc || sudo mount -t proc none $(SANDBOX_UBUNTU)/proc
endef
define SANDBOX_UBUNTU_DOWN
sync
sudo umount $(SANDBOX_UBUNTU)/proc
endef