From b9a580d2c7efbb1817a573c2749aa24d067032d1 Mon Sep 17 00:00:00 2001 From: Alexei Sheplyakov Date: Mon, 22 Dec 2014 12:32:48 +0300 Subject: [PATCH] parallel make fixes: add missing dependencies * packages: - build each package in a separate directory so sereral packages can be built in parallel. - prepare_*_source: make sure git repositories are cloned before using them. * image/centos, bootstrap, docker: add dependency on the locally built nailgun, astute and co. RPMs to make sure RPMs are ready before trying to install them. * docker: - start an instance of the simple http server per a container being built, make sure those processes listen the different ports. - fix sporadic build failures due to the HTTP daemon premature exit. The simple_http_daemon which serves RPM repository exits in 600 seconds by default. However a container might require more time (due to a slow machine, a high load, etc), thus the HTTP server might exit before the required packages have been downloaded. Set the default timeout to infinity to prevent such an obscure error. * repos.mk: several repositories can be cloned at the same time, so writes to version.yaml should be serialized. To keep things simple create version.yaml after cloning the repositories (with Fuel components). While at it polish the rule which clones the repository. * packages/rpm, bootstrap: depend only on CentOS repo (more tasks can run in parallel). blueprint support-ubuntu-trusty Change-Id: I3a72802a7728121581e71227b013bb7aeffaf049 --- bootstrap/module.mk | 7 ++++--- docker/module.mk | 2 ++ image/centos/module.mk | 3 ++- packages/deb/module.mk | 9 ++++++--- packages/module.mk | 4 ++++ packages/rpm/module.mk | 13 ++++++++----- repos.mk | 29 +++++++++++++++++------------ utils/simple_http_daemon.py | 2 +- 8 files changed, 44 insertions(+), 25 deletions(-) diff --git a/bootstrap/module.mk b/bootstrap/module.mk index 3e58167f5..97fc1e505 100644 --- a/bootstrap/module.mk +++ b/bootstrap/module.mk @@ -98,7 +98,7 @@ $(BUILD_DIR)/bootstrap/initramfs.img: \ sudo sh -c "cd $(INITRAMROOT) && find . -xdev | cpio --create \ --format='newc' | gzip -9 > $(BUILD_DIR)/bootstrap/initramfs.img" -$(BUILD_DIR)/bootstrap/linux: $(BUILD_DIR)/mirror/build.done +$(BUILD_DIR)/bootstrap/linux: $(BUILD_DIR)/mirror/centos/build.done mkdir -p $(BUILD_DIR)/bootstrap find $(LOCAL_MIRROR_CENTOS_OS_BASEURL) -name '$(KERNEL_PATTERN)' | xargs rpm2cpio | \ (cd $(BUILD_DIR)/bootstrap/; cpio -imd './boot/vmlinuz*') @@ -114,7 +114,7 @@ $(BUILD_DIR)/bootstrap/etc/yum.conf $(BUILD_DIR)/bootstrap/etc/yum.repos.d/base. $(BUILD_DIR)/bootstrap/customize-initram-root.done: $(call depv,BOOTSTRAP_RPMS_CUSTOM) $(BUILD_DIR)/bootstrap/customize-initram-root.done: \ - $(BUILD_DIR)/packages/build.done \ + $(BUILD_DIR)/packages/rpm/build.done \ $(BUILD_DIR)/bootstrap/prepare-initram-root.done \ $(call find-files,$(SOURCE_DIR)/bootstrap/sync) \ $(BUILD_DIR)/repos/nailgun.done \ @@ -161,7 +161,8 @@ $(BUILD_DIR)/bootstrap/customize-initram-root.done: \ $(BUILD_DIR)/bootstrap/prepare-initram-root.done: $(call depv,BOOTSTRAP_RPMS) $(BUILD_DIR)/bootstrap/prepare-initram-root.done: \ - $(BUILD_DIR)/mirror/build.done \ + $(BUILD_DIR)/mirror/centos/build.done \ + $(BUILD_DIR)/packages/rpm/build.done \ $(BUILD_DIR)/bootstrap/etc/yum.conf \ $(BUILD_DIR)/bootstrap/etc/yum.repos.d/base.repo diff --git a/docker/module.mk b/docker/module.mk index e482ff92b..4c6ffbc2e 100644 --- a/docker/module.mk +++ b/docker/module.mk @@ -34,9 +34,11 @@ define build_container ifndef DOCKER_DEP_FILE $(BUILD_DIR)/docker/build.done: $(BUILD_DIR)/docker/$1.done endif +$(eval RANDOM_PORT:=$(shell echo $$(($(RANDOM_PORT)+1)))) $(BUILD_DIR)/docker/$1.done: \ $(BUILD_DIR)/mirror/build.done \ $(BUILD_DIR)/repos/repos.done \ + $(BUILD_DIR)/packages/build.done \ $(BUILD_DIR)/iso/isoroot-files.done \ $(BUILD_DIR)/docker/base-images.done (cd $(LOCAL_MIRROR_CENTOS) && python $(SOURCE_DIR)/utils/simple_http_daemon.py $(RANDOM_PORT) /tmp/simple_http_daemon_$(RANDOM_PORT).pid) diff --git a/image/centos/module.mk b/image/centos/module.mk index 622433bd7..b1ea7d507 100644 --- a/image/centos/module.mk +++ b/image/centos/module.mk @@ -19,7 +19,8 @@ ifdef TARGET_CENTOS_DEP_FILE $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(TARGET_CENTOS_DEP_FILE) $(ACTION.COPY) else -$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/mirror/build.done +$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/mirror/centos/build.done +$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/packages/rpm/build.done $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): SANDBOX:=$(BUILD_DIR)/image/centos/SANDBOX $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): export SANDBOX_UP:=$(SANDBOX_UP) $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): export SANDBOX_DOWN:=$(SANDBOX_DOWN) diff --git a/packages/deb/module.mk b/packages/deb/module.mk index cdaa97d73..70f4e746f 100644 --- a/packages/deb/module.mk +++ b/packages/deb/module.mk @@ -3,8 +3,11 @@ clean: clean-deb clean-deb: - -sudo umount $(BUILD_DIR)/packages/deb/SANDBOX/proc - -sudo umount $(BUILD_DIR)/packages/deb/SANDBOX/dev + -mount | grep '$(BUILD_DIR)/packages/deb/SANDBOX' | while read entry; do \ + set -- $$entry; \ + mntpt="$$3"; \ + sudo umount $$mntpt; \ + done sudo rm -rf $(BUILD_DIR)/packages/deb # Usage: @@ -14,7 +17,7 @@ $(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1.done $(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1-repocleanup.done $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/mirror/ubuntu/build.done $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/packages/source_$1.done -$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX +$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX/$1 $(BUILD_DIR)/packages/deb/$1.done: SANDBOX_DEB_PKGS:=apt wget bzip2 apt-utils build-essential python-setuptools devscripts debhelper fakeroot $(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_UP:=$$(SANDBOX_UBUNTU_UP) $(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_DOWN:=$$(SANDBOX_UBUNTU_DOWN) diff --git a/packages/module.mk b/packages/module.mk index 94fa041f8..acb8d26b0 100644 --- a/packages/module.mk +++ b/packages/module.mk @@ -3,6 +3,7 @@ # Note: dependencies for deb targets are also specified here to make # sure the source is ready before the build is started. define prepare_file_source +$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2 $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3) mkdir -p $(BUILD_DIR)/packages/sources/$1 @@ -14,6 +15,7 @@ endef # Note: dependencies for deb targets are also specified here to make # sure the source is ready before the build is started. define prepare_python_source +$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2 $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3) mkdir -p $(BUILD_DIR)/packages/sources/$1 @@ -30,6 +32,7 @@ endef # Note: dependencies for deb targets are also specified here to make # sure the source is ready before the build is started. define prepare_tgz_source +$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2 $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3) mkdir -p $(BUILD_DIR)/packages/sources/$1 @@ -41,6 +44,7 @@ endef # Note: dependencies for deb targets are also specified here to make # sure the source is ready before the build is started. define prepare_ruby21_source +$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2 $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3) mkdir -p $(BUILD_DIR)/packages/sources/$1 diff --git a/packages/rpm/module.mk b/packages/rpm/module.mk index 98fe23938..fa876d3d3 100644 --- a/packages/rpm/module.mk +++ b/packages/rpm/module.mk @@ -3,8 +3,11 @@ clean: clean-rpm clean-rpm: - -sudo umount $(BUILD_DIR)/packages/rpm/SANDBOX/proc - -sudo umount $(BUILD_DIR)/packages/rpm/SANDBOX/dev + -mount | grep '$(BUILD_DIR)/packages/rpm/SANDBOX' | while read entry; do \ + set -- $$entry; \ + mntpt="$$3"; \ + sudo umount $$mntpt; \ + done sudo rm -rf $(BUILD_DIR)/packages/rpm RPM_SOURCES:=$(BUILD_DIR)/packages/rpm/SOURCES @@ -19,11 +22,11 @@ $(BUILD_DIR)/packages/rpm/repo.done: $(BUILD_DIR)/packages/rpm/$1-repocleanup.do # It will build astute rpm package $1: $(BUILD_DIR)/packages/rpm/$1.done -$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/mirror/build.done +$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/mirror/centos/build.done $(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/source_$1.done -$(BUILD_DIR)/packages/rpm/$1.done: SANDBOX:=$(BUILD_DIR)/packages/rpm/SANDBOX +$(BUILD_DIR)/packages/rpm/$1.done: SANDBOX:=$(BUILD_DIR)/packages/rpm/SANDBOX/$1 $(BUILD_DIR)/packages/rpm/$1.done: export SANDBOX_UP:=$$(SANDBOX_UP) $(BUILD_DIR)/packages/rpm/$1.done: export SANDBOX_DOWN:=$$(SANDBOX_DOWN) $(BUILD_DIR)/packages/rpm/$1.done: \ @@ -40,7 +43,7 @@ $(BUILD_DIR)/packages/rpm/$1.done: \ sudo sh -c "$$$${SANDBOX_DOWN}" $$(ACTION.TOUCH) -$(BUILD_DIR)/packages/rpm/$1-repocleanup.done: $(BUILD_DIR)/mirror/build.done +$(BUILD_DIR)/packages/rpm/$1-repocleanup.done: $(BUILD_DIR)/mirror/centos/build.done find $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages -regex '.*/$1-[^-]+-[^-]+' -delete $$(ACTION.TOUCH) endef diff --git a/repos.mk b/repos.mk index 37ce02ae3..adf00045a 100644 --- a/repos.mk +++ b/repos.mk @@ -2,16 +2,13 @@ repos: $(BUILD_DIR)/repos/repos.done -$(BUILD_DIR)/repos/repos.done: - sed -i '/^ fuelmain_sha:/d' $(BUILD_DIR)/repos/version.yaml - /bin/echo " fuelmain_sha: \"`git rev-parse --verify HEAD`\"" >> $(BUILD_DIR)/repos/version.yaml - $(ACTION.TOUCH) - +fuel_components_repos:= # Usage: # (eval (call build_repo,repo_name,repo_uri,sha)) define build_repo $(BUILD_DIR)/repos/$1/%: $(BUILD_DIR)/repos/$1.done $(BUILD_DIR)/repos/repos.done: $(BUILD_DIR)/repos/$1.done +fuel_components_repos:=$(fuel_components_repos) $1 $(BUILD_DIR)/repos/$1.done: # Clone repo and checkout required commit @@ -22,17 +19,25 @@ $(BUILD_DIR)/repos/$1.done: git clone $2 $(BUILD_DIR)/repos/$1 && (cd $(BUILD_DIR)/repos/$1 && git checkout -q $3) # Pull gerrit commits if given - $(foreach var,$5, - test "$(var)" = "none" || ( cd $(BUILD_DIR)/repos/$1 && git fetch $4 $(var) && git cherry-pick FETCH_HEAD ) ; + $(foreach var,$(filter-out none,$5), + ( cd $(BUILD_DIR)/repos/$1 && git fetch $4 $(var) && git cherry-pick FETCH_HEAD ) ; ) - # Update versions.yaml - touch $(BUILD_DIR)/repos/version.yaml - sed -i '/^ $1_sha:/d' $(BUILD_DIR)/repos/version.yaml - /bin/echo " $1_sha: \"`cd $(BUILD_DIR)/repos/$1 && git rev-parse --verify HEAD`\"" >> $(BUILD_DIR)/repos/version.yaml - touch $(BUILD_DIR)/repos/$1.done + touch $$@ endef + $(eval $(call build_repo,nailgun,$(NAILGUN_REPO),$(NAILGUN_COMMIT),$(NAILGUN_GERRIT_URL),$(NAILGUN_GERRIT_COMMIT))) $(eval $(call build_repo,astute,$(ASTUTE_REPO),$(ASTUTE_COMMIT),$(ASTUTE_GERRIT_URL),$(ASTUTE_GERRIT_COMMIT))) $(eval $(call build_repo,fuellib,$(FUELLIB_REPO),$(FUELLIB_COMMIT),$(FUELLIB_GERRIT_URL),$(FUELLIB_GERRIT_COMMIT))) $(eval $(call build_repo,ostf,$(OSTF_REPO),$(OSTF_COMMIT),$(OSTF_GERRIT_URL),$(OSTF_GERRIT_COMMIT))) + +$(BUILD_DIR)/repos/repos.done: + version_yaml=$(BUILD_DIR)/repos/version.yaml; \ + for repo in $(strip $(fuel_components_repos)); do \ + repo_commit_id=`git --git-dir=$(BUILD_DIR)/repos/$$repo/.git rev-parse --verify HEAD`; \ + echo " $${repo}_sha: \"$${repo_commit_id}\""; \ + done > $${version_yaml}.tmp; \ + fuel_main_commit_id=`git rev-parse --verify HEAD`; \ + echo " fuelmain_sha: \"${fuelmain_commit_id}\"" >> $${version_yaml}.tmp; \ + mv $${version_yaml}.tmp $${version_yaml} + $(ACTION.TOUCH) diff --git a/utils/simple_http_daemon.py b/utils/simple_http_daemon.py index ca51796d0..bc4fccd75 100755 --- a/utils/simple_http_daemon.py +++ b/utils/simple_http_daemon.py @@ -67,7 +67,7 @@ if __name__ == "__main__": if sys.argv[3:]: timeout = int(sys.argv[3]) else: - timeout = 600 + timeout = sys.maxint server = SimpleHTTPDaemon('0.0.0.0', port, pid, timeout) server.start()