parallel make fixes: add missing dependencies

* packages: - build each package in a separate directory so sereral packages can be built in parallel. - prepare_*_source: make sure git repositories are cloned before using them. * image/centos, bootstrap, docker: add dependency on the locally built nailgun, astute and co. RPMs to make sure RPMs are ready before trying to install them. * docker: - start an instance of the simple http server per a container being built, make sure those processes listen the different ports. - fix sporadic build failures due to the HTTP daemon premature exit. The simple_http_daemon which serves RPM repository exits in 600 seconds by default. However a container might require more time (due to a slow machine, a high load, etc), thus the HTTP server might exit before the required packages have been downloaded. Set the default timeout to infinity to prevent such an obscure error. * repos.mk: several repositories can be cloned at the same time, so writes to version.yaml should be serialized. To keep things simple create version.yaml after cloning the repositories (with Fuel components). While at it polish the rule which clones the repository. * packages/rpm, bootstrap: depend only on CentOS repo (more tasks can run in parallel). blueprint support-ubuntu-trusty Change-Id: I3a72802a7728121581e71227b013bb7aeffaf049
2014-12-22 12:32:48 +03:00 · 2014-12-22 12:32:48 +03:00 · b9a580d2c7
parent c83a59b403
commit b9a580d2c7
8 changed files with 44 additions and 25 deletions
--- a/bootstrap/module.mk
+++ b/bootstrap/module.mk
@ -98,7 +98,7 @@ $(BUILD_DIR)/bootstrap/initramfs.img: \
 	sudo sh -c "cd $(INITRAMROOT) && find . -xdev | cpio --create \
        --format='newc' | gzip -9 > $(BUILD_DIR)/bootstrap/initramfs.img"

-$(BUILD_DIR)/bootstrap/linux: $(BUILD_DIR)/mirror/build.done
+$(BUILD_DIR)/bootstrap/linux: $(BUILD_DIR)/mirror/centos/build.done
 	mkdir -p $(BUILD_DIR)/bootstrap
 	find $(LOCAL_MIRROR_CENTOS_OS_BASEURL) -name '$(KERNEL_PATTERN)' | xargs rpm2cpio | \
 		(cd $(BUILD_DIR)/bootstrap/; cpio -imd './boot/vmlinuz*')
@ -114,7 +114,7 @@ $(BUILD_DIR)/bootstrap/etc/yum.conf $(BUILD_DIR)/bootstrap/etc/yum.repos.d/base.

 $(BUILD_DIR)/bootstrap/customize-initram-root.done: $(call depv,BOOTSTRAP_RPMS_CUSTOM)
 $(BUILD_DIR)/bootstrap/customize-initram-root.done: \
-		$(BUILD_DIR)/packages/build.done \
+		$(BUILD_DIR)/packages/rpm/build.done \
 		$(BUILD_DIR)/bootstrap/prepare-initram-root.done \
 		$(call find-files,$(SOURCE_DIR)/bootstrap/sync) \
 		$(BUILD_DIR)/repos/nailgun.done \
@ -161,7 +161,8 @@ $(BUILD_DIR)/bootstrap/customize-initram-root.done: \

 $(BUILD_DIR)/bootstrap/prepare-initram-root.done: $(call depv,BOOTSTRAP_RPMS)
 $(BUILD_DIR)/bootstrap/prepare-initram-root.done: \
-		$(BUILD_DIR)/mirror/build.done \
+		$(BUILD_DIR)/mirror/centos/build.done \
+		$(BUILD_DIR)/packages/rpm/build.done \
 		$(BUILD_DIR)/bootstrap/etc/yum.conf \
 		$(BUILD_DIR)/bootstrap/etc/yum.repos.d/base.repo

--- a/docker/module.mk
+++ b/docker/module.mk
@ -34,9 +34,11 @@ define build_container
 ifndef DOCKER_DEP_FILE
 $(BUILD_DIR)/docker/build.done: $(BUILD_DIR)/docker/$1.done
 endif
+$(eval RANDOM_PORT:=$(shell echo $$(($(RANDOM_PORT)+1))))
 $(BUILD_DIR)/docker/$1.done: \
 		$(BUILD_DIR)/mirror/build.done \
 		$(BUILD_DIR)/repos/repos.done \
+		$(BUILD_DIR)/packages/build.done \
 		$(BUILD_DIR)/iso/isoroot-files.done \
 		$(BUILD_DIR)/docker/base-images.done
 	(cd $(LOCAL_MIRROR_CENTOS) && python $(SOURCE_DIR)/utils/simple_http_daemon.py $(RANDOM_PORT) /tmp/simple_http_daemon_$(RANDOM_PORT).pid)
--- a/image/centos/module.mk
+++ b/image/centos/module.mk
@ -19,7 +19,8 @@ ifdef TARGET_CENTOS_DEP_FILE
 $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(TARGET_CENTOS_DEP_FILE)
 	$(ACTION.COPY)
 else
-$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/mirror/build.done
+$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/mirror/centos/build.done
+$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/packages/rpm/build.done
 $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): SANDBOX:=$(BUILD_DIR)/image/centos/SANDBOX
 $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): export SANDBOX_UP:=$(SANDBOX_UP)
 $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): export SANDBOX_DOWN:=$(SANDBOX_DOWN)
--- a/packages/deb/module.mk
+++ b/packages/deb/module.mk
@ -3,8 +3,11 @@
 clean: clean-deb

 clean-deb:
-	-sudo umount $(BUILD_DIR)/packages/deb/SANDBOX/proc
-	-sudo umount $(BUILD_DIR)/packages/deb/SANDBOX/dev
+	-mount | grep '$(BUILD_DIR)/packages/deb/SANDBOX' | while read entry; do \
+		set -- $$entry; \
+		mntpt="$$3"; \
+		sudo umount $$mntpt; \
+	done
 	sudo rm -rf $(BUILD_DIR)/packages/deb

 # Usage:
@ -14,7 +17,7 @@ $(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1.done
 $(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1-repocleanup.done
 $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/mirror/ubuntu/build.done
 $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/packages/source_$1.done
-$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX
+$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX/$1
 $(BUILD_DIR)/packages/deb/$1.done: SANDBOX_DEB_PKGS:=apt wget bzip2 apt-utils build-essential python-setuptools devscripts debhelper fakeroot
 $(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_UP:=$$(SANDBOX_UBUNTU_UP)
 $(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_DOWN:=$$(SANDBOX_UBUNTU_DOWN)
--- a/packages/module.mk
+++ b/packages/module.mk
@ -3,6 +3,7 @@
 # Note: dependencies for deb targets are also specified here to make
 # sure the source is ready before the build is started.
 define prepare_file_source
+$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done
 $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2
 $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
 	mkdir -p $(BUILD_DIR)/packages/sources/$1
@ -14,6 +15,7 @@ endef
 # Note: dependencies for deb targets are also specified here to make
 # sure the source is ready before the build is started.
 define prepare_python_source
+$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done
 $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2
 $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
 	mkdir -p $(BUILD_DIR)/packages/sources/$1
@ -30,6 +32,7 @@ endef
 # Note: dependencies for deb targets are also specified here to make
 # sure the source is ready before the build is started.
 define prepare_tgz_source
+$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done
 $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2
 $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
 	mkdir -p $(BUILD_DIR)/packages/sources/$1
@ -41,6 +44,7 @@ endef
 # Note: dependencies for deb targets are also specified here to make
 # sure the source is ready before the build is started.
 define prepare_ruby21_source
+$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done
 $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2
 $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
 	mkdir -p $(BUILD_DIR)/packages/sources/$1
--- a/packages/rpm/module.mk
+++ b/packages/rpm/module.mk
@ -3,8 +3,11 @@
 clean: clean-rpm

 clean-rpm:
-	-sudo umount $(BUILD_DIR)/packages/rpm/SANDBOX/proc
-	-sudo umount $(BUILD_DIR)/packages/rpm/SANDBOX/dev
+	-mount | grep '$(BUILD_DIR)/packages/rpm/SANDBOX' | while read entry; do \
+		set -- $$entry; \
+		mntpt="$$3"; \
+		sudo umount $$mntpt; \
+	done
 	sudo rm -rf $(BUILD_DIR)/packages/rpm

 RPM_SOURCES:=$(BUILD_DIR)/packages/rpm/SOURCES
@ -19,11 +22,11 @@ $(BUILD_DIR)/packages/rpm/repo.done: $(BUILD_DIR)/packages/rpm/$1-repocleanup.do
 # It will build astute rpm package
 $1: $(BUILD_DIR)/packages/rpm/$1.done

-$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/mirror/build.done
+$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/mirror/centos/build.done
 $(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/source_$1.done


-$(BUILD_DIR)/packages/rpm/$1.done: SANDBOX:=$(BUILD_DIR)/packages/rpm/SANDBOX
+$(BUILD_DIR)/packages/rpm/$1.done: SANDBOX:=$(BUILD_DIR)/packages/rpm/SANDBOX/$1
 $(BUILD_DIR)/packages/rpm/$1.done: export SANDBOX_UP:=$$(SANDBOX_UP)
 $(BUILD_DIR)/packages/rpm/$1.done: export SANDBOX_DOWN:=$$(SANDBOX_DOWN)
 $(BUILD_DIR)/packages/rpm/$1.done: \
@ -40,7 +43,7 @@ $(BUILD_DIR)/packages/rpm/$1.done: \
 	sudo sh -c "$$$${SANDBOX_DOWN}"
 	$$(ACTION.TOUCH)

-$(BUILD_DIR)/packages/rpm/$1-repocleanup.done: $(BUILD_DIR)/mirror/build.done
+$(BUILD_DIR)/packages/rpm/$1-repocleanup.done: $(BUILD_DIR)/mirror/centos/build.done
 	find $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages -regex '.*/$1-[^-]+-[^-]+' -delete
 	$$(ACTION.TOUCH)
 endef
--- a/repos.mk
+++ b/repos.mk
@ -2,16 +2,13 @@

 repos: $(BUILD_DIR)/repos/repos.done

-$(BUILD_DIR)/repos/repos.done:
-	sed -i '/^  fuelmain_sha:/d' $(BUILD_DIR)/repos/version.yaml
-	/bin/echo "  fuelmain_sha: \"`git rev-parse --verify HEAD`\"" >> $(BUILD_DIR)/repos/version.yaml
-	$(ACTION.TOUCH)
-
+fuel_components_repos:=
 # Usage:
 # (eval (call build_repo,repo_name,repo_uri,sha))
 define build_repo
 $(BUILD_DIR)/repos/$1/%: $(BUILD_DIR)/repos/$1.done
 $(BUILD_DIR)/repos/repos.done: $(BUILD_DIR)/repos/$1.done
+fuel_components_repos:=$(fuel_components_repos) $1

 $(BUILD_DIR)/repos/$1.done:
 	# Clone repo and checkout required commit
@ -22,17 +19,25 @@ $(BUILD_DIR)/repos/$1.done:
 	git clone $2 $(BUILD_DIR)/repos/$1 && (cd $(BUILD_DIR)/repos/$1 && git checkout -q $3)

 	# Pull gerrit commits if given
-	$(foreach var,$5,
-		test "$(var)" = "none" || ( cd $(BUILD_DIR)/repos/$1 && git fetch $4 $(var) && git cherry-pick FETCH_HEAD ) ;
+	$(foreach var,$(filter-out none,$5),
+		( cd $(BUILD_DIR)/repos/$1 && git fetch $4 $(var) && git cherry-pick FETCH_HEAD ) ;
 	)
-	# Update versions.yaml
-	touch $(BUILD_DIR)/repos/version.yaml
-	sed -i '/^  $1_sha:/d' $(BUILD_DIR)/repos/version.yaml
-	/bin/echo "  $1_sha: \"`cd $(BUILD_DIR)/repos/$1 && git rev-parse --verify HEAD`\"" >> $(BUILD_DIR)/repos/version.yaml
-	touch $(BUILD_DIR)/repos/$1.done
+	touch $$@
 endef

+
 $(eval $(call build_repo,nailgun,$(NAILGUN_REPO),$(NAILGUN_COMMIT),$(NAILGUN_GERRIT_URL),$(NAILGUN_GERRIT_COMMIT)))
 $(eval $(call build_repo,astute,$(ASTUTE_REPO),$(ASTUTE_COMMIT),$(ASTUTE_GERRIT_URL),$(ASTUTE_GERRIT_COMMIT)))
 $(eval $(call build_repo,fuellib,$(FUELLIB_REPO),$(FUELLIB_COMMIT),$(FUELLIB_GERRIT_URL),$(FUELLIB_GERRIT_COMMIT)))
 $(eval $(call build_repo,ostf,$(OSTF_REPO),$(OSTF_COMMIT),$(OSTF_GERRIT_URL),$(OSTF_GERRIT_COMMIT)))
+
+$(BUILD_DIR)/repos/repos.done:
+	version_yaml=$(BUILD_DIR)/repos/version.yaml; \
+	for repo in $(strip $(fuel_components_repos)); do \
+		repo_commit_id=`git --git-dir=$(BUILD_DIR)/repos/$$repo/.git rev-parse --verify HEAD`; \
+		echo "  $${repo}_sha: \"$${repo_commit_id}\""; \
+	done > $${version_yaml}.tmp; \
+	fuel_main_commit_id=`git rev-parse --verify HEAD`; \
+	echo "  fuelmain_sha: \"${fuelmain_commit_id}\"" >> $${version_yaml}.tmp; \
+	mv $${version_yaml}.tmp $${version_yaml}
+	$(ACTION.TOUCH)
--- a/utils/simple_http_daemon.py
+++ b/utils/simple_http_daemon.py
@ -67,7 +67,7 @@ if __name__ == "__main__":
    if sys.argv[3:]:
        timeout = int(sys.argv[3])
    else:
-        timeout = 600
+        timeout = sys.maxint

    server = SimpleHTTPDaemon('0.0.0.0', port, pid, timeout)
    server.start()