openstack
/
fuel-main
Code Issues Proposed changes

parallel make fixes: add missing dependencies 

* packages:
  - build each package in a separate directory so sereral packages can be
    built in parallel.
  - prepare_*_source: make sure git repositories are cloned before using
    them.

* image/centos, bootstrap, docker: add dependency on the locally built
  nailgun, astute and co. RPMs to make sure RPMs are ready before trying
  to install them.

* docker:
  - start an instance of the simple http server per a container being built,
    make sure those processes listen the different ports.
  - fix sporadic build failures due to the HTTP daemon premature exit.
    The simple_http_daemon which serves RPM repository exits in 600 seconds
    by default. However a container might require more time (due to a slow
    machine, a high load, etc), thus the HTTP server might exit before
    the required packages have been downloaded. Set the default timeout
    to infinity to prevent such an obscure error.

* repos.mk: several repositories can be cloned at the same time, so writes
  to version.yaml should be serialized. To keep things simple create
  version.yaml after cloning the repositories (with Fuel components).
  While at it polish the rule which clones the repository.

* packages/rpm, bootstrap: depend only on CentOS repo (more tasks can
  run in parallel).

blueprint support-ubuntu-trusty
Change-Id: I3a72802a7728121581e71227b013bb7aeffaf049
This commit is contained in:
Alexei Sheplyakov 2014-12-22 12:32:48 +03:00
parent c83a59b403
commit b9a580d2c7
8 changed files with 44 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
bootstrap/module.mk
View File

@ -98,7 +98,7 @@ $(BUILD_DIR)/bootstrap/initramfs.img: \
sudo sh -c "cd $(INITRAMROOT) && find . -xdev | cpio --create \ sudo sh -c "cd $(INITRAMROOT) && find . -xdev | cpio --create \
--format='newc' | gzip -9 > $(BUILD_DIR)/bootstrap/initramfs.img" --format='newc' | gzip -9 > $(BUILD_DIR)/bootstrap/initramfs.img"
$(BUILD_DIR)/bootstrap/linux: $(BUILD_DIR)/mirror/build.done $(BUILD_DIR)/bootstrap/linux: $(BUILD_DIR)/mirror/centos/build.done
mkdir -p $(BUILD_DIR)/bootstrap mkdir -p $(BUILD_DIR)/bootstrap
find $(LOCAL_MIRROR_CENTOS_OS_BASEURL) -name '$(KERNEL_PATTERN)' | xargs rpm2cpio | \ find $(LOCAL_MIRROR_CENTOS_OS_BASEURL) -name '$(KERNEL_PATTERN)' | xargs rpm2cpio | \
(cd $(BUILD_DIR)/bootstrap/; cpio -imd './boot/vmlinuz*') (cd $(BUILD_DIR)/bootstrap/; cpio -imd './boot/vmlinuz*')
@ -114,7 +114,7 @@ $(BUILD_DIR)/bootstrap/etc/yum.conf $(BUILD_DIR)/bootstrap/etc/yum.repos.d/base.
$(BUILD_DIR)/bootstrap/customize-initram-root.done: $(call depv,BOOTSTRAP_RPMS_CUSTOM) $(BUILD_DIR)/bootstrap/customize-initram-root.done: $(call depv,BOOTSTRAP_RPMS_CUSTOM)
$(BUILD_DIR)/bootstrap/customize-initram-root.done: \ $(BUILD_DIR)/bootstrap/customize-initram-root.done: \
$(BUILD_DIR)/packages/build.done \ $(BUILD_DIR)/packages/rpm/build.done \
$(BUILD_DIR)/bootstrap/prepare-initram-root.done \ $(BUILD_DIR)/bootstrap/prepare-initram-root.done \
$(call find-files,$(SOURCE_DIR)/bootstrap/sync) \ $(call find-files,$(SOURCE_DIR)/bootstrap/sync) \
$(BUILD_DIR)/repos/nailgun.done \ $(BUILD_DIR)/repos/nailgun.done \
@ -161,7 +161,8 @@ $(BUILD_DIR)/bootstrap/customize-initram-root.done: \
$(BUILD_DIR)/bootstrap/prepare-initram-root.done: $(call depv,BOOTSTRAP_RPMS) $(BUILD_DIR)/bootstrap/prepare-initram-root.done: $(call depv,BOOTSTRAP_RPMS)
$(BUILD_DIR)/bootstrap/prepare-initram-root.done: \ $(BUILD_DIR)/bootstrap/prepare-initram-root.done: \
$(BUILD_DIR)/mirror/build.done \ $(BUILD_DIR)/mirror/centos/build.done \
$(BUILD_DIR)/packages/rpm/build.done \
$(BUILD_DIR)/bootstrap/etc/yum.conf \ $(BUILD_DIR)/bootstrap/etc/yum.conf \
$(BUILD_DIR)/bootstrap/etc/yum.repos.d/base.repo $(BUILD_DIR)/bootstrap/etc/yum.repos.d/base.repo

2
docker/module.mk
View File

@ -34,9 +34,11 @@ define build_container
ifndef DOCKER_DEP_FILE ifndef DOCKER_DEP_FILE
$(BUILD_DIR)/docker/build.done: $(BUILD_DIR)/docker/$1.done $(BUILD_DIR)/docker/build.done: $(BUILD_DIR)/docker/$1.done
endif endif
$(eval RANDOM_PORT:=$(shell echo $$(($(RANDOM_PORT)+1))))
$(BUILD_DIR)/docker/$1.done: \ $(BUILD_DIR)/docker/$1.done: \
$(BUILD_DIR)/mirror/build.done \ $(BUILD_DIR)/mirror/build.done \
$(BUILD_DIR)/repos/repos.done \ $(BUILD_DIR)/repos/repos.done \
$(BUILD_DIR)/packages/build.done \
$(BUILD_DIR)/iso/isoroot-files.done \ $(BUILD_DIR)/iso/isoroot-files.done \
$(BUILD_DIR)/docker/base-images.done $(BUILD_DIR)/docker/base-images.done
(cd $(LOCAL_MIRROR_CENTOS) && python $(SOURCE_DIR)/utils/simple_http_daemon.py $(RANDOM_PORT) /tmp/simple_http_daemon_$(RANDOM_PORT).pid) (cd $(LOCAL_MIRROR_CENTOS) && python $(SOURCE_DIR)/utils/simple_http_daemon.py $(RANDOM_PORT) /tmp/simple_http_daemon_$(RANDOM_PORT).pid)

3
image/centos/module.mk
View File

@ -19,7 +19,8 @@ ifdef TARGET_CENTOS_DEP_FILE
$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(TARGET_CENTOS_DEP_FILE) $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(TARGET_CENTOS_DEP_FILE)
$(ACTION.COPY) $(ACTION.COPY)
else else
$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/mirror/build.done $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/mirror/centos/build.done
$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): $(BUILD_DIR)/packages/rpm/build.done
$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): SANDBOX:=$(BUILD_DIR)/image/centos/SANDBOX $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): SANDBOX:=$(BUILD_DIR)/image/centos/SANDBOX
$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): export SANDBOX_UP:=$(SANDBOX_UP) $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): export SANDBOX_UP:=$(SANDBOX_UP)
$(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): export SANDBOX_DOWN:=$(SANDBOX_DOWN) $(BUILD_DIR)/images/$(TARGET_CENTOS_IMG_ART_NAME): export SANDBOX_DOWN:=$(SANDBOX_DOWN)

9
packages/deb/module.mk
View File

@ -3,8 +3,11 @@
clean: clean-deb clean: clean-deb
clean-deb: clean-deb:
-sudo umount $(BUILD_DIR)/packages/deb/SANDBOX/proc -mount | grep '$(BUILD_DIR)/packages/deb/SANDBOX' | while read entry; do \
-sudo umount $(BUILD_DIR)/packages/deb/SANDBOX/dev set -- $$entry; \
mntpt="$$3"; \
sudo umount $$mntpt; \
done
sudo rm -rf $(BUILD_DIR)/packages/deb sudo rm -rf $(BUILD_DIR)/packages/deb
# Usage: # Usage:
@ -14,7 +17,7 @@ $(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1.done
$(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1-repocleanup.done $(BUILD_DIR)/packages/deb/repo.done: $(BUILD_DIR)/packages/deb/$1-repocleanup.done
$(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/mirror/ubuntu/build.done $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/mirror/ubuntu/build.done
$(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/packages/source_$1.done $(BUILD_DIR)/packages/deb/$1.done: $(BUILD_DIR)/packages/source_$1.done
$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX $(BUILD_DIR)/packages/deb/$1.done: SANDBOX_UBUNTU:=$(BUILD_DIR)/packages/deb/SANDBOX/$1
$(BUILD_DIR)/packages/deb/$1.done: SANDBOX_DEB_PKGS:=apt wget bzip2 apt-utils build-essential python-setuptools devscripts debhelper fakeroot $(BUILD_DIR)/packages/deb/$1.done: SANDBOX_DEB_PKGS:=apt wget bzip2 apt-utils build-essential python-setuptools devscripts debhelper fakeroot
$(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_UP:=$$(SANDBOX_UBUNTU_UP) $(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_UP:=$$(SANDBOX_UBUNTU_UP)
$(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_DOWN:=$$(SANDBOX_UBUNTU_DOWN) $(BUILD_DIR)/packages/deb/$1.done: export SANDBOX_UBUNTU_DOWN:=$$(SANDBOX_UBUNTU_DOWN)

4
packages/module.mk
View File

@ -3,6 +3,7 @@
# Note: dependencies for deb targets are also specified here to make # Note: dependencies for deb targets are also specified here to make
# sure the source is ready before the build is started. # sure the source is ready before the build is started.
define prepare_file_source define prepare_file_source
$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done
$(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2 $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2
$(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3) $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/sources/$1 mkdir -p $(BUILD_DIR)/packages/sources/$1
@ -14,6 +15,7 @@ endef
# Note: dependencies for deb targets are also specified here to make # Note: dependencies for deb targets are also specified here to make
# sure the source is ready before the build is started. # sure the source is ready before the build is started.
define prepare_python_source define prepare_python_source
$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done
$(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2 $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2
$(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3) $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/sources/$1 mkdir -p $(BUILD_DIR)/packages/sources/$1
@ -30,6 +32,7 @@ endef
# Note: dependencies for deb targets are also specified here to make # Note: dependencies for deb targets are also specified here to make
# sure the source is ready before the build is started. # sure the source is ready before the build is started.
define prepare_tgz_source define prepare_tgz_source
$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done
$(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2 $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2
$(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3) $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/sources/$1 mkdir -p $(BUILD_DIR)/packages/sources/$1
@ -41,6 +44,7 @@ endef
# Note: dependencies for deb targets are also specified here to make # Note: dependencies for deb targets are also specified here to make
# sure the source is ready before the build is started. # sure the source is ready before the build is started.
define prepare_ruby21_source define prepare_ruby21_source
$(BUILD_DIR)/packages/sources/$1/$2: $(BUILD_DIR)/repos/repos.done
$(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2 $(BUILD_DIR)/packages/source_$1.done: $(BUILD_DIR)/packages/sources/$1/$2
$(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3) $(BUILD_DIR)/packages/sources/$1/$2: $(call find-files,$3)
mkdir -p $(BUILD_DIR)/packages/sources/$1 mkdir -p $(BUILD_DIR)/packages/sources/$1

13
packages/rpm/module.mk
View File

@ -3,8 +3,11 @@
clean: clean-rpm clean: clean-rpm
clean-rpm: clean-rpm:
-sudo umount $(BUILD_DIR)/packages/rpm/SANDBOX/proc -mount | grep '$(BUILD_DIR)/packages/rpm/SANDBOX' | while read entry; do \
-sudo umount $(BUILD_DIR)/packages/rpm/SANDBOX/dev set -- $$entry; \
mntpt="$$3"; \
sudo umount $$mntpt; \
done
sudo rm -rf $(BUILD_DIR)/packages/rpm sudo rm -rf $(BUILD_DIR)/packages/rpm
RPM_SOURCES:=$(BUILD_DIR)/packages/rpm/SOURCES RPM_SOURCES:=$(BUILD_DIR)/packages/rpm/SOURCES
@ -19,11 +22,11 @@ $(BUILD_DIR)/packages/rpm/repo.done: $(BUILD_DIR)/packages/rpm/$1-repocleanup.do
# It will build astute rpm package # It will build astute rpm package
$1: $(BUILD_DIR)/packages/rpm/$1.done $1: $(BUILD_DIR)/packages/rpm/$1.done
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/mirror/build.done $(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/mirror/centos/build.done
$(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/source_$1.done $(BUILD_DIR)/packages/rpm/$1.done: $(BUILD_DIR)/packages/source_$1.done
$(BUILD_DIR)/packages/rpm/$1.done: SANDBOX:=$(BUILD_DIR)/packages/rpm/SANDBOX $(BUILD_DIR)/packages/rpm/$1.done: SANDBOX:=$(BUILD_DIR)/packages/rpm/SANDBOX/$1
$(BUILD_DIR)/packages/rpm/$1.done: export SANDBOX_UP:=$$(SANDBOX_UP) $(BUILD_DIR)/packages/rpm/$1.done: export SANDBOX_UP:=$$(SANDBOX_UP)
$(BUILD_DIR)/packages/rpm/$1.done: export SANDBOX_DOWN:=$$(SANDBOX_DOWN) $(BUILD_DIR)/packages/rpm/$1.done: export SANDBOX_DOWN:=$$(SANDBOX_DOWN)
$(BUILD_DIR)/packages/rpm/$1.done: \ $(BUILD_DIR)/packages/rpm/$1.done: \
@ -40,7 +43,7 @@ $(BUILD_DIR)/packages/rpm/$1.done: \
sudo sh -c "$$$${SANDBOX_DOWN}" sudo sh -c "$$$${SANDBOX_DOWN}"
$$(ACTION.TOUCH) $$(ACTION.TOUCH)
$(BUILD_DIR)/packages/rpm/$1-repocleanup.done: $(BUILD_DIR)/mirror/build.done $(BUILD_DIR)/packages/rpm/$1-repocleanup.done: $(BUILD_DIR)/mirror/centos/build.done
find $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages -regex '.*/$1-[^-]+-[^-]+' -delete find $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages -regex '.*/$1-[^-]+-[^-]+' -delete
$$(ACTION.TOUCH) $$(ACTION.TOUCH)
endef endef

29
repos.mk
View File

@ -2,16 +2,13 @@
repos: $(BUILD_DIR)/repos/repos.done repos: $(BUILD_DIR)/repos/repos.done
$(BUILD_DIR)/repos/repos.done: fuel_components_repos:=
sed -i '/^ fuelmain_sha:/d' $(BUILD_DIR)/repos/version.yaml
/bin/echo " fuelmain_sha: \"`git rev-parse --verify HEAD`\"" >> $(BUILD_DIR)/repos/version.yaml
$(ACTION.TOUCH)
# Usage: # Usage:
# (eval (call build_repo,repo_name,repo_uri,sha)) # (eval (call build_repo,repo_name,repo_uri,sha))
define build_repo define build_repo
$(BUILD_DIR)/repos/$1/%: $(BUILD_DIR)/repos/$1.done $(BUILD_DIR)/repos/$1/%: $(BUILD_DIR)/repos/$1.done
$(BUILD_DIR)/repos/repos.done: $(BUILD_DIR)/repos/$1.done $(BUILD_DIR)/repos/repos.done: $(BUILD_DIR)/repos/$1.done
fuel_components_repos:=$(fuel_components_repos) $1
$(BUILD_DIR)/repos/$1.done: $(BUILD_DIR)/repos/$1.done:
# Clone repo and checkout required commit # Clone repo and checkout required commit
@ -22,17 +19,25 @@ $(BUILD_DIR)/repos/$1.done:
git clone $2 $(BUILD_DIR)/repos/$1 && (cd $(BUILD_DIR)/repos/$1 && git checkout -q $3) git clone $2 $(BUILD_DIR)/repos/$1 && (cd $(BUILD_DIR)/repos/$1 && git checkout -q $3)
# Pull gerrit commits if given # Pull gerrit commits if given
$(foreach var,$5, $(foreach var,$(filter-out none,$5),
test "$(var)" = "none" || ( cd $(BUILD_DIR)/repos/$1 && git fetch $4 $(var) && git cherry-pick FETCH_HEAD ) ; ( cd $(BUILD_DIR)/repos/$1 && git fetch $4 $(var) && git cherry-pick FETCH_HEAD ) ;
) )
# Update versions.yaml touch $$@
touch $(BUILD_DIR)/repos/version.yaml
sed -i '/^ $1_sha:/d' $(BUILD_DIR)/repos/version.yaml
/bin/echo " $1_sha: \"`cd $(BUILD_DIR)/repos/$1 && git rev-parse --verify HEAD`\"" >> $(BUILD_DIR)/repos/version.yaml
touch $(BUILD_DIR)/repos/$1.done
endef endef
$(eval $(call build_repo,nailgun,$(NAILGUN_REPO),$(NAILGUN_COMMIT),$(NAILGUN_GERRIT_URL),$(NAILGUN_GERRIT_COMMIT))) $(eval $(call build_repo,nailgun,$(NAILGUN_REPO),$(NAILGUN_COMMIT),$(NAILGUN_GERRIT_URL),$(NAILGUN_GERRIT_COMMIT)))
$(eval $(call build_repo,astute,$(ASTUTE_REPO),$(ASTUTE_COMMIT),$(ASTUTE_GERRIT_URL),$(ASTUTE_GERRIT_COMMIT))) $(eval $(call build_repo,astute,$(ASTUTE_REPO),$(ASTUTE_COMMIT),$(ASTUTE_GERRIT_URL),$(ASTUTE_GERRIT_COMMIT)))
$(eval $(call build_repo,fuellib,$(FUELLIB_REPO),$(FUELLIB_COMMIT),$(FUELLIB_GERRIT_URL),$(FUELLIB_GERRIT_COMMIT))) $(eval $(call build_repo,fuellib,$(FUELLIB_REPO),$(FUELLIB_COMMIT),$(FUELLIB_GERRIT_URL),$(FUELLIB_GERRIT_COMMIT)))
$(eval $(call build_repo,ostf,$(OSTF_REPO),$(OSTF_COMMIT),$(OSTF_GERRIT_URL),$(OSTF_GERRIT_COMMIT))) $(eval $(call build_repo,ostf,$(OSTF_REPO),$(OSTF_COMMIT),$(OSTF_GERRIT_URL),$(OSTF_GERRIT_COMMIT)))
$(BUILD_DIR)/repos/repos.done:
version_yaml=$(BUILD_DIR)/repos/version.yaml; \
for repo in $(strip $(fuel_components_repos)); do \
repo_commit_id=`git --git-dir=$(BUILD_DIR)/repos/$$repo/.git rev-parse --verify HEAD`; \
echo " $${repo}_sha: \"$${repo_commit_id}\""; \
done > $${version_yaml}.tmp; \
fuel_main_commit_id=`git rev-parse --verify HEAD`; \
echo " fuelmain_sha: \"${fuelmain_commit_id}\"" >> $${version_yaml}.tmp; \
mv $${version_yaml}.tmp $${version_yaml}
$(ACTION.TOUCH)

2
utils/simple_http_daemon.py
View File

@ -67,7 +67,7 @@ if __name__ == "__main__":
if sys.argv[3:]: if sys.argv[3:]:
timeout = int(sys.argv[3]) timeout = int(sys.argv[3])
else: else:
timeout = 600 timeout = sys.maxint
server = SimpleHTTPDaemon('0.0.0.0', port, pid, timeout) server = SimpleHTTPDaemon('0.0.0.0', port, pid, timeout)
server.start() server.start()