Building OpenStack packages from master

Blueprint: openstack-from-master
Change-Id: Iedd5a1fb625dfc2e89e10ef073d5584546b63ea3

Makefile

@@ -69,6 +69,7 @@ include $(SOURCE_DIR)/sandbox.mk
 include $(SOURCE_DIR)/repos.mk
 include $(SOURCE_DIR)/mirror/module.mk
 include $(SOURCE_DIR)/packages/module.mk
+include $(SOURCE_DIR)/packages/openstack/module.mk
 include $(SOURCE_DIR)/docker/module.mk
 include $(SOURCE_DIR)/bootstrap/module.mk
 include $(SOURCE_DIR)/iso/module.mk

config.mk

@@ -35,6 +35,28 @@ IMG_PATH:=$(ISO_DIR)/$(ISO_NAME).img
 # Rebuld packages locally (do not use upstream versions)
 BUILD_PACKAGES?=1
 
+# Build OpenStack packages from external sources (do not use prepackaged versions)
+# Enter the comma-separated list of OpenStack packages to build, or '0' otherwise.
+# Example: BUILD_OPENSTACK_PACKAGES=neutron,keystone
+BUILD_OPENSTACK_PACKAGES?=0
+
+# Define a set of defaults for each OpenStack package
+# For each component defined in BUILD_OPENSTACK_PACKAGES variable, this routine will set
+# the following variables (i.e. for 'BUILD_OPENSTACK_PACKAGES=neutron'):
+# NEUTRON_REPO, NEUTRON_COMMIT, NEUTRON_SPEC_REPO, NEUTRON_SPEC_COMMIT,
+# NEUTRON_GERRIT_URL, NEUTRON_GERRIT_COMMIT, NEUTRON_GERRIT_URL,
+# NEUTRON_SPEC_GERRIT_URL, NEUTRON_SPEC_GERRIT_COMMIT
+define set_vars
+    $(call uc,$(1))_REPO?=https://github.com/openstack/$(1).git
+    $(call uc,$(1))_COMMIT?=master
+    $(call uc,$(1))_SPEC_REPO?=https://osci-gerrit.mirantis.com/openstack-ci/openstack/$(1)-build.git
+    $(call uc,$(1))_SPEC_COMMIT?=master
+    $(call uc,$(1))_GERRIT_URL=https://review.openstack.org/openstack/$(1)
+    $(call uc,$(1))_GERRIT_COMMIT?=none
+    $(call uc,$(1))_SPEC_GERRIT_URL?=https://osci-gerrit.mirantis.com/openstack-ci/openstack/$(1)-build.git
+    $(call uc,$(1))_SPEC_GERRIT_COMMIT?=none
+endef
+
 # Do not compress javascript and css files
 NO_UI_OPTIMIZE:=0
 

packages/openstack/module.mk

@@ -0,0 +1,13 @@
+include $(SOURCE_DIR)/packages/openstack/rpm/module.mk
+
+.PHONY: openstack
+
+ifneq ($(BUILD_OPENSTACK_PACKAGES),0)
+$(BUILD_DIR)/openstack/build.done: \
+		$(BUILD_DIR)/openstack/rpm/build.done
+endif
+
+$(BUILD_DIR)/openstack/build.done:
+	$(ACTION.TOUCH)
+
+openstack: $(BUILD_DIR)/openstack/build.done

packages/openstack/rpm/module.mk

@@ -0,0 +1,89 @@
+define yum_centos_repo
+[centos]
+name=CentOS-$(CENTOS_RELEASE) - Base
+#mirrorlist=http://mirrorlist.centos.org/?release=$(CENTOS_RELEASE)&arch=$(CENTOS_ARCH)&repo=os
+baseurl=http://mirrors.msk.mirantis.net/centos/$(CENTOS_RELEASE)/os/$(CENTOS_ARCH)
+gpgcheck=0
+enabled=0
+priority=10
+
+[centos-master]
+name=CentOS-master
+baseurl=http://fuel-repository.mirantis.com/repos/centos-fuel-master/centos/
+gpgcheck=0
+enabled=0
+priority=1
+endef
+
+define INSTALL_CENTOS_REPO
+cat > $(SANDBOX)/etc/yum.repos.d/centos.repo <<EOF
+$(yum_centos_repo)
+EOF
+endef
+
+# Usage:
+# (eval (call prepare_openstack_source,package_name,file_name,source_path))
+define prepare_openstack_source
+$(BUILD_DIR)/openstack/rpm/$1.done: $(BUILD_DIR)/openstack/rpm/sources/$1/$2
+$(BUILD_DIR)/openstack/rpm/sources/$1/$2: $(call find-files,$3)
+	mkdir -p $(BUILD_DIR)/openstack/rpm/sources/$1
+	cd $3 && python setup.py sdist -d $(BUILD_DIR)/openstack/rpm/sources/$1 && python setup.py --version sdist > $(BUILD_DIR)/openstack/rpm/$1-version-tag
+endef
+
+# Usage:
+# (eval (call build_openstack_rpm,package_name))
+define build_openstack_rpm
+$(BUILD_DIR)/openstack/rpm/repo.done: $(BUILD_DIR)/openstack/rpm/$1.done
+$(BUILD_DIR)/openstack/rpm/repo.done: $(BUILD_DIR)/openstack/rpm/$1-repocleanup.done
+
+# You can use package name as a target, for example: make neutron
+# It will build neutron rpm package
+$1: $(BUILD_DIR)/openstack/rpm/$1.done
+
+$(BUILD_DIR)/openstack/rpm/$1.done: $(BUILD_DIR)/mirror/build.done
+
+$(BUILD_DIR)/openstack/rpm/$1.done: SANDBOX:=$(BUILD_DIR)/packages/rpm/SANDBOX
+$(BUILD_DIR)/openstack/rpm/$1.done: export SANDBOX_UP:=$$(SANDBOX_UP)
+$(BUILD_DIR)/openstack/rpm/$1.done: export SANDBOX_DOWN:=$$(SANDBOX_DOWN)
+$(BUILD_DIR)/openstack/rpm/$1.done: export INSTALL_CENTOS_REPO:=$$(INSTALL_CENTOS_REPO)
+$(BUILD_DIR)/openstack/rpm/$1.done: \
+	$(BUILD_DIR)/repos/repos.done
+	mkdir -p $(BUILD_DIR)/openstack/rpm/RPMS/x86_64 $(BUILD_DIR)/openstack/rpm/sources/specs
+	sudo sh -c "$$$${SANDBOX_UP}"
+	sudo yum -c $$(SANDBOX)/etc/yum.conf --installroot=$$(SANDBOX) -y --nogpgcheck install ruby rpm-build tar python-setuptools yum-utils
+	sudo mkdir -p $$(SANDBOX)/tmp/SPECS $$(SANDBOX)/tmp/SOURCES $$(SANDBOX)/tmp/BUILD $$(SANDBOX)/tmp/BUILDROOT $$(SANDBOX)/tmp/RPMS $$(SANDBOX)/tmp/SRPMS
+	sudo cp $(BUILD_DIR)/openstack/rpm/sources/$1/*.tar.gz $$(SANDBOX)/tmp/SOURCES/
+	sudo cp -r $(BUILD_DIR)/repos/$1-build/rpm/SOURCES/* $$(SANDBOX)/tmp/SOURCES
+	sed "s/Version:.*/Version:\t`cat $(BUILD_DIR)/openstack/rpm/$1-version-tag`/" $(BUILD_DIR)/repos/$1-build/rpm/SPECS/openstack-$1.spec > $(BUILD_DIR)/openstack/rpm/sources/specs/openstack-$1.spec
+	sed -i "s/Source0:.*/Source0:\t$1-`cat $(BUILD_DIR)/openstack/rpm/$1-version-tag`\.tar\.gz/" $(BUILD_DIR)/openstack/rpm/sources/specs/openstack-$1.spec
+	sudo cp $(BUILD_DIR)/openstack/rpm/sources/specs/openstack-$1.spec $$(SANDBOX)/tmp/
+	sudo chroot $$(SANDBOX) rpmbuild --nodeps -vv --define "_topdir /tmp" -bs /tmp/openstack-$1.spec
+	sudo sh -c "$$$${INSTALL_CENTOS_REPO}"
+	sudo yum-builddep -c $$(SANDBOX)/etc/yum.conf --enablerepo=centos --enablerepo=centos-master --installroot=$$(SANDBOX) -y --nogpgcheck $$(SANDBOX)/tmp/SRPMS/openstack-$1*.rpm
+	sudo rm -rf $$(SANDBOX)/tmp/RPMS
+	sudo chroot $$(SANDBOX) rpmbuild --nodeps -vv --define "_topdir /tmp" -ba /tmp/openstack-$1.spec
+	cp $$(SANDBOX)/tmp/RPMS/*/*$1*.rpm $(BUILD_DIR)/openstack/rpm/RPMS/x86_64
+	sudo sh -c "$$$${SANDBOX_DOWN}"
+	$$(ACTION.TOUCH)
+
+$(BUILD_DIR)/openstack/rpm/$1-repocleanup.done: $(BUILD_DIR)/mirror/build.done
+	find $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages -regex '.*$1-[^-]+-[^-]+.*' -delete
+	$$(ACTION.TOUCH)
+endef
+
+ifneq ($(BUILD_OPENSTACK_PACKAGES),0)
+$(foreach pkg,$(subst $(comma), ,$(BUILD_OPENSTACK_PACKAGES)),$(eval $(call set_vars,$(pkg))))
+$(foreach pkg,$(subst $(comma), ,$(BUILD_OPENSTACK_PACKAGES)),$(eval $(call build_repo,$(pkg),$($(call uc,$(pkg))_REPO),$($(call uc,$(pkg))_COMMIT),$($(call uc,$(pkg))_GERRIT_URL),$($(call uc,$(pkg))_GERRIT_COMMIT))))
+$(foreach pkg,$(subst $(comma), ,$(BUILD_OPENSTACK_PACKAGES)),$(eval $(call build_repo,$(pkg)-build,$($(call uc,$(pkg))_SPEC_REPO),$($(call uc,$(pkg))_SPEC_COMMIT),$($(call uc,$(pkg))_SPEC_GERRIT_URL),$($(call uc,$(pkg))_SPEC_GERRIT_COMMIT))))
+$(foreach pkg,$(subst $(comma), ,$(BUILD_OPENSTACK_PACKAGES)),$(eval $(call prepare_openstack_source,$(pkg),$(pkg)-2014.1.tar.gz,$(BUILD_DIR)/repos/$(pkg))))
+$(foreach pkg,$(subst $(comma), ,$(BUILD_OPENSTACK_PACKAGES)),$(eval $(call build_openstack_rpm,$(pkg))))
+endif
+
+$(BUILD_DIR)/openstack/rpm/repo.done:
+	find $(BUILD_DIR)/openstack/rpm/RPMS -name '*.rpm' -exec cp -u {} $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages \;
+	createrepo -g $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml \
+		-o $(LOCAL_MIRROR_CENTOS_OS_BASEURL) $(LOCAL_MIRROR_CENTOS_OS_BASEURL)
+	$(ACTION.TOUCH)
+
+$(BUILD_DIR)/openstack/rpm/build.done: $(BUILD_DIR)/openstack/rpm/repo.done
+	$(ACTION.TOUCH)

rules.mk

@@ -35,3 +35,9 @@ $(BUILD_DIR)/%/.dir:
 
 assert-variable=$(if $($1),,$(error Variable $1 need to be defined))
 find-files=$(shell test -e $1 && find $1 -type f 2> /dev/null)
+
+# uppercase conversion routine
+# usage: UPPER_VAR = $(call uc,$(VAR))
+uc = $(shell echo $(1) | tr a-z A-Z)
+
+comma:=,