diff --git a/Makefile b/Makefile index ed16079ae..ff1779776 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ $(BUILD_DIR)/current_version: $(call depv,CURRENT_VERSION) upgrade-versions: $(BUILD_DIR)/upgrade_versions $(BUILD_DIR)/upgrade_versions: $(call depv,UPGRADE_VERSIONS) echo -n > $@ - $(foreach diff,$(UPGRADE_VERSIONS),echo $(diff) >> $@;) + $(foreach upgrade_version,$(UPGRADE_VERSIONS),echo $(upgrade_version) >> $@;) # Macroses for make include $(SOURCE_DIR)/rules.mk diff --git a/config.mk b/config.mk index 4db60bd1b..a3bc6f063 100644 --- a/config.mk +++ b/config.mk @@ -33,38 +33,8 @@ PRODUCT_NAME:=mos # to come from DEPS_DIR "as is" CURRENT_VERSION:=$(PRODUCT_VERSION) -# This variable is used for building diff repos. -# If it is not set then diff repo will not be built. -# If it is set then diff $(BASE_VERSION)->$(CURRENT_VERSION) -BASE_VERSION:=6.0 - PACKAGE_VERSION=6.1.0 -# UPGRADE_VERSIONS?=\ -# 6.0:5.1 \ -# 5.1 \ -# 5.0.3:5.0 -# -# It means we need to put into a tarball -# -# 0) 5.1 -> 6.0 diff mirror and other 6.0 files -# 1) 5.1 full mirror and other 5.1 files -# 2) 5.0 -> 5.0.3 diff mirror and other 5.0.3 files -# -# * It is prohibited to have multiple bundles for -# a particular version. E.g. 6.0 bundle can be one of -# ** 6.0 full bundle -# ** X.Y.Z -> 6.0 diff bundle -# -# * If a key (version before colon) is -# the same as $(CURRENT_VERSION) then -# a mirror (full or diff) will be built. -# -# * If a key does not match $(CURRENT_VERSION) then -# a mirror (full or diff) is supposed to be -# available as an artifact from a previous build job. -# -UPGRADE_VERSIONS?=\ - $(CURRENT_VERSION) +UPGRADE_VERSIONS?=$(CURRENT_VERSION) # Path to pre-built artifacts DEPS_DIR_CURRENT?=$(DEPS_DIR)/$(CURRENT_VERSION) @@ -80,10 +50,6 @@ DOCKER_ART_NAME?=fuel-images.tar.lrz VERSION_YAML_ART_NAME?=version.yaml CENTOS_REPO_ART_NAME?=centos-repo.tar UBUNTU_REPO_ART_NAME?=ubuntu-repo.tar -# actual name for a diff repo will be -# $(DIFF_CENTOS_REPO_ART_BASE)-NEWVERSION-OLDVERSION.tar -DIFF_CENTOS_REPO_ART_BASE?=diff-centos-repo -DIFF_UBUNTU_REPO_ART_BASE?=diff-ubuntu-repo PUPPET_ART_NAME?=puppet.tgz OPENSTACK_YAML_ART_NAME?=openstack.yaml TARGET_UBUNTU_IMG_ART_NAME?=ubuntu_target_images.tar @@ -177,8 +143,6 @@ LOCAL_MIRROR_UBUNTU:=$(LOCAL_MIRROR)/ubuntu LOCAL_MIRROR_UBUNTU_OS_BASEURL:=$(LOCAL_MIRROR_UBUNTU) LOCAL_MIRROR_DOCKER:=$(LOCAL_MIRROR)/docker LOCAL_MIRROR_DOCKER_BASEURL:=$(LOCAL_MIRROR_DOCKER) -DIFF_MIRROR_CENTOS_BASE:=$(LOCAL_MIRROR)/centos_updates -DIFF_MIRROR_UBUNTU_BASE:=$(LOCAL_MIRROR)/ubuntu_updates # Use download.mirantis.com mirror by default. Other possible values are # 'msk', 'srt', 'usa', 'hrk'. diff --git a/mirror/create_diff_mirrors.sh b/mirror/create_diff_mirrors.sh deleted file mode 100755 index 4623358df..000000000 --- a/mirror/create_diff_mirrors.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -NEWER=$1 -OLDER=$2 -DIFF=$3 - -# we need to check if OLDER dir exists -# if not, nothing to compare return exit 1 -test -d ${OLDER} || { echo "${OLDER} path doesn't exist, nothing to compare"; exit 1; } - -test -d ${DIFF} || mkdir -p ${DIFF} - -function copy_by_checksum (){ - # compare files by checksum - CHECKSUM1=$(md5sum -- $1 | awk '{print $1}') - CHECKSUM2=$(md5sum -- $2 | awk '{print $1}') - - # if checksum is different then copy - [ $CHECKSUM1 = $CHECKSUM2 ] || cp -av $1 $3 -} - -export -f copy_by_checksum - -# if file doesn't exists in old, then copy it to diff -# else compare by checksum -find ${NEWER} -type f -printf "%f\n" | xargs -i bash -c "test -f ${OLDER}/{} \ - && copy_by_checksum ${NEWER}/{} ${OLDER}/{} ${DIFF} \ - || cp -av ${NEWER}/{} ${DIFF}/{}" diff --git a/mirror/diff_mirror_module.mk b/mirror/diff_mirror_module.mk deleted file mode 100644 index f0e470abc..000000000 --- a/mirror/diff_mirror_module.mk +++ /dev/null @@ -1,101 +0,0 @@ -############################# -# CENTOS DIFF MIRROR ARTIFACT -############################# -ifneq ($(BASE_VERSION),) -.PHONY: centos-diff-repo - -DIFF_CENTOS_REPO_ART_NAME:=$(DIFF_CENTOS_REPO_ART_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION).tar -centos-diff-repo: $(ARTS_DIR)/$(DIFF_CENTOS_REPO_ART_NAME) - -$(ARTS_DIR)/$(DIFF_CENTOS_REPO_ART_NAME): $(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME) - $(ACTION.COPY) - -DIFF_CENTOS_REPO_DEP_FILE:=$(call find-files,$(DEPS_DIR_CURRENT)/$(DIFF_CENTOS_REPO_ART_NAME)) - -ifneq ($(DIFF_CENTOS_REPO_DEP_FILE),) -$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): $(DIFF_CENTOS_REPO_DEP_FILE) - $(ACTION.COPY) -else -.DELETE_ON_ERROR: $(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME) -CURRENT_CENTOS_REPO_DEP_FILE:=$(call find-files,$(DEPS_DIR_CURRENT)/$(CENTOS_REPO_ART_NAME)) -ifneq ($(CURRENT_CENTOS_REPO_DEP_FILE),) -$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): CURDIR=$(BUILD_DIR)/mirror/$(CURRENT_VERSION)/centos-repo/Packages -$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): $(BUILD_DIR)/mirror/centos_repo_current.done -$(BUILD_DIR)/mirror/centos_repo_current.done: $(CURRENT_CENTOS_REPO_DEP_FILE) - mkdir -p $(BUILD_DIR)/mirror/$(CURRENT_VERSION) - tar xf $(CURRENT_CENTOS_REPO_DEP_FILE) -C $(BUILD_DIR)/mirror/$(CURRENT_VERSION) - $(ACTION.TOUCH) -else -$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): CURDIR=$(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages -$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): $(BUILD_DIR)/mirror/centos_repo_current.done -$(BUILD_DIR)/mirror/centos_repo_current.done: \ - $(BUILD_DIR)/mirror/build.done \ - $(BUILD_DIR)/packages/build.done \ - $(BUILD_DIR)/openstack/build.done - $(ACTION.TOUCH) - -endif -$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): BASEDIR=$(BUILD_DIR)/mirror/$(BASE_VERSION)/centos-repo/Packages -$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): DIFFDIR=$(DIFF_MIRROR_CENTOS_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION)/Packages -$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): | $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml -# unpacking old version centos mirror - mkdir -p $(BUILD_DIR)/mirror/$(BASE_VERSION) - tar xf $(DEPS_DIR)/$(BASE_VERSION)/$(CENTOS_REPO_ART_NAME) -C $(BUILD_DIR)/mirror/$(BASE_VERSION) -# copying packages which differ from those in base version - mkdir -p $(DIFFDIR) - /bin/bash $(SOURCE_DIR)/mirror/create_diff_mirrors.sh $(CURDIR) $(BASEDIR) $(DIFFDIR) -# creating diff mirror - cp $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml $(DIFFDIR)/../comps.xml - createrepo -g $(DIFFDIR)/../comps.xml -o $(DIFFDIR)/../ $(DIFFDIR)/../ - rpm -qi -p $(DIFFDIR)/*.rpm | $(SOURCE_DIR)/iso/pkg-versions.awk > $(DIFF_MIRROR_CENTOS_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION)/centos-versions.yaml - tar cf $@ -C $(DIFF_MIRROR_CENTOS_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION) --xform s:^:centos_updates-$(CURRENT_VERSION)-$(BASE_VERSION)/: . -endif # ifneq ($(DIFF_CENTOS_REPO_DEP_FILE),) -endif # ifneq ($(BASE_VERSION),) - - -############################# -# UBUNTU DIFF MIRROR ARTIFACT -############################# -ifneq ($(BASE_VERSION),) -.PHONY: ubuntu-diff-repo - -DIFF_UBUNTU_REPO_ART_NAME:=$(DIFF_UBUNTU_REPO_ART_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION).tar -ubuntu-diff-repo: $(ARTS_DIR)/$(DIFF_UBUNTU_REPO_ART_NAME) - -$(ARTS_DIR)/$(DIFF_UBUNTU_REPO_ART_NAME): $(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME) - $(ACTION.COPY) - -DIFF_UBUNTU_REPO_DEP_FILE:=$(call find-files,$(DEPS_DIR_CURRENT)/$(DIFF_UBUNTU_REPO_ART_NAME)) - -ifneq ($(DIFF_UBUNTU_REPO_DEP_FILE),) -$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): $(DIFF_UBUNTU_REPO_DEP_FILE) - $(ACTION.COPY) -else -.DELETE_ON_ERROR: $(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME) -CURRENT_UBUNTU_REPO_DEP_FILE:=$(call find-files,$(DEPS_DIR_CURRENT)/$(UBUNTU_REPO_ART_NAME)) -ifneq ($(CURRENT_UBUNTU_REPO_DEP_FILE),) -$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): CURDIR=$(BUILD_DIR)/mirror/$(CURRENT_VERSION)/ubuntu-repo -$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): $(BUILD_DIR)/mirror/ubuntu_repo_current.done -$(BUILD_DIR)/mirror/ubuntu_repo_current.done: $(CURRENT_UBUNTU_REPO_DEP_FILE) - mkdir -p $(BUILD_DIR)/mirror/$(CURRENT_VERSION) - tar xf $(CURRENT_UBUNTU_REPO_DEP_FILE) -C $(BUILD_DIR)/mirror/$(CURRENT_VERSION) - $(ACTION.TOUCH) -else -$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): CURDIR=$(LOCAL_MIRROR_UBUNTU_OS_BASEURL) -$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): $(BUILD_DIR)/mirror/ubuntu_repo_current.done -$(BUILD_DIR)/mirror/ubuntu_repo_current.done: \ - $(BUILD_DIR)/mirror/build.done \ - $(BUILD_DIR)/packages/build.done \ - $(BUILD_DIR)/openstack/build.done - $(ACTION.TOUCH) - -endif -$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): BASEDIR=$(BUILD_DIR)/mirror/$(BASE_VERSION)/ubuntu-repo -$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): DIFFDIR=$(DIFF_MIRROR_UBUNTU_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION) -$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): - mkdir -p $(DIFFDIR) - rsync -av $(CURDIR)/* $(DIFFDIR) - zcat $(DIFFDIR)/dists/$(UBUNTU_RELEASE)/main/binary-amd64/Packages.gz | $(SOURCE_DIR)/iso/pkg-versions.awk > $(DIFF_MIRROR_UBUNTU_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION)/ubuntu-versions.yaml - tar cf $@ -C $(DIFF_MIRROR_UBUNTU_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION) --xform s:^:ubuntu_updates-$(CURRENT_VERSION)-$(BASE_VERSION)/: . -endif # ifneq ($(DIFF_UBUNTU_REPO_DEP_FILE),) -endif # ifneq ($(BASE_VERSION),) diff --git a/mirror/module.mk b/mirror/module.mk index fceffa90f..0bc810bea 100644 --- a/mirror/module.mk +++ b/mirror/module.mk @@ -11,7 +11,6 @@ clean-mirror: include $(SOURCE_DIR)/mirror/centos/module.mk include $(SOURCE_DIR)/mirror/ubuntu/module.mk include $(SOURCE_DIR)/mirror/docker/module.mk -include $(SOURCE_DIR)/mirror/diff_mirror_module.mk $(BUILD_DIR)/mirror/build.done: \ $(BUILD_DIR)/mirror/centos/build.done \ diff --git a/upgrade/module.mk b/upgrade/module.mk index 99c06d0b0..0580f7ffa 100644 --- a/upgrade/module.mk +++ b/upgrade/module.mk @@ -73,31 +73,16 @@ endif $(BUILD_DIR)/upgrade/openstack_version_$1: $$(ARTS_DIR_$1)/$(OPENSTACK_YAML_ART_NAME) python -c "import yaml; print filter(lambda r: r['fields'].get('name'), yaml.load(open('$$(ARTS_DIR_$1)/$(OPENSTACK_YAML_ART_NAME)')))[0]['fields']['version']" > $$@ -ifneq ($2,) -$(BUILD_DIR)/upgrade/openstack_version_$2: $(DEPS_DIR)/$2/$(OPENSTACK_YAML_ART_NAME) - python -c "import yaml; print filter(lambda r: r['fields'].get('name'), yaml.load(open('$(DEPS_DIR)/$2/$(OPENSTACK_YAML_ART_NAME)')))[0]['fields']['version']" > $$@ -endif - $(BUILD_DIR)/upgrade/openstack-part.done: $(BUILD_DIR)/upgrade/openstack-$1-part.done .DELETE_ON_ERROR: $(BUILD_DIR)/upgrade/openstack-part.tar -ifneq ($2,) -$(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_REPO_ART=$(DIFF_CENTOS_REPO_ART_BASE)-$1-$2.tar -$(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_REPO_ART_TOPDIR=centos_updates-$1-$2 -$(BUILD_DIR)/upgrade/openstack-$1-part.done: UBUNTU_REPO_ART=$(DIFF_UBUNTU_REPO_ART_BASE)-$1-$2.tar -$(BUILD_DIR)/upgrade/openstack-$1-part.done: UBUNTU_REPO_ART_TOPDIR=ubuntu_updates-$1-$2 -$(BUILD_DIR)/upgrade/openstack-$1-part.done: $(BUILD_DIR)/upgrade/openstack_version_$2 -$(BUILD_DIR)/upgrade/openstack-$1-part.done: $$(ARTS_DIR_$1)/$(DIFF_CENTOS_REPO_ART_BASE)-$1-$2.tar -$(BUILD_DIR)/upgrade/openstack-$1-part.done: $$(ARTS_DIR_$1)/$(DIFF_UBUNTU_REPO_ART_BASE)-$1-$2.tar -else $(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_REPO_ART=$(CENTOS_REPO_ART_NAME) $(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_REPO_ART_TOPDIR=centos-repo $(BUILD_DIR)/upgrade/openstack-$1-part.done: UBUNTU_REPO_ART=$(UBUNTU_REPO_ART_NAME) $(BUILD_DIR)/upgrade/openstack-$1-part.done: UBUNTU_REPO_ART_TOPDIR=ubuntu-repo $(BUILD_DIR)/upgrade/openstack-$1-part.done: $$(ARTS_DIR_$1)/$(CENTOS_REPO_ART_NAME) $(BUILD_DIR)/upgrade/openstack-$1-part.done: $$(ARTS_DIR_$1)/$(UBUNTU_REPO_ART_NAME) -endif $(BUILD_DIR)/upgrade/openstack-$1-part.done: BASE=$(BUILD_DIR)/upgrade/openstack-$1-part $(BUILD_DIR)/upgrade/openstack-$1-part.done: OPENSTACK_VERSION=$$(shell cat $(BUILD_DIR)/upgrade/openstack_version_$1) $(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_BASE=$$(BASE)/upgrade/repos/$$(OPENSTACK_VERSION)/centos/x86_64 @@ -119,11 +104,6 @@ $(BUILD_DIR)/upgrade/openstack-$1-part.done: \ # OPENSTACK-YAML mkdir -p $$(RELEASES_BASE) cp $$(ARTS_DIR_$1)/$(OPENSTACK_YAML_ART_NAME) $$(RELEASES_BASE)/$$(OPENSTACK_VERSION).yaml -# METADATA-YAML -ifneq ($2,) - grep -q "diff_releases:" $(BUILD_DIR)/upgrade/metadata.yaml 2>/dev/null || echo "diff_releases:" > $(BUILD_DIR)/upgrade/metadata.yaml - echo " $$(OPENSTACK_VERSION): $$(shell cat $(BUILD_DIR)/upgrade/openstack_version_$2)" >> $(BUILD_DIR)/upgrade/metadata.yaml -endif # VERSION-YAML mkdir -p $$(RELEASE_VERSIONS_BASE) cp $$(ARTS_DIR_$1)/$(VERSION_YAML_ART_NAME) $$(RELEASE_VERSIONS_BASE)/$$(OPENSTACK_VERSION).yaml @@ -134,10 +114,12 @@ endif endef -$(foreach diff,$(UPGRADE_VERSIONS),$(eval $(call build_openstack_part,$(shell echo $(diff) | awk -F':' '{print $$1}'),$(shell echo $(diff) | awk -F':' '{print $$2}')))) +$(foreach upgrade_version,$(UPGRADE_VERSIONS),$(eval $(call build_openstack_part,$(upgrade_version)))) $(BUILD_DIR)/upgrade/openstack-part.done: +# This is for backward compatibility with upgrade script. +# It tries to figure out whether a particular update bundle diffirential or not. grep -q "diff_releases:" $(BUILD_DIR)/upgrade/metadata.yaml 2>/dev/null || echo "diff_releases: {}" > $(BUILD_DIR)/upgrade/metadata.yaml tar rf $(BUILD_DIR)/upgrade/openstack-part.tar -C $(BUILD_DIR)/upgrade/ metadata.yaml --xform s:^:upgrade/releases/: $(ACTION.TOUCH)