openstack
/
fuel-main
Code Issues Proposed changes

Remove differential (diff) repos 

We used to create differential Ubuntu and Centos
repositories for new Fuel versions to make them
smaller. However, this technique contradicts
our current upgrade/patching approach. So, this
patch removes this functionality.

Closes-Bug: #1447135
Related-Bug: #1446293

Change-Id: I6c9288e77c6ccc5872056a8119230ec725846bba
This commit is contained in:
Vladimir Kozhukalov 2015-04-22 15:02:40 +03:00
parent ea80e2c7df
commit 3dea637b2e
6 changed files with 5 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -75,7 +75,7 @@ $(BUILD_DIR)/current_version: $(call depv,CURRENT_VERSION)
upgrade-versions: $(BUILD_DIR)/upgrade_versions
$(BUILD_DIR)/upgrade_versions: $(call depv,UPGRADE_VERSIONS)
echo -n > $@
$(foreach diff,$(UPGRADE_VERSIONS),echo $(diff) >> $@;)
$(foreach upgrade_version,$(UPGRADE_VERSIONS),echo $(upgrade_version) >> $@;)
# Macroses for make
include $(SOURCE_DIR)/rules.mk

38
config.mk
View File

@ -33,38 +33,8 @@ PRODUCT_NAME:=mos
# to come from DEPS_DIR "as is"
CURRENT_VERSION:=$(PRODUCT_VERSION)
# This variable is used for building diff repos.
# If it is not set then diff repo will not be built.
# If it is set then diff $(BASE_VERSION)->$(CURRENT_VERSION)
BASE_VERSION:=6.0
PACKAGE_VERSION=6.1.0
# UPGRADE_VERSIONS?=\
# 6.0:5.1 \
# 5.1 \
# 5.0.3:5.0
#
# It means we need to put into a tarball
#
# 0) 5.1 -> 6.0 diff mirror and other 6.0 files
# 1) 5.1 full mirror and other 5.1 files
# 2) 5.0 -> 5.0.3 diff mirror and other 5.0.3 files
#
# * It is prohibited to have multiple bundles for
# a particular version. E.g. 6.0 bundle can be one of
# ** 6.0 full bundle
# ** X.Y.Z -> 6.0 diff bundle
#
# * If a key (version before colon) is
# the same as $(CURRENT_VERSION) then
# a mirror (full or diff) will be built.
#
# * If a key does not match $(CURRENT_VERSION) then
# a mirror (full or diff) is supposed to be
# available as an artifact from a previous build job.
#
UPGRADE_VERSIONS?=\
$(CURRENT_VERSION)
UPGRADE_VERSIONS?=$(CURRENT_VERSION)
# Path to pre-built artifacts
DEPS_DIR_CURRENT?=$(DEPS_DIR)/$(CURRENT_VERSION)
@ -80,10 +50,6 @@ DOCKER_ART_NAME?=fuel-images.tar.lrz
VERSION_YAML_ART_NAME?=version.yaml
CENTOS_REPO_ART_NAME?=centos-repo.tar
UBUNTU_REPO_ART_NAME?=ubuntu-repo.tar
# actual name for a diff repo will be
# $(DIFF_CENTOS_REPO_ART_BASE)-NEWVERSION-OLDVERSION.tar
DIFF_CENTOS_REPO_ART_BASE?=diff-centos-repo
DIFF_UBUNTU_REPO_ART_BASE?=diff-ubuntu-repo
PUPPET_ART_NAME?=puppet.tgz
OPENSTACK_YAML_ART_NAME?=openstack.yaml
TARGET_UBUNTU_IMG_ART_NAME?=ubuntu_target_images.tar
@ -177,8 +143,6 @@ LOCAL_MIRROR_UBUNTU:=$(LOCAL_MIRROR)/ubuntu
LOCAL_MIRROR_UBUNTU_OS_BASEURL:=$(LOCAL_MIRROR_UBUNTU)
LOCAL_MIRROR_DOCKER:=$(LOCAL_MIRROR)/docker
LOCAL_MIRROR_DOCKER_BASEURL:=$(LOCAL_MIRROR_DOCKER)
DIFF_MIRROR_CENTOS_BASE:=$(LOCAL_MIRROR)/centos_updates
DIFF_MIRROR_UBUNTU_BASE:=$(LOCAL_MIRROR)/ubuntu_updates
# Use download.mirantis.com mirror by default. Other possible values are
# 'msk', 'srt', 'usa', 'hrk'.

28
mirror/create_diff_mirrors.sh
View File

@ -1,28 +0,0 @@
#!/bin/bash
NEWER=$1
OLDER=$2
DIFF=$3
# we need to check if OLDER dir exists
# if not, nothing to compare return exit 1
test -d ${OLDER} || { echo "${OLDER} path doesn't exist, nothing to compare"; exit 1; }
test -d ${DIFF} || mkdir -p ${DIFF}
function copy_by_checksum (){
# compare files by checksum
CHECKSUM1=$(md5sum -- $1 | awk '{print $1}')
CHECKSUM2=$(md5sum -- $2 | awk '{print $1}')
# if checksum is different then copy
[ $CHECKSUM1 = $CHECKSUM2 ] || cp -av $1 $3
}
export -f copy_by_checksum
# if file doesn't exists in old, then copy it to diff
# else compare by checksum
find ${NEWER} -type f -printf "%f\n" | xargs -i bash -c "test -f ${OLDER}/{} \
&& copy_by_checksum ${NEWER}/{} ${OLDER}/{} ${DIFF} \
|| cp -av ${NEWER}/{} ${DIFF}/{}"

101
mirror/diff_mirror_module.mk
View File

@ -1,101 +0,0 @@
#############################
# CENTOS DIFF MIRROR ARTIFACT
#############################
ifneq ($(BASE_VERSION),)
.PHONY: centos-diff-repo
DIFF_CENTOS_REPO_ART_NAME:=$(DIFF_CENTOS_REPO_ART_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION).tar
centos-diff-repo: $(ARTS_DIR)/$(DIFF_CENTOS_REPO_ART_NAME)
$(ARTS_DIR)/$(DIFF_CENTOS_REPO_ART_NAME): $(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME)
$(ACTION.COPY)
DIFF_CENTOS_REPO_DEP_FILE:=$(call find-files,$(DEPS_DIR_CURRENT)/$(DIFF_CENTOS_REPO_ART_NAME))
ifneq ($(DIFF_CENTOS_REPO_DEP_FILE),)
$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): $(DIFF_CENTOS_REPO_DEP_FILE)
$(ACTION.COPY)
else
.DELETE_ON_ERROR: $(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME)
CURRENT_CENTOS_REPO_DEP_FILE:=$(call find-files,$(DEPS_DIR_CURRENT)/$(CENTOS_REPO_ART_NAME))
ifneq ($(CURRENT_CENTOS_REPO_DEP_FILE),)
$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): CURDIR=$(BUILD_DIR)/mirror/$(CURRENT_VERSION)/centos-repo/Packages
$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): $(BUILD_DIR)/mirror/centos_repo_current.done
$(BUILD_DIR)/mirror/centos_repo_current.done: $(CURRENT_CENTOS_REPO_DEP_FILE)
mkdir -p $(BUILD_DIR)/mirror/$(CURRENT_VERSION)
tar xf $(CURRENT_CENTOS_REPO_DEP_FILE) -C $(BUILD_DIR)/mirror/$(CURRENT_VERSION)
$(ACTION.TOUCH)
else
$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): CURDIR=$(LOCAL_MIRROR_CENTOS_OS_BASEURL)/Packages
$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): $(BUILD_DIR)/mirror/centos_repo_current.done
$(BUILD_DIR)/mirror/centos_repo_current.done: \
$(BUILD_DIR)/mirror/build.done \
$(BUILD_DIR)/packages/build.done \
$(BUILD_DIR)/openstack/build.done
$(ACTION.TOUCH)
endif
$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): BASEDIR=$(BUILD_DIR)/mirror/$(BASE_VERSION)/centos-repo/Packages
$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): DIFFDIR=$(DIFF_MIRROR_CENTOS_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION)/Packages
$(BUILD_DIR)/mirror/$(DIFF_CENTOS_REPO_ART_NAME): | $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml
# unpacking old version centos mirror
mkdir -p $(BUILD_DIR)/mirror/$(BASE_VERSION)
tar xf $(DEPS_DIR)/$(BASE_VERSION)/$(CENTOS_REPO_ART_NAME) -C $(BUILD_DIR)/mirror/$(BASE_VERSION)
# copying packages which differ from those in base version
mkdir -p $(DIFFDIR)
/bin/bash $(SOURCE_DIR)/mirror/create_diff_mirrors.sh $(CURDIR) $(BASEDIR) $(DIFFDIR)
# creating diff mirror
cp $(LOCAL_MIRROR_CENTOS_OS_BASEURL)/comps.xml $(DIFFDIR)/../comps.xml
createrepo -g $(DIFFDIR)/../comps.xml -o $(DIFFDIR)/../ $(DIFFDIR)/../
rpm -qi -p $(DIFFDIR)/*.rpm | $(SOURCE_DIR)/iso/pkg-versions.awk > $(DIFF_MIRROR_CENTOS_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION)/centos-versions.yaml
tar cf $@ -C $(DIFF_MIRROR_CENTOS_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION) --xform s:^:centos_updates-$(CURRENT_VERSION)-$(BASE_VERSION)/: .
endif # ifneq ($(DIFF_CENTOS_REPO_DEP_FILE),)
endif # ifneq ($(BASE_VERSION),)
#############################
# UBUNTU DIFF MIRROR ARTIFACT
#############################
ifneq ($(BASE_VERSION),)
.PHONY: ubuntu-diff-repo
DIFF_UBUNTU_REPO_ART_NAME:=$(DIFF_UBUNTU_REPO_ART_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION).tar
ubuntu-diff-repo: $(ARTS_DIR)/$(DIFF_UBUNTU_REPO_ART_NAME)
$(ARTS_DIR)/$(DIFF_UBUNTU_REPO_ART_NAME): $(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME)
$(ACTION.COPY)
DIFF_UBUNTU_REPO_DEP_FILE:=$(call find-files,$(DEPS_DIR_CURRENT)/$(DIFF_UBUNTU_REPO_ART_NAME))
ifneq ($(DIFF_UBUNTU_REPO_DEP_FILE),)
$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): $(DIFF_UBUNTU_REPO_DEP_FILE)
$(ACTION.COPY)
else
.DELETE_ON_ERROR: $(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME)
CURRENT_UBUNTU_REPO_DEP_FILE:=$(call find-files,$(DEPS_DIR_CURRENT)/$(UBUNTU_REPO_ART_NAME))
ifneq ($(CURRENT_UBUNTU_REPO_DEP_FILE),)
$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): CURDIR=$(BUILD_DIR)/mirror/$(CURRENT_VERSION)/ubuntu-repo
$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): $(BUILD_DIR)/mirror/ubuntu_repo_current.done
$(BUILD_DIR)/mirror/ubuntu_repo_current.done: $(CURRENT_UBUNTU_REPO_DEP_FILE)
mkdir -p $(BUILD_DIR)/mirror/$(CURRENT_VERSION)
tar xf $(CURRENT_UBUNTU_REPO_DEP_FILE) -C $(BUILD_DIR)/mirror/$(CURRENT_VERSION)
$(ACTION.TOUCH)
else
$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): CURDIR=$(LOCAL_MIRROR_UBUNTU_OS_BASEURL)
$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): $(BUILD_DIR)/mirror/ubuntu_repo_current.done
$(BUILD_DIR)/mirror/ubuntu_repo_current.done: \
$(BUILD_DIR)/mirror/build.done \
$(BUILD_DIR)/packages/build.done \
$(BUILD_DIR)/openstack/build.done
$(ACTION.TOUCH)
endif
$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): BASEDIR=$(BUILD_DIR)/mirror/$(BASE_VERSION)/ubuntu-repo
$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME): DIFFDIR=$(DIFF_MIRROR_UBUNTU_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION)
$(BUILD_DIR)/mirror/$(DIFF_UBUNTU_REPO_ART_NAME):
mkdir -p $(DIFFDIR)
rsync -av $(CURDIR)/* $(DIFFDIR)
zcat $(DIFFDIR)/dists/$(UBUNTU_RELEASE)/main/binary-amd64/Packages.gz | $(SOURCE_DIR)/iso/pkg-versions.awk > $(DIFF_MIRROR_UBUNTU_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION)/ubuntu-versions.yaml
tar cf $@ -C $(DIFF_MIRROR_UBUNTU_BASE)-$(CURRENT_VERSION)-$(BASE_VERSION) --xform s:^:ubuntu_updates-$(CURRENT_VERSION)-$(BASE_VERSION)/: .
endif # ifneq ($(DIFF_UBUNTU_REPO_DEP_FILE),)
endif # ifneq ($(BASE_VERSION),)

1
mirror/module.mk
View File

@ -11,7 +11,6 @@ clean-mirror:
include $(SOURCE_DIR)/mirror/centos/module.mk
include $(SOURCE_DIR)/mirror/ubuntu/module.mk
include $(SOURCE_DIR)/mirror/docker/module.mk
include $(SOURCE_DIR)/mirror/diff_mirror_module.mk
$(BUILD_DIR)/mirror/build.done: \
$(BUILD_DIR)/mirror/centos/build.done \

24
upgrade/module.mk
View File

@ -73,31 +73,16 @@ endif
$(BUILD_DIR)/upgrade/openstack_version_$1: $$(ARTS_DIR_$1)/$(OPENSTACK_YAML_ART_NAME)
python -c "import yaml; print filter(lambda r: r['fields'].get('name'), yaml.load(open('$$(ARTS_DIR_$1)/$(OPENSTACK_YAML_ART_NAME)')))[0]['fields']['version']" > $$@
ifneq ($2,)
$(BUILD_DIR)/upgrade/openstack_version_$2: $(DEPS_DIR)/$2/$(OPENSTACK_YAML_ART_NAME)
python -c "import yaml; print filter(lambda r: r['fields'].get('name'), yaml.load(open('$(DEPS_DIR)/$2/$(OPENSTACK_YAML_ART_NAME)')))[0]['fields']['version']" > $$@
endif
$(BUILD_DIR)/upgrade/openstack-part.done: $(BUILD_DIR)/upgrade/openstack-$1-part.done
.DELETE_ON_ERROR: $(BUILD_DIR)/upgrade/openstack-part.tar
ifneq ($2,)
$(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_REPO_ART=$(DIFF_CENTOS_REPO_ART_BASE)-$1-$2.tar
$(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_REPO_ART_TOPDIR=centos_updates-$1-$2
$(BUILD_DIR)/upgrade/openstack-$1-part.done: UBUNTU_REPO_ART=$(DIFF_UBUNTU_REPO_ART_BASE)-$1-$2.tar
$(BUILD_DIR)/upgrade/openstack-$1-part.done: UBUNTU_REPO_ART_TOPDIR=ubuntu_updates-$1-$2
$(BUILD_DIR)/upgrade/openstack-$1-part.done: $(BUILD_DIR)/upgrade/openstack_version_$2
$(BUILD_DIR)/upgrade/openstack-$1-part.done: $$(ARTS_DIR_$1)/$(DIFF_CENTOS_REPO_ART_BASE)-$1-$2.tar
$(BUILD_DIR)/upgrade/openstack-$1-part.done: $$(ARTS_DIR_$1)/$(DIFF_UBUNTU_REPO_ART_BASE)-$1-$2.tar
else
$(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_REPO_ART=$(CENTOS_REPO_ART_NAME)
$(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_REPO_ART_TOPDIR=centos-repo
$(BUILD_DIR)/upgrade/openstack-$1-part.done: UBUNTU_REPO_ART=$(UBUNTU_REPO_ART_NAME)
$(BUILD_DIR)/upgrade/openstack-$1-part.done: UBUNTU_REPO_ART_TOPDIR=ubuntu-repo
$(BUILD_DIR)/upgrade/openstack-$1-part.done: $$(ARTS_DIR_$1)/$(CENTOS_REPO_ART_NAME)
$(BUILD_DIR)/upgrade/openstack-$1-part.done: $$(ARTS_DIR_$1)/$(UBUNTU_REPO_ART_NAME)
endif
$(BUILD_DIR)/upgrade/openstack-$1-part.done: BASE=$(BUILD_DIR)/upgrade/openstack-$1-part
$(BUILD_DIR)/upgrade/openstack-$1-part.done: OPENSTACK_VERSION=$$(shell cat $(BUILD_DIR)/upgrade/openstack_version_$1)
$(BUILD_DIR)/upgrade/openstack-$1-part.done: CENTOS_BASE=$$(BASE)/upgrade/repos/$$(OPENSTACK_VERSION)/centos/x86_64
@ -119,11 +104,6 @@ $(BUILD_DIR)/upgrade/openstack-$1-part.done: \
# OPENSTACK-YAML
mkdir -p $$(RELEASES_BASE)
cp $$(ARTS_DIR_$1)/$(OPENSTACK_YAML_ART_NAME) $$(RELEASES_BASE)/$$(OPENSTACK_VERSION).yaml
# METADATA-YAML
ifneq ($2,)
grep -q "diff_releases:" $(BUILD_DIR)/upgrade/metadata.yaml 2>/dev/null || echo "diff_releases:" > $(BUILD_DIR)/upgrade/metadata.yaml
echo " $$(OPENSTACK_VERSION): $$(shell cat $(BUILD_DIR)/upgrade/openstack_version_$2)" >> $(BUILD_DIR)/upgrade/metadata.yaml
endif
# VERSION-YAML
mkdir -p $$(RELEASE_VERSIONS_BASE)
cp $$(ARTS_DIR_$1)/$(VERSION_YAML_ART_NAME) $$(RELEASE_VERSIONS_BASE)/$$(OPENSTACK_VERSION).yaml
@ -134,10 +114,12 @@ endif
endef
$(foreach diff,$(UPGRADE_VERSIONS),$(eval $(call build_openstack_part,$(shell echo $(diff) | awk -F':' '{print $$1}'),$(shell echo $(diff) | awk -F':' '{print $$2}'))))
$(foreach upgrade_version,$(UPGRADE_VERSIONS),$(eval $(call build_openstack_part,$(upgrade_version))))
$(BUILD_DIR)/upgrade/openstack-part.done:
# This is for backward compatibility with upgrade script.
# It tries to figure out whether a particular update bundle diffirential or not.
grep -q "diff_releases:" $(BUILD_DIR)/upgrade/metadata.yaml 2>/dev/null || echo "diff_releases: {}" > $(BUILD_DIR)/upgrade/metadata.yaml
tar rf $(BUILD_DIR)/upgrade/openstack-part.tar -C $(BUILD_DIR)/upgrade/ metadata.yaml --xform s:^:upgrade/releases/:
$(ACTION.TOUCH)