From e6b165748a8771a9f15d3416fc0cd05b6e40b63a Mon Sep 17 00:00:00 2001 From: Vladimir Kozhukalov Date: Thu, 26 May 2016 08:15:17 +0300 Subject: [PATCH] Modify default repos for fuel and fuelmenu In fuel (nailgun) and fuelmenu we have hardcoded default repos which we should be able to change mostly for test purposes. Default ISO build flow uses fuel packages rebuild approach, which modifies ubuntu suit as well, eg: sync(mos-master) --> (iso)mos10.0 --> deploy(mos10.0) To use this feature one must use BUILD_PACKAGES=0, this flag turns off: * fuel packages building and * ubuntu suit update. so We consume the same suit as we use during debmirroring, eg: sync(mos-master) --> iso(mos-master) --> deploy(mos-master) DocImpact Change-Id: Ia3cefa7c87e35ecd9244a4026b86e772bf569ca9 Closes-bug: #1556125 --- iso/bootstrap_admin_node.sh | 12 +++ iso/fix_default_repos.py | 206 ++++++++++++++++++++++++++++++++++++ iso/ks.template | 3 + iso/module.mk | 19 +++- specs/fuel-main.spec | 3 +- 5 files changed, 241 insertions(+), 2 deletions(-) create mode 100755 iso/fix_default_repos.py diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh index 59114da5d..655bc51ae 100755 --- a/iso/bootstrap_admin_node.sh +++ b/iso/bootstrap_admin_node.sh @@ -61,6 +61,7 @@ wget \ ASTUTE_YAML='/etc/fuel/astute.yaml' BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" +CUSTOM_REPOS="/root/default_deb_repos.yaml" bs_build_log='/var/log/fuel-bootstrap-image-build.log' bs_status=0 # Backup network configs to this folder. Folder will be created only if @@ -328,6 +329,11 @@ if (virt-what | fgrep -q "virtualbox") ; then done fi +# change default repo path in fuel-menu before starting any deployment steps +if [ -f "${CUSTOM_REPOS}" ]; then + fix_default_repos.py fuelmenu --repositories-file "${CUSTOM_REPOS}" || fail +fi + fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail set +x echo "Done!" @@ -517,6 +523,12 @@ fi # apply puppet /etc/puppet/modules/fuel/examples/deploy.sh || fail +# Update default repo path +if [ -f "${CUSTOM_REPOS}" ]; then + fix_default_repos.py fuel \ + --repositories-file "${CUSTOM_REPOS}" \ + --release-version "${OPENSTACK_VERSION}" || fail +fi # Sync time systemctl stop ntpd diff --git a/iso/fix_default_repos.py b/iso/fix_default_repos.py new file mode 100755 index 000000000..8bd060cbf --- /dev/null +++ b/iso/fix_default_repos.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import argparse +import os + +import six +import yaml + +from fuelclient import objects + + +FUELMENU_DEFAULT_SETTINGS_PATH = \ + "/usr/lib/python2.7/site-packages/fuelmenu/settings.yaml" + + +def is_subdict(dict1, dict2): + """Checks that dict1 is subdict of dict2. + + >>> is_subdict({"a": 1}, {'a': 1, 'b': 1}) + True + + :param dict1: the candidate + :param dict2: the super dict + :return: True if all keys from dict1 are present + and has same value in dict2 otherwise False + """ + for k, v in six.iteritems(dict1): + if k not in dict2 or dict2[k] != v: + return False + return True + + +def lists_merge(main, patch, key): + """Merges the list of dicts with same keys. + + >>> lists_merge([{"a": 1, "c": 2}], [{"a": 1, "c": 3}], key="a") + [{'a': 1, 'c': 3}] + + :param main: the main list + :type main: list + :param patch: the list of additional elements + :type patch: list + :param key: the key for compare + """ + main_idx = dict( + (x[key], i) for i, x in enumerate(main) + ) + + patch_idx = dict( + (x[key], i) for i, x in enumerate(patch) + ) + + for k in sorted(patch_idx): + if k in main_idx: + main[main_idx[k]].update(patch[patch_idx[k]]) + else: + main.append(patch[patch_idx[k]]) + return main + + +def update_release_repos(repositories, + release_match, + replace_repos=False): + """Applies repositories for existing default settings. + :param repositories: the meta information of repositories + :param release_match: The pattern to check Fuel Release + """ + releases = six.moves.filter( + lambda x: is_subdict(release_match, x.data), + objects.Release.get_all() + ) + for release in releases: + modified = _update_repository_settings( + release.data["attributes_metadata"], + repositories, + replace_repos=replace_repos) + if modified: + release.data["attributes_metadata"] = modified + print "Try to update the Release '%s'" % release.data['name'] + release.connection.put_request( + release.instance_api_path.format(release.id), + release.data + ) + + +def _update_repository_settings(settings, + repositories, + replace_repos=False): + """Updates repository settings. + :param settings: the target settings + :param repositories: the meta of repositories + """ + editable = settings["editable"] + if 'repo_setup' not in editable: + return + + repos_attr = editable["repo_setup"]["repos"] + if replace_repos: + repos_attr['value'] = repositories + else: + lists_merge(repos_attr['value'], repositories, "name") + + settings["editable"]["repo_setup"]["repos"] = repos_attr + + return settings + + +def fix_fuel_repos(address, port, user, password, + release_version, release_os, repositories): + os.environ["SERVER_ADDRESS"] = address + os.environ["LISTEN_PORT"] = port + os.environ["KEYSTONE_USER"] = user + os.environ["KEYSTONE_PASS"] = password + + release_match = { + "version": release_version, + "operating_system": release_os + } + + update_release_repos(repositories, release_match) + + +def fix_fuelmenu_repos(repositories, replace_repos=False): + print "Try to update default fuelmenu settings" + with open(FUELMENU_DEFAULT_SETTINGS_PATH) as f: + settings = yaml.safe_load(f) + if replace_repos: + settings["BOOTSTRAP"]["repos"] = repositories + else: + lists_merge(settings["BOOTSTRAP"]["repos"], repositories, "name") + with open(FUELMENU_DEFAULT_SETTINGS_PATH, "w") as f: + f.write(yaml.safe_dump(settings, default_flow_style=False)) + + +def main(): + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers( + dest="action", help='actions' + ) + fuel_parser = subparsers.add_parser( + 'fuel', help='fix fuel repos' + ) + fuel_parser.add_argument( + '--release-version', dest='release_version', action='store', + type=str, help='release version', default='newton-10.0' + ) + fuel_parser.add_argument( + '--release-os', dest='release_os', action='store', + type=str, help='release operating system', default='Ubuntu' + ) + fuel_parser.add_argument( + '--repositories-file', dest='repositories_file', action='store', + type=str, help='file where repositories are defined', required=True + ) + fuel_parser.add_argument( + '-a', '--address', dest='address', action='store', type=str, + help='fuel address', default='127.0.0.1' + ) + fuel_parser.add_argument( + '-p', '--port', dest='port', action='store', type=str, + help='fuel port', default='8000' + ) + fuel_parser.add_argument( + '--user', dest='user', action='store', type=str, + help='fuel user', default='admin' + ) + fuel_parser.add_argument( + '--password', dest='password', action='store', type=str, + help='fuel password', default='admin' + ) + fuelmenu_parser = subparsers.add_parser( + 'fuelmenu', help='fix fuelmenu repos' + ) + fuelmenu_parser.add_argument( + '--repositories-file', dest='repositories_file', action='store', + type=str, help='file where repositories are defined', required=True + ) + params, other_params = parser.parse_known_args() + + with open(params.repositories_file) as f: + repositories = yaml.safe_load(f) + + if params.action == 'fuel': + fix_fuel_repos(params.address, params.port, + params.user, params.password, + params.release_version, params.release_os, + repositories) + else: + fix_fuelmenu_repos(repositories) + + +if __name__ == "__main__": + main() diff --git a/iso/ks.template b/iso/ks.template index 8db3759af..5bf201ca0 100644 --- a/iso/ks.template +++ b/iso/ks.template @@ -390,6 +390,9 @@ OPENSTACK_VERSION=`rpm2cpio ${SOURCE}/mos-centos/Packages/fuel-openstack-metadat test -e ${SOURCE}/fuel_build_number && cp ${SOURCE}/fuel_build_number /etc/fuel_build_number test -e ${SOURCE}/fuel_build_id && cp ${SOURCE}/fuel_build_id /etc/fuel_build_id +# Copy repos config +test -e ${SOURCE}/default_deb_repos.yaml && cp ${SOURCE}/default_deb_repos.yaml /root/default_deb_repos.yaml + # ---------------------- # UNPACKING REPOSITORIES # ---------------------- diff --git a/iso/module.mk b/iso/module.mk index 8995378fa..af883dcdf 100644 --- a/iso/module.mk +++ b/iso/module.mk @@ -31,6 +31,24 @@ $(BUILD_DIR)/iso/isoroot.done: $(ISOROOT)/fuel_build_id $(ISOROOT)/fuel_build_id: echo "$(BUILD_ID)" > $@ +############## +# CUSTOM REPOS +############## + +define default_deb_repos +- name: mos + suite: $(MIRROR_MOS_UBUNTU_SUITE) +endef + +# if we are not building packages and sync repos only, we MUST use +# the same suit as we use during debmirroring +ifeq ($(BUILD_PACKAGES),0) +$(BUILD_DIR)/iso/isoroot.done: $(ISOROOT)/default_deb_repos.yaml +endif +$(ISOROOT)/default_deb_repos.yaml: export default_deb_repos_content:=$(default_deb_repos) +$(ISOROOT)/default_deb_repos.yaml: + /bin/echo -e "$${default_deb_repos_content}\n" > $@ + ############### # CENTOS MIRROR ############### @@ -88,7 +106,6 @@ $(ISOROOT)/ks.cfg: $(SOURCE_DIR)/iso/ks.template $(SOURCE_DIR)/iso/ks.py $(ISORO python $(SOURCE_DIR)/iso/ks.py \ -t $(SOURCE_DIR)/iso/ks.template \ -c $(ISOROOT)/ks.yaml \ - -u '{"CENTOS_RELEASE": "$(CENTOS_RELEASE)", "PRODUCT_VERSION": "$(PRODUCT_VERSION)"}' \ -o $@.tmp mv $@.tmp $@ diff --git a/specs/fuel-main.spec b/specs/fuel-main.spec index 7f6b0c02d..5053255c3 100644 --- a/specs/fuel-main.spec +++ b/specs/fuel-main.spec @@ -48,6 +48,7 @@ for file in %{_builddir}/%{name}-%{version}/fuel-release/*.repo ; do install -D -m 644 "$file" %{buildroot}/etc/yum.repos.d done install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/bootstrap_admin_node.sh %{buildroot}%{_sbindir}/bootstrap_admin_node.sh +install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/fix_default_repos.py %{buildroot}%{_sbindir}/fix_default_repos.py %clean rm -rf %{buildroot} @@ -92,4 +93,4 @@ This packages provides script to deploy Fuel components. %files -n fuel-setup %defattr(-,root,root) %{_sbindir}/bootstrap_admin_node.sh - +%{_sbindir}/fix_default_repos.py