Modify default repos for fuel and fuelmenu

In fuel (nailgun) and fuelmenu we have hardcoded
default repos which we should be able to change
mostly for test purposes.

Default ISO build flow uses fuel packages rebuild
approach, which modifies ubuntu suit as well, eg:
   sync(mos-master) --> (iso)mos10.0 --> deploy(mos10.0)

To use this feature one must use BUILD_PACKAGES=0,
this flag turns off:
  * fuel packages building and
  * ubuntu suit update.
so We consume the same suit as we use during
debmirroring, eg:
   sync(mos-master) --> iso(mos-master) --> deploy(mos-master)

DocImpact
Change-Id: Ia3cefa7c87e35ecd9244a4026b86e772bf569ca9
Closes-bug: #1556125
This commit is contained in:
Vladimir Kozhukalov 2016-05-26 08:15:17 +03:00 committed by Sergey Kulanov
parent a0d7115479
commit e6b165748a
5 changed files with 241 additions and 2 deletions

View File

@ -61,6 +61,7 @@ wget \
ASTUTE_YAML='/etc/fuel/astute.yaml'
BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
CUSTOM_REPOS="/root/default_deb_repos.yaml"
bs_build_log='/var/log/fuel-bootstrap-image-build.log'
bs_status=0
# Backup network configs to this folder. Folder will be created only if
@ -328,6 +329,11 @@ if (virt-what | fgrep -q "virtualbox") ; then
done
fi
# change default repo path in fuel-menu before starting any deployment steps
if [ -f "${CUSTOM_REPOS}" ]; then
fix_default_repos.py fuelmenu --repositories-file "${CUSTOM_REPOS}" || fail
fi
fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail
set +x
echo "Done!"
@ -517,6 +523,12 @@ fi
# apply puppet
/etc/puppet/modules/fuel/examples/deploy.sh || fail
# Update default repo path
if [ -f "${CUSTOM_REPOS}" ]; then
fix_default_repos.py fuel \
--repositories-file "${CUSTOM_REPOS}" \
--release-version "${OPENSTACK_VERSION}" || fail
fi
# Sync time
systemctl stop ntpd

206
iso/fix_default_repos.py Executable file
View File

@ -0,0 +1,206 @@
#!/usr/bin/env python
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import argparse
import os
import six
import yaml
from fuelclient import objects
FUELMENU_DEFAULT_SETTINGS_PATH = \
"/usr/lib/python2.7/site-packages/fuelmenu/settings.yaml"
def is_subdict(dict1, dict2):
"""Checks that dict1 is subdict of dict2.
>>> is_subdict({"a": 1}, {'a': 1, 'b': 1})
True
:param dict1: the candidate
:param dict2: the super dict
:return: True if all keys from dict1 are present
and has same value in dict2 otherwise False
"""
for k, v in six.iteritems(dict1):
if k not in dict2 or dict2[k] != v:
return False
return True
def lists_merge(main, patch, key):
"""Merges the list of dicts with same keys.
>>> lists_merge([{"a": 1, "c": 2}], [{"a": 1, "c": 3}], key="a")
[{'a': 1, 'c': 3}]
:param main: the main list
:type main: list
:param patch: the list of additional elements
:type patch: list
:param key: the key for compare
"""
main_idx = dict(
(x[key], i) for i, x in enumerate(main)
)
patch_idx = dict(
(x[key], i) for i, x in enumerate(patch)
)
for k in sorted(patch_idx):
if k in main_idx:
main[main_idx[k]].update(patch[patch_idx[k]])
else:
main.append(patch[patch_idx[k]])
return main
def update_release_repos(repositories,
release_match,
replace_repos=False):
"""Applies repositories for existing default settings.
:param repositories: the meta information of repositories
:param release_match: The pattern to check Fuel Release
"""
releases = six.moves.filter(
lambda x: is_subdict(release_match, x.data),
objects.Release.get_all()
)
for release in releases:
modified = _update_repository_settings(
release.data["attributes_metadata"],
repositories,
replace_repos=replace_repos)
if modified:
release.data["attributes_metadata"] = modified
print "Try to update the Release '%s'" % release.data['name']
release.connection.put_request(
release.instance_api_path.format(release.id),
release.data
)
def _update_repository_settings(settings,
repositories,
replace_repos=False):
"""Updates repository settings.
:param settings: the target settings
:param repositories: the meta of repositories
"""
editable = settings["editable"]
if 'repo_setup' not in editable:
return
repos_attr = editable["repo_setup"]["repos"]
if replace_repos:
repos_attr['value'] = repositories
else:
lists_merge(repos_attr['value'], repositories, "name")
settings["editable"]["repo_setup"]["repos"] = repos_attr
return settings
def fix_fuel_repos(address, port, user, password,
release_version, release_os, repositories):
os.environ["SERVER_ADDRESS"] = address
os.environ["LISTEN_PORT"] = port
os.environ["KEYSTONE_USER"] = user
os.environ["KEYSTONE_PASS"] = password
release_match = {
"version": release_version,
"operating_system": release_os
}
update_release_repos(repositories, release_match)
def fix_fuelmenu_repos(repositories, replace_repos=False):
print "Try to update default fuelmenu settings"
with open(FUELMENU_DEFAULT_SETTINGS_PATH) as f:
settings = yaml.safe_load(f)
if replace_repos:
settings["BOOTSTRAP"]["repos"] = repositories
else:
lists_merge(settings["BOOTSTRAP"]["repos"], repositories, "name")
with open(FUELMENU_DEFAULT_SETTINGS_PATH, "w") as f:
f.write(yaml.safe_dump(settings, default_flow_style=False))
def main():
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(
dest="action", help='actions'
)
fuel_parser = subparsers.add_parser(
'fuel', help='fix fuel repos'
)
fuel_parser.add_argument(
'--release-version', dest='release_version', action='store',
type=str, help='release version', default='newton-10.0'
)
fuel_parser.add_argument(
'--release-os', dest='release_os', action='store',
type=str, help='release operating system', default='Ubuntu'
)
fuel_parser.add_argument(
'--repositories-file', dest='repositories_file', action='store',
type=str, help='file where repositories are defined', required=True
)
fuel_parser.add_argument(
'-a', '--address', dest='address', action='store', type=str,
help='fuel address', default='127.0.0.1'
)
fuel_parser.add_argument(
'-p', '--port', dest='port', action='store', type=str,
help='fuel port', default='8000'
)
fuel_parser.add_argument(
'--user', dest='user', action='store', type=str,
help='fuel user', default='admin'
)
fuel_parser.add_argument(
'--password', dest='password', action='store', type=str,
help='fuel password', default='admin'
)
fuelmenu_parser = subparsers.add_parser(
'fuelmenu', help='fix fuelmenu repos'
)
fuelmenu_parser.add_argument(
'--repositories-file', dest='repositories_file', action='store',
type=str, help='file where repositories are defined', required=True
)
params, other_params = parser.parse_known_args()
with open(params.repositories_file) as f:
repositories = yaml.safe_load(f)
if params.action == 'fuel':
fix_fuel_repos(params.address, params.port,
params.user, params.password,
params.release_version, params.release_os,
repositories)
else:
fix_fuelmenu_repos(repositories)
if __name__ == "__main__":
main()

View File

@ -390,6 +390,9 @@ OPENSTACK_VERSION=`rpm2cpio ${SOURCE}/mos-centos/Packages/fuel-openstack-metadat
test -e ${SOURCE}/fuel_build_number && cp ${SOURCE}/fuel_build_number /etc/fuel_build_number
test -e ${SOURCE}/fuel_build_id && cp ${SOURCE}/fuel_build_id /etc/fuel_build_id
# Copy repos config
test -e ${SOURCE}/default_deb_repos.yaml && cp ${SOURCE}/default_deb_repos.yaml /root/default_deb_repos.yaml
# ----------------------
# UNPACKING REPOSITORIES
# ----------------------

View File

@ -31,6 +31,24 @@ $(BUILD_DIR)/iso/isoroot.done: $(ISOROOT)/fuel_build_id
$(ISOROOT)/fuel_build_id:
echo "$(BUILD_ID)" > $@
##############
# CUSTOM REPOS
##############
define default_deb_repos
- name: mos
suite: $(MIRROR_MOS_UBUNTU_SUITE)
endef
# if we are not building packages and sync repos only, we MUST use
# the same suit as we use during debmirroring
ifeq ($(BUILD_PACKAGES),0)
$(BUILD_DIR)/iso/isoroot.done: $(ISOROOT)/default_deb_repos.yaml
endif
$(ISOROOT)/default_deb_repos.yaml: export default_deb_repos_content:=$(default_deb_repos)
$(ISOROOT)/default_deb_repos.yaml:
/bin/echo -e "$${default_deb_repos_content}\n" > $@
###############
# CENTOS MIRROR
###############
@ -88,7 +106,6 @@ $(ISOROOT)/ks.cfg: $(SOURCE_DIR)/iso/ks.template $(SOURCE_DIR)/iso/ks.py $(ISORO
python $(SOURCE_DIR)/iso/ks.py \
-t $(SOURCE_DIR)/iso/ks.template \
-c $(ISOROOT)/ks.yaml \
-u '{"CENTOS_RELEASE": "$(CENTOS_RELEASE)", "PRODUCT_VERSION": "$(PRODUCT_VERSION)"}' \
-o $@.tmp
mv $@.tmp $@

View File

@ -48,6 +48,7 @@ for file in %{_builddir}/%{name}-%{version}/fuel-release/*.repo ; do
install -D -m 644 "$file" %{buildroot}/etc/yum.repos.d
done
install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/bootstrap_admin_node.sh %{buildroot}%{_sbindir}/bootstrap_admin_node.sh
install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/fix_default_repos.py %{buildroot}%{_sbindir}/fix_default_repos.py
%clean
rm -rf %{buildroot}
@ -92,4 +93,4 @@ This packages provides script to deploy Fuel components.
%files -n fuel-setup
%defattr(-,root,root)
%{_sbindir}/bootstrap_admin_node.sh
%{_sbindir}/fix_default_repos.py