summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorVladimir Kozhukalov <vkozhukalov@mirantis.com>2016-05-26 08:15:17 +0300
committerSergey Kulanov <skulanov@mirantis.com>2016-05-31 15:10:01 +0300
commite6b165748a8771a9f15d3416fc0cd05b6e40b63a (patch)
tree2f2e766da4e955022ffd189f9c0d96515ce7fbb5
parenta0d71154796c960da6f1d3d3d95b0a22ab300e21 (diff)
Modify default repos for fuel and fuelmenu
In fuel (nailgun) and fuelmenu we have hardcoded default repos which we should be able to change mostly for test purposes. Default ISO build flow uses fuel packages rebuild approach, which modifies ubuntu suit as well, eg: sync(mos-master) --> (iso)mos10.0 --> deploy(mos10.0) To use this feature one must use BUILD_PACKAGES=0, this flag turns off: * fuel packages building and * ubuntu suit update. so We consume the same suit as we use during debmirroring, eg: sync(mos-master) --> iso(mos-master) --> deploy(mos-master) DocImpact Change-Id: Ia3cefa7c87e35ecd9244a4026b86e772bf569ca9 Closes-bug: #1556125
Notes
Notes (review): Code-Review+1: Maksim Malchuk <mmalchuk@mirantis.com> Code-Review+1: Roman Vyalov <rvyalov@mirantis.com> Code-Review+1: Vitaly Parakhin <vparakhin@mirantis.com> Code-Review+2: Vladimir Kozhukalov <vkozhukalov@mirantis.com> Workflow+1: Vladimir Kozhukalov <vkozhukalov@mirantis.com> Verified+2: Jenkins Submitted-by: Jenkins Submitted-at: Tue, 31 May 2016 17:39:48 +0000 Reviewed-on: https://review.openstack.org/321353 Project: openstack/fuel-main Branch: refs/heads/master
-rwxr-xr-xiso/bootstrap_admin_node.sh12
-rwxr-xr-xiso/fix_default_repos.py206
-rw-r--r--iso/ks.template3
-rw-r--r--iso/module.mk19
-rw-r--r--specs/fuel-main.spec3
5 files changed, 241 insertions, 2 deletions
diff --git a/iso/bootstrap_admin_node.sh b/iso/bootstrap_admin_node.sh
index 59114da..655bc51 100755
--- a/iso/bootstrap_admin_node.sh
+++ b/iso/bootstrap_admin_node.sh
@@ -61,6 +61,7 @@ wget \
61 61
62ASTUTE_YAML='/etc/fuel/astute.yaml' 62ASTUTE_YAML='/etc/fuel/astute.yaml'
63BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf" 63BOOTSTRAP_NODE_CONFIG="/etc/fuel/bootstrap_admin_node.conf"
64CUSTOM_REPOS="/root/default_deb_repos.yaml"
64bs_build_log='/var/log/fuel-bootstrap-image-build.log' 65bs_build_log='/var/log/fuel-bootstrap-image-build.log'
65bs_status=0 66bs_status=0
66# Backup network configs to this folder. Folder will be created only if 67# Backup network configs to this folder. Folder will be created only if
@@ -328,6 +329,11 @@ if (virt-what | fgrep -q "virtualbox") ; then
328 done 329 done
329fi 330fi
330 331
332# change default repo path in fuel-menu before starting any deployment steps
333if [ -f "${CUSTOM_REPOS}" ]; then
334 fix_default_repos.py fuelmenu --repositories-file "${CUSTOM_REPOS}" || fail
335fi
336
331fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail 337fuelmenu --save-only --iface=$ADMIN_INTERFACE || fail
332set +x 338set +x
333echo "Done!" 339echo "Done!"
@@ -517,6 +523,12 @@ fi
517 523
518# apply puppet 524# apply puppet
519/etc/puppet/modules/fuel/examples/deploy.sh || fail 525/etc/puppet/modules/fuel/examples/deploy.sh || fail
526# Update default repo path
527if [ -f "${CUSTOM_REPOS}" ]; then
528 fix_default_repos.py fuel \
529 --repositories-file "${CUSTOM_REPOS}" \
530 --release-version "${OPENSTACK_VERSION}" || fail
531fi
520 532
521# Sync time 533# Sync time
522systemctl stop ntpd 534systemctl stop ntpd
diff --git a/iso/fix_default_repos.py b/iso/fix_default_repos.py
new file mode 100755
index 0000000..8bd060c
--- /dev/null
+++ b/iso/fix_default_repos.py
@@ -0,0 +1,206 @@
1#!/usr/bin/env python
2# Copyright 2016 Mirantis, Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16import argparse
17import os
18
19import six
20import yaml
21
22from fuelclient import objects
23
24
25FUELMENU_DEFAULT_SETTINGS_PATH = \
26 "/usr/lib/python2.7/site-packages/fuelmenu/settings.yaml"
27
28
29def is_subdict(dict1, dict2):
30 """Checks that dict1 is subdict of dict2.
31
32 >>> is_subdict({"a": 1}, {'a': 1, 'b': 1})
33 True
34
35 :param dict1: the candidate
36 :param dict2: the super dict
37 :return: True if all keys from dict1 are present
38 and has same value in dict2 otherwise False
39 """
40 for k, v in six.iteritems(dict1):
41 if k not in dict2 or dict2[k] != v:
42 return False
43 return True
44
45
46def lists_merge(main, patch, key):
47 """Merges the list of dicts with same keys.
48
49 >>> lists_merge([{"a": 1, "c": 2}], [{"a": 1, "c": 3}], key="a")
50 [{'a': 1, 'c': 3}]
51
52 :param main: the main list
53 :type main: list
54 :param patch: the list of additional elements
55 :type patch: list
56 :param key: the key for compare
57 """
58 main_idx = dict(
59 (x[key], i) for i, x in enumerate(main)
60 )
61
62 patch_idx = dict(
63 (x[key], i) for i, x in enumerate(patch)
64 )
65
66 for k in sorted(patch_idx):
67 if k in main_idx:
68 main[main_idx[k]].update(patch[patch_idx[k]])
69 else:
70 main.append(patch[patch_idx[k]])
71 return main
72
73
74def update_release_repos(repositories,
75 release_match,
76 replace_repos=False):
77 """Applies repositories for existing default settings.
78 :param repositories: the meta information of repositories
79 :param release_match: The pattern to check Fuel Release
80 """
81 releases = six.moves.filter(
82 lambda x: is_subdict(release_match, x.data),
83 objects.Release.get_all()
84 )
85 for release in releases:
86 modified = _update_repository_settings(
87 release.data["attributes_metadata"],
88 repositories,
89 replace_repos=replace_repos)
90 if modified:
91 release.data["attributes_metadata"] = modified
92 print "Try to update the Release '%s'" % release.data['name']
93 release.connection.put_request(
94 release.instance_api_path.format(release.id),
95 release.data
96 )
97
98
99def _update_repository_settings(settings,
100 repositories,
101 replace_repos=False):
102 """Updates repository settings.
103 :param settings: the target settings
104 :param repositories: the meta of repositories
105 """
106 editable = settings["editable"]
107 if 'repo_setup' not in editable:
108 return
109
110 repos_attr = editable["repo_setup"]["repos"]
111 if replace_repos:
112 repos_attr['value'] = repositories
113 else:
114 lists_merge(repos_attr['value'], repositories, "name")
115
116 settings["editable"]["repo_setup"]["repos"] = repos_attr
117
118 return settings
119
120
121def fix_fuel_repos(address, port, user, password,
122 release_version, release_os, repositories):
123 os.environ["SERVER_ADDRESS"] = address
124 os.environ["LISTEN_PORT"] = port
125 os.environ["KEYSTONE_USER"] = user
126 os.environ["KEYSTONE_PASS"] = password
127
128 release_match = {
129 "version": release_version,
130 "operating_system": release_os
131 }
132
133 update_release_repos(repositories, release_match)
134
135
136def fix_fuelmenu_repos(repositories, replace_repos=False):
137 print "Try to update default fuelmenu settings"
138 with open(FUELMENU_DEFAULT_SETTINGS_PATH) as f:
139 settings = yaml.safe_load(f)
140 if replace_repos:
141 settings["BOOTSTRAP"]["repos"] = repositories
142 else:
143 lists_merge(settings["BOOTSTRAP"]["repos"], repositories, "name")
144 with open(FUELMENU_DEFAULT_SETTINGS_PATH, "w") as f:
145 f.write(yaml.safe_dump(settings, default_flow_style=False))
146
147
148def main():
149 parser = argparse.ArgumentParser()
150 subparsers = parser.add_subparsers(
151 dest="action", help='actions'
152 )
153 fuel_parser = subparsers.add_parser(
154 'fuel', help='fix fuel repos'
155 )
156 fuel_parser.add_argument(
157 '--release-version', dest='release_version', action='store',
158 type=str, help='release version', default='newton-10.0'
159 )
160 fuel_parser.add_argument(
161 '--release-os', dest='release_os', action='store',
162 type=str, help='release operating system', default='Ubuntu'
163 )
164 fuel_parser.add_argument(
165 '--repositories-file', dest='repositories_file', action='store',
166 type=str, help='file where repositories are defined', required=True
167 )
168 fuel_parser.add_argument(
169 '-a', '--address', dest='address', action='store', type=str,
170 help='fuel address', default='127.0.0.1'
171 )
172 fuel_parser.add_argument(
173 '-p', '--port', dest='port', action='store', type=str,
174 help='fuel port', default='8000'
175 )
176 fuel_parser.add_argument(
177 '--user', dest='user', action='store', type=str,
178 help='fuel user', default='admin'
179 )
180 fuel_parser.add_argument(
181 '--password', dest='password', action='store', type=str,
182 help='fuel password', default='admin'
183 )
184 fuelmenu_parser = subparsers.add_parser(
185 'fuelmenu', help='fix fuelmenu repos'
186 )
187 fuelmenu_parser.add_argument(
188 '--repositories-file', dest='repositories_file', action='store',
189 type=str, help='file where repositories are defined', required=True
190 )
191 params, other_params = parser.parse_known_args()
192
193 with open(params.repositories_file) as f:
194 repositories = yaml.safe_load(f)
195
196 if params.action == 'fuel':
197 fix_fuel_repos(params.address, params.port,
198 params.user, params.password,
199 params.release_version, params.release_os,
200 repositories)
201 else:
202 fix_fuelmenu_repos(repositories)
203
204
205if __name__ == "__main__":
206 main()
diff --git a/iso/ks.template b/iso/ks.template
index 8db3759..5bf201c 100644
--- a/iso/ks.template
+++ b/iso/ks.template
@@ -390,6 +390,9 @@ OPENSTACK_VERSION=`rpm2cpio ${SOURCE}/mos-centos/Packages/fuel-openstack-metadat
390test -e ${SOURCE}/fuel_build_number && cp ${SOURCE}/fuel_build_number /etc/fuel_build_number 390test -e ${SOURCE}/fuel_build_number && cp ${SOURCE}/fuel_build_number /etc/fuel_build_number
391test -e ${SOURCE}/fuel_build_id && cp ${SOURCE}/fuel_build_id /etc/fuel_build_id 391test -e ${SOURCE}/fuel_build_id && cp ${SOURCE}/fuel_build_id /etc/fuel_build_id
392 392
393# Copy repos config
394test -e ${SOURCE}/default_deb_repos.yaml && cp ${SOURCE}/default_deb_repos.yaml /root/default_deb_repos.yaml
395
393# ---------------------- 396# ----------------------
394# UNPACKING REPOSITORIES 397# UNPACKING REPOSITORIES
395# ---------------------- 398# ----------------------
diff --git a/iso/module.mk b/iso/module.mk
index 8995378..af883dc 100644
--- a/iso/module.mk
+++ b/iso/module.mk
@@ -31,6 +31,24 @@ $(BUILD_DIR)/iso/isoroot.done: $(ISOROOT)/fuel_build_id
31$(ISOROOT)/fuel_build_id: 31$(ISOROOT)/fuel_build_id:
32 echo "$(BUILD_ID)" > $@ 32 echo "$(BUILD_ID)" > $@
33 33
34##############
35# CUSTOM REPOS
36##############
37
38define default_deb_repos
39- name: mos
40 suite: $(MIRROR_MOS_UBUNTU_SUITE)
41endef
42
43# if we are not building packages and sync repos only, we MUST use
44# the same suit as we use during debmirroring
45ifeq ($(BUILD_PACKAGES),0)
46$(BUILD_DIR)/iso/isoroot.done: $(ISOROOT)/default_deb_repos.yaml
47endif
48$(ISOROOT)/default_deb_repos.yaml: export default_deb_repos_content:=$(default_deb_repos)
49$(ISOROOT)/default_deb_repos.yaml:
50 /bin/echo -e "$${default_deb_repos_content}\n" > $@
51
34############### 52###############
35# CENTOS MIRROR 53# CENTOS MIRROR
36############### 54###############
@@ -88,7 +106,6 @@ $(ISOROOT)/ks.cfg: $(SOURCE_DIR)/iso/ks.template $(SOURCE_DIR)/iso/ks.py $(ISORO
88 python $(SOURCE_DIR)/iso/ks.py \ 106 python $(SOURCE_DIR)/iso/ks.py \
89 -t $(SOURCE_DIR)/iso/ks.template \ 107 -t $(SOURCE_DIR)/iso/ks.template \
90 -c $(ISOROOT)/ks.yaml \ 108 -c $(ISOROOT)/ks.yaml \
91 -u '{"CENTOS_RELEASE": "$(CENTOS_RELEASE)", "PRODUCT_VERSION": "$(PRODUCT_VERSION)"}' \
92 -o $@.tmp 109 -o $@.tmp
93 mv $@.tmp $@ 110 mv $@.tmp $@
94 111
diff --git a/specs/fuel-main.spec b/specs/fuel-main.spec
index 7f6b0c0..5053255 100644
--- a/specs/fuel-main.spec
+++ b/specs/fuel-main.spec
@@ -48,6 +48,7 @@ for file in %{_builddir}/%{name}-%{version}/fuel-release/*.repo ; do
48 install -D -m 644 "$file" %{buildroot}/etc/yum.repos.d 48 install -D -m 644 "$file" %{buildroot}/etc/yum.repos.d
49done 49done
50install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/bootstrap_admin_node.sh %{buildroot}%{_sbindir}/bootstrap_admin_node.sh 50install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/bootstrap_admin_node.sh %{buildroot}%{_sbindir}/bootstrap_admin_node.sh
51install -D -p -m 755 %{_builddir}/%{name}-%{version}/iso/fix_default_repos.py %{buildroot}%{_sbindir}/fix_default_repos.py
51 52
52%clean 53%clean
53rm -rf %{buildroot} 54rm -rf %{buildroot}
@@ -92,4 +93,4 @@ This packages provides script to deploy Fuel components.
92%files -n fuel-setup 93%files -n fuel-setup
93%defattr(-,root,root) 94%defattr(-,root,root)
94%{_sbindir}/bootstrap_admin_node.sh 95%{_sbindir}/bootstrap_admin_node.sh
95 96%{_sbindir}/fix_default_repos.py