summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2017-05-27 10:22:08 +0000
committerGerrit Code Review <review@openstack.org>2017-05-27 10:22:08 +0000
commit3ac57740c612daf4d87ff2398bce3e94f7ef1636 (patch)
tree5faad53da4f7e5a671dc86ec1a1002451ad5d4f0
parent795ec082ed894f8cc7117a87bffb386db69dbd05 (diff)
parent3fdd9df983dbd23000007c457f50d125166708f9 (diff)
Merge "Move create_graph into config.py"2.5.0
-rw-r--r--diskimage_builder/block_device/blockdevice.py110
-rw-r--r--diskimage_builder/block_device/config.py92
-rw-r--r--diskimage_builder/block_device/tests/test_config.py18
-rw-r--r--diskimage_builder/block_device/tests/test_mount_order.py4
4 files changed, 106 insertions, 118 deletions
diff --git a/diskimage_builder/block_device/blockdevice.py b/diskimage_builder/block_device/blockdevice.py
index 456f4bc..2e80fd7 100644
--- a/diskimage_builder/block_device/blockdevice.py
+++ b/diskimage_builder/block_device/blockdevice.py
@@ -20,16 +20,8 @@ import shutil
20import sys 20import sys
21import yaml 21import yaml
22 22
23import networkx as nx 23from diskimage_builder.block_device.config import config_tree_to_graph
24 24from diskimage_builder.block_device.config import create_graph
25from stevedore import extension
26
27from diskimage_builder.block_device.config import \
28 config_tree_to_graph
29from diskimage_builder.block_device.exception import \
30 BlockDeviceSetupException
31from diskimage_builder.block_device.plugin import NodeBase
32from diskimage_builder.block_device.plugin import PluginBase
33from diskimage_builder.block_device.utils import exec_sudo 25from diskimage_builder.block_device.utils import exec_sudo
34 26
35 27
@@ -147,9 +139,6 @@ class BlockDevice(object):
147 self.params['build-dir'], "states/block-device") 139 self.params['build-dir'], "states/block-device")
148 self.state_json_file_name \ 140 self.state_json_file_name \
149 = os.path.join(self.state_dir, "state.json") 141 = os.path.join(self.state_dir, "state.json")
150 self.plugin_manager = extension.ExtensionManager(
151 namespace='diskimage_builder.block_device.plugin',
152 invoke_on_load=False)
153 self.config_json_file_name \ 142 self.config_json_file_name \
154 = os.path.join(self.state_dir, "config.json") 143 = os.path.join(self.state_dir, "config.json")
155 144
@@ -168,95 +157,8 @@ class BlockDevice(object):
168 with open(self.state_json_file_name, "w") as fd: 157 with open(self.state_json_file_name, "w") as fd:
169 json.dump(state, fd) 158 json.dump(state, fd)
170 159
171 def create_graph(self, config, default_config):
172 """Generate configuration digraph
173
174 Generate the configuration digraph from the config
175
176 :param config: graph configuration file
177 :param default_config: default parameters (from --params)
178 :return: tuple with the graph object, nodes in call order
179 """
180 # This is the directed graph of nodes: each parse method must
181 # add the appropriate nodes and edges.
182 dg = nx.DiGraph()
183
184 for config_entry in config:
185 # this should have been checked by generate_config
186 assert len(config_entry) == 1
187
188 logger.debug("Config entry [%s]" % config_entry)
189 cfg_obj_name = list(config_entry.keys())[0]
190 cfg_obj_val = config_entry[cfg_obj_name]
191
192 # Instantiate a "plugin" object, passing it the
193 # configuration entry
194 # XXX would a "factory" pattern for plugins, where we make
195 # a method call on an object stevedore has instantiated be
196 # better here?
197 if cfg_obj_name not in self.plugin_manager:
198 raise BlockDeviceSetupException(
199 ("Config element [%s] is not implemented" % cfg_obj_name))
200 plugin = self.plugin_manager[cfg_obj_name].plugin
201 assert issubclass(plugin, PluginBase)
202 cfg_obj = plugin(cfg_obj_val, default_config)
203
204 # Ask the plugin for the nodes it would like to insert
205 # into the graph. Some plugins, such as partitioning,
206 # return multiple nodes from one config entry.
207 nodes = cfg_obj.get_nodes()
208 assert isinstance(nodes, list)
209 for node in nodes:
210 # plugins should return nodes...
211 assert isinstance(node, NodeBase)
212 # ensure node names are unique. networkx by default
213 # just appends the attribute to the node dict for
214 # existing nodes, which is not what we want.
215 if node.name in dg.node:
216 raise BlockDeviceSetupException(
217 "Duplicate node name: %s" % (node.name))
218 logger.debug("Adding %s : %s", node.name, node)
219 dg.add_node(node.name, obj=node)
220
221 # Now find edges
222 for name, attr in dg.nodes(data=True):
223 obj = attr['obj']
224 # Unfortunately, we can not determine node edges just from
225 # the configuration file. It's not always simply the
226 # "base:" pointer. So ask nodes for a list of nodes they
227 # want to point to. *mostly* it's just base: ... but
228 # mounting is different.
229 # edges_from are the nodes that point to us
230 # edges_to are the nodes we point to
231 edges_from, edges_to = obj.get_edges()
232 logger.debug("Edges for %s: f:%s t:%s", name,
233 edges_from, edges_to)
234 for edge_from in edges_from:
235 if edge_from not in dg.node:
236 raise BlockDeviceSetupException(
237 "Edge not defined: %s->%s" % (edge_from, name))
238 dg.add_edge(edge_from, name)
239 for edge_to in edges_to:
240 if edge_to not in dg.node:
241 raise BlockDeviceSetupException(
242 "Edge not defined: %s->%s" % (name, edge_to))
243 dg.add_edge(name, edge_to)
244
245 # this can be quite helpful debugging but needs pydotplus.
246 # run "dotty /tmp/out.dot"
247 # XXX: maybe an env var that dumps to a tmpdir or something?
248 # nx.nx_pydot.write_dot(dg, '/tmp/graph_dump.dot')
249
250 # Topological sort (i.e. create a linear array that satisfies
251 # dependencies) and return the object list
252 call_order_nodes = nx.topological_sort(dg)
253 logger.debug("Call order: %s", list(call_order_nodes))
254 call_order = [dg.node[n]['obj'] for n in call_order_nodes]
255
256 return dg, call_order
257
258 def create(self, result, rollback): 160 def create(self, result, rollback):
259 dg, call_order = self.create_graph(self.config, self.params) 161 dg, call_order = create_graph(self.config, self.params)
260 for node in call_order: 162 for node in call_order:
261 node.create(result, rollback) 163 node.create(result, rollback)
262 164
@@ -413,7 +315,7 @@ class BlockDevice(object):
413 return 0 315 return 0
414 316
415 # Deleting must be done in reverse order 317 # Deleting must be done in reverse order
416 dg, call_order = self.create_graph(self.config, self.params) 318 dg, call_order = create_graph(self.config, self.params)
417 reverse_order = reversed(call_order) 319 reverse_order = reversed(call_order)
418 320
419 if dg is None: 321 if dg is None:
@@ -427,7 +329,7 @@ class BlockDevice(object):
427 """Cleanup all remaining relicts - in good case""" 329 """Cleanup all remaining relicts - in good case"""
428 330
429 # Deleting must be done in reverse order 331 # Deleting must be done in reverse order
430 dg, call_order = self.create_graph(self.config, self.params) 332 dg, call_order = create_graph(self.config, self.params)
431 reverse_order = reversed(call_order) 333 reverse_order = reversed(call_order)
432 334
433 for node in reverse_order: 335 for node in reverse_order:
@@ -442,7 +344,7 @@ class BlockDevice(object):
442 """Cleanup all remaining relicts - in case of an error""" 344 """Cleanup all remaining relicts - in case of an error"""
443 345
444 # Deleting must be done in reverse order 346 # Deleting must be done in reverse order
445 dg, call_order = self.create_graph(self.config, self.params) 347 dg, call_order = create_graph(self.config, self.params)
446 reverse_order = reversed(call_order) 348 reverse_order = reversed(call_order)
447 349
448 for node in reverse_order: 350 for node in reverse_order:
diff --git a/diskimage_builder/block_device/config.py b/diskimage_builder/block_device/config.py
index 1fef2a0..61f8b36 100644
--- a/diskimage_builder/block_device/config.py
+++ b/diskimage_builder/block_device/config.py
@@ -11,11 +11,14 @@
11# under the License. 11# under the License.
12 12
13import logging 13import logging
14import networkx as nx
14 15
15from stevedore import extension 16from stevedore import extension
16 17
17from diskimage_builder.block_device.exception import \ 18from diskimage_builder.block_device.exception import \
18 BlockDeviceSetupException 19 BlockDeviceSetupException
20from diskimage_builder.block_device.plugin import NodeBase
21from diskimage_builder.block_device.plugin import PluginBase
19 22
20 23
21logger = logging.getLogger(__name__) 24logger = logging.getLogger(__name__)
@@ -138,6 +141,95 @@ def config_tree_to_graph(config):
138 return output 141 return output
139 142
140 143
144def create_graph(config, default_config):
145 """Generate configuration digraph
146
147 Generate the configuration digraph from the config
148
149 :param config: graph configuration file
150 :param default_config: default parameters (from --params)
151 :return: tuple with the graph object (a :class:`nx.Digraph`),
152 ordered list of :class:`NodeBase` objects
153
154 """
155 # This is the directed graph of nodes: each parse method must
156 # add the appropriate nodes and edges.
157 dg = nx.DiGraph()
158
159 for config_entry in config:
160 # this should have been checked by generate_config
161 assert len(config_entry) == 1
162
163 logger.debug("Config entry [%s]" % config_entry)
164 cfg_obj_name = list(config_entry.keys())[0]
165 cfg_obj_val = config_entry[cfg_obj_name]
166
167 # Instantiate a "plugin" object, passing it the
168 # configuration entry
169 # XXX : would a "factory" pattern for plugins, where we
170 # make a method call on an object stevedore has instantiated
171 # be better here?
172 if not is_a_plugin(cfg_obj_name):
173 raise BlockDeviceSetupException(
174 ("Config element [%s] is not implemented" % cfg_obj_name))
175 plugin = _extensions[cfg_obj_name].plugin
176 assert issubclass(plugin, PluginBase)
177 cfg_obj = plugin(cfg_obj_val, default_config)
178
179 # Ask the plugin for the nodes it would like to insert
180 # into the graph. Some plugins, such as partitioning,
181 # return multiple nodes from one config entry.
182 nodes = cfg_obj.get_nodes()
183 assert isinstance(nodes, list)
184 for node in nodes:
185 # plugins should return nodes...
186 assert isinstance(node, NodeBase)
187 # ensure node names are unique. networkx by default
188 # just appends the attribute to the node dict for
189 # existing nodes, which is not what we want.
190 if node.name in dg.node:
191 raise BlockDeviceSetupException(
192 "Duplicate node name: %s" % (node.name))
193 logger.debug("Adding %s : %s", node.name, node)
194 dg.add_node(node.name, obj=node)
195
196 # Now find edges
197 for name, attr in dg.nodes(data=True):
198 obj = attr['obj']
199 # Unfortunately, we can not determine node edges just from
200 # the configuration file. It's not always simply the
201 # "base:" pointer. So ask nodes for a list of nodes they
202 # want to point to. *mostly* it's just base: ... but
203 # mounting is different.
204 # edges_from are the nodes that point to us
205 # edges_to are the nodes we point to
206 edges_from, edges_to = obj.get_edges()
207 logger.debug("Edges for %s: f:%s t:%s", name,
208 edges_from, edges_to)
209 for edge_from in edges_from:
210 if edge_from not in dg.node:
211 raise BlockDeviceSetupException(
212 "Edge not defined: %s->%s" % (edge_from, name))
213 dg.add_edge(edge_from, name)
214 for edge_to in edges_to:
215 if edge_to not in dg.node:
216 raise BlockDeviceSetupException(
217 "Edge not defined: %s->%s" % (name, edge_to))
218 dg.add_edge(name, edge_to)
219
220 # this can be quite helpful debugging but needs pydotplus.
221 # run "dotty /tmp/out.dot"
222 # XXX: maybe an env var that dumps to a tmpdir or something?
223 # nx.nx_pydot.write_dot(dg, '/tmp/graph_dump.dot')
224
225 # Topological sort (i.e. create a linear array that satisfies
226 # dependencies) and return the object list
227 call_order_nodes = nx.topological_sort(dg)
228 logger.debug("Call order: %s", list(call_order_nodes))
229 call_order = [dg.node[n]['obj'] for n in call_order_nodes]
230
231 return dg, call_order
232
141# 233#
142# On partitioning: objects 234# On partitioning: objects
143# 235#
diff --git a/diskimage_builder/block_device/tests/test_config.py b/diskimage_builder/block_device/tests/test_config.py
index 911deb7..c2bd0ba 100644
--- a/diskimage_builder/block_device/tests/test_config.py
+++ b/diskimage_builder/block_device/tests/test_config.py
@@ -16,10 +16,8 @@ import os
16import testtools 16import testtools
17import yaml 17import yaml
18 18
19from diskimage_builder.block_device.blockdevice \ 19from diskimage_builder.block_device.config import config_tree_to_graph
20 import BlockDevice 20from diskimage_builder.block_device.config import create_graph
21from diskimage_builder.block_device.config \
22 import config_tree_to_graph
23from diskimage_builder.block_device.exception import \ 21from diskimage_builder.block_device.exception import \
24 BlockDeviceSetupException 22 BlockDeviceSetupException
25 23
@@ -63,8 +61,6 @@ class TestGraphGeneration(TestConfig):
63 'mount-base': '/fake', 61 'mount-base': '/fake',
64 } 62 }
65 63
66 self.bd = BlockDevice(self.fake_default_config)
67
68 64
69class TestConfigParsing(TestConfig): 65class TestConfigParsing(TestConfig):
70 """Test parsing config file into a graph""" 66 """Test parsing config file into a graph"""
@@ -121,7 +117,7 @@ class TestCreateGraph(TestGraphGeneration):
121 config = self.load_config_file('bad_edge_graph.yaml') 117 config = self.load_config_file('bad_edge_graph.yaml')
122 self.assertRaisesRegexp(BlockDeviceSetupException, 118 self.assertRaisesRegexp(BlockDeviceSetupException,
123 "Edge not defined: this_is_not_a_node", 119 "Edge not defined: this_is_not_a_node",
124 self.bd.create_graph, 120 create_graph,
125 config, self.fake_default_config) 121 config, self.fake_default_config)
126 122
127 # Test a graph with bad edge pointing to an invalid node 123 # Test a graph with bad edge pointing to an invalid node
@@ -130,15 +126,14 @@ class TestCreateGraph(TestGraphGeneration):
130 self.assertRaisesRegexp(BlockDeviceSetupException, 126 self.assertRaisesRegexp(BlockDeviceSetupException,
131 "Duplicate node name: " 127 "Duplicate node name: "
132 "this_is_a_duplicate", 128 "this_is_a_duplicate",
133 self.bd.create_graph, 129 create_graph,
134 config, self.fake_default_config) 130 config, self.fake_default_config)
135 131
136 # Test digraph generation from deep_graph config file 132 # Test digraph generation from deep_graph config file
137 def test_deep_graph_generator(self): 133 def test_deep_graph_generator(self):
138 config = self.load_config_file('deep_graph.yaml') 134 config = self.load_config_file('deep_graph.yaml')
139 135
140 graph, call_order = self.bd.create_graph(config, 136 graph, call_order = create_graph(config, self.fake_default_config)
141 self.fake_default_config)
142 137
143 call_order_list = [n.name for n in call_order] 138 call_order_list = [n.name for n in call_order]
144 139
@@ -155,8 +150,7 @@ class TestCreateGraph(TestGraphGeneration):
155 def test_multiple_partitions_graph_generator(self): 150 def test_multiple_partitions_graph_generator(self):
156 config = self.load_config_file('multiple_partitions_graph.yaml') 151 config = self.load_config_file('multiple_partitions_graph.yaml')
157 152
158 graph, call_order = self.bd.create_graph(config, 153 graph, call_order = create_graph(config, self.fake_default_config)
159 self.fake_default_config)
160 call_order_list = [n.name for n in call_order] 154 call_order_list = [n.name for n in call_order]
161 155
162 # The sort creating call_order_list is unstable. 156 # The sort creating call_order_list is unstable.
diff --git a/diskimage_builder/block_device/tests/test_mount_order.py b/diskimage_builder/block_device/tests/test_mount_order.py
index 8c948e4..8631536 100644
--- a/diskimage_builder/block_device/tests/test_mount_order.py
+++ b/diskimage_builder/block_device/tests/test_mount_order.py
@@ -15,6 +15,7 @@ import mock
15 15
16import diskimage_builder.block_device.tests.test_config as tc 16import diskimage_builder.block_device.tests.test_config as tc
17 17
18from diskimage_builder.block_device.config import create_graph
18from diskimage_builder.block_device.level3.mount import MountPointNode 19from diskimage_builder.block_device.level3.mount import MountPointNode
19 20
20logger = logging.getLogger(__name__) 21logger = logging.getLogger(__name__)
@@ -27,8 +28,7 @@ class TestMountOrder(tc.TestGraphGeneration):
27 28
28 config = self.load_config_file('multiple_partitions_graph.yaml') 29 config = self.load_config_file('multiple_partitions_graph.yaml')
29 30
30 graph, call_order = self.bd.create_graph(config, 31 graph, call_order = create_graph(config, self.fake_default_config)
31 self.fake_default_config)
32 32
33 result = {} 33 result = {}
34 result['filesys'] = {} 34 result['filesys'] = {}