Fixed merging capabilities.

Instead of previously having merging which was not backwards compatible
with the 0.7.1 and prior methods this patch works to ensure said backwards
compatible while at the same time making the new merging functionality
work in a more customizable manner.

cloudinit/handlers/cloud_config.py

@@ -30,7 +30,25 @@ from cloudinit.settings import (PER_ALWAYS)
 LOG = logging.getLogger(__name__)
 
 MERGE_HEADER = 'Merge-Type'
-DEF_MERGERS = mergers.default_mergers()
+
+# Due to the way the loading of yaml configuration was done previously,
+# where previously each cloud config part was appended to a larger yaml
+# file and then finally that file was loaded as one big yaml file we need
+# to mimic that behavior by altering the default strategy to be replacing
+# keys of prior merges.
+#
+#
+# For example
+# #file 1
+# a: 3
+# #file 2
+# a: 22
+# #combined file (comments not included)
+# a: 3
+# a: 22
+#
+# This gets loaded into yaml with final result {'a': 22}
+DEF_MERGERS = mergers.string_extract_mergers('dict(replace)+list()+str()')
 
 
 class CloudConfigPartHandler(handlers.Handler):
@@ -39,7 +57,6 @@ class CloudConfigPartHandler(handlers.Handler):
         self.cloud_buf = None
         self.cloud_fn = paths.get_ipath("cloud_config")
         self.file_names = []
-        self.mergers = [DEF_MERGERS]
 
     def list_types(self):
         return [
@@ -54,6 +71,8 @@ class CloudConfigPartHandler(handlers.Handler):
         if self.file_names:
             file_lines.append("# from %s files" % (len(self.file_names)))
             for fn in self.file_names:
+                if not fn:
+                    fn = '?'
                 file_lines.append("# %s" % (fn))
             file_lines.append("")
         if self.cloud_buf is not None:
@@ -86,26 +105,20 @@ class CloudConfigPartHandler(handlers.Handler):
         all_mergers.extend(mergers_header)
         if not all_mergers:
             all_mergers = DEF_MERGERS
-        return all_mergers
+        return (payload_yaml, all_mergers)
 
     def _merge_part(self, payload, headers):
-        next_mergers = self._extract_mergers(payload, headers)
-        # Use the merger list from the last call, since it is the one
-        # that will be defining how to merge with the next payload.
-        curr_mergers = list(self.mergers[-1])
-        LOG.debug("Merging by applying %s", curr_mergers)
-        self.mergers.append(next_mergers)
-        merger = mergers.construct(curr_mergers)
+        (payload_yaml, my_mergers) = self._extract_mergers(payload, headers)
+        LOG.debug("Merging by applying %s", my_mergers)
+        merger = mergers.construct(my_mergers)
         if self.cloud_buf is None:
             # First time through, merge with an empty dict...
             self.cloud_buf = {}
-        self.cloud_buf = merger.merge(self.cloud_buf,
-                                      util.load_yaml(payload))
+        self.cloud_buf = merger.merge(self.cloud_buf, payload_yaml)
 
     def _reset(self):
         self.file_names = []
         self.cloud_buf = None
-        self.mergers = [DEF_MERGERS]
 
     def handle_part(self, _data, ctype, filename,  # pylint: disable=W0221
                     payload, _frequency, headers):  # pylint: disable=W0613
@@ -118,7 +131,10 @@ class CloudConfigPartHandler(handlers.Handler):
             return
         try:
             self._merge_part(payload, headers)
-            self.file_names.append(filename)
+            # Ensure filename is ok to store
+            for i in ("\n", "\r", "\t"):
+                filename = filename.replace(i, " ")
+            self.file_names.append(filename.strip())
         except:
             util.logexc(LOG, "Failed at merging in cloud config part from %s",
                         filename)

cloudinit/mergers/__init__.py

@@ -55,6 +55,9 @@ class UnknownMerger(object):
         if not meth:
             meth = self._handle_unknown
             args.insert(0, method_name)
+        LOG.debug("Merging '%s' into '%s' using method '%s' of '%s'",
+                  type_name, type_utils.obj_name(merge_with),
+                  meth.__name__, self)
         return meth(*args)
 
 
@@ -66,6 +69,9 @@ class LookupMerger(UnknownMerger):
         else:
             self._lookups = lookups
 
+    def __str__(self):
+        return 'LookupMerger: (%s)' % (len(self._lookups))
+
     # For items which can not be merged by the parent this object
     # will lookup in a internally maintained set of objects and
     # find which one of those objects can perform the merge. If
@@ -78,6 +84,8 @@ class LookupMerger(UnknownMerger):
                 # First one that has that method/attr gets to be
                 # the one that will be called
                 meth = getattr(merger, meth_wanted)
+                LOG.debug(("Merging using located merger '%s'"
+                            " since it had method '%s'"), merger, meth_wanted)
                 break
         if not meth:
             return UnknownMerger._handle_unknown(self, meth_wanted,
@@ -87,9 +95,9 @@ class LookupMerger(UnknownMerger):
 
 def dict_extract_mergers(config):
     parsed_mergers = []
-    raw_mergers = config.get('merge_how')
+    raw_mergers = config.pop('merge_how', None)
     if raw_mergers is None:
-        raw_mergers = config.get('merge_type')
+        raw_mergers = config.pop('merge_type', None)
     if raw_mergers is None:
         return parsed_mergers
     if isinstance(raw_mergers, (str, basestring)):

cloudinit/mergers/m_dict.py

@@ -16,33 +16,71 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+DEF_MERGE_TYPE = 'no_replace'
+MERGE_TYPES = ('replace', DEF_MERGE_TYPE,)
+
+
+def _has_any(what, *keys):
+    for k in keys:
+        if k in what:
+            return True
+    return False
+
 
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
-        self._overwrite = 'overwrite' in opts
+        # Affects merging behavior...
+        self._method = DEF_MERGE_TYPE
+        for m in MERGE_TYPES:
+            if m in opts:
+                self._method = m
+                break
+        # Affect how recursive merging is done on other primitives.
+        self._recurse_str = 'recurse_str' in opts
+        self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list')
+        self._allow_delete = 'allow_delete' in opts
+        # Backwards compat require this to be on.
+        self._recurse_dict = True
+
+    def __str__(self):
+        s = ('DictMerger: (method=%s,recurse_str=%s,'
+             'recurse_dict=%s,recurse_array=%s,allow_delete=%s)')
+        s = s % (self._method, self._recurse_str,
+                 self._recurse_dict, self._recurse_array, self._allow_delete)
+        return s
+
+    def _do_dict_replace(self, value, merge_with, do_replace):
+
+        def merge_same_key(old_v, new_v):
+            if do_replace:
+                return new_v
+            if isinstance(new_v, (list, tuple)) and self._recurse_array:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (basestring)) and self._recurse_str:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (dict)) and self._recurse_dict:
+                return self._merger.merge(old_v, new_v)
+            # Otherwise leave it be...
+            return old_v
+
+        for (k, v) in merge_with.items():
+            if k in value:
+                if v is None and self._allow_delete:
+                    value.pop(k)
+                else:
+                    value[k] = merge_same_key(value[k], v)
+            else:
+                value[k] = v
+        return value
 
-    # This merging algorithm will attempt to merge with
-    # another dictionary, on encountering any other type of object
-    # it will not merge with said object, but will instead return
-    # the original value
-    #
-    # On encountering a dictionary, it will create a new dictionary
-    # composed of the original and the one to merge with, if 'overwrite'
-    # is enabled then keys that exist in the original will be overwritten
-    # by keys in the one to merge with (and associated values). Otherwise
-    # if not in overwrite mode the 2 conflicting keys themselves will
-    # be merged.
     def _on_dict(self, value, merge_with):
         if not isinstance(merge_with, (dict)):
             return value
-        merged = dict(value)
-        for (k, v) in merge_with.items():
-            if k in merged:
-                if not self._overwrite:
-                    merged[k] = self._merger.merge(merged[k], v)
-                else:
-                    merged[k] = v
-            else:
-                merged[k] = v
+        if self._method == 'replace':
+            merged = self._do_dict_replace(dict(value), merge_with, True)
+        elif self._method == 'no_replace':
+            merged = self._do_dict_replace(dict(value), merge_with, False)
+        else:
+            raise NotImplementedError("Unknown merge type %s" % (self._method))
         return merged

cloudinit/mergers/m_list.py

@@ -16,35 +16,71 @@
 #    You should have received a copy of the GNU General Public License
 #    along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+DEF_MERGE_TYPE = 'replace'
+MERGE_TYPES = ('append', 'prepend', DEF_MERGE_TYPE, 'no_replace')
+
+def _has_any(what, *keys):
+    for k in keys:
+        if k in what:
+            return True
+    return False
+
 
 class Merger(object):
     def __init__(self, merger, opts):
         self._merger = merger
-        self._discard_non = 'discard_non_list' in opts
-        self._extend = 'extend' in opts
+        # Affects merging behavior...
+        self._method = DEF_MERGE_TYPE
+        for m in MERGE_TYPES:
+            if m in opts:
+                self._method = m
+                break
+        # Affect how recursive merging is done on other primitives
+        self._recurse_str = _has_any(opts, 'recurse_str')
+        self._recurse_dict = _has_any(opts, 'recurse_dict')
+        self._recurse_array = _has_any(opts, 'recurse_array', 'recurse_list')
+
+    def __str__(self):
+        return ('ListMerger: (method=%s,recurse_str=%s,'
+                'recurse_dict=%s,recurse_array=%s)') % (self._method,
+                                                        self._recurse_str,
+                                                        self._recurse_dict,
+                                                        self._recurse_array)
 
     def _on_tuple(self, value, merge_with):
-        return self._on_list(list(value), merge_with)
+        return tuple(self._on_list(list(value), merge_with))
 
-    # On encountering a list or tuple type this action will be applied
-    # a new list will be returned, if the value to merge with is itself
-    # a list and we have been told to 'extend', then the value here will
-    # be extended with the other list. If in 'extend' mode then we will
-    # attempt to merge instead, which means that values from the list
-    # to merge with will replace values in te original list (they will
-    # also be merged recursively).
-    #
-    # If the value to merge with is not a list, and we are set to discared
-    # then no modifications will take place, otherwise we will just append
-    # the value to merge with onto the end of our own list.
     def _on_list(self, value, merge_with):
-        new_value = list(value)
-        if isinstance(merge_with, (tuple, list)):
-            if self._extend:
-                new_value.extend(merge_with)
-            else:
-                return new_value
-        else:
-            if not self._discard_non:
-                new_value.append(merge_with)
-        return new_value
+        if (self._method == 'replace' and
+            not isinstance(merge_with, (tuple, list))):
+            return merge_with
+
+        # Ok we now know that what we are merging with is a list or tuple.
+        merged_list = []
+        if self._method == 'prepend':
+            merged_list.extend(merge_with)
+            merged_list.extend(value)
+            return merged_list
+        elif self._method == 'append':
+            merged_list.extend(value)
+            merged_list.extend(merge_with)
+            return merged_list
+
+        def merge_same_index(old_v, new_v):
+            if self._method == 'no_replace':
+                # Leave it be...
+                return old_v
+            if isinstance(new_v, (list, tuple)) and self._recurse_array:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (str, basestring)) and self._recurse_str:
+                return self._merger.merge(old_v, new_v)
+            if isinstance(new_v, (dict)) and self._recurse_dict:
+                return self._merger.merge(old_v, new_v)
+            return new_v
+
+        # Ok now we are replacing same indexes
+        merged_list.extend(value)
+        common_len = min(len(merged_list), len(merge_with))
+        for i in xrange(0, common_len):
+            merged_list[i] = merge_same_index(merged_list[i], merge_with[i])
+        return merged_list

cloudinit/mergers/m_str.py

@@ -1,26 +1,30 @@
+# -*- coding: utf-8 -*-
 # vi: ts=4 expandtab
 #
-#    Copyright (C) 2012 Yahoo! Inc.
+# Copyright (C) 2012 Yahoo! Inc.
 #
-#    Author: Joshua Harlow <harlowja@yahoo-inc.com>
+# Author: Joshua Harlow <harlowja@yahoo-inc.com>
 #
-#    This program is free software: you can redistribute it and/or modify
-#    it under the terms of the GNU General Public License version 3, as
-#    published by the Free Software Foundation.
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License version 3, as
+# published by the Free Software Foundation.
 #
-#    This program is distributed in the hope that it will be useful,
-#    but WITHOUT ANY WARRANTY; without even the implied warranty of
-#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#    GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
 #
-#    You should have received a copy of the GNU General Public License
-#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 
 class Merger(object):
     def __init__(self, _merger, opts):
         self._append = 'append' in opts
 
+    def __str__(self):
+        return 'StringMerger: (append=%s)' % (self._append)
+
     # On encountering a unicode object to merge value with
     # we will for now just proxy into the string method to let it handle it.
     def _on_unicode(self, value, merge_with):
@@ -30,10 +34,11 @@ class Merger(object):
     # perform the following action, if appending we will
     # merge them together, otherwise we will just return value.
     def _on_str(self, value, merge_with):
+        if not isinstance(value, (basestring)):
+            return merge_with
         if not self._append:
-            return value
+            return merge_with
+        if isinstance(value, unicode):
+            return value + unicode(merge_with)
         else:
-            if isinstance(value, (unicode)):
-                return value + unicode(merge_with)
-            else:
-                return value + str(merge_with)
+            return value + str(merge_with)

cloudinit/util.py

@@ -543,21 +543,16 @@ def make_url(scheme, host, port=None,
 def mergemanydict(srcs, reverse=False):
     if reverse:
         srcs = reversed(srcs)
-    m_cfg = {}
-    merge_how = [mergers.default_mergers()]
-    for a_cfg in srcs:
-        if a_cfg:
-            # Take the last merger as the one that
-            # will define how to merge next...
-            mergers_to_apply = list(merge_how[-1])
+    merged_cfg = {}
+    for cfg in srcs:
+        if cfg:
+            # Figure out which mergers to apply...
+            mergers_to_apply = mergers.dict_extract_mergers(cfg)
+            if not mergers_to_apply:
+                mergers_to_apply = mergers.default_mergers()
             merger = mergers.construct(mergers_to_apply)
-            m_cfg = merger.merge(m_cfg, a_cfg)
-            # If the config has now has new merger set,
-            # extract them to be used next time...
-            new_mergers = mergers.dict_extract_mergers(m_cfg)
-            if new_mergers:
-                merge_how.append(new_mergers)
-    return m_cfg
+            merged_cfg = merger.merge(merged_cfg, cfg)
+    return merged_cfg
 
 
 @contextlib.contextmanager

tests/data/merge_sources/expected1.yaml

@@ -0,0 +1 @@
+Blah: ['blah2', 'b']

tests/data/merge_sources/expected10.yaml

@@ -0,0 +1,7 @@
+#cloud-config
+
+power_state:
+  delay: 30
+  mode: poweroff
+  message: [Bye, Bye, Pew, Pew]
+  

tests/data/merge_sources/expected11.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+a: 22
+b: 4
+c: 3

tests/data/merge_sources/expected12.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+a:
+  e:
+    y: 2

tests/data/merge_sources/expected2.yaml

@@ -0,0 +1,3 @@
+Blah: 3
+Blah2: 2
+Blah3: [1]

tests/data/merge_sources/expected3.yaml

@@ -0,0 +1 @@
+Blah: [blah2, 'blah1']

tests/data/merge_sources/expected4.yaml

@@ -0,0 +1,2 @@
+#cloud-config
+Blah: {}

tests/data/merge_sources/expected5.yaml

@@ -0,0 +1,7 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]
+
+

tests/data/merge_sources/expected6.yaml

@@ -0,0 +1,9 @@
+#cloud-config
+
+run_cmds:
+   - bash
+   - top
+   - ps
+   - vi
+   - emacs
+

tests/data/merge_sources/expected7.yaml

@@ -0,0 +1,38 @@
+#cloud-config
+
+users:
+  - default
+  - name: foobar
+    gecos: Foo B. Bar
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+  - name: barfoo
+    gecos: Bar B. Foo
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: users, admin
+    ssh-import-id: None
+    lock-passwd: true
+    ssh-authorized-keys:
+      - <ssh pub key 1>
+      - <ssh pub key 2>
+  - name: cloudy
+    gecos: Magic Cloud App Daemon User
+    inactive: true
+    system: true
+  - bob
+  - joe
+  - sue
+  - name: foobar_jr
+    gecos: Foo B. Bar Jr
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/

tests/data/merge_sources/expected8.yaml

@@ -0,0 +1,7 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral22, /mnt, auto, "defaults,noexec" ]
+ - [ sdc, /opt/data ]
+ - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ]
+ - [ dd, /dev/zero ]

tests/data/merge_sources/expected9.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+phone_home:
+ url: http://my.example.com/$INSTANCE_ID/$BLAH_BLAH
+ post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ]

tests/data/merge_sources/source1-1.yaml

@@ -0,0 +1,3 @@
+#cloud-config
+Blah: ['blah2']
+

tests/data/merge_sources/source1-2.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+Blah: ['b']
+
+merge_how: 'dict(recurse_array,no_replace)+list(append)'

tests/data/merge_sources/source10-1.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+
+power_state:
+ delay: 30
+ mode: poweroff
+ message: [Bye, Bye]

tests/data/merge_sources/source10-2.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+
+power_state:
+  message: [Pew, Pew]
+  
+merge_how: 'dict(recurse_list)+list(append)'

tests/data/merge_sources/source11-1.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+a: 1
+b: 2
+c: 3

tests/data/merge_sources/source11-2.yaml

@@ -0,0 +1,3 @@
+#cloud-config
+
+b: 4

tests/data/merge_sources/source11-3.yaml

@@ -0,0 +1,3 @@
+#cloud-config
+
+a: 22

tests/data/merge_sources/source12-1.yaml

@@ -0,0 +1,8 @@
+#cloud-config
+
+a:
+  c: 1
+  d: 2
+  e:
+    z: a
+    y: b

tests/data/merge_sources/source12-2.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+a:
+  e:
+    y: 2

tests/data/merge_sources/source2-1.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+
+
+Blah: 1
+Blah2: 2
+Blah3: 3

tests/data/merge_sources/source2-2.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]

tests/data/merge_sources/source3-1.yaml

@@ -0,0 +1,4 @@
+#cloud-config
+Blah: ['blah1']
+
+

tests/data/merge_sources/source3-2.yaml

@@ -0,0 +1,4 @@
+#cloud-config
+Blah: ['blah2']
+
+merge_how: 'dict(recurse_array,no_replace)+list(prepend)'

tests/data/merge_sources/source4-1.yaml

@@ -0,0 +1,3 @@
+#cloud-config
+Blah:
+  b: 1

tests/data/merge_sources/source4-2.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+Blah:
+  b: null
+
+
+merge_how: 'dict(allow_delete,no_replace)+list()'

tests/data/merge_sources/source5-1.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+
+
+Blah: 1
+Blah2: 2
+Blah3: 3

tests/data/merge_sources/source5-2.yaml

@@ -0,0 +1,8 @@
+#cloud-config
+
+Blah: 3
+Blah2: 2
+Blah3: [1]
+
+
+merge_how: 'dict(replace)+list(append)'

tests/data/merge_sources/source6-1.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+run_cmds:
+  - bash
+  - top

tests/data/merge_sources/source6-2.yaml

@@ -0,0 +1,8 @@
+#cloud-config
+
+run_cmds:
+  - ps
+  - vi
+  - emacs
+
+merge_type: 'list(append)+dict(recurse_array)+str()'

tests/data/merge_sources/source7-1.yaml

@@ -0,0 +1,27 @@
+#cloud-config
+
+users:
+  - default
+  - name: foobar
+    gecos: Foo B. Bar
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+  - name: barfoo
+    gecos: Bar B. Foo
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    groups: users, admin
+    ssh-import-id: None
+    lock-passwd: true
+    ssh-authorized-keys:
+      - <ssh pub key 1>
+      - <ssh pub key 2>
+  - name: cloudy
+    gecos: Magic Cloud App Daemon User
+    inactive: true
+    system: true
+

tests/data/merge_sources/source7-2.yaml

@@ -0,0 +1,17 @@
+#cloud-config
+
+users:
+  - bob
+  - joe
+  - sue
+  - name: foobar_jr
+    gecos: Foo B. Bar Jr
+    primary-group: foobar
+    groups: users
+    selinux-user: staff_u
+    expiredate: 2012-09-01
+    ssh-import-id: foobar
+    lock-passwd: false
+    passwd: $6$j212wezy$7H/1LT4f9/N3wpgNunhsIqtMj62OKiS3nyNwuizouQc3u7MbYCarYeAHWYPYb2FT.lbioDm2RrkJPb9BZMN1O/
+
+merge_how: "dict(recurse_array)+list(append)"

tests/data/merge_sources/source8-1.yaml

@@ -0,0 +1,7 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral0, /mnt, auto, "defaults,noexec" ]
+ - [ sdc, /opt/data ]
+ - [ xvdh, /opt/data, "auto", "defaults,nobootwait", "0", "0" ]
+ - [ dd, /dev/zero ]

tests/data/merge_sources/source8-2.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+
+mounts:
+ - [ ephemeral22, /mnt, auto, "defaults,noexec" ]
+
+merge_how: 'dict(recurse_array)+list(recurse_list,recurse_str)+str()'

tests/data/merge_sources/source9-1.yaml

@@ -0,0 +1,5 @@
+#cloud-config
+
+phone_home:
+ url: http://my.example.com/$INSTANCE_ID/
+ post: [ pub_key_dsa, pub_key_rsa, pub_key_ecdsa, instance_id ]

tests/data/merge_sources/source9-2.yaml

@@ -0,0 +1,6 @@
+#cloud-config
+
+phone_home:
+ url: $BLAH_BLAH
+
+merge_how: 'dict(recurse_str)+str(append)'

tests/unittests/test_merging.py

@@ -1,142 +1,255 @@
 from tests.unittests import helpers
 
-from cloudinit import mergers
+from cloudinit.handlers import cloud_config
+from cloudinit.handlers import (CONTENT_START, CONTENT_END)
+
+from cloudinit import helpers as c_helpers
+from cloudinit import util
+
+import collections
+import glob
+import os
+import random
+import re
+import string  # pylint: disable=W0402
+
+SOURCE_PAT = "source*.*yaml"
+EXPECTED_PAT = "expected%s.yaml"
+TYPES = [long, int, dict, str, list, tuple, None]
 
 
-class TestSimpleRun(helpers.MockerTestCase):
-    def test_basic_merge(self):
-        source = {
-            'Blah': ['blah2'],
-            'Blah3': 'c',
-        }
-        merge_with = {
-            'Blah2': ['blah3'],
-            'Blah3': 'b',
-            'Blah': ['123'],
-        }
-        # Basic merge should not do thing special
-        merge_how = "list()+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], ['blah2'])
-        self.assertEquals(merged['Blah2'], ['blah3'])
-        self.assertEquals(merged['Blah3'], 'c')
+def _old_mergedict(src, cand):
+    """
+    Merge values from C{cand} into C{src}.
+    If C{src} has a key C{cand} will not override.
+    Nested dictionaries are merged recursively.
+    """
+    if isinstance(src, dict) and isinstance(cand, dict):
+        for (k, v) in cand.iteritems():
+            if k not in src:
+                src[k] = v
+            else:
+                src[k] = _old_mergedict(src[k], v)
+    return src
 
-    def test_dict_overwrite(self):
-        source = {
-            'Blah': ['blah2'],
-        }
-        merge_with = {
-            'Blah': ['123'],
-        }
-        # Now lets try a dict overwrite
-        merge_how = "list()+dict(overwrite)+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], ['123'])
 
-    def test_string_append(self):
-        source = {
-            'Blah': 'blah2',
-        }
-        merge_with = {
-            'Blah': '345',
-        }
-        merge_how = "list()+dict()+str(append)"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah'], 'blah2345')
+def _old_mergemanydict(*args):
+    out = {}
+    for a in args:
+        out = _old_mergedict(out, a)
+    return out
 
-    def test_list_extend(self):
-        source = ['abc']
-        merge_with = ['123']
-        merge_how = "list(extend)+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged, ['abc', '123'])
 
-    def test_deep_merge(self):
-        source = {
-            'a': [1, 'b', 2],
-            'b': 'blahblah',
-            'c': {
-                'e': [1, 2, 3],
-                'f': 'bigblobof',
-                'iamadict': {
-                    'ok': 'ok',
-                }
-            },
-            'run': [
-                'runme',
-                'runme2',
-            ],
-            'runmereally': [
-                'e', ['a'], 'd',
-            ],
-        }
-        merge_with = {
-            'a': ['e', 'f', 'g'],
-            'b': 'more',
-            'c': {
-                'a': 'b',
-                'f': 'stuff',
-            },
-            'run': [
-                'morecmd',
-                'moremoremore',
-            ],
-            'runmereally': [
-                'blah', ['b'], 'e',
-            ],
-        }
-        merge_how = "list(extend)+dict()+str(append)"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['a'], [1, 'b', 2, 'e', 'f', 'g'])
-        self.assertEquals(merged['b'], 'blahblahmore')
-        self.assertEquals(merged['c']['f'], 'bigblobofstuff')
-        self.assertEquals(merged['run'], ['runme', 'runme2', 'morecmd',
-                                          'moremoremore'])
-        self.assertEquals(merged['runmereally'], ['e', ['a'], 'd', 'blah',
-                                                  ['b'], 'e'])
+def _random_str(rand):
+    base = ''
+    for _i in xrange(rand.randint(1, 2 ** 8)):
+        base += rand.choice(string.letters + string.digits)
+    return base
 
-    def test_dict_overwrite_layered(self):
-        source = {
-            'Blah3': {
-                'f': '3',
-                'g': {
-                    'a': 'b',
+
+class _NoMoreException(Exception):
+    pass
+
+
+def _make_dict(current_depth, max_depth, rand):
+    if current_depth >= max_depth:
+        raise _NoMoreException()
+    if current_depth == 0:
+        t = dict
+    else:
+        t = rand.choice(TYPES)
+    base = None
+    if t in [None]:
+        return base
+    if t in [dict, list, tuple]:
+        if t in [dict]:
+            amount = rand.randint(0, 5)
+            keys = [_random_str(rand) for _i in xrange(0, amount)]
+            base = {}
+            for k in keys:
+                try:
+                    base[k] = _make_dict(current_depth + 1, max_depth, rand)
+                except _NoMoreException:
+                    pass
+        elif t in [list, tuple]:
+            base = []
+            amount = rand.randint(0, 5)
+            for _i in xrange(0, amount):
+                try:
+                    base.append(_make_dict(current_depth + 1, max_depth, rand))
+                except _NoMoreException:
+                    pass
+            if t in [tuple]:
+                base = tuple(base)
+    elif t in [long, int]:
+        base = rand.randint(0, 2 ** 8)
+    elif t in [str]:
+        base = _random_str(rand)
+    return base
+
+
+def make_dict(max_depth, seed=None):
+    max_depth = max(1, max_depth)
+    rand = random.Random(seed)
+    return _make_dict(0, max_depth, rand)
+
+
+class TestSimpleRun(helpers.ResourceUsingTestCase):
+    def _load_merge_files(self):
+        merge_root = self.resourceLocation('merge_sources')
+        tests = []
+        source_ids = collections.defaultdict(list)
+        expected_files = {}
+        for fn in glob.glob(os.path.join(merge_root, SOURCE_PAT)):
+            base_fn = os.path.basename(fn)
+            file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn)
+            if not file_id:
+                raise IOError("File %s does not have a numeric identifier"
+                              % (fn))
+            file_id = int(file_id.group(1))
+            source_ids[file_id].append(fn)
+            expected_fn = os.path.join(merge_root, EXPECTED_PAT % (file_id))
+            if not os.path.isfile(expected_fn):
+                raise IOError("No expected file found at %s" % (expected_fn))
+            expected_files[file_id] = expected_fn
+        for i in sorted(source_ids.keys()):
+            source_file_contents = []
+            for fn in sorted(source_ids[i]):
+                source_file_contents.append([fn, util.load_file(fn)])
+            expected = util.load_yaml(util.load_file(expected_files[i]))
+            entry = [source_file_contents, [expected, expected_files[i]]]
+            tests.append(entry)
+        return tests
+
+    def test_seed_runs(self):
+        test_dicts = []
+        for i in range(1, 50):
+            base_dicts = []
+            for j in range(1, 50):
+                base_dicts.append(make_dict(5, i * j))
+            test_dicts.append(base_dicts)
+        for test in test_dicts:
+            c = _old_mergemanydict(*test)
+            d = util.mergemanydict(test)
+            self.assertEquals(c, d)
+
+    def test_merge_cc_samples(self):
+        tests = self._load_merge_files()
+        paths = c_helpers.Paths({})
+        cc_handler = cloud_config.CloudConfigPartHandler(paths)
+        cc_handler.cloud_fn = None
+        for (payloads, (expected_merge, expected_fn)) in tests:
+            cc_handler.handle_part(None, CONTENT_START, None,
+                                   None, None, None)
+            merging_fns = []
+            for (fn, contents) in payloads:
+                cc_handler.handle_part(None, None, "%s.yaml" % (fn),
+                                       contents, None, {})
+                merging_fns.append(fn)
+            merged_buf = cc_handler.cloud_buf
+            cc_handler.handle_part(None, CONTENT_END, None,
+                                   None, None, None)
+            fail_msg = "Equality failure on checking %s with %s: %s != %s"
+            fail_msg = fail_msg % (expected_fn,
+                                   ",".join(merging_fns), merged_buf,
+                                   expected_merge)
+            self.assertEquals(expected_merge, merged_buf, msg=fail_msg)
+
+    def test_compat_merges_dict(self):
+        a = {
+            '1': '2',
+            'b': 'c',
+        }
+        b = {
+            'b': 'e',
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merges_dict2(self):
+        a = {
+            'Blah': 1,
+            'Blah2': 2,
+            'Blah3': 3,
+        }
+        b = {
+            'Blah': 1,
+            'Blah2': 2,
+            'Blah3': [1],
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merges_list(self):
+        a = {'b': [1, 2, 3]}
+        b = {'b': [4, 5]}
+        c = {'b': [6, 7]}
+        e = _old_mergemanydict(a, b, c)
+        f = util.mergemanydict([a, b, c])
+        self.assertEquals(e, f)
+
+    def test_compat_merges_str(self):
+        a = {'b': "hi"}
+        b = {'b': "howdy"}
+        c = {'b': "hallo"}
+        e = _old_mergemanydict(a, b, c)
+        f = util.mergemanydict([a, b, c])
+        self.assertEquals(e, f)
+
+    def test_compat_merge_sub_dict(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': 'g',
+                'e': 'c',
+                'h': 'd',
+                'hh': {
+                    '1': 2,
+                },
+            }
+        }
+        b = {
+            'b': {
+                'e': 'c',
+                'hh': {
+                    '3': 4,
                 }
             }
         }
-        merge_with = {
-            'Blah3': {
-                'e': '2',
-                'g': {
-                    'e': 'f',
-                }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merge_sub_dict2(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': 'g',
             }
         }
-        merge_how = "list()+dict()+str()"
-        merger_set = mergers.string_extract_mergers(merge_how)
-        self.assertEquals(3, len(merger_set))
-        merger = mergers.construct(merger_set)
-        merged = merger.merge(source, merge_with)
-        self.assertEquals(merged['Blah3'], {
-                'e': '2',
-                'f': '3',
-                'g': {
-                    'a': 'b',
-                    'e': 'f',
-                }
-        })
+        b = {
+            'b': {
+                'e': 'c',
+            }
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)
+
+    def test_compat_merge_sub_list(self):
+        a = {
+            '1': '2',
+            'b': {
+                'f': ['1'],
+            }
+        }
+        b = {
+            'b': {
+                'f': [],
+            }
+        }
+        c = _old_mergedict(a, b)
+        d = util.mergemanydict([a, b])
+        self.assertEquals(c, d)

tests/unittests/test_userdata.py

@@ -60,7 +60,6 @@ run:
  - c
 '''
         message1 = MIMEBase("text", "cloud-config")
-        message1['Merge-Type'] = 'dict()+list(extend)+str(append)'
         message1.set_payload(blob)
 
         blob2 = '''
@@ -72,7 +71,8 @@ run:
  - morestuff
 '''
         message2 = MIMEBase("text", "cloud-config")
-        message2['X-Merge-Type'] = 'dict()+list(extend)+str()'
+        message2['X-Merge-Type'] = ('dict(recurse_array,'
+                                    'recurse_str)+list(append)+str(append)')
         message2.set_payload(blob2)
 
         blob3 = '''
@@ -84,7 +84,6 @@ e:
 p: 1
 '''
         message3 = MIMEBase("text", "cloud-config")
-        message3['Merge-Type'] = 'dict()+list()+str()'
         message3.set_payload(blob3)
 
         messages = [message1, message2, message3]
@@ -109,7 +108,7 @@ p: 1
         contents = util.load_yaml(contents)
         self.assertEquals(contents['run'], ['b', 'c', 'stuff', 'morestuff'])
         self.assertEquals(contents['a'], 'be')
-        self.assertEquals(contents['e'], 'fg')
+        self.assertEquals(contents['e'], [1, 2, 3])
         self.assertEquals(contents['p'], 1)
 
     def test_unhandled_type_warning(self):