improvements for launch index, one fix for cloud-archive

1. Docs for launch-index + examples
2. Tests for launch-index + data files
3. Fixing a bug with cloud-archive yaml types allowed (likes a tuple not a list
   for some reason) (LP: #1044594)
4. Setting the 'part' content-type if what we actually use is different.

cloudinit/filters/launch_index.py

@@ -44,7 +44,7 @@ class Filter(object):
         return True
 
     def _do_filter(self, message):
-        # Don't use walk() here since we want to do the reforming of the 
+        # Don't use walk() here since we want to do the reforming of the
         # messages ourselves and not flatten the message listings...
         if not self._select(message):
             return None

cloudinit/user_data.py

@@ -54,7 +54,7 @@ ATTACHMENT_FIELD = 'Number-Attachments'
 
 # Only the following content types can have there launch index examined
 # in there payload, evey other content type can still provide a header
-EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config", "text/cloud-config-archive"]
+EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"]
 
 
 class UserDataProcessor(object):
@@ -84,6 +84,12 @@ class UserDataProcessor(object):
             if ctype is None:
                 ctype = ctype_orig
 
+            if ctype != ctype_orig:
+                if CONTENT_TYPE in part:
+                    part.replace_header(CONTENT_TYPE, ctype)
+                else:
+                    part[CONTENT_TYPE] = ctype
+
             if ctype in INCLUDE_TYPES:
                 self._do_include(payload, append_msg)
                 continue
@@ -92,6 +98,8 @@ class UserDataProcessor(object):
                 self._explode_archive(payload, append_msg)
                 continue
 
+            # Should this be happening, shouldn't
+            # the part header be modified and not the base?
             if CONTENT_TYPE in base_msg:
                 base_msg.replace_header(CONTENT_TYPE, ctype)
             else:
@@ -180,7 +188,7 @@ class UserDataProcessor(object):
                 self._process_msg(new_msg, append_msg)
 
     def _explode_archive(self, archive, append_msg):
-        entries = util.load_yaml(archive, default=[], allowed=[list, set])
+        entries = util.load_yaml(archive, default=[], allowed=(list, set))
         for ent in entries:
             # ent can be one of:
             #  dict { 'filename' : 'value', 'content' :

doc/examples/cloud-config-archive-launch-index.txt

@@ -0,0 +1,30 @@
+#cloud-config-archive
+
+# This is an example of a cloud archive
+# format which includes a set of launch indexes
+# that will be filtered on (thus only showing
+# up in instances with that launch index), this
+# is done by adding the 'launch-index' key which
+# maps to the integer 'launch-index' that the 
+# corresponding content should be used with.
+#
+# It is possible to leave this value out which
+# will mean that the content will be applicable
+# for all instances
+
+- type: foo/wark
+  filename: bar
+  content: |
+    This is my payload
+    hello
+  launch-index: 1  # I will only be used on launch-index 1
+- this is also payload
+- |
+  multi line payload
+  here
+-
+  type: text/upstart-job
+  filename: my-upstart.conf
+  content: |
+   whats this, yo?
+  launch-index: 0 # I will only be used on launch-index 0

doc/examples/cloud-config-launch-index.txt

@@ -0,0 +1,23 @@
+#cloud-config
+# vim: syntax=yaml
+
+#
+# This is the configuration syntax that can be provided to have
+# a given set of cloud config data show up on a certain launch
+# index (and not other launches) by provided a key here which
+# will act as a filter on the instances userdata. When
+# this key is left out (or non-integer) then the content
+# of this file will always be used for all launch-indexes 
+# (ie the previous behavior).
+launch-index: 5
+
+# Upgrade the instance on first boot
+# (ie run apt-get upgrade)
+#
+# Default: false
+#
+apt_upgrade: true
+
+# Other yaml keys below...
+# .......
+# .......

tests/data/filter_cloud_multipart.yaml

@@ -0,0 +1,30 @@
+#cloud-config-archive
+---
+-   content: "\n        blah: true\n        launch-index: 3\n"
+    type: text/cloud-config
+-   content: "\n        blah: true\n        launch-index: 4\n"
+    type: text/cloud-config
+-   content: The quick brown fox jumps over the lazy dog
+    filename: b0.txt
+    launch-index: 0
+    type: plain/text
+-   content: The quick brown fox jumps over the lazy dog
+    filename: b3.txt
+    launch-index: 3
+    type: plain/text
+-   content: The quick brown fox jumps over the lazy dog
+    filename: b2.txt
+    launch-index: 2
+    type: plain/text
+-   content: '#!/bin/bash \n echo "stuff"'
+    filename: b2.txt
+    launch-index: 2
+-   content: '#!/bin/bash \n echo "stuff"'
+    filename: b2.txt
+    launch-index: 1
+-   content: '#!/bin/bash \n echo "stuff"'
+    filename: b2.txt
+    # Use a string to see if conversion works
+    launch-index: "1"
+...
+

tests/data/filter_cloud_multipart_1.email

@@ -0,0 +1,11 @@
+From nobody Fri Aug 31 17:17:00 2012
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+
+#cloud-config
+b: c
+launch-index: 2
+
+

tests/data/filter_cloud_multipart_2.email

@@ -0,0 +1,39 @@
+From nobody Fri Aug 31 17:43:04 2012
+Content-Type: multipart/mixed; boundary="===============1668325974=="
+MIME-Version: 1.0
+
+--===============1668325974==
+Content-Type: text/cloud-config; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+
+#cloud-config
+b: c
+launch-index: 2
+
+
+--===============1668325974==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+
+#cloud-config-archive
+-   content: The quick brown fox jumps over the lazy dog
+    filename: b3.txt
+    launch-index: 3
+    type: plain/text
+
+--===============1668325974==
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+
+#cloud-config
+b: c
+launch-index: 2
+
+
+--===============1668325974==--

tests/data/filter_cloud_multipart_header.email

@@ -0,0 +1,11 @@
+From nobody Fri Aug 31 17:17:00 2012
+Content-Type: text/plain; charset="us-ascii"
+MIME-Version: 1.0
+Launch-Index: 5
+Content-Transfer-Encoding: 7bit
+
+
+#cloud-config
+b: c
+
+

tests/unittests/helpers.py

@@ -0,0 +1,42 @@
+import os
+
+from mocker import MockerTestCase
+
+from cloudinit import helpers as ch
+
+
+class ResourceUsingTestCase(MockerTestCase):
+    def __init__(self, methodName="runTest"):
+        MockerTestCase.__init__(self, methodName)
+        self.resource_path = None
+
+    def resourceLocation(self, subname=None):
+        if self.resource_path is None:
+            paths = [
+                os.path.join('tests', 'data'),
+                os.path.join('data'),
+                os.path.join(os.pardir, 'tests', 'data'),
+                os.path.join(os.pardir, 'data'),
+            ]
+            for p in paths:
+                if os.path.isdir(p):
+                    self.resource_path = p
+                    break
+        self.assertTrue((self.resource_path and
+                         os.path.isdir(self.resource_path)),
+                        msg="Unable to locate test resource data path!")
+        if not subname:
+            return self.resource_path
+        return os.path.join(self.resource_path, subname)
+
+    def readResource(self, name):
+        where = self.resourceLocation(name)
+        with open(where, 'r') as fh:
+            return fh.read()
+
+    def getCloudPaths(self):
+        cp = ch.Paths({
+            'cloud_dir': self.makeDir(),
+            'templates_dir': self.resourceLocation(),
+        })
+        return cp

tests/unittests/test_filters/test_launch_index.py

@@ -0,0 +1,134 @@
+import copy
+
+import helpers as th
+
+import itertools
+
+from cloudinit.filters import launch_index
+from cloudinit import user_data as ud
+from cloudinit import util
+
+
+def count_messages(root):
+    am = 0
+    for m in root.walk():
+        if ud.is_skippable(m):
+            continue
+        am += 1
+    return am
+
+
+class TestLaunchFilter(th.ResourceUsingTestCase):
+
+    def assertCounts(self, message, expected_counts):
+        orig_message = copy.deepcopy(message)
+        for (index, count) in expected_counts.items():
+            index = util.safe_int(index)
+            filtered_message = launch_index.Filter(index).apply(message)
+            self.assertEquals(count_messages(filtered_message), count)
+        # Ensure original message still ok/not modified
+        self.assertTrue(self.equivalentMessage(message, orig_message))
+
+    def equivalentMessage(self, msg1, msg2):
+        msg1_count = count_messages(msg1)
+        msg2_count = count_messages(msg2)
+        if msg1_count != msg2_count:
+            return False
+        # Do some basic payload checking
+        msg1_msgs = [m for m in msg1.walk()]
+        msg1_msgs = [m for m in
+                     itertools.ifilterfalse(ud.is_skippable, msg1_msgs)]
+        msg2_msgs = [m for m in msg2.walk()]
+        msg2_msgs = [m for m in
+                     itertools.ifilterfalse(ud.is_skippable, msg2_msgs)]
+        for i in range(0, len(msg2_msgs)):
+            m1_msg = msg1_msgs[i]
+            m2_msg = msg2_msgs[i]
+            if m1_msg.get_charset() != m2_msg.get_charset():
+                return False
+            if m1_msg.is_multipart() != m2_msg.is_multipart():
+                return False
+            m1_py = m1_msg.get_payload(decode=True)
+            m2_py = m2_msg.get_payload(decode=True)
+            if m1_py != m2_py:
+                return False
+        return True
+
+    def testMultiEmailIndex(self):
+        test_data = self.readResource('filter_cloud_multipart_2.email')
+        ud_proc = ud.UserDataProcessor(self.getCloudPaths())
+        message = ud_proc.process(test_data)
+        self.assertTrue(count_messages(message) > 0)
+        # This file should have the following
+        # indexes -> amount mapping in it
+        expected_counts = {
+            3: 1,
+            2: 2,
+            None: 3,
+            -1: 0,
+        }
+        self.assertCounts(message, expected_counts)
+
+    def testHeaderEmailIndex(self):
+        test_data = self.readResource('filter_cloud_multipart_header.email')
+        ud_proc = ud.UserDataProcessor(self.getCloudPaths())
+        message = ud_proc.process(test_data)
+        self.assertTrue(count_messages(message) > 0)
+        # This file should have the following
+        # indexes -> amount mapping in it
+        expected_counts = {
+            5: 1,
+            -1: 0,
+            'c': 1,
+            None: 1,
+        }
+        self.assertCounts(message, expected_counts)
+
+    def testConfigEmailIndex(self):
+        test_data = self.readResource('filter_cloud_multipart_1.email')
+        ud_proc = ud.UserDataProcessor(self.getCloudPaths())
+        message = ud_proc.process(test_data)
+        self.assertTrue(count_messages(message) > 0)
+        # This file should have the following
+        # indexes -> amount mapping in it
+        expected_counts = {
+            2: 1,
+            -1: 0,
+            None: 1,
+        }
+        self.assertCounts(message, expected_counts)
+
+    def testNoneIndex(self):
+        test_data = self.readResource('filter_cloud_multipart.yaml')
+        ud_proc = ud.UserDataProcessor(self.getCloudPaths())
+        message = ud_proc.process(test_data)
+        start_count = count_messages(message)
+        self.assertTrue(start_count > 0)
+        filtered_message = launch_index.Filter(None).apply(message)
+        self.assertTrue(self.equivalentMessage(message, filtered_message))
+
+    def testIndexes(self):
+        test_data = self.readResource('filter_cloud_multipart.yaml')
+        ud_proc = ud.UserDataProcessor(self.getCloudPaths())
+        message = ud_proc.process(test_data)
+        start_count = count_messages(message)
+        self.assertTrue(start_count > 0)
+        # This file should have the following
+        # indexes -> amount mapping in it
+        expected_counts = {
+            2: 2,
+            3: 2,
+            1: 2,
+            0: 1,
+            4: 1,
+            7: 0,
+            -1: 0,
+            100: 0,
+            # None should just give all back
+            None: start_count,
+            # Non ints should be ignored
+            'c': start_count,
+            # Strings should be converted
+            '1': 2,
+        }
+        self.assertCounts(message, expected_counts)