1. Add an importer function that will search for a given module in a set of search module 'prefixes'

that also has a potential set of required attributes.
2. Use this new importer to find the distro class, the userdata handler modules, the config modules
   and the datasource modules, if none can be found error out accordingly.

cloudinit/config/__init__.py

@@ -51,10 +51,6 @@ def fixup_module(mod, def_freq=PER_INSTANCE):
         freq = mod.frequency
         if freq and freq not in FREQUENCIES:
             LOG.warn("Module %s has an unknown frequency %s", mod, freq)
-    if not hasattr(mod, 'handle'):
-        def empty_handle(_name, _cfg, _cloud, _log, _args):
-            pass
-        setattr(mod, 'handle', empty_handle)
     if not hasattr(mod, 'distros'):
         setattr(mod, 'distros', None)
     return mod

cloudinit/distros/__init__.py

@@ -144,16 +144,13 @@ class Distro(object):
             return False
 
 
-def fetch(distro_name, mods=(__name__, )):
-    mod = None
-    for m in mods:
-        mod_name = "%s.%s" % (m, distro_name)
-        try:
-            mod = importer.import_module(mod_name)
-        except ImportError:
-            pass
-    if not mod:
-        raise RuntimeError("No distribution found for distro %s"
-                           % (distro_name))
-    distro_cls = getattr(mod, 'Distro')
-    return distro_cls
+def fetch(name):
+    locs = importer.find_module(name,
+                                ['', __name__],
+                                ['Distro'])
+    if not locs:
+        raise ImportError("No distribution found for distro %s"
+                           % (name))
+    mod = importer.import_module(locs[0])
+    cls = getattr(mod, 'Distro')
+    return cls

cloudinit/handlers/__init__.py

@@ -202,20 +202,12 @@ def walk(msg, callback, data):
 def fixup_handler(mod, def_freq=PER_INSTANCE):
     if not hasattr(mod, "handler_version"):
         setattr(mod, "handler_version", 1)
-    if not hasattr(mod, 'list_types'):
-        def empty_types():
-            return []
-        setattr(mod, 'list_types', empty_types)
     if not hasattr(mod, 'frequency'):
         setattr(mod, 'frequency', def_freq)
     else:
         freq = mod.frequency
         if freq and freq not in FREQUENCIES:
             LOG.warn("Handler %s has an unknown frequency %s", mod, freq)
-    if not hasattr(mod, 'handle_part'):
-        def empty_handler(_data, _ctype, _filename, _payload):
-            pass
-        setattr(mod, 'handle_part', empty_handler)
     return mod
 
 

cloudinit/importer.py

@@ -23,17 +23,43 @@
 import sys
 
 from cloudinit import log as logging
-from cloudinit import util
 
 LOG = logging.getLogger(__name__)
 
 
-# Simple wrapper that allows us to add more logging in...
 def import_module(module_name):
-    try:
-        LOG.debug("Attempting to import module %s", module_name)
-        __import__(module_name)
-        return sys.modules[module_name]
-    except:
-        util.logexc(LOG, 'Failed at importing %s', module_name)
-        raise
+    __import__(module_name)
+    return sys.modules[module_name]
+
+
+def find_module(base_name, search_paths, required_attrs=None):
+    found_places = []
+    if not required_attrs:
+        required_attrs = []
+    real_paths = []
+    for path in search_paths:
+        real_path = []
+        if path:
+            real_path.extend(path.split("."))
+        real_path.append(base_name)
+        full_path = '.'.join(real_path)
+        real_paths.append(full_path)
+    LOG.debug("Looking for modules %s that have attributes %s",
+              real_paths, required_attrs)
+    for full_path in real_paths:
+        mod = None
+        try:
+            mod = import_module(full_path)
+        except ImportError:
+            pass
+        if not mod:
+            continue
+        found_attrs = 0
+        for attr in required_attrs:
+            if hasattr(mod, attr):
+                found_attrs += 1
+        if found_attrs == len(required_attrs):
+            found_places.append(full_path)
+    LOG.debug("Found %s with attributes %s in %s", base_name,
+              required_attrs, found_places)
+    return found_places

cloudinit/sources/__init__.py

@@ -191,31 +191,19 @@ def list_sources(cfg_list, depends, pkg_list):
     LOG.info(("Looking for for data source in: %s,"
               " via packages %s that matches dependencies %s"),
              cfg_list, pkg_list, depends)
-    for ds_coll in cfg_list:
-        ds_name = str(ds_coll)
+    for ds_name in cfg_list:
         if not ds_name.startswith(DS_PREFIX):
             ds_name = '%s%s' % (DS_PREFIX, ds_name)
-        for pkg in pkg_list:
-            pkg_name = []
-            if pkg:
-                # Any package name given, this affects
-                # the lookup path
-                pkg_name.append(str(pkg))
-            pkg_name.append(ds_name)
-            try:
-                mod = importer.import_module(".".join(pkg_name))
-            except ImportError:
-                continue
-            lister = getattr(mod, "get_datasource_list", None)
-            if not lister:
-                continue
-            cls_matches = lister(depends)
-            if not cls_matches:
-                continue
-            src_list.extend(cls_matches)
-            LOG.debug(("Found a match"
-                       " in %s with matches %s"), mod, cls_matches)
-            break
+        m_locs = importer.find_module(ds_name,
+                                      pkg_list,
+                                      ['get_datasource_list'])
+        for m_loc in m_locs:
+            mod = importer.import_module(m_loc)
+            lister = getattr(mod, "get_datasource_list")
+            matches = lister(depends)
+            if matches:
+                src_list.extend(matches)
+                break
     return src_list
 
 

cloudinit/stages.py

@@ -205,7 +205,7 @@ class Init(object):
         # Any config provided???
         pkg_list = self.cfg.get('datasource_pkg_list') or []
         # Add the defaults at the end
-        for n in [util.obj_name(sources), '']:
+        for n in ['', util.obj_name(sources)]:
             if n not in pkg_list:
                 pkg_list.append(n)
         cfg_list = self.cfg.get('datasource_list') or []
@@ -334,9 +334,17 @@ class Init(object):
 
         # Add handlers in cdir
         potential_handlers = util.find_modules(cdir)
-        for (fname, modname) in potential_handlers.iteritems():
+        for (fname, mod_name) in potential_handlers.iteritems():
             try:
-                mod = handlers.fixup_handler(importer.import_module(modname))
+                mod_locs = importer.find_module(mod_name, [''],
+                                                ['list_types',
+                                                 'handle_part'])
+                if not mod_locs:
+                    LOG.warn(("Could not find a valid user-data handler"
+                              " named %s in file %s"), mod_name, fname)
+                    continue
+                mod = importer.import_module(mod_locs[0])
+                mod = handlers.fixup_handler(mod)
                 types = c_handlers.register(mod)
                 LOG.debug("Added handler for %s from %s", types, fname)
             except:
@@ -482,7 +490,13 @@ class Modules(object):
                           " has an unknown frequency %s"), raw_name, freq)
                 # Reset it so when ran it will get set to a known value
                 freq = None
-            mod = config.fixup_module(importer.import_module(mod_name))
+            mod_locs = importer.find_module(mod_name,
+                                            ['', util.obj_name(config)],
+                                            ['handle'])
+            if not mod_locs:
+                LOG.warn("Could not find module named %s", mod_name)
+                continue
+            mod = config.fixup_module(importer.import_module(mod_locs[0]))
             mostly_mods.append([mod, raw_name, freq, run_args])
         return mostly_mods