Make entropy suitable for pypi distribution, part 0

We need to separate out engine code and audit and repair scripts if
pypi distribution is reqd. This is handled in this commit.

Further commits will remove the two variables hardcoded currently,
and expose __main__ as a cmdline script on installation.

Use imp for module finding and loading, use full path to script in
audit and repair cfg files.

Move audit and repair scripts to examples. Make some changes to hardcoded
stuff to account for this

Update gitignore
Change-Id: I50831003c6f7272967dbeb5c558b76b0183c91be

.gitignore

@@ -56,10 +56,11 @@ ChangeLog
 env
 
 # Project-related
+# json files
+*.json
+*.log
 arch/
 docs/_build
 entropy/test
-entropy/logs
-entropy/audit/*.json
-entropy/repair/*.json
-entropy/cfg/*.cfg
+entropy/examples/logs
+entropy/examples/cfg/

entropy/__main__.py

@@ -31,8 +31,10 @@ from entropy import utils
 LOG = logging.getLogger(__name__)
 
 # TODO(praneshp): Only hardcoded stuff in the project. Find a way to move
-engine_cfg = os.path.join(os.getcwd(), 'entropy', 'cfg', 'engines.cfg')
-log_file = os.path.join(os.getcwd(), 'entropy', 'logs', 'entropy.log')
+engine_cfg = os.path.join(os.getcwd(), 'entropy', 'examples',
+                          'cfg', 'engines.cfg')
+log_file = os.path.join(os.getcwd(), 'entropy', 'examples',
+                        'logs', 'entropy.log')
 
 
 def get_cfg_file(engine, script_type):

entropy/audit/audit.json

@@ -1,13 +0,0 @@
-{
-    "name" : "audit",
-    "hostname" : "localhost",
-    "schedule" :"*/3 * * * *",
-    "username" : "praneshp",
-    "ssh-key" : "id_rsa",
-    "mq_host": "localhost",
-    "mq_port": "5672",
-    "mq_user": "guest",
-    "mq_password": "guest",
-    "script": "runthis.sh",
-    "module": "audit"
-}

entropy/engine.py

@@ -25,7 +25,7 @@ from kombu import Exchange
 import pause
 
 from entropy import utils
-
+import imp
 LOG = logging.getLogger(__name__)
 
 
@@ -98,7 +98,9 @@ class Engine(object):
 
         for script in scripts:
             if script['name'] not in running_scripts:
-                futures.append(setup_func(script))
+                future = setup_func(script)
+                if future is not None:
+                    futures.append(future)
         LOG.info('Running %s scripts %s', script_type,
                  ', '.join(running_scripts))
         return futures
@@ -109,28 +111,25 @@ class Engine(object):
         # Pick out relevant info
         data = dict(utils.load_yaml(script['conf']).next())
         react_script = data['script']
-
-        available_modules = utils.find_module(react_script, ['repair'])
+        search_path, reactor = utils.get_filename_and_path(react_script)
+        available_modules = imp.find_module(reactor, [search_path])
         LOG.info('Found these modules: %s', available_modules)
-        if not available_modules:
-            LOG.error('No module to load')
-        else:
-            imported_module = utils.import_module(available_modules[0])
+        try:
+            imported_module = imp.load_module(react_script, *available_modules)
             kwargs = data
             kwargs['conf'] = script['conf']
-
             # add this job to list of running audits
             self.running_repairs.append(script['name'])
-
             future = self.executor.submit(imported_module.main, **kwargs)
             return future
+        except Exception:
+            LOG.exception("Could not setup %s", kwargs['name'])
+            return None
 
     def setup_audit(self, script):
         LOG.info('Setting up audit script %s', script['name'])
-
         # add this job to list of running audits
         self.running_audits.append(script['name'])
-
         # start a process for this audit script
         future = self.executor.submit(self.start_audit, script)
         return future
@@ -163,9 +162,11 @@ class Engine(object):
         # Put a message on the mq
         #TODO(praneshp): this should be the path with register-audit
         try:
-            available_modules = utils.find_module(kwargs['module'], ['audit'])
+            audit_script = kwargs['module']
+            search_path, auditor = utils.get_filename_and_path(audit_script)
+            available_modules = imp.find_module(auditor, [search_path])
             LOG.info('Found these modules: %s', available_modules)
-            imported_module = utils.import_module(available_modules[0])
+            imported_module = imp.load_module(audit_script, *available_modules)
             audit_obj = imported_module.Audit(**kwargs)
             audit_obj.send_message(**kwargs)
         except Exception:

entropy/examples/audit/audit.py

@@ -19,26 +19,20 @@ from kombu import BrokerConnection
 from kombu.common import maybe_declare
 from kombu.pools import producers
 
-import base
-from entropy.queues import entropy_exchange
-from entropy.queues import PASS_KEY
+from entropy.audit import base
 
 LOG = logging.getLogger(__name__)
 
 
 class Audit(base.AuditBase):
-
-    def test(self):
-        LOG.info('hello world')
-
     def send_message(self, **kwargs):
         connection = BrokerConnection('amqp://%(mq_user)s:%(mq_password)s@'
                                       '%(mq_host)s:%(mq_port)s//' % kwargs)
         message = {'From': __file__,
                    'Date': str(datetime.datetime.now().isoformat())}
         with producers[connection].acquire(block=True) as producer:
-            maybe_declare(entropy_exchange, producer.channel)
+            maybe_declare(kwargs['exchange'], producer.channel)
             producer.publish(message,
-                             exchange=entropy_exchange,
-                             routing_key=PASS_KEY,
+                             exchange=kwargs['exchange'],
+                             routing_key=kwargs['routing_key'],
                              serializer='json')

entropy/examples/audit/vm_count.py

@@ -20,8 +20,7 @@ from kombu.common import maybe_declare
 from kombu.pools import producers
 
 
-import base
-from entropy.queues import entropy_exchange
+from entropy.audit import base
 import libvirt
 
 LOG = logging.getLogger(__name__)
@@ -51,10 +50,10 @@ class Audit(base.AuditBase):
         message = {'From': __name__,
                    'Date': str(datetime.datetime.now().isoformat())}
         with producers[connection].acquire(block=True) as producer:
-            maybe_declare(entropy_exchange, producer.channel)
+            maybe_declare(kwargs['exchange'], producer.channel)
             msg_args = {'vm_count': self.get_vm_count(**kwargs)}
             message['payload'] = msg_args
             producer.publish(message,
-                             exchange=entropy_exchange,
-                             routing_key='vmcount',
+                             exchange=kwargs['exchange'],
+                             routing_key=kwargs['routing_key'],
                              serializer='json')

entropy/examples/audit/vmbooter.py

@@ -23,7 +23,7 @@ from kombu.pools import producers
 from novaclient.client import Client
 import paramiko
 
-import base
+from entropy.audit import base
 
 LOG = logging.getLogger(__name__)
 

entropy/repair/react.json

@@ -1,11 +0,0 @@
-{
-    "name" : "react",
-    "script": "react",
-    "hostname" : "localhost",
-    "username" : "praneshp",
-    "ssh-key" : "id_rsa",
-    "mq_host": "localhost",
-    "mq_port": "5672",
-    "mq_user": "guest",
-    "mq_password": "guest"
-}

entropy/repair/vm_count_react.json

@@ -1,12 +0,0 @@
-{
-    "name" : "vmcount",
-    "script": "vm_count_react",
-    "hostname" : "localhost",
-    "username" : "praneshp",
-    "ssh-key" : "id_rsa",
-    "mq_host": "localhost",
-    "mq_port": "5672",
-    "mq_user": "guest",
-    "mq_password": "guest",
-    "limit": 0
-}

entropy/utils.py

@@ -25,6 +25,10 @@ import yaml
 LOG = logging.getLogger(__name__)
 
 
+def get_filename_and_path(path):
+    return os.path.dirname(path), os.path.basename(path)
+
+
 def get_key_path():
     home_dir = os.path.expanduser("~")
     ssh_dir = os.path.join(home_dir, ".ssh")