From e4fd9aa1f245ef9fc70ea4fbda6bcafa7676e61d Mon Sep 17 00:00:00 2001 From: pran1990 Date: Thu, 10 Apr 2014 13:07:47 -0700 Subject: [PATCH] Make entropy suitable for pypi distribution, part 0 We need to separate out engine code and audit and repair scripts if pypi distribution is reqd. This is handled in this commit. Further commits will remove the two variables hardcoded currently, and expose __main__ as a cmdline script on installation. Use imp for module finding and loading, use full path to script in audit and repair cfg files. Move audit and repair scripts to examples. Make some changes to hardcoded stuff to account for this Update gitignore Change-Id: I50831003c6f7272967dbeb5c558b76b0183c91be --- .gitignore | 9 +++--- entropy/__main__.py | 6 ++-- entropy/audit/audit.json | 13 --------- entropy/engine.py | 29 ++++++++++--------- entropy/{ => examples}/audit/audit.py | 14 +++------ entropy/{ => examples}/audit/vm_count.py | 9 +++--- entropy/{ => examples}/audit/vmbooter.py | 2 +- .../{ => examples}/audit/vmbooter.template | 0 entropy/{ => examples}/repair/react.py | 0 .../{ => examples}/repair/vm_count_react.py | 0 entropy/{ => examples}/repair/vmbooter.py | 0 .../{ => examples}/repair/vmbooter.template | 0 entropy/repair/react.json | 11 ------- entropy/repair/vm_count_react.json | 12 -------- entropy/utils.py | 4 +++ 15 files changed, 37 insertions(+), 72 deletions(-) delete mode 100644 entropy/audit/audit.json rename entropy/{ => examples}/audit/audit.py (79%) rename entropy/{ => examples}/audit/vm_count.py (90%) rename entropy/{ => examples}/audit/vmbooter.py (99%) rename entropy/{ => examples}/audit/vmbooter.template (100%) rename entropy/{ => examples}/repair/react.py (100%) rename entropy/{ => examples}/repair/vm_count_react.py (100%) rename entropy/{ => examples}/repair/vmbooter.py (100%) rename entropy/{ => examples}/repair/vmbooter.template (100%) delete mode 100644 entropy/repair/react.json delete mode 100644 entropy/repair/vm_count_react.json diff --git a/.gitignore b/.gitignore index 584e085..797452a 100644 --- a/.gitignore +++ b/.gitignore @@ -56,10 +56,11 @@ ChangeLog env # Project-related +# json files +*.json +*.log arch/ docs/_build entropy/test -entropy/logs -entropy/audit/*.json -entropy/repair/*.json -entropy/cfg/*.cfg +entropy/examples/logs +entropy/examples/cfg/ diff --git a/entropy/__main__.py b/entropy/__main__.py index 3b02d3b..f9ce6f0 100644 --- a/entropy/__main__.py +++ b/entropy/__main__.py @@ -31,8 +31,10 @@ from entropy import utils LOG = logging.getLogger(__name__) # TODO(praneshp): Only hardcoded stuff in the project. Find a way to move -engine_cfg = os.path.join(os.getcwd(), 'entropy', 'cfg', 'engines.cfg') -log_file = os.path.join(os.getcwd(), 'entropy', 'logs', 'entropy.log') +engine_cfg = os.path.join(os.getcwd(), 'entropy', 'examples', + 'cfg', 'engines.cfg') +log_file = os.path.join(os.getcwd(), 'entropy', 'examples', + 'logs', 'entropy.log') def get_cfg_file(engine, script_type): diff --git a/entropy/audit/audit.json b/entropy/audit/audit.json deleted file mode 100644 index ce6543d..0000000 --- a/entropy/audit/audit.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "name" : "audit", - "hostname" : "localhost", - "schedule" :"*/3 * * * *", - "username" : "praneshp", - "ssh-key" : "id_rsa", - "mq_host": "localhost", - "mq_port": "5672", - "mq_user": "guest", - "mq_password": "guest", - "script": "runthis.sh", - "module": "audit" -} diff --git a/entropy/engine.py b/entropy/engine.py index c4f0c8e..e81a890 100644 --- a/entropy/engine.py +++ b/entropy/engine.py @@ -25,7 +25,7 @@ from kombu import Exchange import pause from entropy import utils - +import imp LOG = logging.getLogger(__name__) @@ -98,7 +98,9 @@ class Engine(object): for script in scripts: if script['name'] not in running_scripts: - futures.append(setup_func(script)) + future = setup_func(script) + if future is not None: + futures.append(future) LOG.info('Running %s scripts %s', script_type, ', '.join(running_scripts)) return futures @@ -109,28 +111,25 @@ class Engine(object): # Pick out relevant info data = dict(utils.load_yaml(script['conf']).next()) react_script = data['script'] - - available_modules = utils.find_module(react_script, ['repair']) + search_path, reactor = utils.get_filename_and_path(react_script) + available_modules = imp.find_module(reactor, [search_path]) LOG.info('Found these modules: %s', available_modules) - if not available_modules: - LOG.error('No module to load') - else: - imported_module = utils.import_module(available_modules[0]) + try: + imported_module = imp.load_module(react_script, *available_modules) kwargs = data kwargs['conf'] = script['conf'] - # add this job to list of running audits self.running_repairs.append(script['name']) - future = self.executor.submit(imported_module.main, **kwargs) return future + except Exception: + LOG.exception("Could not setup %s", kwargs['name']) + return None def setup_audit(self, script): LOG.info('Setting up audit script %s', script['name']) - # add this job to list of running audits self.running_audits.append(script['name']) - # start a process for this audit script future = self.executor.submit(self.start_audit, script) return future @@ -163,9 +162,11 @@ class Engine(object): # Put a message on the mq #TODO(praneshp): this should be the path with register-audit try: - available_modules = utils.find_module(kwargs['module'], ['audit']) + audit_script = kwargs['module'] + search_path, auditor = utils.get_filename_and_path(audit_script) + available_modules = imp.find_module(auditor, [search_path]) LOG.info('Found these modules: %s', available_modules) - imported_module = utils.import_module(available_modules[0]) + imported_module = imp.load_module(audit_script, *available_modules) audit_obj = imported_module.Audit(**kwargs) audit_obj.send_message(**kwargs) except Exception: diff --git a/entropy/audit/audit.py b/entropy/examples/audit/audit.py similarity index 79% rename from entropy/audit/audit.py rename to entropy/examples/audit/audit.py index d713b35..87fdd98 100644 --- a/entropy/audit/audit.py +++ b/entropy/examples/audit/audit.py @@ -19,26 +19,20 @@ from kombu import BrokerConnection from kombu.common import maybe_declare from kombu.pools import producers -import base -from entropy.queues import entropy_exchange -from entropy.queues import PASS_KEY +from entropy.audit import base LOG = logging.getLogger(__name__) class Audit(base.AuditBase): - - def test(self): - LOG.info('hello world') - def send_message(self, **kwargs): connection = BrokerConnection('amqp://%(mq_user)s:%(mq_password)s@' '%(mq_host)s:%(mq_port)s//' % kwargs) message = {'From': __file__, 'Date': str(datetime.datetime.now().isoformat())} with producers[connection].acquire(block=True) as producer: - maybe_declare(entropy_exchange, producer.channel) + maybe_declare(kwargs['exchange'], producer.channel) producer.publish(message, - exchange=entropy_exchange, - routing_key=PASS_KEY, + exchange=kwargs['exchange'], + routing_key=kwargs['routing_key'], serializer='json') diff --git a/entropy/audit/vm_count.py b/entropy/examples/audit/vm_count.py similarity index 90% rename from entropy/audit/vm_count.py rename to entropy/examples/audit/vm_count.py index 8ce9cbc..91903d5 100644 --- a/entropy/audit/vm_count.py +++ b/entropy/examples/audit/vm_count.py @@ -20,8 +20,7 @@ from kombu.common import maybe_declare from kombu.pools import producers -import base -from entropy.queues import entropy_exchange +from entropy.audit import base import libvirt LOG = logging.getLogger(__name__) @@ -51,10 +50,10 @@ class Audit(base.AuditBase): message = {'From': __name__, 'Date': str(datetime.datetime.now().isoformat())} with producers[connection].acquire(block=True) as producer: - maybe_declare(entropy_exchange, producer.channel) + maybe_declare(kwargs['exchange'], producer.channel) msg_args = {'vm_count': self.get_vm_count(**kwargs)} message['payload'] = msg_args producer.publish(message, - exchange=entropy_exchange, - routing_key='vmcount', + exchange=kwargs['exchange'], + routing_key=kwargs['routing_key'], serializer='json') diff --git a/entropy/audit/vmbooter.py b/entropy/examples/audit/vmbooter.py similarity index 99% rename from entropy/audit/vmbooter.py rename to entropy/examples/audit/vmbooter.py index 4612a5d..a39190b 100644 --- a/entropy/audit/vmbooter.py +++ b/entropy/examples/audit/vmbooter.py @@ -23,7 +23,7 @@ from kombu.pools import producers from novaclient.client import Client import paramiko -import base +from entropy.audit import base LOG = logging.getLogger(__name__) diff --git a/entropy/audit/vmbooter.template b/entropy/examples/audit/vmbooter.template similarity index 100% rename from entropy/audit/vmbooter.template rename to entropy/examples/audit/vmbooter.template diff --git a/entropy/repair/react.py b/entropy/examples/repair/react.py similarity index 100% rename from entropy/repair/react.py rename to entropy/examples/repair/react.py diff --git a/entropy/repair/vm_count_react.py b/entropy/examples/repair/vm_count_react.py similarity index 100% rename from entropy/repair/vm_count_react.py rename to entropy/examples/repair/vm_count_react.py diff --git a/entropy/repair/vmbooter.py b/entropy/examples/repair/vmbooter.py similarity index 100% rename from entropy/repair/vmbooter.py rename to entropy/examples/repair/vmbooter.py diff --git a/entropy/repair/vmbooter.template b/entropy/examples/repair/vmbooter.template similarity index 100% rename from entropy/repair/vmbooter.template rename to entropy/examples/repair/vmbooter.template diff --git a/entropy/repair/react.json b/entropy/repair/react.json deleted file mode 100644 index ba6ed62..0000000 --- a/entropy/repair/react.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "name" : "react", - "script": "react", - "hostname" : "localhost", - "username" : "praneshp", - "ssh-key" : "id_rsa", - "mq_host": "localhost", - "mq_port": "5672", - "mq_user": "guest", - "mq_password": "guest" -} \ No newline at end of file diff --git a/entropy/repair/vm_count_react.json b/entropy/repair/vm_count_react.json deleted file mode 100644 index edfd515..0000000 --- a/entropy/repair/vm_count_react.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name" : "vmcount", - "script": "vm_count_react", - "hostname" : "localhost", - "username" : "praneshp", - "ssh-key" : "id_rsa", - "mq_host": "localhost", - "mq_port": "5672", - "mq_user": "guest", - "mq_password": "guest", - "limit": 0 -} \ No newline at end of file diff --git a/entropy/utils.py b/entropy/utils.py index 1cd01ac..0849737 100644 --- a/entropy/utils.py +++ b/entropy/utils.py @@ -25,6 +25,10 @@ import yaml LOG = logging.getLogger(__name__) +def get_filename_and_path(path): + return os.path.dirname(path), os.path.basename(path) + + def get_key_path(): home_dir = os.path.expanduser("~") ssh_dir = os.path.join(home_dir, ".ssh")