From b7ce04cdf5f2dc6573f912815d08cca2921dc514 Mon Sep 17 00:00:00 2001 From: Stan Lagun Date: Thu, 3 Oct 2013 19:59:12 +0400 Subject: [PATCH] Python Agent initial release Change-Id: Id0702fea250048427744b831f8a8bc2dff5ba98d --- python-agent/etc/agent.conf | 35 ++ python-agent/muranoagent/__init__.py | 0 python-agent/muranoagent/app.py | 186 ++++++ python-agent/muranoagent/cmd/__init__.py | 0 python-agent/muranoagent/cmd/run.py | 49 ++ python-agent/muranoagent/config.py | 115 ++++ python-agent/muranoagent/exceptions.py | 35 ++ .../muranoagent/execution_plan_queue.py | 86 +++ .../muranoagent/execution_plan_runner.py | 77 +++ python-agent/muranoagent/execution_result.py | 64 ++ .../muranoagent/executors/__init__.py | 38 ++ .../executors/application/__init__.py | 73 +++ python-agent/muranoagent/files_manager.py | 71 +++ .../muranoagent/openstack/__init__.py | 0 .../muranoagent/openstack/common/__init__.py | 0 .../openstack/common/config/__init__.py | 0 .../openstack/common/config/generator.py | 260 ++++++++ .../openstack/common/eventlet_backdoor.py | 146 +++++ .../openstack/common/gettextutils.py | 365 +++++++++++ .../openstack/common/importutils.py | 68 +++ .../muranoagent/openstack/common/jsonutils.py | 180 ++++++ .../muranoagent/openstack/common/local.py | 47 ++ .../muranoagent/openstack/common/log.py | 566 ++++++++++++++++++ .../openstack/common/loopingcall.py | 147 +++++ .../muranoagent/openstack/common/service.py | 459 ++++++++++++++ .../openstack/common/threadgroup.py | 121 ++++ .../muranoagent/openstack/common/timeutils.py | 197 ++++++ python-agent/muranoagent/script_runner.py | 59 ++ python-agent/muranoagent/win32.py | 27 + python-agent/openstack-common.conf | 14 + python-agent/requirements.txt | 5 + python-agent/setup.cfg | 44 ++ python-agent/setup.py | 22 + python-agent/tools/config/generate_sample.sh | 92 +++ python-agent/tools/install_venv_common.py | 213 +++++++ python-agent/tox.ini | 58 ++ .../ExecutionPlanGenerator}/App.config | 0 .../ExecutionPlanGenerator.csproj | 0 .../ExecutionPlanGenerator}/Program.cs | 0 .../Properties/AssemblyInfo.cs | 0 .../ExecutionPlanGenerator}/packages.config | 0 {Tools => windows-agent/Tools}/NuGet.exe | Bin .../WindowsAgent.sln | 0 .../WindowsAgent}/App.config | 0 .../WindowsAgent}/ExecutionPlan.cs | 0 .../WindowsAgent}/MqMessage.cs | 0 .../WindowsAgent}/PlanExecutor.cs | 0 .../WindowsAgent}/Program.cs | 0 .../WindowsAgent}/Properties/AssemblyInfo.cs | 0 .../WindowsAgent}/RabbitMqClient.cs | 0 .../WindowsAgent}/SampleExecutionPlan.json | 0 .../WindowsAgent}/ServiceManager.cs | 0 .../WindowsAgent}/WindowsAgent.csproj | 0 .../WindowsAgent}/WindowsService.cs | 0 .../WindowsAgent}/WindowsServiceInstaller.cs | 0 .../WindowsAgent}/packages.config | 0 .../packages}/repositories.config | 0 57 files changed, 3919 insertions(+) create mode 100644 python-agent/etc/agent.conf create mode 100644 python-agent/muranoagent/__init__.py create mode 100644 python-agent/muranoagent/app.py create mode 100644 python-agent/muranoagent/cmd/__init__.py create mode 100644 python-agent/muranoagent/cmd/run.py create mode 100644 python-agent/muranoagent/config.py create mode 100644 python-agent/muranoagent/exceptions.py create mode 100644 python-agent/muranoagent/execution_plan_queue.py create mode 100644 python-agent/muranoagent/execution_plan_runner.py create mode 100644 python-agent/muranoagent/execution_result.py create mode 100644 python-agent/muranoagent/executors/__init__.py create mode 100644 python-agent/muranoagent/executors/application/__init__.py create mode 100644 python-agent/muranoagent/files_manager.py create mode 100644 python-agent/muranoagent/openstack/__init__.py create mode 100644 python-agent/muranoagent/openstack/common/__init__.py create mode 100644 python-agent/muranoagent/openstack/common/config/__init__.py create mode 100644 python-agent/muranoagent/openstack/common/config/generator.py create mode 100644 python-agent/muranoagent/openstack/common/eventlet_backdoor.py create mode 100644 python-agent/muranoagent/openstack/common/gettextutils.py create mode 100644 python-agent/muranoagent/openstack/common/importutils.py create mode 100644 python-agent/muranoagent/openstack/common/jsonutils.py create mode 100644 python-agent/muranoagent/openstack/common/local.py create mode 100644 python-agent/muranoagent/openstack/common/log.py create mode 100644 python-agent/muranoagent/openstack/common/loopingcall.py create mode 100644 python-agent/muranoagent/openstack/common/service.py create mode 100644 python-agent/muranoagent/openstack/common/threadgroup.py create mode 100644 python-agent/muranoagent/openstack/common/timeutils.py create mode 100644 python-agent/muranoagent/script_runner.py create mode 100644 python-agent/muranoagent/win32.py create mode 100644 python-agent/openstack-common.conf create mode 100644 python-agent/requirements.txt create mode 100644 python-agent/setup.cfg create mode 100644 python-agent/setup.py create mode 100644 python-agent/tools/config/generate_sample.sh create mode 100644 python-agent/tools/install_venv_common.py create mode 100644 python-agent/tox.ini rename {ExecutionPlanGenerator => windows-agent/ExecutionPlanGenerator}/App.config (100%) rename {ExecutionPlanGenerator => windows-agent/ExecutionPlanGenerator}/ExecutionPlanGenerator.csproj (100%) rename {ExecutionPlanGenerator => windows-agent/ExecutionPlanGenerator}/Program.cs (100%) rename {ExecutionPlanGenerator => windows-agent/ExecutionPlanGenerator}/Properties/AssemblyInfo.cs (100%) rename {ExecutionPlanGenerator => windows-agent/ExecutionPlanGenerator}/packages.config (100%) rename {Tools => windows-agent/Tools}/NuGet.exe (100%) rename WindowsAgent.sln => windows-agent/WindowsAgent.sln (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/App.config (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/ExecutionPlan.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/MqMessage.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/PlanExecutor.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/Program.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/Properties/AssemblyInfo.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/RabbitMqClient.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/SampleExecutionPlan.json (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/ServiceManager.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/WindowsAgent.csproj (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/WindowsService.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/WindowsServiceInstaller.cs (100%) rename {WindowsAgent => windows-agent/WindowsAgent}/packages.config (100%) rename {packages => windows-agent/packages}/repositories.config (100%) diff --git a/python-agent/etc/agent.conf b/python-agent/etc/agent.conf new file mode 100644 index 00000000..a47832c6 --- /dev/null +++ b/python-agent/etc/agent.conf @@ -0,0 +1,35 @@ +[DEFAULT] +debug=True +verbose=True + +storage=. + +[rabbitmq] + +# Input queue name +input_queue = agent-tasks + +# Output routing key (usually queue name) +result_routing_key = agent-results + +# Connection parameters to RabbitMQ service + +# Hostname or IP address where RabbitMQ is located. +host = localhost + +# RabbitMQ port (5672 is a default) +port = 5672 + +# Use SSL for RabbitMQ connections (True or False) +ssl = False + +# Path to SSL CA certificate or empty to allow self signed server certificate +ca_certs = + +# RabbitMQ credentials. Fresh RabbitMQ installation has "guest" account with "guest" password. +login = guest +password = guest + +# RabbitMQ virtual host (vhost). Fresh RabbitMQ installation has "/" vhost preconfigured. +virtual_host = / + diff --git a/python-agent/muranoagent/__init__.py b/python-agent/muranoagent/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python-agent/muranoagent/app.py b/python-agent/muranoagent/app.py new file mode 100644 index 00000000..5df56357 --- /dev/null +++ b/python-agent/muranoagent/app.py @@ -0,0 +1,186 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import win32 +import sys +import os +from execution_plan_runner import ExecutionPlanRunner +from execution_plan_queue import ExecutionPlanQueue +from execution_result import ExecutionResult +from openstack.common import log as logging +from openstack.common import service +from config import CONF +from muranocommon.messaging import MqClient, Message +from exceptions import AgentException +from time import sleep +from bunch import Bunch +import semver +import types + +log = logging.getLogger(__name__) +format_version = '2.0.0' + +class MuranoAgent(service.Service): + def __init__(self): + self._queue = ExecutionPlanQueue() + super(MuranoAgent, self).__init__() + + @staticmethod + def _load_package(name): + try: + log.debug('Loading plugin %s', name) + __import__(name) + except Exception, ex: + log.warn('Cannot load package %s', name, exc_info=1) + pass + + def _load(self): + path = os.path.join(os.path.dirname(__file__), 'executors') + sys.path.insert(1, path) + for entry in os.listdir(path): + package_path = os.path.join(path, entry) + if os.path.isdir(package_path): + MuranoAgent._load_package(entry) + + def start(self): + self._load() + while True: + try: + self._loop_func() + except Exception as ex: + log.exception(ex) + sleep(5) + + def _loop_func(self): + result, timestamp = self._queue.get_execution_plan_result() + if result is not None: + if self._send_result(result): + self._queue.remove(timestamp) + return + + plan = self._queue.get_execution_plan() + if plan is not None: + self._run(plan) + return + + self._wait_plan() + + def _run(self, plan): + with ExecutionPlanRunner(plan) as runner: + try: + result = runner.run() + execution_result = ExecutionResult.from_result(result, plan) + self._queue.put_execution_result(execution_result, plan) + except Exception, ex: + execution_result = ExecutionResult.from_error(ex, plan) + self._queue.put_execution_result(execution_result, plan) + + def _send_result(self, result): + with self._create_rmq_client() as mq: + msg = Message() + msg.body = result + msg.id = result.get('SourceID') + mq.send(message=msg, + key=CONF.rabbitmq.result_routing_key, + exchange=CONF.rabbitmq.result_exchange) + return True + + def _create_rmq_client(self): + rabbitmq = CONF.rabbitmq + connection_params = { + 'login': rabbitmq.login, + 'password': rabbitmq.password, + 'host': rabbitmq.host, + 'port': rabbitmq.port, + 'virtual_host': rabbitmq.virtual_host, + 'ssl': rabbitmq.ssl, + 'ca_certs': rabbitmq.ca_certs.strip() or None + } + return MqClient(**connection_params) + + def _wait_plan(self): + with self._create_rmq_client() as mq: + with mq.open(CONF.rabbitmq.input_queue, + prefetch_count=1) as subscription: + msg = subscription.get_message(timeout=5) + if msg is not None and isinstance(msg.body, dict): + if 'ID' not in msg.body and msg.id: + msg.body['ID'] = msg.id + err = self._verify_plan(msg.body) + if err is None: + self._queue.put_execution_plan(msg.body) + else: + try: + execution_result = ExecutionResult.from_error( + err, Bunch(msg.body)) + self._send_result(execution_result) + except ValueError: + log.warn('Execution result is not produced') + + if msg is not None: + msg.ack() + + def _verify_plan(self, plan): + plan_format_version = plan.get('FormatVersion', '1.0.0') + if semver.compare(plan_format_version, '2.0.0') > 0 or \ + semver.compare(plan_format_version, format_version) < 0: + range_str = 'in range 2.0.0-{0}'.format(plan_format_version) \ + if format_version != '2.0.0' \ + else 'equal to {0}'.format(format_version) + return AgentException( + 3, + 'Unsupported format version {0} (must be {1})'.format( + plan_format_version, range_str)) + + for attr in ('Scripts', 'Files', 'Options'): + if attr is plan and not isinstance( + plan[attr], types.DictionaryType): + return AgentException( + 2, '{0} is not a dictionary'.format(attr)) + + for name, script in plan.get('Scripts', {}).items(): + for attr in ('Type', 'EntryPoint'): + if attr not in script or not isinstance( + script[attr], types.StringTypes): + return AgentException( + 2, 'Incorrect {0} entry in script {1}'.format( + attr, name)) + if not isinstance(script.get('Options', {}), types.DictionaryType): + return AgentException( + 2, 'Incorrect Options entry in script {0}'.format(name)) + + if script['EntryPoint'] not in plan.get('Files', {}): + return AgentException( + 2, 'Script {0} misses entry point {1}'.format( + name, script['EntryPoint'])) + + for additional_file in script.get('Files', []): + if additional_file not in plan.get('Files', {}): + return AgentException( + 2, 'Script {0} misses file {1}'.format( + name, additional_file)) + + for key, plan_file in plan.get('Files', {}).items(): + for attr in ('BodyType', 'Body', 'Name'): + if attr not in plan_file: + return AgentException( + 2, 'Incorrect {0} entry in file {1}'.format( + attr, key)) + + if plan_file['BodyType'] not in ('Text', 'Base64'): + return AgentException( + 2, 'Incorrect BodyType in file {1}'.format(key)) + + return None diff --git a/python-agent/muranoagent/cmd/__init__.py b/python-agent/muranoagent/cmd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python-agent/muranoagent/cmd/run.py b/python-agent/muranoagent/cmd/run.py new file mode 100644 index 00000000..65a8820b --- /dev/null +++ b/python-agent/muranoagent/cmd/run.py @@ -0,0 +1,49 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +import os + +# If ../muranoagent/__init__.py exists, add ../ to Python search path, so +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(__file__), + os.pardir, + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, + 'muranoagent', + '__init__.py')): + sys.path.insert(0, possible_topdir) + +from muranoagent import config +from muranoagent.openstack.common import log +from muranoagent.openstack.common import service +from muranoagent.app import MuranoAgent + + +def main(): + try: + config.parse_args() + log.setup('muranoagent') + launcher = service.ServiceLauncher() + launcher.launch_service(MuranoAgent()) + launcher.wait() + except RuntimeError, e: + sys.stderr.write("ERROR: %s\n" % e) + sys.exit(1) + + +if __name__ == '__main__': + main() diff --git a/python-agent/muranoagent/config.py b/python-agent/muranoagent/config.py new file mode 100644 index 00000000..989efd25 --- /dev/null +++ b/python-agent/muranoagent/config.py @@ -0,0 +1,115 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Routines for configuring Glance +""" + +import logging +import logging.config +import logging.handlers +import os +import sys + +from oslo.config import cfg + +#from muranoagent import __version__ as version + + +CONF = cfg.CONF +CONF.register_cli_opt(cfg.StrOpt('storage')) + +rabbit_opts = [ + cfg.StrOpt('host', default='localhost'), + cfg.IntOpt('port', default=5672), + cfg.StrOpt('login', default='guest'), + cfg.StrOpt('password', default='guest'), + cfg.StrOpt('virtual_host', default='/'), + cfg.BoolOpt('ssl', default=False), + cfg.StrOpt('ca_certs', default=''), + cfg.StrOpt('result_routing_key'), + cfg.StrOpt('result_exchange', default=''), + cfg.StrOpt('input_queue', default='') + +] + +CONF.register_opts(rabbit_opts, group='rabbitmq') + + +CONF.import_opt('verbose', 'muranoagent.openstack.common.log') +CONF.import_opt('debug', 'muranoagent.openstack.common.log') +CONF.import_opt('log_dir', 'muranoagent.openstack.common.log') +CONF.import_opt('log_file', 'muranoagent.openstack.common.log') +CONF.import_opt('log_config', 'muranoagent.openstack.common.log') +CONF.import_opt('log_format', 'muranoagent.openstack.common.log') +CONF.import_opt('log_date_format', 'muranoagent.openstack.common.log') +CONF.import_opt('use_syslog', 'muranoagent.openstack.common.log') +CONF.import_opt('syslog_log_facility', 'muranoagent.openstack.common.log') + + +def parse_args(args=None, usage=None, default_config_files=None): + CONF(args=args, + project='muranoagent', + #version=version, + usage=usage, + default_config_files=default_config_files) + + +def setup_logging(): + """ + Sets up the logging options for a log with supplied name + """ + + if CONF.log_config: + # Use a logging configuration file for all settings... + if os.path.exists(CONF.log_config): + logging.config.fileConfig(CONF.log_config) + return + else: + raise RuntimeError("Unable to locate specified logging " + "config file: %s" % CONF.log_config) + + root_logger = logging.root + if CONF.debug: + root_logger.setLevel(logging.DEBUG) + elif CONF.verbose: + root_logger.setLevel(logging.INFO) + else: + root_logger.setLevel(logging.WARNING) + + formatter = logging.Formatter(CONF.log_format, CONF.log_date_format) + + if CONF.use_syslog: + try: + facility = getattr(logging.handlers.SysLogHandler, + CONF.syslog_log_facility) + except AttributeError: + raise ValueError(_("Invalid syslog facility")) + + handler = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) + elif CONF.log_file: + logfile = CONF.log_file + if CONF.log_dir: + logfile = os.path.join(CONF.log_dir, logfile) + handler = logging.handlers.WatchedFileHandler(logfile) + else: + handler = logging.StreamHandler(sys.stdout) + + handler.setFormatter(formatter) + root_logger.addHandler(handler) + + + diff --git a/python-agent/muranoagent/exceptions.py b/python-agent/muranoagent/exceptions.py new file mode 100644 index 00000000..5359ebcc --- /dev/null +++ b/python-agent/muranoagent/exceptions.py @@ -0,0 +1,35 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class AgentException(Exception): + def __init__(self, code, message=None, additional_data=None): + self._error_code = code + self._additional_data = additional_data + super(AgentException, self).__init__(message) + + @property + def error_code(self): + return self._error_code + + @property + def additional_data(self): + return self._additional_data + + +class CustomException(AgentException): + def __init__(self, code, message=None, additional_data=None): + super(CustomException, self).__init__( + code + 100, message, additional_data) + diff --git a/python-agent/muranoagent/execution_plan_queue.py b/python-agent/muranoagent/execution_plan_queue.py new file mode 100644 index 00000000..9e226a1e --- /dev/null +++ b/python-agent/muranoagent/execution_plan_queue.py @@ -0,0 +1,86 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import shutil +import time +from bunch import Bunch +from config import CONF + + +class ExecutionPlanQueue(object): + plan_filename = 'plan.json' + result_filename = 'result.json' + + def __init__(self): + self._plans_folder = os.path.join(CONF.storage, 'plans') + if not os.path.exists(self._plans_folder): + os.makedirs(self._plans_folder) + + def put_execution_plan(self, execution_plan): + timestamp = str(int(time.time() * 10000)) + #execution_plan['_timestamp'] = timestamp + folder_path = os.path.join(self._plans_folder, timestamp) + os.mkdir(folder_path) + file_path = os.path.join( + folder_path, ExecutionPlanQueue.plan_filename) + with open(file_path, 'w') as out_file: + out_file.write(json.dumps(execution_plan)) + + def _get_first_timestamp(self, filename): + def predicate(folder): + path = os.path.join(self._plans_folder, folder, filename) + return os.path.exists(path) + + timestamps = [ + name for name in os.listdir(self._plans_folder) + if predicate(name) + ] + timestamps.sort() + return None if len(timestamps) == 0 else timestamps[0] + + def _get_first_file(self, filename): + timestamp = self._get_first_timestamp(filename) + if not timestamp: + return None, None + path = os.path.join(self._plans_folder, timestamp, filename) + with open(path) as json_file: + return json.loads(json_file.read()), timestamp + + def get_execution_plan(self): + ep, timestamp = self._get_first_file(ExecutionPlanQueue.plan_filename) + if ep is None: + return None + ep['_timestamp'] = timestamp + return Bunch(ep) + + def put_execution_result(self, result, execution_plan): + timestamp = execution_plan['_timestamp'] + path = os.path.join( + self._plans_folder, timestamp, + ExecutionPlanQueue.result_filename) + with open(path, 'w') as out_file: + out_file.write(json.dumps(result)) + + def remove(self, timestamp): + path = os.path.join(self._plans_folder, timestamp) + shutil.rmtree(path) + + def get_execution_plan_result(self): + return self._get_first_file( + ExecutionPlanQueue.result_filename) + + diff --git a/python-agent/muranoagent/execution_plan_runner.py b/python-agent/muranoagent/execution_plan_runner.py new file mode 100644 index 00000000..a2a56167 --- /dev/null +++ b/python-agent/muranoagent/execution_plan_runner.py @@ -0,0 +1,77 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from bunch import Bunch +from files_manager import FilesManager +from script_runner import ScriptRunner + + +class ExecutionPlanRunner(object): + def __init__(self, execution_plan): + self._execution_plan = execution_plan + self._main_script = self._prepare_script(execution_plan.Body) + self._script_funcs = {} + self._files_manager = FilesManager(execution_plan) + self._prepare_executors(execution_plan) + + def run(self): + script_globals = { + "args": Bunch(self._execution_plan.get('Parameters') or {}) + } + script_globals.update(self._script_funcs) + exec self._main_script in script_globals + if '__execution_plan_exception' in script_globals: + raise script_globals['__execution_plan_exception'] + return script_globals['__execution_plan_result'] + + @staticmethod + def _unindent(script, initial_indent): + lines = script.expandtabs(4).split('\n') + min_indent = sys.maxint + for line in lines: + indent = -1 + for i, c in enumerate(line): + if c != ' ': + indent = i + break + if 0 <= indent < min_indent: + min_indent = indent + return '\n'.join([' ' * initial_indent + line[min_indent:] + for line in lines]) + + def _prepare_executors(self, execution_plan): + for key, value in execution_plan.Scripts.items(): + self._script_funcs[key] = ScriptRunner( + key, Bunch(value), self._files_manager) + + @staticmethod + def _prepare_script(body): + script = 'def __execution_plan_main():\n' + script += ExecutionPlanRunner._unindent(body, 4) + script += """ +try: + __execution_plan_result = __execution_plan_main() +except Exception as e: + __execution_plan_exception = e +""" + return script + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self._files_manager.clear() + return False diff --git a/python-agent/muranoagent/execution_result.py b/python-agent/muranoagent/execution_result.py new file mode 100644 index 00000000..fe7454c8 --- /dev/null +++ b/python-agent/muranoagent/execution_result.py @@ -0,0 +1,64 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from exceptions import AgentException +import uuid +from openstack.common import timeutils + + +class ExecutionResult(object): + @staticmethod + def from_result(result, execution_plan): + if 'ID' not in execution_plan: + raise ValueError('ID attribute is missing from execution plan') + + return { + 'FormatVersion': '2.0.0', + 'ID': uuid.uuid4().hex, + 'SourceID': execution_plan.ID, + 'Action': 'Execution:Result', + 'ErrorCode': 0, + 'Body': result, + 'Time': str(timeutils.utcnow()) + } + + @staticmethod + def from_error(error, execution_plan): + if 'ID' not in execution_plan: + raise ValueError('ID attribute is missing from execution plan') + + error_code = 1 + additional_info = None + message = None + if isinstance(error, int): + error_code = error + elif isinstance(error, Exception): + message = error.message + if isinstance(error, AgentException): + error_code = error.error_code + additional_info = error.additional_data + + return { + 'FormatVersion': '2.0.0', + 'ID': uuid.uuid4().hex, + 'SourceID': execution_plan.ID, + 'Action': 'Execution:Result', + 'ErrorCode': error_code, + 'Body': { + 'Message': message, + 'AdditionalInfo': additional_info + }, + 'Time': str(timeutils.utcnow()) + } diff --git a/python-agent/muranoagent/executors/__init__.py b/python-agent/muranoagent/executors/__init__.py new file mode 100644 index 00000000..662400ec --- /dev/null +++ b/python-agent/muranoagent/executors/__init__.py @@ -0,0 +1,38 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import wraps + + +class ExecutorsRepo(object): + def __init__(self): + self._executors = {} + + def register_executor(self, name, cls): + self._executors[name] = cls + + def create_executor(self, type, name): + if type not in self._executors: + return None + return self._executors[type](name) + +Executors = ExecutorsRepo() + + +def executor(name): + def wrapper(cls): + Executors.register_executor(name, cls) + return cls + return wrapper diff --git a/python-agent/muranoagent/executors/application/__init__.py b/python-agent/muranoagent/executors/application/__init__.py new file mode 100644 index 00000000..9a583b66 --- /dev/null +++ b/python-agent/muranoagent/executors/application/__init__.py @@ -0,0 +1,73 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import stat +import subprocess +import sys +from muranoagent.executors import executor +import muranoagent.exceptions +from bunch import Bunch + + +@executor('Application') +class ApplicationExecutor(object): + def __init__(self, name): + self._capture_stdout = False + self._capture_stderr = False + self._verify_exitcode = True + self._name = name + + def load(self, path, options): + self._path = path + self._capture_stdout = options.get('captureStdout', False) + self._capture_stderr = options.get('captureStderr', False) + self._verify_exitcode = options.get('verifyExitcode', True) + + def run(self, function, commandline, input=None): + dir_name = os.path.dirname(self._path) + os.chdir(dir_name) + app = '"{0}" {1}'.format(os.path.basename(self._path), commandline) + + if not sys.platform == 'win32': + os.chmod(self._path, stat.S_IEXEC | stat.S_IREAD) + app = './' + app + + stdout = subprocess.PIPE if self._capture_stdout else None + stderr = subprocess.PIPE if self._capture_stderr else None + + process = subprocess.Popen( + app, + stdout=stdout, + stderr=stderr, + universal_newlines=True, + cwd=dir_name, + shell=True) + stdout, stderr = process.communicate(input) + retcode = process.poll() + + result = { + 'exitCode': retcode, + 'stdout': stdout.strip() if stdout else None, + 'stderr': stderr.strip() if stderr else None + } + if self._verify_exitcode and retcode != 0: + raise muranoagent.exceptions.CustomException( + 0, + message='Script {0} returned error code'.format(self._name), + additional_data= result) + + return Bunch(result) + diff --git a/python-agent/muranoagent/files_manager.py b/python-agent/muranoagent/files_manager.py new file mode 100644 index 00000000..aa4c1b5d --- /dev/null +++ b/python-agent/muranoagent/files_manager.py @@ -0,0 +1,71 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import base64 +import shutil +from config import CONF + + +class FilesManager(object): + def __init__(self, execution_pan): + self._fetched_files = {} + self._files = execution_pan.get('Files') or {} + + self._cache_folder = os.path.join( + CONF.storage, 'files', execution_pan.ID) + if os.path.exists(self._cache_folder): + self.clear() + os.makedirs(self._cache_folder) + + def put_file(self, file_id, script): + cache_path = self._fetch_file(file_id) + + script_folder = os.path.join(self._cache_folder, script) + if not os.path.exists(script_folder): + os.mkdir(script_folder) + + filedef = self._files[file_id] + filename = filedef['Name'] + + file_folder = os.path.join(script_folder, os.path.dirname(filename)) + if not os.path.exists(file_folder): + os.makedirs(file_folder) + + script_path = os.path.join(script_folder, filename) + + os.symlink(cache_path, script_path) + return script_path + + def _fetch_file(self, file_id): + if file_id in self._fetched_files: + return self._fetched_files[file_id] + + filedef = self._files[file_id] + out_path = os.path.join(self._cache_folder, file_id) + body_type = filedef.get('BodyType', 'Text') + with open(out_path, 'w') as out_file: + if body_type == 'Text': + out_file.write(filedef['Body']) + elif body_type == 'Base64': + out_file.write(base64.b64decode(filedef['Body'])) + + self._fetched_files[file_id] = out_path + return out_path + + def clear(self): + os.chdir(os.path.dirname(self._cache_folder)) + shutil.rmtree(self._cache_folder, ignore_errors=True) + shutil.rmtree(self._cache_folder, ignore_errors=True) diff --git a/python-agent/muranoagent/openstack/__init__.py b/python-agent/muranoagent/openstack/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python-agent/muranoagent/openstack/common/__init__.py b/python-agent/muranoagent/openstack/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python-agent/muranoagent/openstack/common/config/__init__.py b/python-agent/muranoagent/openstack/common/config/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python-agent/muranoagent/openstack/common/config/generator.py b/python-agent/muranoagent/openstack/common/config/generator.py new file mode 100644 index 00000000..44358f88 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/config/generator.py @@ -0,0 +1,260 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 SINA Corporation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +"""Extracts OpenStack config option info from module(s).""" + +from __future__ import print_function + +import imp +import os +import re +import socket +import sys +import textwrap + +from oslo.config import cfg + +from muranoagent.openstack.common import gettextutils +from muranoagent.openstack.common import importutils + +gettextutils.install('muranoagent') + +STROPT = "StrOpt" +BOOLOPT = "BoolOpt" +INTOPT = "IntOpt" +FLOATOPT = "FloatOpt" +LISTOPT = "ListOpt" +MULTISTROPT = "MultiStrOpt" + +OPT_TYPES = { + STROPT: 'string value', + BOOLOPT: 'boolean value', + INTOPT: 'integer value', + FLOATOPT: 'floating point value', + LISTOPT: 'list value', + MULTISTROPT: 'multi valued', +} + +OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT, + FLOATOPT, LISTOPT, + MULTISTROPT])) + +PY_EXT = ".py" +BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__), + "../../../../")) +WORDWRAP_WIDTH = 60 + + +def generate(srcfiles): + mods_by_pkg = dict() + for filepath in srcfiles: + pkg_name = filepath.split(os.sep)[1] + mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]), + os.path.basename(filepath).split('.')[0]]) + mods_by_pkg.setdefault(pkg_name, list()).append(mod_str) + # NOTE(lzyeval): place top level modules before packages + pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys()) + pkg_names.sort() + ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys()) + ext_names.sort() + pkg_names.extend(ext_names) + + # opts_by_group is a mapping of group name to an options list + # The options list is a list of (module, options) tuples + opts_by_group = {'DEFAULT': []} + + for module_name in os.getenv( + "OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','): + module = _import_module(module_name) + if module: + for group, opts in _list_opts(module): + opts_by_group.setdefault(group, []).append((module_name, opts)) + + for pkg_name in pkg_names: + mods = mods_by_pkg.get(pkg_name) + mods.sort() + for mod_str in mods: + if mod_str.endswith('.__init__'): + mod_str = mod_str[:mod_str.rfind(".")] + + mod_obj = _import_module(mod_str) + if not mod_obj: + continue + + for group, opts in _list_opts(mod_obj): + opts_by_group.setdefault(group, []).append((mod_str, opts)) + + print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', [])) + for group, opts in opts_by_group.items(): + print_group_opts(group, opts) + + +def _import_module(mod_str): + try: + if mod_str.startswith('bin.'): + imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:])) + return sys.modules[mod_str[4:]] + else: + return importutils.import_module(mod_str) + except ImportError as ie: + sys.stderr.write("%s\n" % str(ie)) + return None + except Exception: + return None + + +def _is_in_group(opt, group): + "Check if opt is in group." + for key, value in group._opts.items(): + if value['opt'] == opt: + return True + return False + + +def _guess_groups(opt, mod_obj): + # is it in the DEFAULT group? + if _is_in_group(opt, cfg.CONF): + return 'DEFAULT' + + # what other groups is it in? + for key, value in cfg.CONF.items(): + if isinstance(value, cfg.CONF.GroupAttr): + if _is_in_group(opt, value._group): + return value._group.name + + raise RuntimeError( + "Unable to find group for option %s, " + "maybe it's defined twice in the same group?" + % opt.name + ) + + +def _list_opts(obj): + def is_opt(o): + return (isinstance(o, cfg.Opt) and + not isinstance(o, cfg.SubCommandOpt)) + + opts = list() + for attr_str in dir(obj): + attr_obj = getattr(obj, attr_str) + if is_opt(attr_obj): + opts.append(attr_obj) + elif (isinstance(attr_obj, list) and + all(map(lambda x: is_opt(x), attr_obj))): + opts.extend(attr_obj) + + ret = {} + for opt in opts: + ret.setdefault(_guess_groups(opt, obj), []).append(opt) + return ret.items() + + +def print_group_opts(group, opts_by_module): + print("[%s]" % group) + print('') + for mod, opts in opts_by_module: + print('#') + print('# Options defined in %s' % mod) + print('#') + print('') + for opt in opts: + _print_opt(opt) + print('') + + +def _get_my_ip(): + try: + csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + csock.connect(('8.8.8.8', 80)) + (addr, port) = csock.getsockname() + csock.close() + return addr + except socket.error: + return None + + +def _sanitize_default(name, value): + """Set up a reasonably sensible default for pybasedir, my_ip and host.""" + if value.startswith(sys.prefix): + # NOTE(jd) Don't use os.path.join, because it is likely to think the + # second part is an absolute pathname and therefore drop the first + # part. + value = os.path.normpath("/usr/" + value[len(sys.prefix):]) + elif value.startswith(BASEDIR): + return value.replace(BASEDIR, '/usr/lib/python/site-packages') + elif BASEDIR in value: + return value.replace(BASEDIR, '') + elif value == _get_my_ip(): + return '10.0.0.1' + elif value == socket.gethostname() and 'host' in name: + return 'muranoagent' + elif value.strip() != value: + return '"%s"' % value + return value + + +def _print_opt(opt): + opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help + if not opt_help: + sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name) + opt_help = "" + opt_type = None + try: + opt_type = OPTION_REGEX.search(str(type(opt))).group(0) + except (ValueError, AttributeError) as err: + sys.stderr.write("%s\n" % str(err)) + sys.exit(1) + opt_help += ' (' + OPT_TYPES[opt_type] + ')' + print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH))) + try: + if opt_default is None: + print('#%s=' % opt_name) + elif opt_type == STROPT: + assert(isinstance(opt_default, basestring)) + print('#%s=%s' % (opt_name, _sanitize_default(opt_name, + opt_default))) + elif opt_type == BOOLOPT: + assert(isinstance(opt_default, bool)) + print('#%s=%s' % (opt_name, str(opt_default).lower())) + elif opt_type == INTOPT: + assert(isinstance(opt_default, int) and + not isinstance(opt_default, bool)) + print('#%s=%s' % (opt_name, opt_default)) + elif opt_type == FLOATOPT: + assert(isinstance(opt_default, float)) + print('#%s=%s' % (opt_name, opt_default)) + elif opt_type == LISTOPT: + assert(isinstance(opt_default, list)) + print('#%s=%s' % (opt_name, ','.join(opt_default))) + elif opt_type == MULTISTROPT: + assert(isinstance(opt_default, list)) + if not opt_default: + opt_default = [''] + for default in opt_default: + print('#%s=%s' % (opt_name, default)) + print('') + except Exception: + sys.stderr.write('Error in option "%s"\n' % opt_name) + sys.exit(1) + + +def main(): + generate(sys.argv[1:]) + +if __name__ == '__main__': + main() diff --git a/python-agent/muranoagent/openstack/common/eventlet_backdoor.py b/python-agent/muranoagent/openstack/common/eventlet_backdoor.py new file mode 100644 index 00000000..3c5aef37 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/eventlet_backdoor.py @@ -0,0 +1,146 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2012 OpenStack Foundation. +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function + +import errno +import gc +import os +import pprint +import socket +import sys +import traceback + +import eventlet +import eventlet.backdoor +import greenlet +from oslo.config import cfg + +from muranoagent.openstack.common.gettextutils import _ # noqa +from muranoagent.openstack.common import log as logging + +help_for_backdoor_port = ( + "Acceptable values are 0, , and :, where 0 results " + "in listening on a random tcp port number; results in listening " + "on the specified port number (and not enabling backdoor if that port " + "is in use); and : results in listening on the smallest " + "unused port number within the specified range of port numbers. The " + "chosen port is displayed in the service's log file.") +eventlet_backdoor_opts = [ + cfg.StrOpt('backdoor_port', + default=None, + help="Enable eventlet backdoor. %s" % help_for_backdoor_port) +] + +CONF = cfg.CONF +CONF.register_opts(eventlet_backdoor_opts) +LOG = logging.getLogger(__name__) + + +class EventletBackdoorConfigValueError(Exception): + def __init__(self, port_range, help_msg, ex): + msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. ' + '%(help)s' % + {'range': port_range, 'ex': ex, 'help': help_msg}) + super(EventletBackdoorConfigValueError, self).__init__(msg) + self.port_range = port_range + + +def _dont_use_this(): + print("Don't use this, just disconnect instead") + + +def _find_objects(t): + return filter(lambda o: isinstance(o, t), gc.get_objects()) + + +def _print_greenthreads(): + for i, gt in enumerate(_find_objects(greenlet.greenlet)): + print(i, gt) + traceback.print_stack(gt.gr_frame) + print() + + +def _print_nativethreads(): + for threadId, stack in sys._current_frames().items(): + print(threadId) + traceback.print_stack(stack) + print() + + +def _parse_port_range(port_range): + if ':' not in port_range: + start, end = port_range, port_range + else: + start, end = port_range.split(':', 1) + try: + start, end = int(start), int(end) + if end < start: + raise ValueError + return start, end + except ValueError as ex: + raise EventletBackdoorConfigValueError(port_range, ex, + help_for_backdoor_port) + + +def _listen(host, start_port, end_port, listen_func): + try_port = start_port + while True: + try: + return listen_func((host, try_port)) + except socket.error as exc: + if (exc.errno != errno.EADDRINUSE or + try_port >= end_port): + raise + try_port += 1 + + +def initialize_if_enabled(): + backdoor_locals = { + 'exit': _dont_use_this, # So we don't exit the entire process + 'quit': _dont_use_this, # So we don't exit the entire process + 'fo': _find_objects, + 'pgt': _print_greenthreads, + 'pnt': _print_nativethreads, + } + + if CONF.backdoor_port is None: + return None + + start_port, end_port = _parse_port_range(str(CONF.backdoor_port)) + + # NOTE(johannes): The standard sys.displayhook will print the value of + # the last expression and set it to __builtin__._, which overwrites + # the __builtin__._ that gettext sets. Let's switch to using pprint + # since it won't interact poorly with gettext, and it's easier to + # read the output too. + def displayhook(val): + if val is not None: + pprint.pprint(val) + sys.displayhook = displayhook + + sock = _listen('localhost', start_port, end_port, eventlet.listen) + + # In the case of backdoor port being zero, a port number is assigned by + # listen(). In any case, pull the port number out here. + port = sock.getsockname()[1] + LOG.info(_('Eventlet backdoor listening on %(port)s for process %(pid)d') % + {'port': port, 'pid': os.getpid()}) + eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock, + locals=backdoor_locals) + return port diff --git a/python-agent/muranoagent/openstack/common/gettextutils.py b/python-agent/muranoagent/openstack/common/gettextutils.py new file mode 100644 index 00000000..aac8b8b1 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/gettextutils.py @@ -0,0 +1,365 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Red Hat, Inc. +# Copyright 2013 IBM Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +gettext for openstack-common modules. + +Usual usage in an openstack.common module: + + from muranoagent.openstack.common.gettextutils import _ +""" + +import copy +import gettext +import logging +import os +import re +try: + import UserString as _userString +except ImportError: + import collections as _userString + +from babel import localedata +import six + +_localedir = os.environ.get('muranoagent'.upper() + '_LOCALEDIR') +_t = gettext.translation('muranoagent', localedir=_localedir, fallback=True) + +_AVAILABLE_LANGUAGES = {} +USE_LAZY = False + + +def enable_lazy(): + """Convenience function for configuring _() to use lazy gettext + + Call this at the start of execution to enable the gettextutils._ + function to use lazy gettext functionality. This is useful if + your project is importing _ directly instead of using the + gettextutils.install() way of importing the _ function. + """ + global USE_LAZY + USE_LAZY = True + + +def _(msg): + if USE_LAZY: + return Message(msg, 'muranoagent') + else: + if six.PY3: + return _t.gettext(msg) + return _t.ugettext(msg) + + +def install(domain, lazy=False): + """Install a _() function using the given translation domain. + + Given a translation domain, install a _() function using gettext's + install() function. + + The main difference from gettext.install() is that we allow + overriding the default localedir (e.g. /usr/share/locale) using + a translation-domain-specific environment variable (e.g. + NOVA_LOCALEDIR). + + :param domain: the translation domain + :param lazy: indicates whether or not to install the lazy _() function. + The lazy _() introduces a way to do deferred translation + of messages by installing a _ that builds Message objects, + instead of strings, which can then be lazily translated into + any available locale. + """ + if lazy: + # NOTE(mrodden): Lazy gettext functionality. + # + # The following introduces a deferred way to do translations on + # messages in OpenStack. We override the standard _() function + # and % (format string) operation to build Message objects that can + # later be translated when we have more information. + # + # Also included below is an example LocaleHandler that translates + # Messages to an associated locale, effectively allowing many logs, + # each with their own locale. + + def _lazy_gettext(msg): + """Create and return a Message object. + + Lazy gettext function for a given domain, it is a factory method + for a project/module to get a lazy gettext function for its own + translation domain (i.e. nova, glance, cinder, etc.) + + Message encapsulates a string so that we can translate + it later when needed. + """ + return Message(msg, domain) + + from six import moves + moves.builtins.__dict__['_'] = _lazy_gettext + else: + localedir = '%s_LOCALEDIR' % domain.upper() + if six.PY3: + gettext.install(domain, + localedir=os.environ.get(localedir)) + else: + gettext.install(domain, + localedir=os.environ.get(localedir), + unicode=True) + + +class Message(_userString.UserString, object): + """Class used to encapsulate translatable messages.""" + def __init__(self, msg, domain): + # _msg is the gettext msgid and should never change + self._msg = msg + self._left_extra_msg = '' + self._right_extra_msg = '' + self._locale = None + self.params = None + self.domain = domain + + @property + def data(self): + # NOTE(mrodden): this should always resolve to a unicode string + # that best represents the state of the message currently + + localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR') + if self.locale: + lang = gettext.translation(self.domain, + localedir=localedir, + languages=[self.locale], + fallback=True) + else: + # use system locale for translations + lang = gettext.translation(self.domain, + localedir=localedir, + fallback=True) + + if six.PY3: + ugettext = lang.gettext + else: + ugettext = lang.ugettext + + full_msg = (self._left_extra_msg + + ugettext(self._msg) + + self._right_extra_msg) + + if self.params is not None: + full_msg = full_msg % self.params + + return six.text_type(full_msg) + + @property + def locale(self): + return self._locale + + @locale.setter + def locale(self, value): + self._locale = value + if not self.params: + return + + # This Message object may have been constructed with one or more + # Message objects as substitution parameters, given as a single + # Message, or a tuple or Map containing some, so when setting the + # locale for this Message we need to set it for those Messages too. + if isinstance(self.params, Message): + self.params.locale = value + return + if isinstance(self.params, tuple): + for param in self.params: + if isinstance(param, Message): + param.locale = value + return + if isinstance(self.params, dict): + for param in self.params.values(): + if isinstance(param, Message): + param.locale = value + + def _save_dictionary_parameter(self, dict_param): + full_msg = self.data + # look for %(blah) fields in string; + # ignore %% and deal with the + # case where % is first character on the line + keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg) + + # if we don't find any %(blah) blocks but have a %s + if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg): + # apparently the full dictionary is the parameter + params = copy.deepcopy(dict_param) + else: + params = {} + for key in keys: + try: + params[key] = copy.deepcopy(dict_param[key]) + except TypeError: + # cast uncopyable thing to unicode string + params[key] = six.text_type(dict_param[key]) + + return params + + def _save_parameters(self, other): + # we check for None later to see if + # we actually have parameters to inject, + # so encapsulate if our parameter is actually None + if other is None: + self.params = (other, ) + elif isinstance(other, dict): + self.params = self._save_dictionary_parameter(other) + else: + # fallback to casting to unicode, + # this will handle the problematic python code-like + # objects that cannot be deep-copied + try: + self.params = copy.deepcopy(other) + except TypeError: + self.params = six.text_type(other) + + return self + + # overrides to be more string-like + def __unicode__(self): + return self.data + + def __str__(self): + if six.PY3: + return self.__unicode__() + return self.data.encode('utf-8') + + def __getstate__(self): + to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg', + 'domain', 'params', '_locale'] + new_dict = self.__dict__.fromkeys(to_copy) + for attr in to_copy: + new_dict[attr] = copy.deepcopy(self.__dict__[attr]) + + return new_dict + + def __setstate__(self, state): + for (k, v) in state.items(): + setattr(self, k, v) + + # operator overloads + def __add__(self, other): + copied = copy.deepcopy(self) + copied._right_extra_msg += other.__str__() + return copied + + def __radd__(self, other): + copied = copy.deepcopy(self) + copied._left_extra_msg += other.__str__() + return copied + + def __mod__(self, other): + # do a format string to catch and raise + # any possible KeyErrors from missing parameters + self.data % other + copied = copy.deepcopy(self) + return copied._save_parameters(other) + + def __mul__(self, other): + return self.data * other + + def __rmul__(self, other): + return other * self.data + + def __getitem__(self, key): + return self.data[key] + + def __getslice__(self, start, end): + return self.data.__getslice__(start, end) + + def __getattribute__(self, name): + # NOTE(mrodden): handle lossy operations that we can't deal with yet + # These override the UserString implementation, since UserString + # uses our __class__ attribute to try and build a new message + # after running the inner data string through the operation. + # At that point, we have lost the gettext message id and can just + # safely resolve to a string instead. + ops = ['capitalize', 'center', 'decode', 'encode', + 'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip', + 'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill'] + if name in ops: + return getattr(self.data, name) + else: + return _userString.UserString.__getattribute__(self, name) + + +def get_available_languages(domain): + """Lists the available languages for the given translation domain. + + :param domain: the domain to get languages for + """ + if domain in _AVAILABLE_LANGUAGES: + return copy.copy(_AVAILABLE_LANGUAGES[domain]) + + localedir = '%s_LOCALEDIR' % domain.upper() + find = lambda x: gettext.find(domain, + localedir=os.environ.get(localedir), + languages=[x]) + + # NOTE(mrodden): en_US should always be available (and first in case + # order matters) since our in-line message strings are en_US + language_list = ['en_US'] + # NOTE(luisg): Babel <1.0 used a function called list(), which was + # renamed to locale_identifiers() in >=1.0, the requirements master list + # requires >=0.9.6, uncapped, so defensively work with both. We can remove + # this check when the master list updates to >=1.0, and all projects udpate + list_identifiers = (getattr(localedata, 'list', None) or + getattr(localedata, 'locale_identifiers')) + locale_identifiers = list_identifiers() + for i in locale_identifiers: + if find(i) is not None: + language_list.append(i) + _AVAILABLE_LANGUAGES[domain] = language_list + return copy.copy(language_list) + + +def get_localized_message(message, user_locale): + """Gets a localized version of the given message in the given locale.""" + if isinstance(message, Message): + if user_locale: + message.locale = user_locale + return six.text_type(message) + else: + return message + + +class LocaleHandler(logging.Handler): + """Handler that can have a locale associated to translate Messages. + + A quick example of how to utilize the Message class above. + LocaleHandler takes a locale and a target logging.Handler object + to forward LogRecord objects to after translating the internal Message. + """ + + def __init__(self, locale, target): + """Initialize a LocaleHandler + + :param locale: locale to use for translating messages + :param target: logging.Handler object to forward + LogRecord objects to after translation + """ + logging.Handler.__init__(self) + self.locale = locale + self.target = target + + def emit(self, record): + if isinstance(record.msg, Message): + # set the locale and resolve to a string + record.msg.locale = self.locale + + self.target.emit(record) diff --git a/python-agent/muranoagent/openstack/common/importutils.py b/python-agent/muranoagent/openstack/common/importutils.py new file mode 100644 index 00000000..7a303f93 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/importutils.py @@ -0,0 +1,68 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Import related utilities and helper functions. +""" + +import sys +import traceback + + +def import_class(import_str): + """Returns a class from a string including module and class.""" + mod_str, _sep, class_str = import_str.rpartition('.') + try: + __import__(mod_str) + return getattr(sys.modules[mod_str], class_str) + except (ValueError, AttributeError): + raise ImportError('Class %s cannot be found (%s)' % + (class_str, + traceback.format_exception(*sys.exc_info()))) + + +def import_object(import_str, *args, **kwargs): + """Import a class and return an instance of it.""" + return import_class(import_str)(*args, **kwargs) + + +def import_object_ns(name_space, import_str, *args, **kwargs): + """Tries to import object from default namespace. + + Imports a class and return an instance of it, first by trying + to find the class in a default namespace, then failing back to + a full path if not found in the default namespace. + """ + import_value = "%s.%s" % (name_space, import_str) + try: + return import_class(import_value)(*args, **kwargs) + except ImportError: + return import_class(import_str)(*args, **kwargs) + + +def import_module(import_str): + """Import a module.""" + __import__(import_str) + return sys.modules[import_str] + + +def try_import(import_str, default=None): + """Try to import a module and if it fails return default.""" + try: + return import_module(import_str) + except ImportError: + return default diff --git a/python-agent/muranoagent/openstack/common/jsonutils.py b/python-agent/muranoagent/openstack/common/jsonutils.py new file mode 100644 index 00000000..8bc418b1 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/jsonutils.py @@ -0,0 +1,180 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +''' +JSON related utilities. + +This module provides a few things: + + 1) A handy function for getting an object down to something that can be + JSON serialized. See to_primitive(). + + 2) Wrappers around loads() and dumps(). The dumps() wrapper will + automatically use to_primitive() for you if needed. + + 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson + is available. +''' + + +import datetime +import functools +import inspect +import itertools +import json +try: + import xmlrpclib +except ImportError: + # NOTE(jd): xmlrpclib is not shipped with Python 3 + xmlrpclib = None + +import six + +from muranoagent.openstack.common import gettextutils +from muranoagent.openstack.common import importutils +from muranoagent.openstack.common import timeutils + +netaddr = importutils.try_import("netaddr") + +_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod, + inspect.isfunction, inspect.isgeneratorfunction, + inspect.isgenerator, inspect.istraceback, inspect.isframe, + inspect.iscode, inspect.isbuiltin, inspect.isroutine, + inspect.isabstract] + +_simple_types = (six.string_types + six.integer_types + + (type(None), bool, float)) + + +def to_primitive(value, convert_instances=False, convert_datetime=True, + level=0, max_depth=3): + """Convert a complex object into primitives. + + Handy for JSON serialization. We can optionally handle instances, + but since this is a recursive function, we could have cyclical + data structures. + + To handle cyclical data structures we could track the actual objects + visited in a set, but not all objects are hashable. Instead we just + track the depth of the object inspections and don't go too deep. + + Therefore, convert_instances=True is lossy ... be aware. + + """ + # handle obvious types first - order of basic types determined by running + # full tests on nova project, resulting in the following counts: + # 572754 + # 460353 + # 379632 + # 274610 + # 199918 + # 114200 + # 51817 + # 26164 + # 6491 + # 283 + # 19 + if isinstance(value, _simple_types): + return value + + if isinstance(value, datetime.datetime): + if convert_datetime: + return timeutils.strtime(value) + else: + return value + + # value of itertools.count doesn't get caught by nasty_type_tests + # and results in infinite loop when list(value) is called. + if type(value) == itertools.count: + return six.text_type(value) + + # FIXME(vish): Workaround for LP bug 852095. Without this workaround, + # tests that raise an exception in a mocked method that + # has a @wrap_exception with a notifier will fail. If + # we up the dependency to 0.5.4 (when it is released) we + # can remove this workaround. + if getattr(value, '__module__', None) == 'mox': + return 'mock' + + if level > max_depth: + return '?' + + # The try block may not be necessary after the class check above, + # but just in case ... + try: + recursive = functools.partial(to_primitive, + convert_instances=convert_instances, + convert_datetime=convert_datetime, + level=level, + max_depth=max_depth) + if isinstance(value, dict): + return dict((k, recursive(v)) for k, v in value.iteritems()) + elif isinstance(value, (list, tuple)): + return [recursive(lv) for lv in value] + + # It's not clear why xmlrpclib created their own DateTime type, but + # for our purposes, make it a datetime type which is explicitly + # handled + if xmlrpclib and isinstance(value, xmlrpclib.DateTime): + value = datetime.datetime(*tuple(value.timetuple())[:6]) + + if convert_datetime and isinstance(value, datetime.datetime): + return timeutils.strtime(value) + elif isinstance(value, gettextutils.Message): + return value.data + elif hasattr(value, 'iteritems'): + return recursive(dict(value.iteritems()), level=level + 1) + elif hasattr(value, '__iter__'): + return recursive(list(value)) + elif convert_instances and hasattr(value, '__dict__'): + # Likely an instance of something. Watch for cycles. + # Ignore class member vars. + return recursive(value.__dict__, level=level + 1) + elif netaddr and isinstance(value, netaddr.IPAddress): + return six.text_type(value) + else: + if any(test(value) for test in _nasty_type_tests): + return six.text_type(value) + return value + except TypeError: + # Class objects are tricky since they may define something like + # __iter__ defined but it isn't callable as list(). + return six.text_type(value) + + +def dumps(value, default=to_primitive, **kwargs): + return json.dumps(value, default=default, **kwargs) + + +def loads(s): + return json.loads(s) + + +def load(s): + return json.load(s) + + +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append((__name__, 'dumps', TypeError, + 'loads', ValueError, 'load')) + anyjson.force_implementation(__name__) diff --git a/python-agent/muranoagent/openstack/common/local.py b/python-agent/muranoagent/openstack/common/local.py new file mode 100644 index 00000000..e82f17d0 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/local.py @@ -0,0 +1,47 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Local storage of variables using weak references""" + +import threading +import weakref + + +class WeakLocal(threading.local): + def __getattribute__(self, attr): + rval = super(WeakLocal, self).__getattribute__(attr) + if rval: + # NOTE(mikal): this bit is confusing. What is stored is a weak + # reference, not the value itself. We therefore need to lookup + # the weak reference and return the inner value here. + rval = rval() + return rval + + def __setattr__(self, attr, value): + value = weakref.ref(value) + return super(WeakLocal, self).__setattr__(attr, value) + + +# NOTE(mikal): the name "store" should be deprecated in the future +store = WeakLocal() + +# A "weak" store uses weak references and allows an object to fall out of scope +# when it falls out of scope in the code that uses the thread local storage. A +# "strong" store will hold a reference to the object so that it never falls out +# of scope. +weak_store = WeakLocal() +strong_store = threading.local() diff --git a/python-agent/muranoagent/openstack/common/log.py b/python-agent/muranoagent/openstack/common/log.py new file mode 100644 index 00000000..7b2d99eb --- /dev/null +++ b/python-agent/muranoagent/openstack/common/log.py @@ -0,0 +1,566 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Openstack logging handler. + +This module adds to logging functionality by adding the option to specify +a context object when calling the various log methods. If the context object +is not specified, default formatting is used. Additionally, an instance uuid +may be passed as part of the log message, which is intended to make it easier +for admins to find messages related to a specific instance. + +It also allows setting of formatting information through conf. + +""" + +import inspect +import itertools +import logging +import logging.config +import logging.handlers +import os +import sys +import traceback + +from oslo.config import cfg +from six import moves + +from muranoagent.openstack.common.gettextutils import _ # noqa +from muranoagent.openstack.common import importutils +from muranoagent.openstack.common import jsonutils +from muranoagent.openstack.common import local + + +_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S" + +common_cli_opts = [ + cfg.BoolOpt('debug', + short='d', + default=False, + help='Print debugging output (set logging level to ' + 'DEBUG instead of default WARNING level).'), + cfg.BoolOpt('verbose', + short='v', + default=False, + help='Print more verbose output (set logging level to ' + 'INFO instead of default WARNING level).'), +] + +logging_cli_opts = [ + cfg.StrOpt('log-config', + metavar='PATH', + help='If this option is specified, the logging configuration ' + 'file specified is used and overrides any other logging ' + 'options specified. Please see the Python logging module ' + 'documentation for details on logging configuration ' + 'files.'), + cfg.StrOpt('log-format', + default=None, + metavar='FORMAT', + help='DEPRECATED. ' + 'A logging.Formatter log message format string which may ' + 'use any of the available logging.LogRecord attributes. ' + 'This option is deprecated. Please use ' + 'logging_context_format_string and ' + 'logging_default_format_string instead.'), + cfg.StrOpt('log-date-format', + default=_DEFAULT_LOG_DATE_FORMAT, + metavar='DATE_FORMAT', + help='Format string for %%(asctime)s in log records. ' + 'Default: %(default)s'), + cfg.StrOpt('log-file', + metavar='PATH', + deprecated_name='logfile', + help='(Optional) Name of log file to output to. ' + 'If no default is set, logging will go to stdout.'), + cfg.StrOpt('log-dir', + deprecated_name='logdir', + help='(Optional) The base directory used for relative ' + '--log-file paths'), + cfg.BoolOpt('use-syslog', + default=False, + help='Use syslog for logging.'), + cfg.StrOpt('syslog-log-facility', + default='LOG_USER', + help='syslog facility to receive log lines') +] + +generic_log_opts = [ + cfg.BoolOpt('use_stderr', + default=True, + help='Log output to standard error') +] + +log_opts = [ + cfg.StrOpt('logging_context_format_string', + default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' + '%(name)s [%(request_id)s %(user)s %(tenant)s] ' + '%(instance)s%(message)s', + help='format string to use for log messages with context'), + cfg.StrOpt('logging_default_format_string', + default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s ' + '%(name)s [-] %(instance)s%(message)s', + help='format string to use for log messages without context'), + cfg.StrOpt('logging_debug_format_suffix', + default='%(funcName)s %(pathname)s:%(lineno)d', + help='data to append to log format when level is DEBUG'), + cfg.StrOpt('logging_exception_prefix', + default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s ' + '%(instance)s', + help='prefix each line of exception output with this format'), + cfg.ListOpt('default_log_levels', + default=[ + 'amqplib=WARN', + 'sqlalchemy=WARN', + 'boto=WARN', + 'suds=INFO', + 'keystone=INFO', + 'eventlet.wsgi.server=WARN' + ], + help='list of logger=LEVEL pairs'), + cfg.BoolOpt('publish_errors', + default=False, + help='publish error events'), + cfg.BoolOpt('fatal_deprecations', + default=False, + help='make deprecations fatal'), + + # NOTE(mikal): there are two options here because sometimes we are handed + # a full instance (and could include more information), and other times we + # are just handed a UUID for the instance. + cfg.StrOpt('instance_format', + default='[instance: %(uuid)s] ', + help='If an instance is passed with the log message, format ' + 'it like this'), + cfg.StrOpt('instance_uuid_format', + default='[instance: %(uuid)s] ', + help='If an instance UUID is passed with the log message, ' + 'format it like this'), +] + +CONF = cfg.CONF +CONF.register_cli_opts(common_cli_opts) +CONF.register_cli_opts(logging_cli_opts) +CONF.register_opts(generic_log_opts) +CONF.register_opts(log_opts) + +# our new audit level +# NOTE(jkoelker) Since we synthesized an audit level, make the logging +# module aware of it so it acts like other levels. +logging.AUDIT = logging.INFO + 1 +logging.addLevelName(logging.AUDIT, 'AUDIT') + + +try: + NullHandler = logging.NullHandler +except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7 + class NullHandler(logging.Handler): + def handle(self, record): + pass + + def emit(self, record): + pass + + def createLock(self): + self.lock = None + + +def _dictify_context(context): + if context is None: + return None + if not isinstance(context, dict) and getattr(context, 'to_dict', None): + context = context.to_dict() + return context + + +def _get_binary_name(): + return os.path.basename(inspect.stack()[-1][1]) + + +def _get_log_file_path(binary=None): + logfile = CONF.log_file + logdir = CONF.log_dir + + if logfile and not logdir: + return logfile + + if logfile and logdir: + return os.path.join(logdir, logfile) + + if logdir: + binary = binary or _get_binary_name() + return '%s.log' % (os.path.join(logdir, binary),) + + +class BaseLoggerAdapter(logging.LoggerAdapter): + + def audit(self, msg, *args, **kwargs): + self.log(logging.AUDIT, msg, *args, **kwargs) + + +class LazyAdapter(BaseLoggerAdapter): + def __init__(self, name='unknown', version='unknown'): + self._logger = None + self.extra = {} + self.name = name + self.version = version + + @property + def logger(self): + if not self._logger: + self._logger = getLogger(self.name, self.version) + return self._logger + + +class ContextAdapter(BaseLoggerAdapter): + warn = logging.LoggerAdapter.warning + + def __init__(self, logger, project_name, version_string): + self.logger = logger + self.project = project_name + self.version = version_string + + @property + def handlers(self): + return self.logger.handlers + + def deprecated(self, msg, *args, **kwargs): + stdmsg = _("Deprecated: %s") % msg + if CONF.fatal_deprecations: + self.critical(stdmsg, *args, **kwargs) + raise DeprecatedConfig(msg=stdmsg) + else: + self.warn(stdmsg, *args, **kwargs) + + def process(self, msg, kwargs): + # NOTE(mrodden): catch any Message/other object and + # coerce to unicode before they can get + # to the python logging and possibly + # cause string encoding trouble + if not isinstance(msg, basestring): + msg = unicode(msg) + + if 'extra' not in kwargs: + kwargs['extra'] = {} + extra = kwargs['extra'] + + context = kwargs.pop('context', None) + if not context: + context = getattr(local.store, 'context', None) + if context: + extra.update(_dictify_context(context)) + + instance = kwargs.pop('instance', None) + instance_uuid = (extra.get('instance_uuid', None) or + kwargs.pop('instance_uuid', None)) + instance_extra = '' + if instance: + instance_extra = CONF.instance_format % instance + elif instance_uuid: + instance_extra = (CONF.instance_uuid_format + % {'uuid': instance_uuid}) + extra.update({'instance': instance_extra}) + + extra.update({"project": self.project}) + extra.update({"version": self.version}) + extra['extra'] = extra.copy() + return msg, kwargs + + +class JSONFormatter(logging.Formatter): + def __init__(self, fmt=None, datefmt=None): + # NOTE(jkoelker) we ignore the fmt argument, but its still there + # since logging.config.fileConfig passes it. + self.datefmt = datefmt + + def formatException(self, ei, strip_newlines=True): + lines = traceback.format_exception(*ei) + if strip_newlines: + lines = [itertools.ifilter( + lambda x: x, + line.rstrip().splitlines()) for line in lines] + lines = list(itertools.chain(*lines)) + return lines + + def format(self, record): + message = {'message': record.getMessage(), + 'asctime': self.formatTime(record, self.datefmt), + 'name': record.name, + 'msg': record.msg, + 'args': record.args, + 'levelname': record.levelname, + 'levelno': record.levelno, + 'pathname': record.pathname, + 'filename': record.filename, + 'module': record.module, + 'lineno': record.lineno, + 'funcname': record.funcName, + 'created': record.created, + 'msecs': record.msecs, + 'relative_created': record.relativeCreated, + 'thread': record.thread, + 'thread_name': record.threadName, + 'process_name': record.processName, + 'process': record.process, + 'traceback': None} + + if hasattr(record, 'extra'): + message['extra'] = record.extra + + if record.exc_info: + message['traceback'] = self.formatException(record.exc_info) + + return jsonutils.dumps(message) + + +def _create_logging_excepthook(product_name): + def logging_excepthook(type, value, tb): + extra = {} + if CONF.verbose: + extra['exc_info'] = (type, value, tb) + getLogger(product_name).critical(str(value), **extra) + return logging_excepthook + + +class LogConfigError(Exception): + + message = _('Error loading logging config %(log_config)s: %(err_msg)s') + + def __init__(self, log_config, err_msg): + self.log_config = log_config + self.err_msg = err_msg + + def __str__(self): + return self.message % dict(log_config=self.log_config, + err_msg=self.err_msg) + + +def _load_log_config(log_config): + try: + logging.config.fileConfig(log_config) + except moves.configparser.Error as exc: + raise LogConfigError(log_config, str(exc)) + + +def setup(product_name): + """Setup logging.""" + if CONF.log_config: + _load_log_config(CONF.log_config) + else: + _setup_logging_from_conf() + sys.excepthook = _create_logging_excepthook(product_name) + + +def set_defaults(logging_context_format_string): + cfg.set_defaults(log_opts, + logging_context_format_string= + logging_context_format_string) + + +def _find_facility_from_conf(): + facility_names = logging.handlers.SysLogHandler.facility_names + facility = getattr(logging.handlers.SysLogHandler, + CONF.syslog_log_facility, + None) + + if facility is None and CONF.syslog_log_facility in facility_names: + facility = facility_names.get(CONF.syslog_log_facility) + + if facility is None: + valid_facilities = facility_names.keys() + consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON', + 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS', + 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP', + 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3', + 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7'] + valid_facilities.extend(consts) + raise TypeError(_('syslog facility must be one of: %s') % + ', '.join("'%s'" % fac + for fac in valid_facilities)) + + return facility + + +def _setup_logging_from_conf(): + log_root = getLogger(None).logger + for handler in log_root.handlers: + log_root.removeHandler(handler) + + if CONF.use_syslog: + facility = _find_facility_from_conf() + syslog = logging.handlers.SysLogHandler(address='/dev/log', + facility=facility) + log_root.addHandler(syslog) + + logpath = _get_log_file_path() + if logpath: + filelog = logging.handlers.WatchedFileHandler(logpath) + log_root.addHandler(filelog) + + if CONF.use_stderr: + streamlog = ColorHandler() + log_root.addHandler(streamlog) + + elif not CONF.log_file: + # pass sys.stdout as a positional argument + # python2.6 calls the argument strm, in 2.7 it's stream + streamlog = logging.StreamHandler(sys.stdout) + log_root.addHandler(streamlog) + + if CONF.publish_errors: + handler = importutils.import_object( + "muranoagent.openstack.common.log_handler.PublishErrorsHandler", + logging.ERROR) + log_root.addHandler(handler) + + datefmt = CONF.log_date_format + for handler in log_root.handlers: + # NOTE(alaski): CONF.log_format overrides everything currently. This + # should be deprecated in favor of context aware formatting. + if CONF.log_format: + handler.setFormatter(logging.Formatter(fmt=CONF.log_format, + datefmt=datefmt)) + log_root.info('Deprecated: log_format is now deprecated and will ' + 'be removed in the next release') + else: + handler.setFormatter(ContextFormatter(datefmt=datefmt)) + + if CONF.debug: + log_root.setLevel(logging.DEBUG) + elif CONF.verbose: + log_root.setLevel(logging.INFO) + else: + log_root.setLevel(logging.WARNING) + + for pair in CONF.default_log_levels: + mod, _sep, level_name = pair.partition('=') + level = logging.getLevelName(level_name) + logger = logging.getLogger(mod) + logger.setLevel(level) + +_loggers = {} + + +def getLogger(name='unknown', version='unknown'): + if name not in _loggers: + _loggers[name] = ContextAdapter(logging.getLogger(name), + name, + version) + return _loggers[name] + + +def getLazyLogger(name='unknown', version='unknown'): + """Returns lazy logger. + + Creates a pass-through logger that does not create the real logger + until it is really needed and delegates all calls to the real logger + once it is created. + """ + return LazyAdapter(name, version) + + +class WritableLogger(object): + """A thin wrapper that responds to `write` and logs.""" + + def __init__(self, logger, level=logging.INFO): + self.logger = logger + self.level = level + + def write(self, msg): + self.logger.log(self.level, msg) + + +class ContextFormatter(logging.Formatter): + """A context.RequestContext aware formatter configured through flags. + + The flags used to set format strings are: logging_context_format_string + and logging_default_format_string. You can also specify + logging_debug_format_suffix to append extra formatting if the log level is + debug. + + For information about what variables are available for the formatter see: + http://docs.python.org/library/logging.html#formatter + + """ + + def format(self, record): + """Uses contextstring if request_id is set, otherwise default.""" + # NOTE(sdague): default the fancier formating params + # to an empty string so we don't throw an exception if + # they get used + for key in ('instance', 'color'): + if key not in record.__dict__: + record.__dict__[key] = '' + + if record.__dict__.get('request_id', None): + self._fmt = CONF.logging_context_format_string + else: + self._fmt = CONF.logging_default_format_string + + if (record.levelno == logging.DEBUG and + CONF.logging_debug_format_suffix): + self._fmt += " " + CONF.logging_debug_format_suffix + + # Cache this on the record, Logger will respect our formated copy + if record.exc_info: + record.exc_text = self.formatException(record.exc_info, record) + return logging.Formatter.format(self, record) + + def formatException(self, exc_info, record=None): + """Format exception output with CONF.logging_exception_prefix.""" + if not record: + return logging.Formatter.formatException(self, exc_info) + + stringbuffer = moves.StringIO() + traceback.print_exception(exc_info[0], exc_info[1], exc_info[2], + None, stringbuffer) + lines = stringbuffer.getvalue().split('\n') + stringbuffer.close() + + if CONF.logging_exception_prefix.find('%(asctime)') != -1: + record.asctime = self.formatTime(record, self.datefmt) + + formatted_lines = [] + for line in lines: + pl = CONF.logging_exception_prefix % record.__dict__ + fl = '%s%s' % (pl, line) + formatted_lines.append(fl) + return '\n'.join(formatted_lines) + + +class ColorHandler(logging.StreamHandler): + LEVEL_COLORS = { + logging.DEBUG: '\033[00;32m', # GREEN + logging.INFO: '\033[00;36m', # CYAN + logging.AUDIT: '\033[01;36m', # BOLD CYAN + logging.WARN: '\033[01;33m', # BOLD YELLOW + logging.ERROR: '\033[01;31m', # BOLD RED + logging.CRITICAL: '\033[01;31m', # BOLD RED + } + + def format(self, record): + record.color = self.LEVEL_COLORS[record.levelno] + return logging.StreamHandler.format(self, record) + + +class DeprecatedConfig(Exception): + message = _("Fatal call to deprecated config: %(msg)s") + + def __init__(self, msg): + super(Exception, self).__init__(self.message % dict(msg=msg)) diff --git a/python-agent/muranoagent/openstack/common/loopingcall.py b/python-agent/muranoagent/openstack/common/loopingcall.py new file mode 100644 index 00000000..1f65be63 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/loopingcall.py @@ -0,0 +1,147 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from eventlet import event +from eventlet import greenthread + +from muranoagent.openstack.common.gettextutils import _ # noqa +from muranoagent.openstack.common import log as logging +from muranoagent.openstack.common import timeutils + +LOG = logging.getLogger(__name__) + + +class LoopingCallDone(Exception): + """Exception to break out and stop a LoopingCall. + + The poll-function passed to LoopingCall can raise this exception to + break out of the loop normally. This is somewhat analogous to + StopIteration. + + An optional return-value can be included as the argument to the exception; + this return-value will be returned by LoopingCall.wait() + + """ + + def __init__(self, retvalue=True): + """:param retvalue: Value that LoopingCall.wait() should return.""" + self.retvalue = retvalue + + +class LoopingCallBase(object): + def __init__(self, f=None, *args, **kw): + self.args = args + self.kw = kw + self.f = f + self._running = False + self.done = None + + def stop(self): + self._running = False + + def wait(self): + return self.done.wait() + + +class FixedIntervalLoopingCall(LoopingCallBase): + """A fixed interval looping call.""" + + def start(self, interval, initial_delay=None): + self._running = True + done = event.Event() + + def _inner(): + if initial_delay: + greenthread.sleep(initial_delay) + + try: + while self._running: + start = timeutils.utcnow() + self.f(*self.args, **self.kw) + end = timeutils.utcnow() + if not self._running: + break + delay = interval - timeutils.delta_seconds(start, end) + if delay <= 0: + LOG.warn(_('task run outlasted interval by %s sec') % + -delay) + greenthread.sleep(delay if delay > 0 else 0) + except LoopingCallDone as e: + self.stop() + done.send(e.retvalue) + except Exception: + LOG.exception(_('in fixed duration looping call')) + done.send_exception(*sys.exc_info()) + return + else: + done.send(True) + + self.done = done + + greenthread.spawn_n(_inner) + return self.done + + +# TODO(mikal): this class name is deprecated in Havana and should be removed +# in the I release +LoopingCall = FixedIntervalLoopingCall + + +class DynamicLoopingCall(LoopingCallBase): + """A looping call which sleeps until the next known event. + + The function called should return how long to sleep for before being + called again. + """ + + def start(self, initial_delay=None, periodic_interval_max=None): + self._running = True + done = event.Event() + + def _inner(): + if initial_delay: + greenthread.sleep(initial_delay) + + try: + while self._running: + idle = self.f(*self.args, **self.kw) + if not self._running: + break + + if periodic_interval_max is not None: + idle = min(idle, periodic_interval_max) + LOG.debug(_('Dynamic looping call sleeping for %.02f ' + 'seconds'), idle) + greenthread.sleep(idle) + except LoopingCallDone as e: + self.stop() + done.send(e.retvalue) + except Exception: + LOG.exception(_('in dynamic looping call')) + done.send_exception(*sys.exc_info()) + return + else: + done.send(True) + + self.done = done + + greenthread.spawn(_inner) + return self.done diff --git a/python-agent/muranoagent/openstack/common/service.py b/python-agent/muranoagent/openstack/common/service.py new file mode 100644 index 00000000..1e98d102 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/service.py @@ -0,0 +1,459 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Generic Node base class for all workers that run on hosts.""" + +import errno +import os +import random +import signal +import sys +import time + +import eventlet +from eventlet import event +import logging as std_logging +from oslo.config import cfg + +from muranoagent.openstack.common import eventlet_backdoor +from muranoagent.openstack.common.gettextutils import _ # noqa +from muranoagent.openstack.common import importutils +from muranoagent.openstack.common import log as logging +from muranoagent.openstack.common import threadgroup + + +rpc = importutils.try_import('muranoagent.openstack.common.rpc') +CONF = cfg.CONF +LOG = logging.getLogger(__name__) + + +def _sighup_supported(): + return hasattr(signal, 'SIGHUP') + + +def _is_sighup(signo): + return _sighup_supported() and signo == signal.SIGHUP + + +def _signo_to_signame(signo): + signals = {signal.SIGTERM: 'SIGTERM', + signal.SIGINT: 'SIGINT'} + if _sighup_supported(): + signals[signal.SIGHUP] = 'SIGHUP' + return signals[signo] + + +def _set_signals_handler(handler): + signal.signal(signal.SIGTERM, handler) + signal.signal(signal.SIGINT, handler) + if _sighup_supported(): + signal.signal(signal.SIGHUP, handler) + + +class Launcher(object): + """Launch one or more services and wait for them to complete.""" + + def __init__(self): + """Initialize the service launcher. + + :returns: None + + """ + self.services = Services() + self.backdoor_port = eventlet_backdoor.initialize_if_enabled() + + def launch_service(self, service): + """Load and start the given service. + + :param service: The service you would like to start. + :returns: None + + """ + service.backdoor_port = self.backdoor_port + self.services.add(service) + + def stop(self): + """Stop all services which are currently running. + + :returns: None + + """ + self.services.stop() + + def wait(self): + """Waits until all services have been stopped, and then returns. + + :returns: None + + """ + self.services.wait() + + def restart(self): + """Reload config files and restart service. + + :returns: None + + """ + cfg.CONF.reload_config_files() + self.services.restart() + + +class SignalExit(SystemExit): + def __init__(self, signo, exccode=1): + super(SignalExit, self).__init__(exccode) + self.signo = signo + + +class ServiceLauncher(Launcher): + def _handle_signal(self, signo, frame): + # Allow the process to be killed again and die from natural causes + _set_signals_handler(signal.SIG_DFL) + raise SignalExit(signo) + + def handle_signal(self): + _set_signals_handler(self._handle_signal) + + def _wait_for_exit_or_signal(self): + status = None + signo = 0 + + LOG.debug(_('Full set of CONF:')) + CONF.log_opt_values(LOG, std_logging.DEBUG) + + try: + super(ServiceLauncher, self).wait() + except SignalExit as exc: + signame = _signo_to_signame(exc.signo) + LOG.info(_('Caught %s, exiting'), signame) + status = exc.code + signo = exc.signo + except SystemExit as exc: + status = exc.code + finally: + self.stop() + if rpc: + try: + rpc.cleanup() + except Exception: + # We're shutting down, so it doesn't matter at this point. + LOG.exception(_('Exception during rpc cleanup.')) + + return status, signo + + def wait(self): + while True: + self.handle_signal() + status, signo = self._wait_for_exit_or_signal() + if not _is_sighup(signo): + return status + self.restart() + + +class ServiceWrapper(object): + def __init__(self, service, workers): + self.service = service + self.workers = workers + self.children = set() + self.forktimes = [] + + +class ProcessLauncher(object): + def __init__(self): + self.children = {} + self.sigcaught = None + self.running = True + rfd, self.writepipe = os.pipe() + self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r') + self.handle_signal() + + def handle_signal(self): + _set_signals_handler(self._handle_signal) + + def _handle_signal(self, signo, frame): + self.sigcaught = signo + self.running = False + + # Allow the process to be killed again and die from natural causes + _set_signals_handler(signal.SIG_DFL) + + def _pipe_watcher(self): + # This will block until the write end is closed when the parent + # dies unexpectedly + self.readpipe.read() + + LOG.info(_('Parent process has died unexpectedly, exiting')) + + sys.exit(1) + + def _child_process_handle_signal(self): + # Setup child signal handlers differently + def _sigterm(*args): + signal.signal(signal.SIGTERM, signal.SIG_DFL) + raise SignalExit(signal.SIGTERM) + + def _sighup(*args): + signal.signal(signal.SIGHUP, signal.SIG_DFL) + raise SignalExit(signal.SIGHUP) + + signal.signal(signal.SIGTERM, _sigterm) + if _sighup_supported(): + signal.signal(signal.SIGHUP, _sighup) + # Block SIGINT and let the parent send us a SIGTERM + signal.signal(signal.SIGINT, signal.SIG_IGN) + + def _child_wait_for_exit_or_signal(self, launcher): + status = None + signo = 0 + + # NOTE(johannes): All exceptions are caught to ensure this + # doesn't fallback into the loop spawning children. It would + # be bad for a child to spawn more children. + try: + launcher.wait() + except SignalExit as exc: + signame = _signo_to_signame(exc.signo) + LOG.info(_('Caught %s, exiting'), signame) + status = exc.code + signo = exc.signo + except SystemExit as exc: + status = exc.code + except BaseException: + LOG.exception(_('Unhandled exception')) + status = 2 + finally: + launcher.stop() + + return status, signo + + def _child_process(self, service): + self._child_process_handle_signal() + + # Reopen the eventlet hub to make sure we don't share an epoll + # fd with parent and/or siblings, which would be bad + eventlet.hubs.use_hub() + + # Close write to ensure only parent has it open + os.close(self.writepipe) + # Create greenthread to watch for parent to close pipe + eventlet.spawn_n(self._pipe_watcher) + + # Reseed random number generator + random.seed() + + launcher = Launcher() + launcher.launch_service(service) + return launcher + + def _start_child(self, wrap): + if len(wrap.forktimes) > wrap.workers: + # Limit ourselves to one process a second (over the period of + # number of workers * 1 second). This will allow workers to + # start up quickly but ensure we don't fork off children that + # die instantly too quickly. + if time.time() - wrap.forktimes[0] < wrap.workers: + LOG.info(_('Forking too fast, sleeping')) + time.sleep(1) + + wrap.forktimes.pop(0) + + wrap.forktimes.append(time.time()) + + pid = os.fork() + if pid == 0: + launcher = self._child_process(wrap.service) + while True: + self._child_process_handle_signal() + status, signo = self._child_wait_for_exit_or_signal(launcher) + if not _is_sighup(signo): + break + launcher.restart() + + os._exit(status) + + LOG.info(_('Started child %d'), pid) + + wrap.children.add(pid) + self.children[pid] = wrap + + return pid + + def launch_service(self, service, workers=1): + wrap = ServiceWrapper(service, workers) + + LOG.info(_('Starting %d workers'), wrap.workers) + while self.running and len(wrap.children) < wrap.workers: + self._start_child(wrap) + + def _wait_child(self): + try: + # Don't block if no child processes have exited + pid, status = os.waitpid(0, os.WNOHANG) + if not pid: + return None + except OSError as exc: + if exc.errno not in (errno.EINTR, errno.ECHILD): + raise + return None + + if os.WIFSIGNALED(status): + sig = os.WTERMSIG(status) + LOG.info(_('Child %(pid)d killed by signal %(sig)d'), + dict(pid=pid, sig=sig)) + else: + code = os.WEXITSTATUS(status) + LOG.info(_('Child %(pid)s exited with status %(code)d'), + dict(pid=pid, code=code)) + + if pid not in self.children: + LOG.warning(_('pid %d not in child list'), pid) + return None + + wrap = self.children.pop(pid) + wrap.children.remove(pid) + return wrap + + def _respawn_children(self): + while self.running: + wrap = self._wait_child() + if not wrap: + # Yield to other threads if no children have exited + # Sleep for a short time to avoid excessive CPU usage + # (see bug #1095346) + eventlet.greenthread.sleep(.01) + continue + while self.running and len(wrap.children) < wrap.workers: + self._start_child(wrap) + + def wait(self): + """Loop waiting on children to die and respawning as necessary.""" + + LOG.debug(_('Full set of CONF:')) + CONF.log_opt_values(LOG, std_logging.DEBUG) + + while True: + self.handle_signal() + self._respawn_children() + if self.sigcaught: + signame = _signo_to_signame(self.sigcaught) + LOG.info(_('Caught %s, stopping children'), signame) + if not _is_sighup(self.sigcaught): + break + + for pid in self.children: + os.kill(pid, signal.SIGHUP) + self.running = True + self.sigcaught = None + + for pid in self.children: + try: + os.kill(pid, signal.SIGTERM) + except OSError as exc: + if exc.errno != errno.ESRCH: + raise + + # Wait for children to die + if self.children: + LOG.info(_('Waiting on %d children to exit'), len(self.children)) + while self.children: + self._wait_child() + + +class Service(object): + """Service object for binaries running on hosts.""" + + def __init__(self, threads=1000): + self.tg = threadgroup.ThreadGroup(threads) + + # signal that the service is done shutting itself down: + self._done = event.Event() + + def reset(self): + # NOTE(Fengqian): docs for Event.reset() recommend against using it + self._done = event.Event() + + def start(self): + pass + + def stop(self): + self.tg.stop() + self.tg.wait() + # Signal that service cleanup is done: + if not self._done.ready(): + self._done.send() + + def wait(self): + self._done.wait() + + +class Services(object): + + def __init__(self): + self.services = [] + self.tg = threadgroup.ThreadGroup() + self.done = event.Event() + + def add(self, service): + self.services.append(service) + self.tg.add_thread(self.run_service, service, self.done) + + def stop(self): + # wait for graceful shutdown of services: + for service in self.services: + service.stop() + service.wait() + + # Each service has performed cleanup, now signal that the run_service + # wrapper threads can now die: + if not self.done.ready(): + self.done.send() + + # reap threads: + self.tg.stop() + + def wait(self): + self.tg.wait() + + def restart(self): + self.stop() + self.done = event.Event() + for restart_service in self.services: + restart_service.reset() + self.tg.add_thread(self.run_service, restart_service, self.done) + + @staticmethod + def run_service(service, done): + """Service start wrapper. + + :param service: service to run + :param done: event to wait on until a shutdown is triggered + :returns: None + + """ + service.start() + done.wait() + + +def launch(service, workers=None): + if workers: + launcher = ProcessLauncher() + launcher.launch_service(service, workers=workers) + else: + launcher = ServiceLauncher() + launcher.launch_service(service) + return launcher diff --git a/python-agent/muranoagent/openstack/common/threadgroup.py b/python-agent/muranoagent/openstack/common/threadgroup.py new file mode 100644 index 00000000..5f24f1d5 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/threadgroup.py @@ -0,0 +1,121 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import eventlet +from eventlet import greenpool +from eventlet import greenthread + +from muranoagent.openstack.common import log as logging +from muranoagent.openstack.common import loopingcall + + +LOG = logging.getLogger(__name__) + + +def _thread_done(gt, *args, **kwargs): + """Callback function to be passed to GreenThread.link() when we spawn() + Calls the :class:`ThreadGroup` to notify if. + + """ + kwargs['group'].thread_done(kwargs['thread']) + + +class Thread(object): + """Wrapper around a greenthread, that holds a reference to the + :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when + it has done so it can be removed from the threads list. + """ + def __init__(self, thread, group): + self.thread = thread + self.thread.link(_thread_done, group=group, thread=self) + + def stop(self): + self.thread.kill() + + def wait(self): + return self.thread.wait() + + +class ThreadGroup(object): + """The point of the ThreadGroup classis to: + + * keep track of timers and greenthreads (making it easier to stop them + when need be). + * provide an easy API to add timers. + """ + def __init__(self, thread_pool_size=10): + self.pool = greenpool.GreenPool(thread_pool_size) + self.threads = [] + self.timers = [] + + def add_dynamic_timer(self, callback, initial_delay=None, + periodic_interval_max=None, *args, **kwargs): + timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs) + timer.start(initial_delay=initial_delay, + periodic_interval_max=periodic_interval_max) + self.timers.append(timer) + + def add_timer(self, interval, callback, initial_delay=None, + *args, **kwargs): + pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs) + pulse.start(interval=interval, + initial_delay=initial_delay) + self.timers.append(pulse) + + def add_thread(self, callback, *args, **kwargs): + gt = self.pool.spawn(callback, *args, **kwargs) + th = Thread(gt, self) + self.threads.append(th) + + def thread_done(self, thread): + self.threads.remove(thread) + + def stop(self): + current = greenthread.getcurrent() + for x in self.threads: + if x is current: + # don't kill the current thread. + continue + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + + for x in self.timers: + try: + x.stop() + except Exception as ex: + LOG.exception(ex) + self.timers = [] + + def wait(self): + for x in self.timers: + try: + x.wait() + except eventlet.greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) + current = greenthread.getcurrent() + for x in self.threads: + if x is current: + continue + try: + x.wait() + except eventlet.greenlet.GreenletExit: + pass + except Exception as ex: + LOG.exception(ex) diff --git a/python-agent/muranoagent/openstack/common/timeutils.py b/python-agent/muranoagent/openstack/common/timeutils.py new file mode 100644 index 00000000..98d877d5 --- /dev/null +++ b/python-agent/muranoagent/openstack/common/timeutils.py @@ -0,0 +1,197 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Time related utilities and helper functions. +""" + +import calendar +import datetime +import time + +import iso8601 +import six + + +# ISO 8601 extended time format with microseconds +_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f' +_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S' +PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND + + +def isotime(at=None, subsecond=False): + """Stringify time in ISO 8601 format.""" + if not at: + at = utcnow() + st = at.strftime(_ISO8601_TIME_FORMAT + if not subsecond + else _ISO8601_TIME_FORMAT_SUBSECOND) + tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC' + st += ('Z' if tz == 'UTC' else tz) + return st + + +def parse_isotime(timestr): + """Parse time from ISO 8601 format.""" + try: + return iso8601.parse_date(timestr) + except iso8601.ParseError as e: + raise ValueError(unicode(e)) + except TypeError as e: + raise ValueError(unicode(e)) + + +def strtime(at=None, fmt=PERFECT_TIME_FORMAT): + """Returns formatted utcnow.""" + if not at: + at = utcnow() + return at.strftime(fmt) + + +def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT): + """Turn a formatted time back into a datetime.""" + return datetime.datetime.strptime(timestr, fmt) + + +def normalize_time(timestamp): + """Normalize time in arbitrary timezone to UTC naive object.""" + offset = timestamp.utcoffset() + if offset is None: + return timestamp + return timestamp.replace(tzinfo=None) - offset + + +def is_older_than(before, seconds): + """Return True if before is older than seconds.""" + if isinstance(before, six.string_types): + before = parse_strtime(before).replace(tzinfo=None) + return utcnow() - before > datetime.timedelta(seconds=seconds) + + +def is_newer_than(after, seconds): + """Return True if after is newer than seconds.""" + if isinstance(after, six.string_types): + after = parse_strtime(after).replace(tzinfo=None) + return after - utcnow() > datetime.timedelta(seconds=seconds) + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + if utcnow.override_time is None: + # NOTE(kgriffs): This is several times faster + # than going through calendar.timegm(...) + return int(time.time()) + + return calendar.timegm(utcnow().timetuple()) + + +def utcnow(): + """Overridable version of utils.utcnow.""" + if utcnow.override_time: + try: + return utcnow.override_time.pop(0) + except AttributeError: + return utcnow.override_time + return datetime.datetime.utcnow() + + +def iso8601_from_timestamp(timestamp): + """Returns a iso8601 formated date from timestamp.""" + return isotime(datetime.datetime.utcfromtimestamp(timestamp)) + + +utcnow.override_time = None + + +def set_time_override(override_time=None): + """Overrides utils.utcnow. + + Make it return a constant time or a list thereof, one at a time. + + :param override_time: datetime instance or list thereof. If not + given, defaults to the current UTC time. + """ + utcnow.override_time = override_time or datetime.datetime.utcnow() + + +def advance_time_delta(timedelta): + """Advance overridden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + try: + for dt in utcnow.override_time: + dt += timedelta + except TypeError: + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overridden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None + + +def marshall_now(now=None): + """Make an rpc-safe datetime with microseconds. + + Note: tzinfo is stripped, but not required for relative times. + """ + if not now: + now = utcnow() + return dict(day=now.day, month=now.month, year=now.year, hour=now.hour, + minute=now.minute, second=now.second, + microsecond=now.microsecond) + + +def unmarshall_time(tyme): + """Unmarshall a datetime dict.""" + return datetime.datetime(day=tyme['day'], + month=tyme['month'], + year=tyme['year'], + hour=tyme['hour'], + minute=tyme['minute'], + second=tyme['second'], + microsecond=tyme['microsecond']) + + +def delta_seconds(before, after): + """Return the difference between two timing objects. + + Compute the difference in seconds between two date, time, or + datetime objects (as a float, to microsecond resolution). + """ + delta = after - before + try: + return delta.total_seconds() + except AttributeError: + return ((delta.days * 24 * 3600) + delta.seconds + + float(delta.microseconds) / (10 ** 6)) + + +def is_soon(dt, window): + """Determines if time is going to happen in the next window seconds. + + :params dt: the time + :params window: minimum seconds to remain to consider the time not soon + + :return: True if expiration is within the given duration + """ + soon = (utcnow() + datetime.timedelta(seconds=window)) + return normalize_time(dt) <= soon diff --git a/python-agent/muranoagent/script_runner.py b/python-agent/muranoagent/script_runner.py new file mode 100644 index 00000000..f6236030 --- /dev/null +++ b/python-agent/muranoagent/script_runner.py @@ -0,0 +1,59 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from muranoagent.executors import Executors + + +class FunctionRunner(object): + def __init__(self, name, script_runner): + self._name = name + self._script_executor = script_runner + + def __call__(self, *args, **kwargs): + return self._script_executor.execute_function( + self._name, *args, **kwargs) + + +class ScriptRunner(object): + def __init__(self, name, script_info, files_manager): + self._name = name + self._executor = Executors.create_executor(script_info.Type, name) + self._script_info = script_info + self._script_loaded = False + self._files_manager = files_manager + + def __call__(self, *args, **kwargs): + return self.execute_function(None, *args, **kwargs) + + def execute_function(self, name, *args, **kwargs): + self._load() + return self._executor.run(name, *args, **kwargs) + + def __getattr__(self, item): + return FunctionRunner(item, self) + + def _load(self): + if not self._script_loaded: + self._executor.load( + self._prepare_files(), + self._script_info.get("Options") or {}) + self._script_loaded = True + + def _prepare_files(self): + for file_id in self._script_info.get('Files', []): + self._files_manager.put_file(file_id, self._name) + + return self._files_manager.put_file( + self._script_info["EntryPoint"], self._name) diff --git a/python-agent/muranoagent/win32.py b/python-agent/muranoagent/win32.py new file mode 100644 index 00000000..54ed3101 --- /dev/null +++ b/python-agent/muranoagent/win32.py @@ -0,0 +1,27 @@ +# Copyright (c) 2013 Mirantis Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + import win32file + import os + + def symlink(source, link_name): + src = os.path.abspath(source) + dest = os.path.abspath(link_name) + win32file.CreateSymbolicLink(dest, src, 0) + + os.symlink = symlink +except ImportError: + pass diff --git a/python-agent/openstack-common.conf b/python-agent/openstack-common.conf new file mode 100644 index 00000000..e52e2937 --- /dev/null +++ b/python-agent/openstack-common.conf @@ -0,0 +1,14 @@ +[DEFAULT] + +# The list of modules to copy from openstack-common +module=exception +module=importutils +module=jsonutils +module=log +module=install_venv_common +module=config +module=service + + +# The base module to hold the copy of openstack.common +base=muranoagent \ No newline at end of file diff --git a/python-agent/requirements.txt b/python-agent/requirements.txt new file mode 100644 index 00000000..20c8fb4d --- /dev/null +++ b/python-agent/requirements.txt @@ -0,0 +1,5 @@ +pbr>=0.5,<0.6 +semver +bunch +oslo.config +murano-common \ No newline at end of file diff --git a/python-agent/setup.cfg b/python-agent/setup.cfg new file mode 100644 index 00000000..247de95a --- /dev/null +++ b/python-agent/setup.cfg @@ -0,0 +1,44 @@ +[metadata] +name = murano-agent +summary = Python Murano Agent +description-file = + README.rst +license = Apache License, Version 2.0 +author = Mirantis, Inc. +author-email = murano-all@lists.openstack.org +home-page = htts://launchpad.net/murano +classifier = + Development Status :: 5 - Production/Stable + Environment :: OpenStack + Intended Audience :: Developers + Intended Audience :: Information Technology + License :: OSI Approved :: Apache Software License + Operating System :: OS Independent + Programming Language :: Python + +[files] +packages = + muranoagent + +[global] +setup-hooks = + pbr.hooks.setup_hook + +[entry_points] +console_scripts = + muranoagent = muranoagent.cmd.run:main + +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + +[nosetests] +# NOTE(jkoelker) To run the test suite under nose install the following +# coverage http://pypi.python.org/pypi/coverage +# tissue http://pypi.python.org/pypi/tissue (pep8 checker) +# openstack-nose https://github.com/jkoelker/openstack-nose +verbosity=2 +cover-package = muranoagent +cover-html = true +cover-erase = true diff --git a/python-agent/setup.py b/python-agent/setup.py new file mode 100644 index 00000000..70c2b3f3 --- /dev/null +++ b/python-agent/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# Copyright (c) 2013 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT +import setuptools + +setuptools.setup( + setup_requires=['pbr'], + pbr=True) diff --git a/python-agent/tools/config/generate_sample.sh b/python-agent/tools/config/generate_sample.sh new file mode 100644 index 00000000..feb1ebb8 --- /dev/null +++ b/python-agent/tools/config/generate_sample.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +print_hint() { + echo "Try \`${0##*/} --help' for more information." >&2 +} + +PARSED_OPTIONS=$(getopt -n "${0##*/}" -o hb:p:o: \ + --long help,base-dir:,package-name:,output-dir: -- "$@") + +if [ $? != 0 ] ; then print_hint ; exit 1 ; fi + +eval set -- "$PARSED_OPTIONS" + +while true; do + case "$1" in + -h|--help) + echo "${0##*/} [options]" + echo "" + echo "options:" + echo "-h, --help show brief help" + echo "-b, --base-dir=DIR project base directory" + echo "-p, --package-name=NAME project package name" + echo "-o, --output-dir=DIR file output directory" + exit 0 + ;; + -b|--base-dir) + shift + BASEDIR=`echo $1 | sed -e 's/\/*$//g'` + shift + ;; + -p|--package-name) + shift + PACKAGENAME=`echo $1` + shift + ;; + -o|--output-dir) + shift + OUTPUTDIR=`echo $1 | sed -e 's/\/*$//g'` + shift + ;; + --) + break + ;; + esac +done + +BASEDIR=${BASEDIR:-`pwd`} +if ! [ -d $BASEDIR ] +then + echo "${0##*/}: missing project base directory" >&2 ; print_hint ; exit 1 +elif [[ $BASEDIR != /* ]] +then + BASEDIR=$(cd "$BASEDIR" && pwd) +fi + +PACKAGENAME=${PACKAGENAME:-${BASEDIR##*/}} +TARGETDIR=$BASEDIR/$PACKAGENAME +if ! [ -d $TARGETDIR ] +then + echo "${0##*/}: invalid project package name" >&2 ; print_hint ; exit 1 +fi + +OUTPUTDIR=${OUTPUTDIR:-$BASEDIR/etc} +# NOTE(bnemec): Some projects put their sample config in etc/, +# some in etc/$PACKAGENAME/ +if [ -d $OUTPUTDIR/$PACKAGENAME ] +then + OUTPUTDIR=$OUTPUTDIR/$PACKAGENAME +elif ! [ -d $OUTPUTDIR ] +then + echo "${0##*/}: cannot access \`$OUTPUTDIR': No such file or directory" >&2 + exit 1 +fi + +BASEDIRESC=`echo $BASEDIR | sed -e 's/\//\\\\\//g'` +FILES=$(find $TARGETDIR -type f -name "*.py" ! -path "*/tests/*" \ + -exec grep -l "Opt(" {} + | sed -e "s/^$BASEDIRESC\///g" | sort -u) + +EXTRA_MODULES_FILE="`dirname $0`/oslo.config.generator.rc" +if test -r "$EXTRA_MODULES_FILE" +then + source "$EXTRA_MODULES_FILE" +fi + +export EVENTLET_NO_GREENDNS=yes + +OS_VARS=$(set | sed -n '/^OS_/s/=[^=]*$//gp' | xargs) +[ "$OS_VARS" ] && eval "unset \$OS_VARS" + +MODULEPATH=muranoagent.openstack.common.config.generator +OUTPUTFILE=$OUTPUTDIR/$PACKAGENAME.conf.sample +python -m $MODULEPATH $FILES > $OUTPUTFILE diff --git a/python-agent/tools/install_venv_common.py b/python-agent/tools/install_venv_common.py new file mode 100644 index 00000000..92d66ae7 --- /dev/null +++ b/python-agent/tools/install_venv_common.py @@ -0,0 +1,213 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 OpenStack Foundation +# Copyright 2013 IBM Corp. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Provides methods needed by installation script for OpenStack development +virtual environments. + +Since this script is used to bootstrap a virtualenv from the system's Python +environment, it should be kept strictly compatible with Python 2.6. + +Synced in from openstack-common +""" + +from __future__ import print_function + +import optparse +import os +import subprocess +import sys + + +class InstallVenv(object): + + def __init__(self, root, venv, requirements, + test_requirements, py_version, + project): + self.root = root + self.venv = venv + self.requirements = requirements + self.test_requirements = test_requirements + self.py_version = py_version + self.project = project + + def die(self, message, *args): + print(message % args, file=sys.stderr) + sys.exit(1) + + def check_python_version(self): + if sys.version_info < (2, 6): + self.die("Need Python Version >= 2.6") + + def run_command_with_code(self, cmd, redirect_output=True, + check_exit_code=True): + """Runs a command in an out-of-process shell. + + Returns the output of that command. Working directory is self.root. + """ + if redirect_output: + stdout = subprocess.PIPE + else: + stdout = None + + proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout) + output = proc.communicate()[0] + if check_exit_code and proc.returncode != 0: + self.die('Command "%s" failed.\n%s', ' '.join(cmd), output) + return (output, proc.returncode) + + def run_command(self, cmd, redirect_output=True, check_exit_code=True): + return self.run_command_with_code(cmd, redirect_output, + check_exit_code)[0] + + def get_distro(self): + if (os.path.exists('/etc/fedora-release') or + os.path.exists('/etc/redhat-release')): + return Fedora( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + else: + return Distro( + self.root, self.venv, self.requirements, + self.test_requirements, self.py_version, self.project) + + def check_dependencies(self): + self.get_distro().install_virtualenv() + + def create_virtualenv(self, no_site_packages=True): + """Creates the virtual environment and installs PIP. + + Creates the virtual environment and installs PIP only into the + virtual environment. + """ + if not os.path.isdir(self.venv): + print('Creating venv...', end=' ') + if no_site_packages: + self.run_command(['virtualenv', '-q', '--no-site-packages', + self.venv]) + else: + self.run_command(['virtualenv', '-q', self.venv]) + print('done.') + else: + print("venv already exists...") + pass + + def pip_install(self, *args): + self.run_command(['tools/with_venv.sh', + 'pip', 'install', '--upgrade'] + list(args), + redirect_output=False) + + def install_dependencies(self): + print('Installing dependencies with pip (this can take a while)...') + + # First things first, make sure our venv has the latest pip and + # setuptools and pbr + self.pip_install('pip>=1.4') + self.pip_install('setuptools') + self.pip_install('pbr') + + self.pip_install('-r', self.requirements, '-r', self.test_requirements) + + def post_process(self): + self.get_distro().post_process() + + def parse_args(self, argv): + """Parses command-line arguments.""" + parser = optparse.OptionParser() + parser.add_option('-n', '--no-site-packages', + action='store_true', + help="Do not inherit packages from global Python " + "install") + return parser.parse_args(argv[1:])[0] + + +class Distro(InstallVenv): + + def check_cmd(self, cmd): + return bool(self.run_command(['which', cmd], + check_exit_code=False).strip()) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if self.check_cmd('easy_install'): + print('Installing virtualenv via easy_install...', end=' ') + if self.run_command(['easy_install', 'virtualenv']): + print('Succeeded') + return + else: + print('Failed') + + self.die('ERROR: virtualenv not found.\n\n%s development' + ' requires virtualenv, please install it using your' + ' favorite package management tool' % self.project) + + def post_process(self): + """Any distribution-specific post-processing gets done here. + + In particular, this is useful for applying patches to code inside + the venv. + """ + pass + + +class Fedora(Distro): + """This covers all Fedora-based distributions. + + Includes: Fedora, RHEL, CentOS, Scientific Linux + """ + + def check_pkg(self, pkg): + return self.run_command_with_code(['rpm', '-q', pkg], + check_exit_code=False)[1] == 0 + + def apply_patch(self, originalfile, patchfile): + self.run_command(['patch', '-N', originalfile, patchfile], + check_exit_code=False) + + def install_virtualenv(self): + if self.check_cmd('virtualenv'): + return + + if not self.check_pkg('python-virtualenv'): + self.die("Please install 'python-virtualenv'.") + + super(Fedora, self).install_virtualenv() + + def post_process(self): + """Workaround for a bug in eventlet. + + This currently affects RHEL6.1, but the fix can safely be + applied to all RHEL and Fedora distributions. + + This can be removed when the fix is applied upstream. + + Nova: https://bugs.launchpad.net/nova/+bug/884915 + Upstream: https://bitbucket.org/eventlet/eventlet/issue/89 + RHEL: https://bugzilla.redhat.com/958868 + """ + + if os.path.exists('contrib/redhat-eventlet.patch'): + # Install "patch" program if it's not there + if not self.check_pkg('patch'): + self.die("Please install 'patch'.") + + # Apply the eventlet patch + self.apply_patch(os.path.join(self.venv, 'lib', self.py_version, + 'site-packages', + 'eventlet/green/subprocess.py'), + 'contrib/redhat-eventlet.patch') diff --git a/python-agent/tox.ini b/python-agent/tox.ini new file mode 100644 index 00000000..15d56f87 --- /dev/null +++ b/python-agent/tox.ini @@ -0,0 +1,58 @@ +[tox] +envlist = py26,py27,pep8,pyflakes + +[testenv] +setenv = VIRTUAL_ENV={envdir} + NOSE_WITH_OPENSTACK=1 + NOSE_OPENSTACK_COLOR=1 + NOSE_OPENSTACK_RED=0.05 + NOSE_OPENSTACK_YELLOW=0.025 + NOSE_OPENSTACK_SHOW_ELAPSED=1 +deps = -r{toxinidir}/requirements.txt + -r{toxinidir}/test-requirements.txt +commands = nosetests + +[testenv:pep8] +deps = pep8==1.3.3 +commands = pep8 --repeat --show-source muranoagent setup.py + +[testenv:venv] +commands = {posargs} + +[testenv:cover] +commands = nosetests --cover-erase --cover-package=muranoagent --with-xcoverage + +[tox:jenkins] +downloadcache = ~/cache/pip + +[testenv:jenkins26] +basepython = python2.6 +setenv = NOSE_WITH_XUNIT=1 +deps = file://{toxinidir}/.cache.bundle + +[testenv:jenkins27] +basepython = python2.7 +setenv = NOSE_WITH_XUNIT=1 +deps = file://{toxinidir}/.cache.bundle + +[testenv:jenkinscover] +deps = file://{toxinidir}/.cache.bundle +setenv = NOSE_WITH_XUNIT=1 +commands = nosetests --cover-erase --cover-package=muranoagent --with-xcoverage + +[testenv:jenkinsvenv] +deps = file://{toxinidir}/.cache.bundle +setenv = NOSE_WITH_XUNIT=1 +commands = {posargs} + +[testenv:pyflakes] +deps = flake8 +commands = flake8 + +[flake8] +# H301 one import per line +# H302 import only modules +ignore = H301,H302,F401 +show-source = true +builtins = _ +exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*,*egg,tools \ No newline at end of file diff --git a/ExecutionPlanGenerator/App.config b/windows-agent/ExecutionPlanGenerator/App.config similarity index 100% rename from ExecutionPlanGenerator/App.config rename to windows-agent/ExecutionPlanGenerator/App.config diff --git a/ExecutionPlanGenerator/ExecutionPlanGenerator.csproj b/windows-agent/ExecutionPlanGenerator/ExecutionPlanGenerator.csproj similarity index 100% rename from ExecutionPlanGenerator/ExecutionPlanGenerator.csproj rename to windows-agent/ExecutionPlanGenerator/ExecutionPlanGenerator.csproj diff --git a/ExecutionPlanGenerator/Program.cs b/windows-agent/ExecutionPlanGenerator/Program.cs similarity index 100% rename from ExecutionPlanGenerator/Program.cs rename to windows-agent/ExecutionPlanGenerator/Program.cs diff --git a/ExecutionPlanGenerator/Properties/AssemblyInfo.cs b/windows-agent/ExecutionPlanGenerator/Properties/AssemblyInfo.cs similarity index 100% rename from ExecutionPlanGenerator/Properties/AssemblyInfo.cs rename to windows-agent/ExecutionPlanGenerator/Properties/AssemblyInfo.cs diff --git a/ExecutionPlanGenerator/packages.config b/windows-agent/ExecutionPlanGenerator/packages.config similarity index 100% rename from ExecutionPlanGenerator/packages.config rename to windows-agent/ExecutionPlanGenerator/packages.config diff --git a/Tools/NuGet.exe b/windows-agent/Tools/NuGet.exe similarity index 100% rename from Tools/NuGet.exe rename to windows-agent/Tools/NuGet.exe diff --git a/WindowsAgent.sln b/windows-agent/WindowsAgent.sln similarity index 100% rename from WindowsAgent.sln rename to windows-agent/WindowsAgent.sln diff --git a/WindowsAgent/App.config b/windows-agent/WindowsAgent/App.config similarity index 100% rename from WindowsAgent/App.config rename to windows-agent/WindowsAgent/App.config diff --git a/WindowsAgent/ExecutionPlan.cs b/windows-agent/WindowsAgent/ExecutionPlan.cs similarity index 100% rename from WindowsAgent/ExecutionPlan.cs rename to windows-agent/WindowsAgent/ExecutionPlan.cs diff --git a/WindowsAgent/MqMessage.cs b/windows-agent/WindowsAgent/MqMessage.cs similarity index 100% rename from WindowsAgent/MqMessage.cs rename to windows-agent/WindowsAgent/MqMessage.cs diff --git a/WindowsAgent/PlanExecutor.cs b/windows-agent/WindowsAgent/PlanExecutor.cs similarity index 100% rename from WindowsAgent/PlanExecutor.cs rename to windows-agent/WindowsAgent/PlanExecutor.cs diff --git a/WindowsAgent/Program.cs b/windows-agent/WindowsAgent/Program.cs similarity index 100% rename from WindowsAgent/Program.cs rename to windows-agent/WindowsAgent/Program.cs diff --git a/WindowsAgent/Properties/AssemblyInfo.cs b/windows-agent/WindowsAgent/Properties/AssemblyInfo.cs similarity index 100% rename from WindowsAgent/Properties/AssemblyInfo.cs rename to windows-agent/WindowsAgent/Properties/AssemblyInfo.cs diff --git a/WindowsAgent/RabbitMqClient.cs b/windows-agent/WindowsAgent/RabbitMqClient.cs similarity index 100% rename from WindowsAgent/RabbitMqClient.cs rename to windows-agent/WindowsAgent/RabbitMqClient.cs diff --git a/WindowsAgent/SampleExecutionPlan.json b/windows-agent/WindowsAgent/SampleExecutionPlan.json similarity index 100% rename from WindowsAgent/SampleExecutionPlan.json rename to windows-agent/WindowsAgent/SampleExecutionPlan.json diff --git a/WindowsAgent/ServiceManager.cs b/windows-agent/WindowsAgent/ServiceManager.cs similarity index 100% rename from WindowsAgent/ServiceManager.cs rename to windows-agent/WindowsAgent/ServiceManager.cs diff --git a/WindowsAgent/WindowsAgent.csproj b/windows-agent/WindowsAgent/WindowsAgent.csproj similarity index 100% rename from WindowsAgent/WindowsAgent.csproj rename to windows-agent/WindowsAgent/WindowsAgent.csproj diff --git a/WindowsAgent/WindowsService.cs b/windows-agent/WindowsAgent/WindowsService.cs similarity index 100% rename from WindowsAgent/WindowsService.cs rename to windows-agent/WindowsAgent/WindowsService.cs diff --git a/WindowsAgent/WindowsServiceInstaller.cs b/windows-agent/WindowsAgent/WindowsServiceInstaller.cs similarity index 100% rename from WindowsAgent/WindowsServiceInstaller.cs rename to windows-agent/WindowsAgent/WindowsServiceInstaller.cs diff --git a/WindowsAgent/packages.config b/windows-agent/WindowsAgent/packages.config similarity index 100% rename from WindowsAgent/packages.config rename to windows-agent/WindowsAgent/packages.config diff --git a/packages/repositories.config b/windows-agent/packages/repositories.config similarity index 100% rename from packages/repositories.config rename to windows-agent/packages/repositories.config