summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStan Lagun <slagun@mirantis.com>2013-10-03 19:59:12 +0400
committerStan Lagun <slagun@mirantis.com>2013-10-04 12:58:31 +0400
commitb7ce04cdf5f2dc6573f912815d08cca2921dc514 (patch)
tree831139e4c66a9deea8119baa4d5283fefb83c80a
parent2819713996fd19054a06b47b72b87b4b1e462c62 (diff)
Python Agent initial release
Notes
Notes (review): Verified+2: Jenkins Approved+1: Alexander Tivelkov <ativelkov@mirantis.com> Code-Review+2: Alexander Tivelkov <ativelkov@mirantis.com> Submitted-by: Jenkins Submitted-at: Fri, 04 Oct 2013 12:11:57 +0000 Reviewed-on: https://review.openstack.org/49581 Project: stackforge/murano-agent Branch: refs/heads/master
-rw-r--r--python-agent/etc/agent.conf35
-rw-r--r--python-agent/muranoagent/__init__.py0
-rw-r--r--python-agent/muranoagent/app.py186
-rw-r--r--python-agent/muranoagent/cmd/__init__.py0
-rw-r--r--python-agent/muranoagent/cmd/run.py49
-rw-r--r--python-agent/muranoagent/config.py115
-rw-r--r--python-agent/muranoagent/exceptions.py35
-rw-r--r--python-agent/muranoagent/execution_plan_queue.py86
-rw-r--r--python-agent/muranoagent/execution_plan_runner.py77
-rw-r--r--python-agent/muranoagent/execution_result.py64
-rw-r--r--python-agent/muranoagent/executors/__init__.py38
-rw-r--r--python-agent/muranoagent/executors/application/__init__.py73
-rw-r--r--python-agent/muranoagent/files_manager.py71
-rw-r--r--python-agent/muranoagent/openstack/__init__.py0
-rw-r--r--python-agent/muranoagent/openstack/common/__init__.py0
-rw-r--r--python-agent/muranoagent/openstack/common/config/__init__.py0
-rw-r--r--python-agent/muranoagent/openstack/common/config/generator.py260
-rw-r--r--python-agent/muranoagent/openstack/common/eventlet_backdoor.py146
-rw-r--r--python-agent/muranoagent/openstack/common/gettextutils.py365
-rw-r--r--python-agent/muranoagent/openstack/common/importutils.py68
-rw-r--r--python-agent/muranoagent/openstack/common/jsonutils.py180
-rw-r--r--python-agent/muranoagent/openstack/common/local.py47
-rw-r--r--python-agent/muranoagent/openstack/common/log.py566
-rw-r--r--python-agent/muranoagent/openstack/common/loopingcall.py147
-rw-r--r--python-agent/muranoagent/openstack/common/service.py459
-rw-r--r--python-agent/muranoagent/openstack/common/threadgroup.py121
-rw-r--r--python-agent/muranoagent/openstack/common/timeutils.py197
-rw-r--r--python-agent/muranoagent/script_runner.py59
-rw-r--r--python-agent/muranoagent/win32.py27
-rw-r--r--python-agent/openstack-common.conf14
-rw-r--r--python-agent/requirements.txt5
-rw-r--r--python-agent/setup.cfg44
-rw-r--r--python-agent/setup.py22
-rw-r--r--python-agent/tools/config/generate_sample.sh92
-rw-r--r--python-agent/tools/install_venv_common.py213
-rw-r--r--python-agent/tox.ini58
-rw-r--r--windows-agent/ExecutionPlanGenerator/App.config (renamed from ExecutionPlanGenerator/App.config)0
-rw-r--r--windows-agent/ExecutionPlanGenerator/ExecutionPlanGenerator.csproj (renamed from ExecutionPlanGenerator/ExecutionPlanGenerator.csproj)0
-rw-r--r--windows-agent/ExecutionPlanGenerator/Program.cs (renamed from ExecutionPlanGenerator/Program.cs)0
-rw-r--r--windows-agent/ExecutionPlanGenerator/Properties/AssemblyInfo.cs (renamed from ExecutionPlanGenerator/Properties/AssemblyInfo.cs)0
-rw-r--r--windows-agent/ExecutionPlanGenerator/packages.config (renamed from ExecutionPlanGenerator/packages.config)0
-rw-r--r--windows-agent/Tools/NuGet.exe (renamed from Tools/NuGet.exe)bin651264 -> 651264 bytes
-rw-r--r--windows-agent/WindowsAgent.sln (renamed from WindowsAgent.sln)0
-rw-r--r--windows-agent/WindowsAgent/App.config (renamed from WindowsAgent/App.config)0
-rw-r--r--windows-agent/WindowsAgent/ExecutionPlan.cs (renamed from WindowsAgent/ExecutionPlan.cs)0
-rw-r--r--windows-agent/WindowsAgent/MqMessage.cs (renamed from WindowsAgent/MqMessage.cs)0
-rw-r--r--windows-agent/WindowsAgent/PlanExecutor.cs (renamed from WindowsAgent/PlanExecutor.cs)0
-rw-r--r--windows-agent/WindowsAgent/Program.cs (renamed from WindowsAgent/Program.cs)0
-rw-r--r--windows-agent/WindowsAgent/Properties/AssemblyInfo.cs (renamed from WindowsAgent/Properties/AssemblyInfo.cs)0
-rw-r--r--windows-agent/WindowsAgent/RabbitMqClient.cs (renamed from WindowsAgent/RabbitMqClient.cs)0
-rw-r--r--windows-agent/WindowsAgent/SampleExecutionPlan.json (renamed from WindowsAgent/SampleExecutionPlan.json)0
-rw-r--r--windows-agent/WindowsAgent/ServiceManager.cs (renamed from WindowsAgent/ServiceManager.cs)0
-rw-r--r--windows-agent/WindowsAgent/WindowsAgent.csproj (renamed from WindowsAgent/WindowsAgent.csproj)0
-rw-r--r--windows-agent/WindowsAgent/WindowsService.cs (renamed from WindowsAgent/WindowsService.cs)0
-rw-r--r--windows-agent/WindowsAgent/WindowsServiceInstaller.cs (renamed from WindowsAgent/WindowsServiceInstaller.cs)0
-rw-r--r--windows-agent/WindowsAgent/packages.config (renamed from WindowsAgent/packages.config)0
-rw-r--r--windows-agent/packages/repositories.config (renamed from packages/repositories.config)0
57 files changed, 3919 insertions, 0 deletions
diff --git a/python-agent/etc/agent.conf b/python-agent/etc/agent.conf
new file mode 100644
index 0000000..a47832c
--- /dev/null
+++ b/python-agent/etc/agent.conf
@@ -0,0 +1,35 @@
1[DEFAULT]
2debug=True
3verbose=True
4
5storage=.
6
7[rabbitmq]
8
9# Input queue name
10input_queue = agent-tasks
11
12# Output routing key (usually queue name)
13result_routing_key = agent-results
14
15# Connection parameters to RabbitMQ service
16
17# Hostname or IP address where RabbitMQ is located.
18host = localhost
19
20# RabbitMQ port (5672 is a default)
21port = 5672
22
23# Use SSL for RabbitMQ connections (True or False)
24ssl = False
25
26# Path to SSL CA certificate or empty to allow self signed server certificate
27ca_certs =
28
29# RabbitMQ credentials. Fresh RabbitMQ installation has "guest" account with "guest" password.
30login = guest
31password = guest
32
33# RabbitMQ virtual host (vhost). Fresh RabbitMQ installation has "/" vhost preconfigured.
34virtual_host = /
35
diff --git a/python-agent/muranoagent/__init__.py b/python-agent/muranoagent/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/python-agent/muranoagent/__init__.py
diff --git a/python-agent/muranoagent/app.py b/python-agent/muranoagent/app.py
new file mode 100644
index 0000000..5df5635
--- /dev/null
+++ b/python-agent/muranoagent/app.py
@@ -0,0 +1,186 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import win32
17import sys
18import os
19from execution_plan_runner import ExecutionPlanRunner
20from execution_plan_queue import ExecutionPlanQueue
21from execution_result import ExecutionResult
22from openstack.common import log as logging
23from openstack.common import service
24from config import CONF
25from muranocommon.messaging import MqClient, Message
26from exceptions import AgentException
27from time import sleep
28from bunch import Bunch
29import semver
30import types
31
32log = logging.getLogger(__name__)
33format_version = '2.0.0'
34
35class MuranoAgent(service.Service):
36 def __init__(self):
37 self._queue = ExecutionPlanQueue()
38 super(MuranoAgent, self).__init__()
39
40 @staticmethod
41 def _load_package(name):
42 try:
43 log.debug('Loading plugin %s', name)
44 __import__(name)
45 except Exception, ex:
46 log.warn('Cannot load package %s', name, exc_info=1)
47 pass
48
49 def _load(self):
50 path = os.path.join(os.path.dirname(__file__), 'executors')
51 sys.path.insert(1, path)
52 for entry in os.listdir(path):
53 package_path = os.path.join(path, entry)
54 if os.path.isdir(package_path):
55 MuranoAgent._load_package(entry)
56
57 def start(self):
58 self._load()
59 while True:
60 try:
61 self._loop_func()
62 except Exception as ex:
63 log.exception(ex)
64 sleep(5)
65
66 def _loop_func(self):
67 result, timestamp = self._queue.get_execution_plan_result()
68 if result is not None:
69 if self._send_result(result):
70 self._queue.remove(timestamp)
71 return
72
73 plan = self._queue.get_execution_plan()
74 if plan is not None:
75 self._run(plan)
76 return
77
78 self._wait_plan()
79
80 def _run(self, plan):
81 with ExecutionPlanRunner(plan) as runner:
82 try:
83 result = runner.run()
84 execution_result = ExecutionResult.from_result(result, plan)
85 self._queue.put_execution_result(execution_result, plan)
86 except Exception, ex:
87 execution_result = ExecutionResult.from_error(ex, plan)
88 self._queue.put_execution_result(execution_result, plan)
89
90 def _send_result(self, result):
91 with self._create_rmq_client() as mq:
92 msg = Message()
93 msg.body = result
94 msg.id = result.get('SourceID')
95 mq.send(message=msg,
96 key=CONF.rabbitmq.result_routing_key,
97 exchange=CONF.rabbitmq.result_exchange)
98 return True
99
100 def _create_rmq_client(self):
101 rabbitmq = CONF.rabbitmq
102 connection_params = {
103 'login': rabbitmq.login,
104 'password': rabbitmq.password,
105 'host': rabbitmq.host,
106 'port': rabbitmq.port,
107 'virtual_host': rabbitmq.virtual_host,
108 'ssl': rabbitmq.ssl,
109 'ca_certs': rabbitmq.ca_certs.strip() or None
110 }
111 return MqClient(**connection_params)
112
113 def _wait_plan(self):
114 with self._create_rmq_client() as mq:
115 with mq.open(CONF.rabbitmq.input_queue,
116 prefetch_count=1) as subscription:
117 msg = subscription.get_message(timeout=5)
118 if msg is not None and isinstance(msg.body, dict):
119 if 'ID' not in msg.body and msg.id:
120 msg.body['ID'] = msg.id
121 err = self._verify_plan(msg.body)
122 if err is None:
123 self._queue.put_execution_plan(msg.body)
124 else:
125 try:
126 execution_result = ExecutionResult.from_error(
127 err, Bunch(msg.body))
128 self._send_result(execution_result)
129 except ValueError:
130 log.warn('Execution result is not produced')
131
132 if msg is not None:
133 msg.ack()
134
135 def _verify_plan(self, plan):
136 plan_format_version = plan.get('FormatVersion', '1.0.0')
137 if semver.compare(plan_format_version, '2.0.0') > 0 or \
138 semver.compare(plan_format_version, format_version) < 0:
139 range_str = 'in range 2.0.0-{0}'.format(plan_format_version) \
140 if format_version != '2.0.0' \
141 else 'equal to {0}'.format(format_version)
142 return AgentException(
143 3,
144 'Unsupported format version {0} (must be {1})'.format(
145 plan_format_version, range_str))
146
147 for attr in ('Scripts', 'Files', 'Options'):
148 if attr is plan and not isinstance(
149 plan[attr], types.DictionaryType):
150 return AgentException(
151 2, '{0} is not a dictionary'.format(attr))
152
153 for name, script in plan.get('Scripts', {}).items():
154 for attr in ('Type', 'EntryPoint'):
155 if attr not in script or not isinstance(
156 script[attr], types.StringTypes):
157 return AgentException(
158 2, 'Incorrect {0} entry in script {1}'.format(
159 attr, name))
160 if not isinstance(script.get('Options', {}), types.DictionaryType):
161 return AgentException(
162 2, 'Incorrect Options entry in script {0}'.format(name))
163
164 if script['EntryPoint'] not in plan.get('Files', {}):
165 return AgentException(
166 2, 'Script {0} misses entry point {1}'.format(
167 name, script['EntryPoint']))
168
169 for additional_file in script.get('Files', []):
170 if additional_file not in plan.get('Files', {}):
171 return AgentException(
172 2, 'Script {0} misses file {1}'.format(
173 name, additional_file))
174
175 for key, plan_file in plan.get('Files', {}).items():
176 for attr in ('BodyType', 'Body', 'Name'):
177 if attr not in plan_file:
178 return AgentException(
179 2, 'Incorrect {0} entry in file {1}'.format(
180 attr, key))
181
182 if plan_file['BodyType'] not in ('Text', 'Base64'):
183 return AgentException(
184 2, 'Incorrect BodyType in file {1}'.format(key))
185
186 return None
diff --git a/python-agent/muranoagent/cmd/__init__.py b/python-agent/muranoagent/cmd/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/python-agent/muranoagent/cmd/__init__.py
diff --git a/python-agent/muranoagent/cmd/run.py b/python-agent/muranoagent/cmd/run.py
new file mode 100644
index 0000000..65a8820
--- /dev/null
+++ b/python-agent/muranoagent/cmd/run.py
@@ -0,0 +1,49 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import sys
17import os
18
19# If ../muranoagent/__init__.py exists, add ../ to Python search path, so
20# it will override what happens to be installed in /usr/(local/)lib/python...
21possible_topdir = os.path.normpath(os.path.join(os.path.abspath(__file__),
22 os.pardir,
23 os.pardir,
24 os.pardir))
25if os.path.exists(os.path.join(possible_topdir,
26 'muranoagent',
27 '__init__.py')):
28 sys.path.insert(0, possible_topdir)
29
30from muranoagent import config
31from muranoagent.openstack.common import log
32from muranoagent.openstack.common import service
33from muranoagent.app import MuranoAgent
34
35
36def main():
37 try:
38 config.parse_args()
39 log.setup('muranoagent')
40 launcher = service.ServiceLauncher()
41 launcher.launch_service(MuranoAgent())
42 launcher.wait()
43 except RuntimeError, e:
44 sys.stderr.write("ERROR: %s\n" % e)
45 sys.exit(1)
46
47
48if __name__ == '__main__':
49 main()
diff --git a/python-agent/muranoagent/config.py b/python-agent/muranoagent/config.py
new file mode 100644
index 0000000..989efd2
--- /dev/null
+++ b/python-agent/muranoagent/config.py
@@ -0,0 +1,115 @@
1# Copyright 2011 OpenStack LLC.
2# All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16"""
17Routines for configuring Glance
18"""
19
20import logging
21import logging.config
22import logging.handlers
23import os
24import sys
25
26from oslo.config import cfg
27
28#from muranoagent import __version__ as version
29
30
31CONF = cfg.CONF
32CONF.register_cli_opt(cfg.StrOpt('storage'))
33
34rabbit_opts = [
35 cfg.StrOpt('host', default='localhost'),
36 cfg.IntOpt('port', default=5672),
37 cfg.StrOpt('login', default='guest'),
38 cfg.StrOpt('password', default='guest'),
39 cfg.StrOpt('virtual_host', default='/'),
40 cfg.BoolOpt('ssl', default=False),
41 cfg.StrOpt('ca_certs', default=''),
42 cfg.StrOpt('result_routing_key'),
43 cfg.StrOpt('result_exchange', default=''),
44 cfg.StrOpt('input_queue', default='')
45
46]
47
48CONF.register_opts(rabbit_opts, group='rabbitmq')
49
50
51CONF.import_opt('verbose', 'muranoagent.openstack.common.log')
52CONF.import_opt('debug', 'muranoagent.openstack.common.log')
53CONF.import_opt('log_dir', 'muranoagent.openstack.common.log')
54CONF.import_opt('log_file', 'muranoagent.openstack.common.log')
55CONF.import_opt('log_config', 'muranoagent.openstack.common.log')
56CONF.import_opt('log_format', 'muranoagent.openstack.common.log')
57CONF.import_opt('log_date_format', 'muranoagent.openstack.common.log')
58CONF.import_opt('use_syslog', 'muranoagent.openstack.common.log')
59CONF.import_opt('syslog_log_facility', 'muranoagent.openstack.common.log')
60
61
62def parse_args(args=None, usage=None, default_config_files=None):
63 CONF(args=args,
64 project='muranoagent',
65 #version=version,
66 usage=usage,
67 default_config_files=default_config_files)
68
69
70def setup_logging():
71 """
72 Sets up the logging options for a log with supplied name
73 """
74
75 if CONF.log_config:
76 # Use a logging configuration file for all settings...
77 if os.path.exists(CONF.log_config):
78 logging.config.fileConfig(CONF.log_config)
79 return
80 else:
81 raise RuntimeError("Unable to locate specified logging "
82 "config file: %s" % CONF.log_config)
83
84 root_logger = logging.root
85 if CONF.debug:
86 root_logger.setLevel(logging.DEBUG)
87 elif CONF.verbose:
88 root_logger.setLevel(logging.INFO)
89 else:
90 root_logger.setLevel(logging.WARNING)
91
92 formatter = logging.Formatter(CONF.log_format, CONF.log_date_format)
93
94 if CONF.use_syslog:
95 try:
96 facility = getattr(logging.handlers.SysLogHandler,
97 CONF.syslog_log_facility)
98 except AttributeError:
99 raise ValueError(_("Invalid syslog facility"))
100
101 handler = logging.handlers.SysLogHandler(address='/dev/log',
102 facility=facility)
103 elif CONF.log_file:
104 logfile = CONF.log_file
105 if CONF.log_dir:
106 logfile = os.path.join(CONF.log_dir, logfile)
107 handler = logging.handlers.WatchedFileHandler(logfile)
108 else:
109 handler = logging.StreamHandler(sys.stdout)
110
111 handler.setFormatter(formatter)
112 root_logger.addHandler(handler)
113
114
115
diff --git a/python-agent/muranoagent/exceptions.py b/python-agent/muranoagent/exceptions.py
new file mode 100644
index 0000000..5359ebc
--- /dev/null
+++ b/python-agent/muranoagent/exceptions.py
@@ -0,0 +1,35 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16class AgentException(Exception):
17 def __init__(self, code, message=None, additional_data=None):
18 self._error_code = code
19 self._additional_data = additional_data
20 super(AgentException, self).__init__(message)
21
22 @property
23 def error_code(self):
24 return self._error_code
25
26 @property
27 def additional_data(self):
28 return self._additional_data
29
30
31class CustomException(AgentException):
32 def __init__(self, code, message=None, additional_data=None):
33 super(CustomException, self).__init__(
34 code + 100, message, additional_data)
35
diff --git a/python-agent/muranoagent/execution_plan_queue.py b/python-agent/muranoagent/execution_plan_queue.py
new file mode 100644
index 0000000..9e226a1
--- /dev/null
+++ b/python-agent/muranoagent/execution_plan_queue.py
@@ -0,0 +1,86 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import json
17import os
18import shutil
19import time
20from bunch import Bunch
21from config import CONF
22
23
24class ExecutionPlanQueue(object):
25 plan_filename = 'plan.json'
26 result_filename = 'result.json'
27
28 def __init__(self):
29 self._plans_folder = os.path.join(CONF.storage, 'plans')
30 if not os.path.exists(self._plans_folder):
31 os.makedirs(self._plans_folder)
32
33 def put_execution_plan(self, execution_plan):
34 timestamp = str(int(time.time() * 10000))
35 #execution_plan['_timestamp'] = timestamp
36 folder_path = os.path.join(self._plans_folder, timestamp)
37 os.mkdir(folder_path)
38 file_path = os.path.join(
39 folder_path, ExecutionPlanQueue.plan_filename)
40 with open(file_path, 'w') as out_file:
41 out_file.write(json.dumps(execution_plan))
42
43 def _get_first_timestamp(self, filename):
44 def predicate(folder):
45 path = os.path.join(self._plans_folder, folder, filename)
46 return os.path.exists(path)
47
48 timestamps = [
49 name for name in os.listdir(self._plans_folder)
50 if predicate(name)
51 ]
52 timestamps.sort()
53 return None if len(timestamps) == 0 else timestamps[0]
54
55 def _get_first_file(self, filename):
56 timestamp = self._get_first_timestamp(filename)
57 if not timestamp:
58 return None, None
59 path = os.path.join(self._plans_folder, timestamp, filename)
60 with open(path) as json_file:
61 return json.loads(json_file.read()), timestamp
62
63 def get_execution_plan(self):
64 ep, timestamp = self._get_first_file(ExecutionPlanQueue.plan_filename)
65 if ep is None:
66 return None
67 ep['_timestamp'] = timestamp
68 return Bunch(ep)
69
70 def put_execution_result(self, result, execution_plan):
71 timestamp = execution_plan['_timestamp']
72 path = os.path.join(
73 self._plans_folder, timestamp,
74 ExecutionPlanQueue.result_filename)
75 with open(path, 'w') as out_file:
76 out_file.write(json.dumps(result))
77
78 def remove(self, timestamp):
79 path = os.path.join(self._plans_folder, timestamp)
80 shutil.rmtree(path)
81
82 def get_execution_plan_result(self):
83 return self._get_first_file(
84 ExecutionPlanQueue.result_filename)
85
86
diff --git a/python-agent/muranoagent/execution_plan_runner.py b/python-agent/muranoagent/execution_plan_runner.py
new file mode 100644
index 0000000..a2a5616
--- /dev/null
+++ b/python-agent/muranoagent/execution_plan_runner.py
@@ -0,0 +1,77 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import sys
17from bunch import Bunch
18from files_manager import FilesManager
19from script_runner import ScriptRunner
20
21
22class ExecutionPlanRunner(object):
23 def __init__(self, execution_plan):
24 self._execution_plan = execution_plan
25 self._main_script = self._prepare_script(execution_plan.Body)
26 self._script_funcs = {}
27 self._files_manager = FilesManager(execution_plan)
28 self._prepare_executors(execution_plan)
29
30 def run(self):
31 script_globals = {
32 "args": Bunch(self._execution_plan.get('Parameters') or {})
33 }
34 script_globals.update(self._script_funcs)
35 exec self._main_script in script_globals
36 if '__execution_plan_exception' in script_globals:
37 raise script_globals['__execution_plan_exception']
38 return script_globals['__execution_plan_result']
39
40 @staticmethod
41 def _unindent(script, initial_indent):
42 lines = script.expandtabs(4).split('\n')
43 min_indent = sys.maxint
44 for line in lines:
45 indent = -1
46 for i, c in enumerate(line):
47 if c != ' ':
48 indent = i
49 break
50 if 0 <= indent < min_indent:
51 min_indent = indent
52 return '\n'.join([' ' * initial_indent + line[min_indent:]
53 for line in lines])
54
55 def _prepare_executors(self, execution_plan):
56 for key, value in execution_plan.Scripts.items():
57 self._script_funcs[key] = ScriptRunner(
58 key, Bunch(value), self._files_manager)
59
60 @staticmethod
61 def _prepare_script(body):
62 script = 'def __execution_plan_main():\n'
63 script += ExecutionPlanRunner._unindent(body, 4)
64 script += """
65try:
66 __execution_plan_result = __execution_plan_main()
67except Exception as e:
68 __execution_plan_exception = e
69"""
70 return script
71
72 def __enter__(self):
73 return self
74
75 def __exit__(self, exc_type, exc_val, exc_tb):
76 self._files_manager.clear()
77 return False
diff --git a/python-agent/muranoagent/execution_result.py b/python-agent/muranoagent/execution_result.py
new file mode 100644
index 0000000..fe7454c
--- /dev/null
+++ b/python-agent/muranoagent/execution_result.py
@@ -0,0 +1,64 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from exceptions import AgentException
17import uuid
18from openstack.common import timeutils
19
20
21class ExecutionResult(object):
22 @staticmethod
23 def from_result(result, execution_plan):
24 if 'ID' not in execution_plan:
25 raise ValueError('ID attribute is missing from execution plan')
26
27 return {
28 'FormatVersion': '2.0.0',
29 'ID': uuid.uuid4().hex,
30 'SourceID': execution_plan.ID,
31 'Action': 'Execution:Result',
32 'ErrorCode': 0,
33 'Body': result,
34 'Time': str(timeutils.utcnow())
35 }
36
37 @staticmethod
38 def from_error(error, execution_plan):
39 if 'ID' not in execution_plan:
40 raise ValueError('ID attribute is missing from execution plan')
41
42 error_code = 1
43 additional_info = None
44 message = None
45 if isinstance(error, int):
46 error_code = error
47 elif isinstance(error, Exception):
48 message = error.message
49 if isinstance(error, AgentException):
50 error_code = error.error_code
51 additional_info = error.additional_data
52
53 return {
54 'FormatVersion': '2.0.0',
55 'ID': uuid.uuid4().hex,
56 'SourceID': execution_plan.ID,
57 'Action': 'Execution:Result',
58 'ErrorCode': error_code,
59 'Body': {
60 'Message': message,
61 'AdditionalInfo': additional_info
62 },
63 'Time': str(timeutils.utcnow())
64 }
diff --git a/python-agent/muranoagent/executors/__init__.py b/python-agent/muranoagent/executors/__init__.py
new file mode 100644
index 0000000..662400e
--- /dev/null
+++ b/python-agent/muranoagent/executors/__init__.py
@@ -0,0 +1,38 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16from functools import wraps
17
18
19class ExecutorsRepo(object):
20 def __init__(self):
21 self._executors = {}
22
23 def register_executor(self, name, cls):
24 self._executors[name] = cls
25
26 def create_executor(self, type, name):
27 if type not in self._executors:
28 return None
29 return self._executors[type](name)
30
31Executors = ExecutorsRepo()
32
33
34def executor(name):
35 def wrapper(cls):
36 Executors.register_executor(name, cls)
37 return cls
38 return wrapper
diff --git a/python-agent/muranoagent/executors/application/__init__.py b/python-agent/muranoagent/executors/application/__init__.py
new file mode 100644
index 0000000..9a583b6
--- /dev/null
+++ b/python-agent/muranoagent/executors/application/__init__.py
@@ -0,0 +1,73 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import os
17import stat
18import subprocess
19import sys
20from muranoagent.executors import executor
21import muranoagent.exceptions
22from bunch import Bunch
23
24
25@executor('Application')
26class ApplicationExecutor(object):
27 def __init__(self, name):
28 self._capture_stdout = False
29 self._capture_stderr = False
30 self._verify_exitcode = True
31 self._name = name
32
33 def load(self, path, options):
34 self._path = path
35 self._capture_stdout = options.get('captureStdout', False)
36 self._capture_stderr = options.get('captureStderr', False)
37 self._verify_exitcode = options.get('verifyExitcode', True)
38
39 def run(self, function, commandline, input=None):
40 dir_name = os.path.dirname(self._path)
41 os.chdir(dir_name)
42 app = '"{0}" {1}'.format(os.path.basename(self._path), commandline)
43
44 if not sys.platform == 'win32':
45 os.chmod(self._path, stat.S_IEXEC | stat.S_IREAD)
46 app = './' + app
47
48 stdout = subprocess.PIPE if self._capture_stdout else None
49 stderr = subprocess.PIPE if self._capture_stderr else None
50
51 process = subprocess.Popen(
52 app,
53 stdout=stdout,
54 stderr=stderr,
55 universal_newlines=True,
56 cwd=dir_name,
57 shell=True)
58 stdout, stderr = process.communicate(input)
59 retcode = process.poll()
60
61 result = {
62 'exitCode': retcode,
63 'stdout': stdout.strip() if stdout else None,
64 'stderr': stderr.strip() if stderr else None
65 }
66 if self._verify_exitcode and retcode != 0:
67 raise muranoagent.exceptions.CustomException(
68 0,
69 message='Script {0} returned error code'.format(self._name),
70 additional_data= result)
71
72 return Bunch(result)
73
diff --git a/python-agent/muranoagent/files_manager.py b/python-agent/muranoagent/files_manager.py
new file mode 100644
index 0000000..aa4c1b5
--- /dev/null
+++ b/python-agent/muranoagent/files_manager.py
@@ -0,0 +1,71 @@
1# Copyright (c) 2013 Mirantis Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12# implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16import os
17import base64
18import shutil
19from config import CONF
20
21
22class FilesManager(object):
23 def __init__(self, execution_pan):
24 self._fetched_files = {}
25 self._files = execution_pan.get('Files') or {}
26
27 self._cache_folder = os.path.join(
28 CONF.storage, 'files', execution_pan.ID)
29 if os.path.exists(self._cache_folder):
30 self.clear()
31 os.makedirs(self._cache_folder)
32
33 def put_file(self, file_id, script):
34 cache_path = self._fetch_file(file_id)
35
36 script_folder = os.path.join(self._cache_folder, script)
37 if not os.path.exists(script_folder):
38 os.mkdir(script_folder)
39
40 filedef = self._files[file_id]
41 filename = filedef['Name']
42
43 file_folder = os.path.join(script_folder, os.path.dirname(filename))
44 if not os.path.exists(file_folder):
45 os.makedirs(file_folder)
46
47 script_path = os.path.join(script_folder, filename)
48
49 os.symlink(cache_path, script_path)
50 return script_path
51
52 def _fetch_file(self, file_id):
53 if file_id in self._fetched_files:
54 return self._fetched_files[file_id]
55
56 filedef = self._files[file_id]
57 out_path = os.path.join(self._cache_folder, file_id)
58 body_type = filedef.get('BodyType', 'Text')
59 with open(out_path, 'w') as out_file:
60 if body_type == 'Text':
61 out_file.write(filedef['Body'])
62 elif body_type == 'Base64':
63 out_file.write(base64.b64decode(filedef['Body']))
64
65 self._fetched_files[file_id] = out_path
66 return out_path
67
68 def clear(self):
69 os.chdir(os.path.dirname(self._cache_folder))
70 shutil.rmtree(self._cache_folder, ignore_errors=True)
71 shutil.rmtree(self._cache_folder, ignore_errors=True)
diff --git a/python-agent/muranoagent/openstack/__init__.py b/python-agent/muranoagent/openstack/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/python-agent/muranoagent/openstack/__init__.py
diff --git a/python-agent/muranoagent/openstack/common/__init__.py b/python-agent/muranoagent/openstack/common/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/__init__.py
diff --git a/python-agent/muranoagent/openstack/common/config/__init__.py b/python-agent/muranoagent/openstack/common/config/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/config/__init__.py
diff --git a/python-agent/muranoagent/openstack/common/config/generator.py b/python-agent/muranoagent/openstack/common/config/generator.py
new file mode 100644
index 0000000..44358f8
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/config/generator.py
@@ -0,0 +1,260 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2012 SINA Corporation
4# All Rights Reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17#
18
19"""Extracts OpenStack config option info from module(s)."""
20
21from __future__ import print_function
22
23import imp
24import os
25import re
26import socket
27import sys
28import textwrap
29
30from oslo.config import cfg
31
32from muranoagent.openstack.common import gettextutils
33from muranoagent.openstack.common import importutils
34
35gettextutils.install('muranoagent')
36
37STROPT = "StrOpt"
38BOOLOPT = "BoolOpt"
39INTOPT = "IntOpt"
40FLOATOPT = "FloatOpt"
41LISTOPT = "ListOpt"
42MULTISTROPT = "MultiStrOpt"
43
44OPT_TYPES = {
45 STROPT: 'string value',
46 BOOLOPT: 'boolean value',
47 INTOPT: 'integer value',
48 FLOATOPT: 'floating point value',
49 LISTOPT: 'list value',
50 MULTISTROPT: 'multi valued',
51}
52
53OPTION_REGEX = re.compile(r"(%s)" % "|".join([STROPT, BOOLOPT, INTOPT,
54 FLOATOPT, LISTOPT,
55 MULTISTROPT]))
56
57PY_EXT = ".py"
58BASEDIR = os.path.abspath(os.path.join(os.path.dirname(__file__),
59 "../../../../"))
60WORDWRAP_WIDTH = 60
61
62
63def generate(srcfiles):
64 mods_by_pkg = dict()
65 for filepath in srcfiles:
66 pkg_name = filepath.split(os.sep)[1]
67 mod_str = '.'.join(['.'.join(filepath.split(os.sep)[:-1]),
68 os.path.basename(filepath).split('.')[0]])
69 mods_by_pkg.setdefault(pkg_name, list()).append(mod_str)
70 # NOTE(lzyeval): place top level modules before packages
71 pkg_names = filter(lambda x: x.endswith(PY_EXT), mods_by_pkg.keys())
72 pkg_names.sort()
73 ext_names = filter(lambda x: x not in pkg_names, mods_by_pkg.keys())
74 ext_names.sort()
75 pkg_names.extend(ext_names)
76
77 # opts_by_group is a mapping of group name to an options list
78 # The options list is a list of (module, options) tuples
79 opts_by_group = {'DEFAULT': []}
80
81 for module_name in os.getenv(
82 "OSLO_CONFIG_GENERATOR_EXTRA_MODULES", "").split(','):
83 module = _import_module(module_name)
84 if module:
85 for group, opts in _list_opts(module):
86 opts_by_group.setdefault(group, []).append((module_name, opts))
87
88 for pkg_name in pkg_names:
89 mods = mods_by_pkg.get(pkg_name)
90 mods.sort()
91 for mod_str in mods:
92 if mod_str.endswith('.__init__'):
93 mod_str = mod_str[:mod_str.rfind(".")]
94
95 mod_obj = _import_module(mod_str)
96 if not mod_obj:
97 continue
98
99 for group, opts in _list_opts(mod_obj):
100 opts_by_group.setdefault(group, []).append((mod_str, opts))
101
102 print_group_opts('DEFAULT', opts_by_group.pop('DEFAULT', []))
103 for group, opts in opts_by_group.items():
104 print_group_opts(group, opts)
105
106
107def _import_module(mod_str):
108 try:
109 if mod_str.startswith('bin.'):
110 imp.load_source(mod_str[4:], os.path.join('bin', mod_str[4:]))
111 return sys.modules[mod_str[4:]]
112 else:
113 return importutils.import_module(mod_str)
114 except ImportError as ie:
115 sys.stderr.write("%s\n" % str(ie))
116 return None
117 except Exception:
118 return None
119
120
121def _is_in_group(opt, group):
122 "Check if opt is in group."
123 for key, value in group._opts.items():
124 if value['opt'] == opt:
125 return True
126 return False
127
128
129def _guess_groups(opt, mod_obj):
130 # is it in the DEFAULT group?
131 if _is_in_group(opt, cfg.CONF):
132 return 'DEFAULT'
133
134 # what other groups is it in?
135 for key, value in cfg.CONF.items():
136 if isinstance(value, cfg.CONF.GroupAttr):
137 if _is_in_group(opt, value._group):
138 return value._group.name
139
140 raise RuntimeError(
141 "Unable to find group for option %s, "
142 "maybe it's defined twice in the same group?"
143 % opt.name
144 )
145
146
147def _list_opts(obj):
148 def is_opt(o):
149 return (isinstance(o, cfg.Opt) and
150 not isinstance(o, cfg.SubCommandOpt))
151
152 opts = list()
153 for attr_str in dir(obj):
154 attr_obj = getattr(obj, attr_str)
155 if is_opt(attr_obj):
156 opts.append(attr_obj)
157 elif (isinstance(attr_obj, list) and
158 all(map(lambda x: is_opt(x), attr_obj))):
159 opts.extend(attr_obj)
160
161 ret = {}
162 for opt in opts:
163 ret.setdefault(_guess_groups(opt, obj), []).append(opt)
164 return ret.items()
165
166
167def print_group_opts(group, opts_by_module):
168 print("[%s]" % group)
169 print('')
170 for mod, opts in opts_by_module:
171 print('#')
172 print('# Options defined in %s' % mod)
173 print('#')
174 print('')
175 for opt in opts:
176 _print_opt(opt)
177 print('')
178
179
180def _get_my_ip():
181 try:
182 csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
183 csock.connect(('8.8.8.8', 80))
184 (addr, port) = csock.getsockname()
185 csock.close()
186 return addr
187 except socket.error:
188 return None
189
190
191def _sanitize_default(name, value):
192 """Set up a reasonably sensible default for pybasedir, my_ip and host."""
193 if value.startswith(sys.prefix):
194 # NOTE(jd) Don't use os.path.join, because it is likely to think the
195 # second part is an absolute pathname and therefore drop the first
196 # part.
197 value = os.path.normpath("/usr/" + value[len(sys.prefix):])
198 elif value.startswith(BASEDIR):
199 return value.replace(BASEDIR, '/usr/lib/python/site-packages')
200 elif BASEDIR in value:
201 return value.replace(BASEDIR, '')
202 elif value == _get_my_ip():
203 return '10.0.0.1'
204 elif value == socket.gethostname() and 'host' in name:
205 return 'muranoagent'
206 elif value.strip() != value:
207 return '"%s"' % value
208 return value
209
210
211def _print_opt(opt):
212 opt_name, opt_default, opt_help = opt.dest, opt.default, opt.help
213 if not opt_help:
214 sys.stderr.write('WARNING: "%s" is missing help string.\n' % opt_name)
215 opt_help = ""
216 opt_type = None
217 try:
218 opt_type = OPTION_REGEX.search(str(type(opt))).group(0)
219 except (ValueError, AttributeError) as err:
220 sys.stderr.write("%s\n" % str(err))
221 sys.exit(1)
222 opt_help += ' (' + OPT_TYPES[opt_type] + ')'
223 print('#', "\n# ".join(textwrap.wrap(opt_help, WORDWRAP_WIDTH)))
224 try:
225 if opt_default is None:
226 print('#%s=<None>' % opt_name)
227 elif opt_type == STROPT:
228 assert(isinstance(opt_default, basestring))
229 print('#%s=%s' % (opt_name, _sanitize_default(opt_name,
230 opt_default)))
231 elif opt_type == BOOLOPT:
232 assert(isinstance(opt_default, bool))
233 print('#%s=%s' % (opt_name, str(opt_default).lower()))
234 elif opt_type == INTOPT:
235 assert(isinstance(opt_default, int) and
236 not isinstance(opt_default, bool))
237 print('#%s=%s' % (opt_name, opt_default))
238 elif opt_type == FLOATOPT:
239 assert(isinstance(opt_default, float))
240 print('#%s=%s' % (opt_name, opt_default))
241 elif opt_type == LISTOPT:
242 assert(isinstance(opt_default, list))
243 print('#%s=%s' % (opt_name, ','.join(opt_default)))
244 elif opt_type == MULTISTROPT:
245 assert(isinstance(opt_default, list))
246 if not opt_default:
247 opt_default = ['']
248 for default in opt_default:
249 print('#%s=%s' % (opt_name, default))
250 print('')
251 except Exception:
252 sys.stderr.write('Error in option "%s"\n' % opt_name)
253 sys.exit(1)
254
255
256def main():
257 generate(sys.argv[1:])
258
259if __name__ == '__main__':
260 main()
diff --git a/python-agent/muranoagent/openstack/common/eventlet_backdoor.py b/python-agent/muranoagent/openstack/common/eventlet_backdoor.py
new file mode 100644
index 0000000..3c5aef3
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/eventlet_backdoor.py
@@ -0,0 +1,146 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright (c) 2012 OpenStack Foundation.
4# Administrator of the National Aeronautics and Space Administration.
5# All Rights Reserved.
6#
7# Licensed under the Apache License, Version 2.0 (the "License"); you may
8# not use this file except in compliance with the License. You may obtain
9# a copy of the License at
10#
11# http://www.apache.org/licenses/LICENSE-2.0
12#
13# Unless required by applicable law or agreed to in writing, software
14# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16# License for the specific language governing permissions and limitations
17# under the License.
18
19from __future__ import print_function
20
21import errno
22import gc
23import os
24import pprint
25import socket
26import sys
27import traceback
28
29import eventlet
30import eventlet.backdoor
31import greenlet
32from oslo.config import cfg
33
34from muranoagent.openstack.common.gettextutils import _ # noqa
35from muranoagent.openstack.common import log as logging
36
37help_for_backdoor_port = (
38 "Acceptable values are 0, <port>, and <start>:<end>, where 0 results "
39 "in listening on a random tcp port number; <port> results in listening "
40 "on the specified port number (and not enabling backdoor if that port "
41 "is in use); and <start>:<end> results in listening on the smallest "
42 "unused port number within the specified range of port numbers. The "
43 "chosen port is displayed in the service's log file.")
44eventlet_backdoor_opts = [
45 cfg.StrOpt('backdoor_port',
46 default=None,
47 help="Enable eventlet backdoor. %s" % help_for_backdoor_port)
48]
49
50CONF = cfg.CONF
51CONF.register_opts(eventlet_backdoor_opts)
52LOG = logging.getLogger(__name__)
53
54
55class EventletBackdoorConfigValueError(Exception):
56 def __init__(self, port_range, help_msg, ex):
57 msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. '
58 '%(help)s' %
59 {'range': port_range, 'ex': ex, 'help': help_msg})
60 super(EventletBackdoorConfigValueError, self).__init__(msg)
61 self.port_range = port_range
62
63
64def _dont_use_this():
65 print("Don't use this, just disconnect instead")
66
67
68def _find_objects(t):
69 return filter(lambda o: isinstance(o, t), gc.get_objects())
70
71
72def _print_greenthreads():
73 for i, gt in enumerate(_find_objects(greenlet.greenlet)):
74 print(i, gt)
75 traceback.print_stack(gt.gr_frame)
76 print()
77
78
79def _print_nativethreads():
80 for threadId, stack in sys._current_frames().items():
81 print(threadId)
82 traceback.print_stack(stack)
83 print()
84
85
86def _parse_port_range(port_range):
87 if ':' not in port_range:
88 start, end = port_range, port_range
89 else:
90 start, end = port_range.split(':', 1)
91 try:
92 start, end = int(start), int(end)
93 if end < start:
94 raise ValueError
95 return start, end
96 except ValueError as ex:
97 raise EventletBackdoorConfigValueError(port_range, ex,
98 help_for_backdoor_port)
99
100
101def _listen(host, start_port, end_port, listen_func):
102 try_port = start_port
103 while True:
104 try:
105 return listen_func((host, try_port))
106 except socket.error as exc:
107 if (exc.errno != errno.EADDRINUSE or
108 try_port >= end_port):
109 raise
110 try_port += 1
111
112
113def initialize_if_enabled():
114 backdoor_locals = {
115 'exit': _dont_use_this, # So we don't exit the entire process
116 'quit': _dont_use_this, # So we don't exit the entire process
117 'fo': _find_objects,
118 'pgt': _print_greenthreads,
119 'pnt': _print_nativethreads,
120 }
121
122 if CONF.backdoor_port is None:
123 return None
124
125 start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
126
127 # NOTE(johannes): The standard sys.displayhook will print the value of
128 # the last expression and set it to __builtin__._, which overwrites
129 # the __builtin__._ that gettext sets. Let's switch to using pprint
130 # since it won't interact poorly with gettext, and it's easier to
131 # read the output too.
132 def displayhook(val):
133 if val is not None:
134 pprint.pprint(val)
135 sys.displayhook = displayhook
136
137 sock = _listen('localhost', start_port, end_port, eventlet.listen)
138
139 # In the case of backdoor port being zero, a port number is assigned by
140 # listen(). In any case, pull the port number out here.
141 port = sock.getsockname()[1]
142 LOG.info(_('Eventlet backdoor listening on %(port)s for process %(pid)d') %
143 {'port': port, 'pid': os.getpid()})
144 eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
145 locals=backdoor_locals)
146 return port
diff --git a/python-agent/muranoagent/openstack/common/gettextutils.py b/python-agent/muranoagent/openstack/common/gettextutils.py
new file mode 100644
index 0000000..aac8b8b
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/gettextutils.py
@@ -0,0 +1,365 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2012 Red Hat, Inc.
4# Copyright 2013 IBM Corp.
5# All Rights Reserved.
6#
7# Licensed under the Apache License, Version 2.0 (the "License"); you may
8# not use this file except in compliance with the License. You may obtain
9# a copy of the License at
10#
11# http://www.apache.org/licenses/LICENSE-2.0
12#
13# Unless required by applicable law or agreed to in writing, software
14# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
15# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
16# License for the specific language governing permissions and limitations
17# under the License.
18
19"""
20gettext for openstack-common modules.
21
22Usual usage in an openstack.common module:
23
24 from muranoagent.openstack.common.gettextutils import _
25"""
26
27import copy
28import gettext
29import logging
30import os
31import re
32try:
33 import UserString as _userString
34except ImportError:
35 import collections as _userString
36
37from babel import localedata
38import six
39
40_localedir = os.environ.get('muranoagent'.upper() + '_LOCALEDIR')
41_t = gettext.translation('muranoagent', localedir=_localedir, fallback=True)
42
43_AVAILABLE_LANGUAGES = {}
44USE_LAZY = False
45
46
47def enable_lazy():
48 """Convenience function for configuring _() to use lazy gettext
49
50 Call this at the start of execution to enable the gettextutils._
51 function to use lazy gettext functionality. This is useful if
52 your project is importing _ directly instead of using the
53 gettextutils.install() way of importing the _ function.
54 """
55 global USE_LAZY
56 USE_LAZY = True
57
58
59def _(msg):
60 if USE_LAZY:
61 return Message(msg, 'muranoagent')
62 else:
63 if six.PY3:
64 return _t.gettext(msg)
65 return _t.ugettext(msg)
66
67
68def install(domain, lazy=False):
69 """Install a _() function using the given translation domain.
70
71 Given a translation domain, install a _() function using gettext's
72 install() function.
73
74 The main difference from gettext.install() is that we allow
75 overriding the default localedir (e.g. /usr/share/locale) using
76 a translation-domain-specific environment variable (e.g.
77 NOVA_LOCALEDIR).
78
79 :param domain: the translation domain
80 :param lazy: indicates whether or not to install the lazy _() function.
81 The lazy _() introduces a way to do deferred translation
82 of messages by installing a _ that builds Message objects,
83 instead of strings, which can then be lazily translated into
84 any available locale.
85 """
86 if lazy:
87 # NOTE(mrodden): Lazy gettext functionality.
88 #
89 # The following introduces a deferred way to do translations on
90 # messages in OpenStack. We override the standard _() function
91 # and % (format string) operation to build Message objects that can
92 # later be translated when we have more information.
93 #
94 # Also included below is an example LocaleHandler that translates
95 # Messages to an associated locale, effectively allowing many logs,
96 # each with their own locale.
97
98 def _lazy_gettext(msg):
99 """Create and return a Message object.
100
101 Lazy gettext function for a given domain, it is a factory method
102 for a project/module to get a lazy gettext function for its own
103 translation domain (i.e. nova, glance, cinder, etc.)
104
105 Message encapsulates a string so that we can translate
106 it later when needed.
107 """
108 return Message(msg, domain)
109
110 from six import moves
111 moves.builtins.__dict__['_'] = _lazy_gettext
112 else:
113 localedir = '%s_LOCALEDIR' % domain.upper()
114 if six.PY3:
115 gettext.install(domain,
116 localedir=os.environ.get(localedir))
117 else:
118 gettext.install(domain,
119 localedir=os.environ.get(localedir),
120 unicode=True)
121
122
123class Message(_userString.UserString, object):
124 """Class used to encapsulate translatable messages."""
125 def __init__(self, msg, domain):
126 # _msg is the gettext msgid and should never change
127 self._msg = msg
128 self._left_extra_msg = ''
129 self._right_extra_msg = ''
130 self._locale = None
131 self.params = None
132 self.domain = domain
133
134 @property
135 def data(self):
136 # NOTE(mrodden): this should always resolve to a unicode string
137 # that best represents the state of the message currently
138
139 localedir = os.environ.get(self.domain.upper() + '_LOCALEDIR')
140 if self.locale:
141 lang = gettext.translation(self.domain,
142 localedir=localedir,
143 languages=[self.locale],
144 fallback=True)
145 else:
146 # use system locale for translations
147 lang = gettext.translation(self.domain,
148 localedir=localedir,
149 fallback=True)
150
151 if six.PY3:
152 ugettext = lang.gettext
153 else:
154 ugettext = lang.ugettext
155
156 full_msg = (self._left_extra_msg +
157 ugettext(self._msg) +
158 self._right_extra_msg)
159
160 if self.params is not None:
161 full_msg = full_msg % self.params
162
163 return six.text_type(full_msg)
164
165 @property
166 def locale(self):
167 return self._locale
168
169 @locale.setter
170 def locale(self, value):
171 self._locale = value
172 if not self.params:
173 return
174
175 # This Message object may have been constructed with one or more
176 # Message objects as substitution parameters, given as a single
177 # Message, or a tuple or Map containing some, so when setting the
178 # locale for this Message we need to set it for those Messages too.
179 if isinstance(self.params, Message):
180 self.params.locale = value
181 return
182 if isinstance(self.params, tuple):
183 for param in self.params:
184 if isinstance(param, Message):
185 param.locale = value
186 return
187 if isinstance(self.params, dict):
188 for param in self.params.values():
189 if isinstance(param, Message):
190 param.locale = value
191
192 def _save_dictionary_parameter(self, dict_param):
193 full_msg = self.data
194 # look for %(blah) fields in string;
195 # ignore %% and deal with the
196 # case where % is first character on the line
197 keys = re.findall('(?:[^%]|^)?%\((\w*)\)[a-z]', full_msg)
198
199 # if we don't find any %(blah) blocks but have a %s
200 if not keys and re.findall('(?:[^%]|^)%[a-z]', full_msg):
201 # apparently the full dictionary is the parameter
202 params = copy.deepcopy(dict_param)
203 else:
204 params = {}
205 for key in keys:
206 try:
207 params[key] = copy.deepcopy(dict_param[key])
208 except TypeError:
209 # cast uncopyable thing to unicode string
210 params[key] = six.text_type(dict_param[key])
211
212 return params
213
214 def _save_parameters(self, other):
215 # we check for None later to see if
216 # we actually have parameters to inject,
217 # so encapsulate if our parameter is actually None
218 if other is None:
219 self.params = (other, )
220 elif isinstance(other, dict):
221 self.params = self._save_dictionary_parameter(other)
222 else:
223 # fallback to casting to unicode,
224 # this will handle the problematic python code-like
225 # objects that cannot be deep-copied
226 try:
227 self.params = copy.deepcopy(other)
228 except TypeError:
229 self.params = six.text_type(other)
230
231 return self
232
233 # overrides to be more string-like
234 def __unicode__(self):
235 return self.data
236
237 def __str__(self):
238 if six.PY3:
239 return self.__unicode__()
240 return self.data.encode('utf-8')
241
242 def __getstate__(self):
243 to_copy = ['_msg', '_right_extra_msg', '_left_extra_msg',
244 'domain', 'params', '_locale']
245 new_dict = self.__dict__.fromkeys(to_copy)
246 for attr in to_copy:
247 new_dict[attr] = copy.deepcopy(self.__dict__[attr])
248
249 return new_dict
250
251 def __setstate__(self, state):
252 for (k, v) in state.items():
253 setattr(self, k, v)
254
255 # operator overloads
256 def __add__(self, other):
257 copied = copy.deepcopy(self)
258 copied._right_extra_msg += other.__str__()
259 return copied
260
261 def __radd__(self, other):
262 copied = copy.deepcopy(self)
263 copied._left_extra_msg += other.__str__()
264 return copied
265
266 def __mod__(self, other):
267 # do a format string to catch and raise
268 # any possible KeyErrors from missing parameters
269 self.data % other
270 copied = copy.deepcopy(self)
271 return copied._save_parameters(other)
272
273 def __mul__(self, other):
274 return self.data * other
275
276 def __rmul__(self, other):
277 return other * self.data
278
279 def __getitem__(self, key):
280 return self.data[key]
281
282 def __getslice__(self, start, end):
283 return self.data.__getslice__(start, end)
284
285 def __getattribute__(self, name):
286 # NOTE(mrodden): handle lossy operations that we can't deal with yet
287 # These override the UserString implementation, since UserString
288 # uses our __class__ attribute to try and build a new message
289 # after running the inner data string through the operation.
290 # At that point, we have lost the gettext message id and can just
291 # safely resolve to a string instead.
292 ops = ['capitalize', 'center', 'decode', 'encode',
293 'expandtabs', 'ljust', 'lstrip', 'replace', 'rjust', 'rstrip',
294 'strip', 'swapcase', 'title', 'translate', 'upper', 'zfill']
295 if name in ops:
296 return getattr(self.data, name)
297 else:
298 return _userString.UserString.__getattribute__(self, name)
299
300
301def get_available_languages(domain):
302 """Lists the available languages for the given translation domain.
303
304 :param domain: the domain to get languages for
305 """
306 if domain in _AVAILABLE_LANGUAGES:
307 return copy.copy(_AVAILABLE_LANGUAGES[domain])
308
309 localedir = '%s_LOCALEDIR' % domain.upper()
310 find = lambda x: gettext.find(domain,
311 localedir=os.environ.get(localedir),
312 languages=[x])
313
314 # NOTE(mrodden): en_US should always be available (and first in case
315 # order matters) since our in-line message strings are en_US
316 language_list = ['en_US']
317 # NOTE(luisg): Babel <1.0 used a function called list(), which was
318 # renamed to locale_identifiers() in >=1.0, the requirements master list
319 # requires >=0.9.6, uncapped, so defensively work with both. We can remove
320 # this check when the master list updates to >=1.0, and all projects udpate
321 list_identifiers = (getattr(localedata, 'list', None) or
322 getattr(localedata, 'locale_identifiers'))
323 locale_identifiers = list_identifiers()
324 for i in locale_identifiers:
325 if find(i) is not None:
326 language_list.append(i)
327 _AVAILABLE_LANGUAGES[domain] = language_list
328 return copy.copy(language_list)
329
330
331def get_localized_message(message, user_locale):
332 """Gets a localized version of the given message in the given locale."""
333 if isinstance(message, Message):
334 if user_locale:
335 message.locale = user_locale
336 return six.text_type(message)
337 else:
338 return message
339
340
341class LocaleHandler(logging.Handler):
342 """Handler that can have a locale associated to translate Messages.
343
344 A quick example of how to utilize the Message class above.
345 LocaleHandler takes a locale and a target logging.Handler object
346 to forward LogRecord objects to after translating the internal Message.
347 """
348
349 def __init__(self, locale, target):
350 """Initialize a LocaleHandler
351
352 :param locale: locale to use for translating messages
353 :param target: logging.Handler object to forward
354 LogRecord objects to after translation
355 """
356 logging.Handler.__init__(self)
357 self.locale = locale
358 self.target = target
359
360 def emit(self, record):
361 if isinstance(record.msg, Message):
362 # set the locale and resolve to a string
363 record.msg.locale = self.locale
364
365 self.target.emit(record)
diff --git a/python-agent/muranoagent/openstack/common/importutils.py b/python-agent/muranoagent/openstack/common/importutils.py
new file mode 100644
index 0000000..7a303f9
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/importutils.py
@@ -0,0 +1,68 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2011 OpenStack Foundation.
4# All Rights Reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
18"""
19Import related utilities and helper functions.
20"""
21
22import sys
23import traceback
24
25
26def import_class(import_str):
27 """Returns a class from a string including module and class."""
28 mod_str, _sep, class_str = import_str.rpartition('.')
29 try:
30 __import__(mod_str)
31 return getattr(sys.modules[mod_str], class_str)
32 except (ValueError, AttributeError):
33 raise ImportError('Class %s cannot be found (%s)' %
34 (class_str,
35 traceback.format_exception(*sys.exc_info())))
36
37
38def import_object(import_str, *args, **kwargs):
39 """Import a class and return an instance of it."""
40 return import_class(import_str)(*args, **kwargs)
41
42
43def import_object_ns(name_space, import_str, *args, **kwargs):
44 """Tries to import object from default namespace.
45
46 Imports a class and return an instance of it, first by trying
47 to find the class in a default namespace, then failing back to
48 a full path if not found in the default namespace.
49 """
50 import_value = "%s.%s" % (name_space, import_str)
51 try:
52 return import_class(import_value)(*args, **kwargs)
53 except ImportError:
54 return import_class(import_str)(*args, **kwargs)
55
56
57def import_module(import_str):
58 """Import a module."""
59 __import__(import_str)
60 return sys.modules[import_str]
61
62
63def try_import(import_str, default=None):
64 """Try to import a module and if it fails return default."""
65 try:
66 return import_module(import_str)
67 except ImportError:
68 return default
diff --git a/python-agent/muranoagent/openstack/common/jsonutils.py b/python-agent/muranoagent/openstack/common/jsonutils.py
new file mode 100644
index 0000000..8bc418b
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/jsonutils.py
@@ -0,0 +1,180 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2010 United States Government as represented by the
4# Administrator of the National Aeronautics and Space Administration.
5# Copyright 2011 Justin Santa Barbara
6# All Rights Reserved.
7#
8# Licensed under the Apache License, Version 2.0 (the "License"); you may
9# not use this file except in compliance with the License. You may obtain
10# a copy of the License at
11#
12# http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17# License for the specific language governing permissions and limitations
18# under the License.
19
20'''
21JSON related utilities.
22
23This module provides a few things:
24
25 1) A handy function for getting an object down to something that can be
26 JSON serialized. See to_primitive().
27
28 2) Wrappers around loads() and dumps(). The dumps() wrapper will
29 automatically use to_primitive() for you if needed.
30
31 3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
32 is available.
33'''
34
35
36import datetime
37import functools
38import inspect
39import itertools
40import json
41try:
42 import xmlrpclib
43except ImportError:
44 # NOTE(jd): xmlrpclib is not shipped with Python 3
45 xmlrpclib = None
46
47import six
48
49from muranoagent.openstack.common import gettextutils
50from muranoagent.openstack.common import importutils
51from muranoagent.openstack.common import timeutils
52
53netaddr = importutils.try_import("netaddr")
54
55_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
56 inspect.isfunction, inspect.isgeneratorfunction,
57 inspect.isgenerator, inspect.istraceback, inspect.isframe,
58 inspect.iscode, inspect.isbuiltin, inspect.isroutine,
59 inspect.isabstract]
60
61_simple_types = (six.string_types + six.integer_types
62 + (type(None), bool, float))
63
64
65def to_primitive(value, convert_instances=False, convert_datetime=True,
66 level=0, max_depth=3):
67 """Convert a complex object into primitives.
68
69 Handy for JSON serialization. We can optionally handle instances,
70 but since this is a recursive function, we could have cyclical
71 data structures.
72
73 To handle cyclical data structures we could track the actual objects
74 visited in a set, but not all objects are hashable. Instead we just
75 track the depth of the object inspections and don't go too deep.
76
77 Therefore, convert_instances=True is lossy ... be aware.
78
79 """
80 # handle obvious types first - order of basic types determined by running
81 # full tests on nova project, resulting in the following counts:
82 # 572754 <type 'NoneType'>
83 # 460353 <type 'int'>
84 # 379632 <type 'unicode'>
85 # 274610 <type 'str'>
86 # 199918 <type 'dict'>
87 # 114200 <type 'datetime.datetime'>
88 # 51817 <type 'bool'>
89 # 26164 <type 'list'>
90 # 6491 <type 'float'>
91 # 283 <type 'tuple'>
92 # 19 <type 'long'>
93 if isinstance(value, _simple_types):
94 return value
95
96 if isinstance(value, datetime.datetime):
97 if convert_datetime:
98 return timeutils.strtime(value)
99 else:
100 return value
101
102 # value of itertools.count doesn't get caught by nasty_type_tests
103 # and results in infinite loop when list(value) is called.
104 if type(value) == itertools.count:
105 return six.text_type(value)
106
107 # FIXME(vish): Workaround for LP bug 852095. Without this workaround,
108 # tests that raise an exception in a mocked method that
109 # has a @wrap_exception with a notifier will fail. If
110 # we up the dependency to 0.5.4 (when it is released) we
111 # can remove this workaround.
112 if getattr(value, '__module__', None) == 'mox':
113 return 'mock'
114
115 if level > max_depth:
116 return '?'
117
118 # The try block may not be necessary after the class check above,
119 # but just in case ...
120 try:
121 recursive = functools.partial(to_primitive,
122 convert_instances=convert_instances,
123 convert_datetime=convert_datetime,
124 level=level,
125 max_depth=max_depth)
126 if isinstance(value, dict):
127 return dict((k, recursive(v)) for k, v in value.iteritems())
128 elif isinstance(value, (list, tuple)):
129 return [recursive(lv) for lv in value]
130
131 # It's not clear why xmlrpclib created their own DateTime type, but
132 # for our purposes, make it a datetime type which is explicitly
133 # handled
134 if xmlrpclib and isinstance(value, xmlrpclib.DateTime):
135 value = datetime.datetime(*tuple(value.timetuple())[:6])
136
137 if convert_datetime and isinstance(value, datetime.datetime):
138 return timeutils.strtime(value)
139 elif isinstance(value, gettextutils.Message):
140 return value.data
141 elif hasattr(value, 'iteritems'):
142 return recursive(dict(value.iteritems()), level=level + 1)
143 elif hasattr(value, '__iter__'):
144 return recursive(list(value))
145 elif convert_instances and hasattr(value, '__dict__'):
146 # Likely an instance of something. Watch for cycles.
147 # Ignore class member vars.
148 return recursive(value.__dict__, level=level + 1)
149 elif netaddr and isinstance(value, netaddr.IPAddress):
150 return six.text_type(value)
151 else:
152 if any(test(value) for test in _nasty_type_tests):
153 return six.text_type(value)
154 return value
155 except TypeError:
156 # Class objects are tricky since they may define something like
157 # __iter__ defined but it isn't callable as list().
158 return six.text_type(value)
159
160
161def dumps(value, default=to_primitive, **kwargs):
162 return json.dumps(value, default=default, **kwargs)
163
164
165def loads(s):
166 return json.loads(s)
167
168
169def load(s):
170 return json.load(s)
171
172
173try:
174 import anyjson
175except ImportError:
176 pass
177else:
178 anyjson._modules.append((__name__, 'dumps', TypeError,
179 'loads', ValueError, 'load'))
180 anyjson.force_implementation(__name__)
diff --git a/python-agent/muranoagent/openstack/common/local.py b/python-agent/muranoagent/openstack/common/local.py
new file mode 100644
index 0000000..e82f17d
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/local.py
@@ -0,0 +1,47 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2011 OpenStack Foundation.
4# All Rights Reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
18"""Local storage of variables using weak references"""
19
20import threading
21import weakref
22
23
24class WeakLocal(threading.local):
25 def __getattribute__(self, attr):
26 rval = super(WeakLocal, self).__getattribute__(attr)
27 if rval:
28 # NOTE(mikal): this bit is confusing. What is stored is a weak
29 # reference, not the value itself. We therefore need to lookup
30 # the weak reference and return the inner value here.
31 rval = rval()
32 return rval
33
34 def __setattr__(self, attr, value):
35 value = weakref.ref(value)
36 return super(WeakLocal, self).__setattr__(attr, value)
37
38
39# NOTE(mikal): the name "store" should be deprecated in the future
40store = WeakLocal()
41
42# A "weak" store uses weak references and allows an object to fall out of scope
43# when it falls out of scope in the code that uses the thread local storage. A
44# "strong" store will hold a reference to the object so that it never falls out
45# of scope.
46weak_store = WeakLocal()
47strong_store = threading.local()
diff --git a/python-agent/muranoagent/openstack/common/log.py b/python-agent/muranoagent/openstack/common/log.py
new file mode 100644
index 0000000..7b2d99e
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/log.py
@@ -0,0 +1,566 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2011 OpenStack Foundation.
4# Copyright 2010 United States Government as represented by the
5# Administrator of the National Aeronautics and Space Administration.
6# All Rights Reserved.
7#
8# Licensed under the Apache License, Version 2.0 (the "License"); you may
9# not use this file except in compliance with the License. You may obtain
10# a copy of the License at
11#
12# http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17# License for the specific language governing permissions and limitations
18# under the License.
19
20"""Openstack logging handler.
21
22This module adds to logging functionality by adding the option to specify
23a context object when calling the various log methods. If the context object
24is not specified, default formatting is used. Additionally, an instance uuid
25may be passed as part of the log message, which is intended to make it easier
26for admins to find messages related to a specific instance.
27
28It also allows setting of formatting information through conf.
29
30"""
31
32import inspect
33import itertools
34import logging
35import logging.config
36import logging.handlers
37import os
38import sys
39import traceback
40
41from oslo.config import cfg
42from six import moves
43
44from muranoagent.openstack.common.gettextutils import _ # noqa
45from muranoagent.openstack.common import importutils
46from muranoagent.openstack.common import jsonutils
47from muranoagent.openstack.common import local
48
49
50_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
51
52common_cli_opts = [
53 cfg.BoolOpt('debug',
54 short='d',
55 default=False,
56 help='Print debugging output (set logging level to '
57 'DEBUG instead of default WARNING level).'),
58 cfg.BoolOpt('verbose',
59 short='v',
60 default=False,
61 help='Print more verbose output (set logging level to '
62 'INFO instead of default WARNING level).'),
63]
64
65logging_cli_opts = [
66 cfg.StrOpt('log-config',
67 metavar='PATH',
68 help='If this option is specified, the logging configuration '
69 'file specified is used and overrides any other logging '
70 'options specified. Please see the Python logging module '
71 'documentation for details on logging configuration '
72 'files.'),
73 cfg.StrOpt('log-format',
74 default=None,
75 metavar='FORMAT',
76 help='DEPRECATED. '
77 'A logging.Formatter log message format string which may '
78 'use any of the available logging.LogRecord attributes. '
79 'This option is deprecated. Please use '
80 'logging_context_format_string and '
81 'logging_default_format_string instead.'),
82 cfg.StrOpt('log-date-format',
83 default=_DEFAULT_LOG_DATE_FORMAT,
84 metavar='DATE_FORMAT',
85 help='Format string for %%(asctime)s in log records. '
86 'Default: %(default)s'),
87 cfg.StrOpt('log-file',
88 metavar='PATH',
89 deprecated_name='logfile',
90 help='(Optional) Name of log file to output to. '
91 'If no default is set, logging will go to stdout.'),
92 cfg.StrOpt('log-dir',
93 deprecated_name='logdir',
94 help='(Optional) The base directory used for relative '
95 '--log-file paths'),
96 cfg.BoolOpt('use-syslog',
97 default=False,
98 help='Use syslog for logging.'),
99 cfg.StrOpt('syslog-log-facility',
100 default='LOG_USER',
101 help='syslog facility to receive log lines')
102]
103
104generic_log_opts = [
105 cfg.BoolOpt('use_stderr',
106 default=True,
107 help='Log output to standard error')
108]
109
110log_opts = [
111 cfg.StrOpt('logging_context_format_string',
112 default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
113 '%(name)s [%(request_id)s %(user)s %(tenant)s] '
114 '%(instance)s%(message)s',
115 help='format string to use for log messages with context'),
116 cfg.StrOpt('logging_default_format_string',
117 default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
118 '%(name)s [-] %(instance)s%(message)s',
119 help='format string to use for log messages without context'),
120 cfg.StrOpt('logging_debug_format_suffix',
121 default='%(funcName)s %(pathname)s:%(lineno)d',
122 help='data to append to log format when level is DEBUG'),
123 cfg.StrOpt('logging_exception_prefix',
124 default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
125 '%(instance)s',
126 help='prefix each line of exception output with this format'),
127 cfg.ListOpt('default_log_levels',
128 default=[
129 'amqplib=WARN',
130 'sqlalchemy=WARN',
131 'boto=WARN',
132 'suds=INFO',
133 'keystone=INFO',
134 'eventlet.wsgi.server=WARN'
135 ],
136 help='list of logger=LEVEL pairs'),
137 cfg.BoolOpt('publish_errors',
138 default=False,
139 help='publish error events'),
140 cfg.BoolOpt('fatal_deprecations',
141 default=False,
142 help='make deprecations fatal'),
143
144 # NOTE(mikal): there are two options here because sometimes we are handed
145 # a full instance (and could include more information), and other times we
146 # are just handed a UUID for the instance.
147 cfg.StrOpt('instance_format',
148 default='[instance: %(uuid)s] ',
149 help='If an instance is passed with the log message, format '
150 'it like this'),
151 cfg.StrOpt('instance_uuid_format',
152 default='[instance: %(uuid)s] ',
153 help='If an instance UUID is passed with the log message, '
154 'format it like this'),
155]
156
157CONF = cfg.CONF
158CONF.register_cli_opts(common_cli_opts)
159CONF.register_cli_opts(logging_cli_opts)
160CONF.register_opts(generic_log_opts)
161CONF.register_opts(log_opts)
162
163# our new audit level
164# NOTE(jkoelker) Since we synthesized an audit level, make the logging
165# module aware of it so it acts like other levels.
166logging.AUDIT = logging.INFO + 1
167logging.addLevelName(logging.AUDIT, 'AUDIT')
168
169
170try:
171 NullHandler = logging.NullHandler
172except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7
173 class NullHandler(logging.Handler):
174 def handle(self, record):
175 pass
176
177 def emit(self, record):
178 pass
179
180 def createLock(self):
181 self.lock = None
182
183
184def _dictify_context(context):
185 if context is None:
186 return None
187 if not isinstance(context, dict) and getattr(context, 'to_dict', None):
188 context = context.to_dict()
189 return context
190
191
192def _get_binary_name():
193 return os.path.basename(inspect.stack()[-1][1])
194
195
196def _get_log_file_path(binary=None):
197 logfile = CONF.log_file
198 logdir = CONF.log_dir
199
200 if logfile and not logdir:
201 return logfile
202
203 if logfile and logdir:
204 return os.path.join(logdir, logfile)
205
206 if logdir:
207 binary = binary or _get_binary_name()
208 return '%s.log' % (os.path.join(logdir, binary),)
209
210
211class BaseLoggerAdapter(logging.LoggerAdapter):
212
213 def audit(self, msg, *args, **kwargs):
214 self.log(logging.AUDIT, msg, *args, **kwargs)
215
216
217class LazyAdapter(BaseLoggerAdapter):
218 def __init__(self, name='unknown', version='unknown'):
219 self._logger = None
220 self.extra = {}
221 self.name = name
222 self.version = version
223
224 @property
225 def logger(self):
226 if not self._logger:
227 self._logger = getLogger(self.name, self.version)
228 return self._logger
229
230
231class ContextAdapter(BaseLoggerAdapter):
232 warn = logging.LoggerAdapter.warning
233
234 def __init__(self, logger, project_name, version_string):
235 self.logger = logger
236 self.project = project_name
237 self.version = version_string
238
239 @property
240 def handlers(self):
241 return self.logger.handlers
242
243 def deprecated(self, msg, *args, **kwargs):
244 stdmsg = _("Deprecated: %s") % msg
245 if CONF.fatal_deprecations:
246 self.critical(stdmsg, *args, **kwargs)
247 raise DeprecatedConfig(msg=stdmsg)
248 else:
249 self.warn(stdmsg, *args, **kwargs)
250
251 def process(self, msg, kwargs):
252 # NOTE(mrodden): catch any Message/other object and
253 # coerce to unicode before they can get
254 # to the python logging and possibly
255 # cause string encoding trouble
256 if not isinstance(msg, basestring):
257 msg = unicode(msg)
258
259 if 'extra' not in kwargs:
260 kwargs['extra'] = {}
261 extra = kwargs['extra']
262
263 context = kwargs.pop('context', None)
264 if not context:
265 context = getattr(local.store, 'context', None)
266 if context:
267 extra.update(_dictify_context(context))
268
269 instance = kwargs.pop('instance', None)
270 instance_uuid = (extra.get('instance_uuid', None) or
271 kwargs.pop('instance_uuid', None))
272 instance_extra = ''
273 if instance:
274 instance_extra = CONF.instance_format % instance
275 elif instance_uuid:
276 instance_extra = (CONF.instance_uuid_format
277 % {'uuid': instance_uuid})
278 extra.update({'instance': instance_extra})
279
280 extra.update({"project": self.project})
281 extra.update({"version": self.version})
282 extra['extra'] = extra.copy()
283 return msg, kwargs
284
285
286class JSONFormatter(logging.Formatter):
287 def __init__(self, fmt=None, datefmt=None):
288 # NOTE(jkoelker) we ignore the fmt argument, but its still there
289 # since logging.config.fileConfig passes it.
290 self.datefmt = datefmt
291
292 def formatException(self, ei, strip_newlines=True):
293 lines = traceback.format_exception(*ei)
294 if strip_newlines:
295 lines = [itertools.ifilter(
296 lambda x: x,
297 line.rstrip().splitlines()) for line in lines]
298 lines = list(itertools.chain(*lines))
299 return lines
300
301 def format(self, record):
302 message = {'message': record.getMessage(),
303 'asctime': self.formatTime(record, self.datefmt),
304 'name': record.name,
305 'msg': record.msg,
306 'args': record.args,
307 'levelname': record.levelname,
308 'levelno': record.levelno,
309 'pathname': record.pathname,
310 'filename': record.filename,
311 'module': record.module,
312 'lineno': record.lineno,
313 'funcname': record.funcName,
314 'created': record.created,
315 'msecs': record.msecs,
316 'relative_created': record.relativeCreated,
317 'thread': record.thread,
318 'thread_name': record.threadName,
319 'process_name': record.processName,
320 'process': record.process,
321 'traceback': None}
322
323 if hasattr(record, 'extra'):
324 message['extra'] = record.extra
325
326 if record.exc_info:
327 message['traceback'] = self.formatException(record.exc_info)
328
329 return jsonutils.dumps(message)
330
331
332def _create_logging_excepthook(product_name):
333 def logging_excepthook(type, value, tb):
334 extra = {}
335 if CONF.verbose:
336 extra['exc_info'] = (type, value, tb)
337 getLogger(product_name).critical(str(value), **extra)
338 return logging_excepthook
339
340
341class LogConfigError(Exception):
342
343 message = _('Error loading logging config %(log_config)s: %(err_msg)s')
344
345 def __init__(self, log_config, err_msg):
346 self.log_config = log_config
347 self.err_msg = err_msg
348
349 def __str__(self):
350 return self.message % dict(log_config=self.log_config,
351 err_msg=self.err_msg)
352
353
354def _load_log_config(log_config):
355 try:
356 logging.config.fileConfig(log_config)
357 except moves.configparser.Error as exc:
358 raise LogConfigError(log_config, str(exc))
359
360
361def setup(product_name):
362 """Setup logging."""
363 if CONF.log_config:
364 _load_log_config(CONF.log_config)
365 else:
366 _setup_logging_from_conf()
367 sys.excepthook = _create_logging_excepthook(product_name)
368
369
370def set_defaults(logging_context_format_string):
371 cfg.set_defaults(log_opts,
372 logging_context_format_string=
373 logging_context_format_string)
374
375
376def _find_facility_from_conf():
377 facility_names = logging.handlers.SysLogHandler.facility_names
378 facility = getattr(logging.handlers.SysLogHandler,
379 CONF.syslog_log_facility,
380 None)
381
382 if facility is None and CONF.syslog_log_facility in facility_names:
383 facility = facility_names.get(CONF.syslog_log_facility)
384
385 if facility is None:
386 valid_facilities = facility_names.keys()
387 consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
388 'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
389 'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
390 'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
391 'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
392 valid_facilities.extend(consts)
393 raise TypeError(_('syslog facility must be one of: %s') %
394 ', '.join("'%s'" % fac
395 for fac in valid_facilities))
396
397 return facility
398
399
400def _setup_logging_from_conf():
401 log_root = getLogger(None).logger
402 for handler in log_root.handlers:
403 log_root.removeHandler(handler)
404
405 if CONF.use_syslog:
406 facility = _find_facility_from_conf()
407 syslog = logging.handlers.SysLogHandler(address='/dev/log',
408 facility=facility)
409 log_root.addHandler(syslog)
410
411 logpath = _get_log_file_path()
412 if logpath:
413 filelog = logging.handlers.WatchedFileHandler(logpath)
414 log_root.addHandler(filelog)
415
416 if CONF.use_stderr:
417 streamlog = ColorHandler()
418 log_root.addHandler(streamlog)
419
420 elif not CONF.log_file:
421 # pass sys.stdout as a positional argument
422 # python2.6 calls the argument strm, in 2.7 it's stream
423 streamlog = logging.StreamHandler(sys.stdout)
424 log_root.addHandler(streamlog)
425
426 if CONF.publish_errors:
427 handler = importutils.import_object(
428 "muranoagent.openstack.common.log_handler.PublishErrorsHandler",
429 logging.ERROR)
430 log_root.addHandler(handler)
431
432 datefmt = CONF.log_date_format
433 for handler in log_root.handlers:
434 # NOTE(alaski): CONF.log_format overrides everything currently. This
435 # should be deprecated in favor of context aware formatting.
436 if CONF.log_format:
437 handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
438 datefmt=datefmt))
439 log_root.info('Deprecated: log_format is now deprecated and will '
440 'be removed in the next release')
441 else:
442 handler.setFormatter(ContextFormatter(datefmt=datefmt))
443
444 if CONF.debug:
445 log_root.setLevel(logging.DEBUG)
446 elif CONF.verbose:
447 log_root.setLevel(logging.INFO)
448 else:
449 log_root.setLevel(logging.WARNING)
450
451 for pair in CONF.default_log_levels:
452 mod, _sep, level_name = pair.partition('=')
453 level = logging.getLevelName(level_name)
454 logger = logging.getLogger(mod)
455 logger.setLevel(level)
456
457_loggers = {}
458
459
460def getLogger(name='unknown', version='unknown'):
461 if name not in _loggers:
462 _loggers[name] = ContextAdapter(logging.getLogger(name),
463 name,
464 version)
465 return _loggers[name]
466
467
468def getLazyLogger(name='unknown', version='unknown'):
469 """Returns lazy logger.
470
471 Creates a pass-through logger that does not create the real logger
472 until it is really needed and delegates all calls to the real logger
473 once it is created.
474 """
475 return LazyAdapter(name, version)
476
477
478class WritableLogger(object):
479 """A thin wrapper that responds to `write` and logs."""
480
481 def __init__(self, logger, level=logging.INFO):
482 self.logger = logger
483 self.level = level
484
485 def write(self, msg):
486 self.logger.log(self.level, msg)
487
488
489class ContextFormatter(logging.Formatter):
490 """A context.RequestContext aware formatter configured through flags.
491
492 The flags used to set format strings are: logging_context_format_string
493 and logging_default_format_string. You can also specify
494 logging_debug_format_suffix to append extra formatting if the log level is
495 debug.
496
497 For information about what variables are available for the formatter see:
498 http://docs.python.org/library/logging.html#formatter
499
500 """
501
502 def format(self, record):
503 """Uses contextstring if request_id is set, otherwise default."""
504 # NOTE(sdague): default the fancier formating params
505 # to an empty string so we don't throw an exception if
506 # they get used
507 for key in ('instance', 'color'):
508 if key not in record.__dict__:
509 record.__dict__[key] = ''
510
511 if record.__dict__.get('request_id', None):
512 self._fmt = CONF.logging_context_format_string
513 else:
514 self._fmt = CONF.logging_default_format_string
515
516 if (record.levelno == logging.DEBUG and
517 CONF.logging_debug_format_suffix):
518 self._fmt += " " + CONF.logging_debug_format_suffix
519
520 # Cache this on the record, Logger will respect our formated copy
521 if record.exc_info:
522 record.exc_text = self.formatException(record.exc_info, record)
523 return logging.Formatter.format(self, record)
524
525 def formatException(self, exc_info, record=None):
526 """Format exception output with CONF.logging_exception_prefix."""
527 if not record:
528 return logging.Formatter.formatException(self, exc_info)
529
530 stringbuffer = moves.StringIO()
531 traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
532 None, stringbuffer)
533 lines = stringbuffer.getvalue().split('\n')
534 stringbuffer.close()
535
536 if CONF.logging_exception_prefix.find('%(asctime)') != -1:
537 record.asctime = self.formatTime(record, self.datefmt)
538
539 formatted_lines = []
540 for line in lines:
541 pl = CONF.logging_exception_prefix % record.__dict__
542 fl = '%s%s' % (pl, line)
543 formatted_lines.append(fl)
544 return '\n'.join(formatted_lines)
545
546
547class ColorHandler(logging.StreamHandler):
548 LEVEL_COLORS = {
549 logging.DEBUG: '\033[00;32m', # GREEN
550 logging.INFO: '\033[00;36m', # CYAN
551 logging.AUDIT: '\033[01;36m', # BOLD CYAN
552 logging.WARN: '\033[01;33m', # BOLD YELLOW
553 logging.ERROR: '\033[01;31m', # BOLD RED
554 logging.CRITICAL: '\033[01;31m', # BOLD RED
555 }
556
557 def format(self, record):
558 record.color = self.LEVEL_COLORS[record.levelno]
559 return logging.StreamHandler.format(self, record)
560
561
562class DeprecatedConfig(Exception):
563 message = _("Fatal call to deprecated config: %(msg)s")
564
565 def __init__(self, msg):
566 super(Exception, self).__init__(self.message % dict(msg=msg))
diff --git a/python-agent/muranoagent/openstack/common/loopingcall.py b/python-agent/muranoagent/openstack/common/loopingcall.py
new file mode 100644
index 0000000..1f65be6
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/loopingcall.py
@@ -0,0 +1,147 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2010 United States Government as represented by the
4# Administrator of the National Aeronautics and Space Administration.
5# Copyright 2011 Justin Santa Barbara
6# All Rights Reserved.
7#
8# Licensed under the Apache License, Version 2.0 (the "License"); you may
9# not use this file except in compliance with the License. You may obtain
10# a copy of the License at
11#
12# http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17# License for the specific language governing permissions and limitations
18# under the License.
19
20import sys
21
22from eventlet import event
23from eventlet import greenthread
24
25from muranoagent.openstack.common.gettextutils import _ # noqa
26from muranoagent.openstack.common import log as logging
27from muranoagent.openstack.common import timeutils
28
29LOG = logging.getLogger(__name__)
30
31
32class LoopingCallDone(Exception):
33 """Exception to break out and stop a LoopingCall.
34
35 The poll-function passed to LoopingCall can raise this exception to
36 break out of the loop normally. This is somewhat analogous to
37 StopIteration.
38
39 An optional return-value can be included as the argument to the exception;
40 this return-value will be returned by LoopingCall.wait()
41
42 """
43
44 def __init__(self, retvalue=True):
45 """:param retvalue: Value that LoopingCall.wait() should return."""
46 self.retvalue = retvalue
47
48
49class LoopingCallBase(object):
50 def __init__(self, f=None, *args, **kw):
51 self.args = args
52 self.kw = kw
53 self.f = f
54 self._running = False
55 self.done = None
56
57 def stop(self):
58 self._running = False
59
60 def wait(self):
61 return self.done.wait()
62
63
64class FixedIntervalLoopingCall(LoopingCallBase):
65 """A fixed interval looping call."""
66
67 def start(self, interval, initial_delay=None):
68 self._running = True
69 done = event.Event()
70
71 def _inner():
72 if initial_delay:
73 greenthread.sleep(initial_delay)
74
75 try:
76 while self._running:
77 start = timeutils.utcnow()
78 self.f(*self.args, **self.kw)
79 end = timeutils.utcnow()
80 if not self._running:
81 break
82 delay = interval - timeutils.delta_seconds(start, end)
83 if delay <= 0:
84 LOG.warn(_('task run outlasted interval by %s sec') %
85 -delay)
86 greenthread.sleep(delay if delay > 0 else 0)
87 except LoopingCallDone as e:
88 self.stop()
89 done.send(e.retvalue)
90 except Exception:
91 LOG.exception(_('in fixed duration looping call'))
92 done.send_exception(*sys.exc_info())
93 return
94 else:
95 done.send(True)
96
97 self.done = done
98
99 greenthread.spawn_n(_inner)
100 return self.done
101
102
103# TODO(mikal): this class name is deprecated in Havana and should be removed
104# in the I release
105LoopingCall = FixedIntervalLoopingCall
106
107
108class DynamicLoopingCall(LoopingCallBase):
109 """A looping call which sleeps until the next known event.
110
111 The function called should return how long to sleep for before being
112 called again.
113 """
114
115 def start(self, initial_delay=None, periodic_interval_max=None):
116 self._running = True
117 done = event.Event()
118
119 def _inner():
120 if initial_delay:
121 greenthread.sleep(initial_delay)
122
123 try:
124 while self._running:
125 idle = self.f(*self.args, **self.kw)
126 if not self._running:
127 break
128
129 if periodic_interval_max is not None:
130 idle = min(idle, periodic_interval_max)
131 LOG.debug(_('Dynamic looping call sleeping for %.02f '
132 'seconds'), idle)
133 greenthread.sleep(idle)
134 except LoopingCallDone as e:
135 self.stop()
136 done.send(e.retvalue)
137 except Exception:
138 LOG.exception(_('in dynamic looping call'))
139 done.send_exception(*sys.exc_info())
140 return
141 else:
142 done.send(True)
143
144 self.done = done
145
146 greenthread.spawn(_inner)
147 return self.done
diff --git a/python-agent/muranoagent/openstack/common/service.py b/python-agent/muranoagent/openstack/common/service.py
new file mode 100644
index 0000000..1e98d10
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/service.py
@@ -0,0 +1,459 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2010 United States Government as represented by the
4# Administrator of the National Aeronautics and Space Administration.
5# Copyright 2011 Justin Santa Barbara
6# All Rights Reserved.
7#
8# Licensed under the Apache License, Version 2.0 (the "License"); you may
9# not use this file except in compliance with the License. You may obtain
10# a copy of the License at
11#
12# http://www.apache.org/licenses/LICENSE-2.0
13#
14# Unless required by applicable law or agreed to in writing, software
15# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
16# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
17# License for the specific language governing permissions and limitations
18# under the License.
19
20"""Generic Node base class for all workers that run on hosts."""
21
22import errno
23import os
24import random
25import signal
26import sys
27import time
28
29import eventlet
30from eventlet import event
31import logging as std_logging
32from oslo.config import cfg
33
34from muranoagent.openstack.common import eventlet_backdoor
35from muranoagent.openstack.common.gettextutils import _ # noqa
36from muranoagent.openstack.common import importutils
37from muranoagent.openstack.common import log as logging
38from muranoagent.openstack.common import threadgroup
39
40
41rpc = importutils.try_import('muranoagent.openstack.common.rpc')
42CONF = cfg.CONF
43LOG = logging.getLogger(__name__)
44
45
46def _sighup_supported():
47 return hasattr(signal, 'SIGHUP')
48
49
50def _is_sighup(signo):
51 return _sighup_supported() and signo == signal.SIGHUP
52
53
54def _signo_to_signame(signo):
55 signals = {signal.SIGTERM: 'SIGTERM',
56 signal.SIGINT: 'SIGINT'}
57 if _sighup_supported():
58 signals[signal.SIGHUP] = 'SIGHUP'
59 return signals[signo]
60
61
62def _set_signals_handler(handler):
63 signal.signal(signal.SIGTERM, handler)
64 signal.signal(signal.SIGINT, handler)
65 if _sighup_supported():
66 signal.signal(signal.SIGHUP, handler)
67
68
69class Launcher(object):
70 """Launch one or more services and wait for them to complete."""
71
72 def __init__(self):
73 """Initialize the service launcher.
74
75 :returns: None
76
77 """
78 self.services = Services()
79 self.backdoor_port = eventlet_backdoor.initialize_if_enabled()
80
81 def launch_service(self, service):
82 """Load and start the given service.
83
84 :param service: The service you would like to start.
85 :returns: None
86
87 """
88 service.backdoor_port = self.backdoor_port
89 self.services.add(service)
90
91 def stop(self):
92 """Stop all services which are currently running.
93
94 :returns: None
95
96 """
97 self.services.stop()
98
99 def wait(self):
100 """Waits until all services have been stopped, and then returns.
101
102 :returns: None
103
104 """
105 self.services.wait()
106
107 def restart(self):
108 """Reload config files and restart service.
109
110 :returns: None
111
112 """
113 cfg.CONF.reload_config_files()
114 self.services.restart()
115
116
117class SignalExit(SystemExit):
118 def __init__(self, signo, exccode=1):
119 super(SignalExit, self).__init__(exccode)
120 self.signo = signo
121
122
123class ServiceLauncher(Launcher):
124 def _handle_signal(self, signo, frame):
125 # Allow the process to be killed again and die from natural causes
126 _set_signals_handler(signal.SIG_DFL)
127 raise SignalExit(signo)
128
129 def handle_signal(self):
130 _set_signals_handler(self._handle_signal)
131
132 def _wait_for_exit_or_signal(self):
133 status = None
134 signo = 0
135
136 LOG.debug(_('Full set of CONF:'))
137 CONF.log_opt_values(LOG, std_logging.DEBUG)
138
139 try:
140 super(ServiceLauncher, self).wait()
141 except SignalExit as exc:
142 signame = _signo_to_signame(exc.signo)
143 LOG.info(_('Caught %s, exiting'), signame)
144 status = exc.code
145 signo = exc.signo
146 except SystemExit as exc:
147 status = exc.code
148 finally:
149 self.stop()
150 if rpc:
151 try:
152 rpc.cleanup()
153 except Exception:
154 # We're shutting down, so it doesn't matter at this point.
155 LOG.exception(_('Exception during rpc cleanup.'))
156
157 return status, signo
158
159 def wait(self):
160 while True:
161 self.handle_signal()
162 status, signo = self._wait_for_exit_or_signal()
163 if not _is_sighup(signo):
164 return status
165 self.restart()
166
167
168class ServiceWrapper(object):
169 def __init__(self, service, workers):
170 self.service = service
171 self.workers = workers
172 self.children = set()
173 self.forktimes = []
174
175
176class ProcessLauncher(object):
177 def __init__(self):
178 self.children = {}
179 self.sigcaught = None
180 self.running = True
181 rfd, self.writepipe = os.pipe()
182 self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')
183 self.handle_signal()
184
185 def handle_signal(self):
186 _set_signals_handler(self._handle_signal)
187
188 def _handle_signal(self, signo, frame):
189 self.sigcaught = signo
190 self.running = False
191
192 # Allow the process to be killed again and die from natural causes
193 _set_signals_handler(signal.SIG_DFL)
194
195 def _pipe_watcher(self):
196 # This will block until the write end is closed when the parent
197 # dies unexpectedly
198 self.readpipe.read()
199
200 LOG.info(_('Parent process has died unexpectedly, exiting'))
201
202 sys.exit(1)
203
204 def _child_process_handle_signal(self):
205 # Setup child signal handlers differently
206 def _sigterm(*args):
207 signal.signal(signal.SIGTERM, signal.SIG_DFL)
208 raise SignalExit(signal.SIGTERM)
209
210 def _sighup(*args):
211 signal.signal(signal.SIGHUP, signal.SIG_DFL)
212 raise SignalExit(signal.SIGHUP)
213
214 signal.signal(signal.SIGTERM, _sigterm)
215 if _sighup_supported():
216 signal.signal(signal.SIGHUP, _sighup)
217 # Block SIGINT and let the parent send us a SIGTERM
218 signal.signal(signal.SIGINT, signal.SIG_IGN)
219
220 def _child_wait_for_exit_or_signal(self, launcher):
221 status = None
222 signo = 0
223
224 # NOTE(johannes): All exceptions are caught to ensure this
225 # doesn't fallback into the loop spawning children. It would
226 # be bad for a child to spawn more children.
227 try:
228 launcher.wait()
229 except SignalExit as exc:
230 signame = _signo_to_signame(exc.signo)
231 LOG.info(_('Caught %s, exiting'), signame)
232 status = exc.code
233 signo = exc.signo
234 except SystemExit as exc:
235 status = exc.code
236 except BaseException:
237 LOG.exception(_('Unhandled exception'))
238 status = 2
239 finally:
240 launcher.stop()
241
242 return status, signo
243
244 def _child_process(self, service):
245 self._child_process_handle_signal()
246
247 # Reopen the eventlet hub to make sure we don't share an epoll
248 # fd with parent and/or siblings, which would be bad
249 eventlet.hubs.use_hub()
250
251 # Close write to ensure only parent has it open
252 os.close(self.writepipe)
253 # Create greenthread to watch for parent to close pipe
254 eventlet.spawn_n(self._pipe_watcher)
255
256 # Reseed random number generator
257 random.seed()
258
259 launcher = Launcher()
260 launcher.launch_service(service)
261 return launcher
262
263 def _start_child(self, wrap):
264 if len(wrap.forktimes) > wrap.workers:
265 # Limit ourselves to one process a second (over the period of
266 # number of workers * 1 second). This will allow workers to
267 # start up quickly but ensure we don't fork off children that
268 # die instantly too quickly.
269 if time.time() - wrap.forktimes[0] < wrap.workers:
270 LOG.info(_('Forking too fast, sleeping'))
271 time.sleep(1)
272
273 wrap.forktimes.pop(0)
274
275 wrap.forktimes.append(time.time())
276
277 pid = os.fork()
278 if pid == 0:
279 launcher = self._child_process(wrap.service)
280 while True:
281 self._child_process_handle_signal()
282 status, signo = self._child_wait_for_exit_or_signal(launcher)
283 if not _is_sighup(signo):
284 break
285 launcher.restart()
286
287 os._exit(status)
288
289 LOG.info(_('Started child %d'), pid)
290
291 wrap.children.add(pid)
292 self.children[pid] = wrap
293
294 return pid
295
296 def launch_service(self, service, workers=1):
297 wrap = ServiceWrapper(service, workers)
298
299 LOG.info(_('Starting %d workers'), wrap.workers)
300 while self.running and len(wrap.children) < wrap.workers:
301 self._start_child(wrap)
302
303 def _wait_child(self):
304 try:
305 # Don't block if no child processes have exited
306 pid, status = os.waitpid(0, os.WNOHANG)
307 if not pid:
308 return None
309 except OSError as exc:
310 if exc.errno not in (errno.EINTR, errno.ECHILD):
311 raise
312 return None
313
314 if os.WIFSIGNALED(status):
315 sig = os.WTERMSIG(status)
316 LOG.info(_('Child %(pid)d killed by signal %(sig)d'),
317 dict(pid=pid, sig=sig))
318 else:
319 code = os.WEXITSTATUS(status)
320 LOG.info(_('Child %(pid)s exited with status %(code)d'),
321 dict(pid=pid, code=code))
322
323 if pid not in self.children:
324 LOG.warning(_('pid %d not in child list'), pid)
325 return None
326
327 wrap = self.children.pop(pid)
328 wrap.children.remove(pid)
329 return wrap
330
331 def _respawn_children(self):
332 while self.running:
333 wrap = self._wait_child()
334 if not wrap:
335 # Yield to other threads if no children have exited
336 # Sleep for a short time to avoid excessive CPU usage
337 # (see bug #1095346)
338 eventlet.greenthread.sleep(.01)
339 continue
340 while self.running and len(wrap.children) < wrap.workers:
341 self._start_child(wrap)
342
343 def wait(self):
344 """Loop waiting on children to die and respawning as necessary."""
345
346 LOG.debug(_('Full set of CONF:'))
347 CONF.log_opt_values(LOG, std_logging.DEBUG)
348
349 while True:
350 self.handle_signal()
351 self._respawn_children()
352 if self.sigcaught:
353 signame = _signo_to_signame(self.sigcaught)
354 LOG.info(_('Caught %s, stopping children'), signame)
355 if not _is_sighup(self.sigcaught):
356 break
357
358 for pid in self.children:
359 os.kill(pid, signal.SIGHUP)
360 self.running = True
361 self.sigcaught = None
362
363 for pid in self.children:
364 try:
365 os.kill(pid, signal.SIGTERM)
366 except OSError as exc:
367 if exc.errno != errno.ESRCH:
368 raise
369
370 # Wait for children to die
371 if self.children:
372 LOG.info(_('Waiting on %d children to exit'), len(self.children))
373 while self.children:
374 self._wait_child()
375
376
377class Service(object):
378 """Service object for binaries running on hosts."""
379
380 def __init__(self, threads=1000):
381 self.tg = threadgroup.ThreadGroup(threads)
382
383 # signal that the service is done shutting itself down:
384 self._done = event.Event()
385
386 def reset(self):
387 # NOTE(Fengqian): docs for Event.reset() recommend against using it
388 self._done = event.Event()
389
390 def start(self):
391 pass
392
393 def stop(self):
394 self.tg.stop()
395 self.tg.wait()
396 # Signal that service cleanup is done:
397 if not self._done.ready():
398 self._done.send()
399
400 def wait(self):
401 self._done.wait()
402
403
404class Services(object):
405
406 def __init__(self):
407 self.services = []
408 self.tg = threadgroup.ThreadGroup()
409 self.done = event.Event()
410
411 def add(self, service):
412 self.services.append(service)
413 self.tg.add_thread(self.run_service, service, self.done)
414
415 def stop(self):
416 # wait for graceful shutdown of services:
417 for service in self.services:
418 service.stop()
419 service.wait()
420
421 # Each service has performed cleanup, now signal that the run_service
422 # wrapper threads can now die:
423 if not self.done.ready():
424 self.done.send()
425
426 # reap threads:
427 self.tg.stop()
428
429 def wait(self):
430 self.tg.wait()
431
432 def restart(self):
433 self.stop()
434 self.done = event.Event()
435 for restart_service in self.services:
436 restart_service.reset()
437 self.tg.add_thread(self.run_service, restart_service, self.done)
438
439 @staticmethod
440 def run_service(service, done):
441 """Service start wrapper.
442
443 :param service: service to run
444 :param done: event to wait on until a shutdown is triggered
445 :returns: None
446
447 """
448 service.start()
449 done.wait()
450
451
452def launch(service, workers=None):
453 if workers:
454 launcher = ProcessLauncher()
455 launcher.launch_service(service, workers=workers)
456 else:
457 launcher = ServiceLauncher()
458 launcher.launch_service(service)
459 return launcher
diff --git a/python-agent/muranoagent/openstack/common/threadgroup.py b/python-agent/muranoagent/openstack/common/threadgroup.py
new file mode 100644
index 0000000..5f24f1d
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/threadgroup.py
@@ -0,0 +1,121 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2012 Red Hat, Inc.
4#
5# Licensed under the Apache License, Version 2.0 (the "License"); you may
6# not use this file except in compliance with the License. You may obtain
7# a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14# License for the specific language governing permissions and limitations
15# under the License.
16
17import eventlet
18from eventlet import greenpool
19from eventlet import greenthread
20
21from muranoagent.openstack.common import log as logging
22from muranoagent.openstack.common import loopingcall
23
24
25LOG = logging.getLogger(__name__)
26
27
28def _thread_done(gt, *args, **kwargs):
29 """Callback function to be passed to GreenThread.link() when we spawn()
30 Calls the :class:`ThreadGroup` to notify if.
31
32 """
33 kwargs['group'].thread_done(kwargs['thread'])
34
35
36class Thread(object):
37 """Wrapper around a greenthread, that holds a reference to the
38 :class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when
39 it has done so it can be removed from the threads list.
40 """
41 def __init__(self, thread, group):
42 self.thread = thread
43 self.thread.link(_thread_done, group=group, thread=self)
44
45 def stop(self):
46 self.thread.kill()
47
48 def wait(self):
49 return self.thread.wait()
50
51
52class ThreadGroup(object):
53 """The point of the ThreadGroup classis to:
54
55 * keep track of timers and greenthreads (making it easier to stop them
56 when need be).
57 * provide an easy API to add timers.
58 """
59 def __init__(self, thread_pool_size=10):
60 self.pool = greenpool.GreenPool(thread_pool_size)
61 self.threads = []
62 self.timers = []
63
64 def add_dynamic_timer(self, callback, initial_delay=None,
65 periodic_interval_max=None, *args, **kwargs):
66 timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs)
67 timer.start(initial_delay=initial_delay,
68 periodic_interval_max=periodic_interval_max)
69 self.timers.append(timer)
70
71 def add_timer(self, interval, callback, initial_delay=None,
72 *args, **kwargs):
73 pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs)
74 pulse.start(interval=interval,
75 initial_delay=initial_delay)
76 self.timers.append(pulse)
77
78 def add_thread(self, callback, *args, **kwargs):
79 gt = self.pool.spawn(callback, *args, **kwargs)
80 th = Thread(gt, self)
81 self.threads.append(th)
82
83 def thread_done(self, thread):
84 self.threads.remove(thread)
85
86 def stop(self):
87 current = greenthread.getcurrent()
88 for x in self.threads:
89 if x is current:
90 # don't kill the current thread.
91 continue
92 try:
93 x.stop()
94 except Exception as ex:
95 LOG.exception(ex)
96
97 for x in self.timers:
98 try:
99 x.stop()
100 except Exception as ex:
101 LOG.exception(ex)
102 self.timers = []
103
104 def wait(self):
105 for x in self.timers:
106 try:
107 x.wait()
108 except eventlet.greenlet.GreenletExit:
109 pass
110 except Exception as ex:
111 LOG.exception(ex)
112 current = greenthread.getcurrent()
113 for x in self.threads:
114 if x is current:
115 continue
116 try:
117 x.wait()
118 except eventlet.greenlet.GreenletExit:
119 pass
120 except Exception as ex:
121 LOG.exception(ex)
diff --git a/python-agent/muranoagent/openstack/common/timeutils.py b/python-agent/muranoagent/openstack/common/timeutils.py
new file mode 100644
index 0000000..98d877d
--- /dev/null
+++ b/python-agent/muranoagent/openstack/common/timeutils.py
@@ -0,0 +1,197 @@
1# vim: tabstop=4 shiftwidth=4 softtabstop=4
2
3# Copyright 2011 OpenStack Foundation.
4# All Rights Reserved.
5#
6# Licensed under the Apache License, Version 2.0 (the "License"); you may
7# not use this file except in compliance with the License. You may obtain
8# a copy of the License at
9#
10# http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15# License for the specific language governing permissions and limitations
16# under the License.
17
18"""
19Time related utilities and helper functions.
20"""
21
22import calendar
23import datetime
24import time
25
26import iso8601
27import six
28
29
30# ISO 8601 extended time format with microseconds
31_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
32_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
33PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
34
35
36def isotime(at=None, subsecond=False):
37 """Stringify time in ISO 8601 format."""
38 if not at:
39 at = utcnow()
40 st = at.strftime(_ISO8601_TIME_FORMAT
41 if not subsecond
42 else _ISO8601_TIME_FORMAT_SUBSECOND)
43 tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
44 st += ('Z' if tz == 'UTC' else tz)
45 return st
46
47
48def parse_isotime(timestr):
49 """Parse time from ISO 8601 format."""
50 try:
51 return iso8601.parse_date(timestr)
52 except iso8601.ParseError as e:
53 raise ValueError(unicode(e))
54 except TypeError as e:
55 raise ValueError(unicode(e))
56
57
58def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
59 """Returns formatted utcnow."""
60 if not at:
61 at = utcnow()
62 return at.strftime(fmt)
63
64
65def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
66 """Turn a formatted time back into a datetime."""
67 return datetime.datetime.strptime(timestr, fmt)
68
69
70def normalize_time(timestamp):
71 """Normalize time in arbitrary timezone to UTC naive object."""
72 offset = timestamp.utcoffset()
73 if offset is None:
74 return timestamp
75 return timestamp.replace(tzinfo=None) - offset
76
77
78def is_older_than(before, seconds):
79 """Return True if before is older than seconds."""
80 if isinstance(before, six.string_types):
81 before = parse_strtime(before).replace(tzinfo=None)
82 return utcnow() - before > datetime.timedelta(seconds=seconds)
83
84
85def is_newer_than(after, seconds):
86 """Return True if after is newer than seconds."""
87 if isinstance(after, six.string_types):
88 after = parse_strtime(after).replace(tzinfo=None)
89 return after - utcnow() > datetime.timedelta(seconds=seconds)
90
91
92def utcnow_ts():
93 """Timestamp version of our utcnow function."""
94 if utcnow.override_time is None:
95 # NOTE(kgriffs): This is several times faster
96 # than going through calendar.timegm(...)
97 return int(time.time())
98
99 return calendar.timegm(utcnow().timetuple())
100
101
102def utcnow():
103 """Overridable version of utils.utcnow."""
104 if utcnow.override_time:
105 try:
106 return utcnow.override_time.pop(0)
107 except AttributeError:
108 return utcnow.override_time
109 return datetime.datetime.utcnow()
110
111
112def iso8601_from_timestamp(timestamp):
113 """Returns a iso8601 formated date from timestamp."""
114 return isotime(datetime.datetime.utcfromtimestamp(timestamp))
115
116
117utcnow.override_time = None
118
119
120def set_time_override(override_time=None):
121 """Overrides utils.utcnow.
122
123 Make it return a constant time or a list thereof, one at a time.
124
125 :param override_time: datetime instance or list thereof. If not
126 given, defaults to the current UTC time.
127 """
128 utcnow.override_time = override_time or datetime.datetime.utcnow()
129
130
131def advance_time_delta(timedelta):
132 """Advance overridden time using a datetime.timedelta."""
133 assert(not utcnow.override_time is None)
134 try:
135 for dt in utcnow.override_time:
136 dt += timedelta
137 except TypeError:
138 utcnow.override_time += timedelta
139
140
141def advance_time_seconds(seconds):
142 """Advance overridden time by seconds."""
143 advance_time_delta(datetime.timedelta(0, seconds))
144
145
146def clear_time_override():
147 """Remove the overridden time."""
148 utcnow.override_time = None
149
150
151def marshall_now(now=None):
152 """Make an rpc-safe datetime with microseconds.
153
154 Note: tzinfo is stripped, but not required for relative times.