initial python implementation
Change-Id: I25ca78414cd26340a4a640cb4f9ee92dda650037
This commit is contained in:
parent
87a56ca2be
commit
ed99a4edb3
|
@ -0,0 +1,8 @@
|
||||||
|
[run]
|
||||||
|
branch = True
|
||||||
|
source = monasca
|
||||||
|
omit = monasca/tests/*
|
||||||
|
|
||||||
|
[report]
|
||||||
|
ignore-errors = True
|
||||||
|
|
|
@ -1,16 +1,31 @@
|
||||||
*.swp
|
*.py[co]
|
||||||
*.iml
|
*~
|
||||||
.DS_Store
|
doc/build/*
|
||||||
.cache
|
dist
|
||||||
.classpath
|
build
|
||||||
.idea
|
cover
|
||||||
.project
|
.coverage
|
||||||
.target/
|
*.egg
|
||||||
debs/*
|
*.egg-info
|
||||||
.settings/
|
.testrepository
|
||||||
target/
|
.tox
|
||||||
test-output/
|
ChangeLog
|
||||||
logs/
|
MANIFEST
|
||||||
log/
|
|
||||||
*config*.yml
|
|
||||||
db/config.yml
|
*.swp
|
||||||
|
*.iml
|
||||||
|
.DS_Store
|
||||||
|
.cache
|
||||||
|
.classpath
|
||||||
|
.idea
|
||||||
|
.project
|
||||||
|
.target/
|
||||||
|
debs/*
|
||||||
|
.settings/
|
||||||
|
target/
|
||||||
|
test-output/
|
||||||
|
logs/
|
||||||
|
log/
|
||||||
|
*config*.yml
|
||||||
|
db/config.yml
|
||||||
|
|
|
@ -0,0 +1,8 @@
|
||||||
|
[DEFAULT]
|
||||||
|
test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
|
||||||
|
OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
|
||||||
|
OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-60} \
|
||||||
|
${PYTHON:-python} -m subunit.run discover monasca $LISTOPT $IDOPTION
|
||||||
|
|
||||||
|
test_id_option=--load-list $IDFILE
|
||||||
|
test_list_option=--list
|
40
README.md
40
README.md
|
@ -37,6 +37,46 @@ Requests flow through the following architectural layers from top to bottom:
|
||||||
|
|
||||||
* API Specification: [/docs/monasca-api-spec.md](/docs/monasca-api-spec.md).
|
* API Specification: [/docs/monasca-api-spec.md](/docs/monasca-api-spec.md).
|
||||||
|
|
||||||
|
|
||||||
|
python monasca api implementation
|
||||||
|
=================================
|
||||||
|
|
||||||
|
To install the python api implementation, git clone the source and run the
|
||||||
|
following command::
|
||||||
|
|
||||||
|
sudo python setup.py install
|
||||||
|
|
||||||
|
If it installs successfully, you will need to make changes to the following
|
||||||
|
two files to reflect your system settings, especially where kafka server is
|
||||||
|
located::
|
||||||
|
|
||||||
|
/etc/monasca/monasca.ini
|
||||||
|
/etc/monasca/monasca.conf
|
||||||
|
|
||||||
|
Once the configurations are modified to match your environment, you can start
|
||||||
|
up the server by following the following instructions.
|
||||||
|
|
||||||
|
To start the server, run the following command:
|
||||||
|
|
||||||
|
Running the server in foreground mode
|
||||||
|
gunicorn -k eventlet --worker-connections=2000 --backlog=1000
|
||||||
|
--paste /etc/monasca/monasca.ini
|
||||||
|
|
||||||
|
Running the server as daemons
|
||||||
|
gunicorn -k eventlet --worker-connections=2000 --backlog=1000
|
||||||
|
--paste /etc/monasca/monasca.ini -D
|
||||||
|
|
||||||
|
To check if the code follows python coding style, run the following command
|
||||||
|
from the root directory of this project
|
||||||
|
|
||||||
|
tox -e pep8
|
||||||
|
|
||||||
|
To run all the unit test cases, run the following command from the root
|
||||||
|
directory of this project
|
||||||
|
|
||||||
|
tox -e py27 (or -e py26, -e py33)
|
||||||
|
|
||||||
|
|
||||||
# License
|
# License
|
||||||
|
|
||||||
Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
|
Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
|
||||||
|
|
|
@ -0,0 +1,33 @@
|
||||||
|
[DEFAULT]
|
||||||
|
#logging, make sure that the user under whom the server runs has permission
|
||||||
|
#to write to the directory.
|
||||||
|
log_file=monasca.log
|
||||||
|
log_dir=/var/log/monasca/
|
||||||
|
log_level=DEBUG
|
||||||
|
|
||||||
|
[dispatcher]
|
||||||
|
driver = kafka
|
||||||
|
|
||||||
|
[kafka]
|
||||||
|
#The endpoint to the kafka server
|
||||||
|
uri = 192.168.1.191:9092
|
||||||
|
#The topic on the kafka server
|
||||||
|
topic = metrics
|
||||||
|
#consumer group name
|
||||||
|
group = metric_group
|
||||||
|
#how many times to try when error occurs
|
||||||
|
max_retry = 1
|
||||||
|
#wait time between tries when kafka goes down
|
||||||
|
wait_time = 1
|
||||||
|
#use synchronized or asynchronized connection to kafka
|
||||||
|
async = False
|
||||||
|
#send messages in bulk or send messages one by one.
|
||||||
|
compact = False
|
||||||
|
#How many partitions this connection should listen messages on, this
|
||||||
|
#parameter is for reading from kafka. If listens on multiple partitions,
|
||||||
|
#For example, if the client should listen on partitions 1 and 3, then the
|
||||||
|
#configuration should look like the following:
|
||||||
|
# partitions = 1
|
||||||
|
# partitions = 3
|
||||||
|
#default to listen on partition 0.
|
||||||
|
partitions = 0
|
|
@ -0,0 +1,25 @@
|
||||||
|
[DEFAULT]
|
||||||
|
name = monasca
|
||||||
|
|
||||||
|
[pipeline:main]
|
||||||
|
# Add validator in the pipeline so the metrics messages can be validated.
|
||||||
|
pipeline = validator api
|
||||||
|
|
||||||
|
[app:api]
|
||||||
|
paste.app_factory = monasca.api.server:api_app
|
||||||
|
|
||||||
|
[filter:login]
|
||||||
|
use = egg: monasca_api#login
|
||||||
|
|
||||||
|
[filter:inspector]
|
||||||
|
use = egg: monasca_api#inspector
|
||||||
|
|
||||||
|
[filter:validator]
|
||||||
|
use = egg: monasca_api#metric_validator
|
||||||
|
|
||||||
|
[server:main]
|
||||||
|
use = egg:gunicorn#main
|
||||||
|
host = 0.0.0.0
|
||||||
|
port = 9090
|
||||||
|
workers = 1
|
||||||
|
proc_name = monasca
|
|
@ -0,0 +1,95 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from monasca.common import resource_api
|
||||||
|
from monasca.openstack.common import log
|
||||||
|
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class V2API(object):
|
||||||
|
def __init__(self, global_conf):
|
||||||
|
LOG.debug('initializing V2API!')
|
||||||
|
self.global_conf = global_conf
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/metrics/{id}')
|
||||||
|
def do_get_metrics(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/metrics/', method='post')
|
||||||
|
def do_post_metrics(self, req, res):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/{version_id}', method='get')
|
||||||
|
def do_get_version(self, req, res, version_id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/metrics/measurements')
|
||||||
|
def do_get_measurements(self, req, res):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/metrics/statistics')
|
||||||
|
def do_get_statistics(self, req, res):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/notification-methods', method='post')
|
||||||
|
def do_post_notification_methods(self, req, res):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/notification-methods/{id}', method='put')
|
||||||
|
def do_put_notification_methods(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/notification-methods/{id}', method='delete')
|
||||||
|
def do_delete_notification_methods(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/notification-methods/{id}')
|
||||||
|
def do_get_notification_methods(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/alarms', method='post')
|
||||||
|
def do_post_alarms(self, req, res):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/alarms/{id}', method='put')
|
||||||
|
def do_put_alarms(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/alarms/{id}', method='patch')
|
||||||
|
def do_patch_alarms(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/alarms/{id}', method='delete')
|
||||||
|
def do_delete_alarms(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/alarms/', method='get')
|
||||||
|
def do_get_alarms(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/alarms/{id}', method='get')
|
||||||
|
def do_get_alarm_by_id(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/alarms/state-history')
|
||||||
|
def do_get_alarms_state_history(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/alarms/{id}/state-history')
|
||||||
|
def do_get_alarm_state_history(self, req, res, id):
|
||||||
|
res.status = '501 Not Implemented'
|
|
@ -0,0 +1,58 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from oslo.config import cfg
|
||||||
|
from stevedore import driver
|
||||||
|
|
||||||
|
from monasca.common import resource_api
|
||||||
|
from monasca.openstack.common import log
|
||||||
|
|
||||||
|
DISPATCHER_NAMESPACE = 'monasca.dispatcher'
|
||||||
|
|
||||||
|
|
||||||
|
cfg.CONF.register_opts(
|
||||||
|
[cfg.StrOpt('driver', default='kafka',
|
||||||
|
help='The name of the dispatcher for the api server'),
|
||||||
|
], group="dispatcher")
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def api_app(conf):
|
||||||
|
|
||||||
|
# Setup logs
|
||||||
|
log_levels = (cfg.CONF.default_log_levels)
|
||||||
|
cfg.set_defaults(log.log_opts, default_log_levels=log_levels)
|
||||||
|
|
||||||
|
cfg.CONF(args=[], project='monasca')
|
||||||
|
log.setup('monasca')
|
||||||
|
|
||||||
|
# Create the application
|
||||||
|
app = resource_api.ResourceAPI()
|
||||||
|
|
||||||
|
# load the driver specified by dispatcher in the monasca.ini file
|
||||||
|
manager = driver.DriverManager(namespace=DISPATCHER_NAMESPACE,
|
||||||
|
name=cfg.CONF.dispatcher.driver,
|
||||||
|
invoke_on_load=True,
|
||||||
|
invoke_args=[conf])
|
||||||
|
|
||||||
|
LOG.debug('Dispatcher driver %s is loaded.' % cfg.CONF.dispatcher.driver)
|
||||||
|
|
||||||
|
# add the driver to the application
|
||||||
|
app.add_route(None, manager.driver)
|
||||||
|
|
||||||
|
LOG.debug('Dispatcher driver has been added to the routes!')
|
||||||
|
return app
|
|
@ -0,0 +1,223 @@
|
||||||
|
#
|
||||||
|
# Copyright 2012-2013 eNovance <licensing@enovance.com>
|
||||||
|
#
|
||||||
|
# Author: Julien Danjou <julien@danjou.info>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from kafka import client
|
||||||
|
from kafka import common
|
||||||
|
from kafka import consumer
|
||||||
|
from kafka import producer
|
||||||
|
from oslo.config import cfg
|
||||||
|
from oslo.config import types
|
||||||
|
import time
|
||||||
|
try:
|
||||||
|
import ujson as json
|
||||||
|
except ImportError:
|
||||||
|
import json
|
||||||
|
|
||||||
|
from monasca.openstack.common import log
|
||||||
|
|
||||||
|
|
||||||
|
OPTS = [
|
||||||
|
cfg.StrOpt('uri',
|
||||||
|
help='Address to kafka server. For example: '
|
||||||
|
'uri=192.168.1.191:9092'),
|
||||||
|
cfg.StrOpt('topic',
|
||||||
|
default='event',
|
||||||
|
help='The topic that this collector will listen on.'),
|
||||||
|
cfg.StrOpt('group',
|
||||||
|
default='event_consumer',
|
||||||
|
help='The group name that this service belongs to.'),
|
||||||
|
cfg.IntOpt('wait_time',
|
||||||
|
default=1,
|
||||||
|
help='The wait time when no messages on kafka queue.'),
|
||||||
|
cfg.IntOpt('ack_time',
|
||||||
|
default=20,
|
||||||
|
help='The ack time back to kafka.'),
|
||||||
|
cfg.IntOpt('max_retry',
|
||||||
|
default=3,
|
||||||
|
help='The number of retry when there is a connection error.'),
|
||||||
|
cfg.BoolOpt('auto_commit',
|
||||||
|
default=False,
|
||||||
|
help='If automatically commmit when consume messages.'),
|
||||||
|
cfg.BoolOpt('async',
|
||||||
|
default=True,
|
||||||
|
help='The type of posting.'),
|
||||||
|
cfg.BoolOpt('compact',
|
||||||
|
default=True,
|
||||||
|
help=('Specify if the message received should be parsed.'
|
||||||
|
'If True, message will not be parsed, otherwise '
|
||||||
|
'messages will be parsed.')),
|
||||||
|
cfg.MultiOpt('partitions',
|
||||||
|
item_type=types.Integer(),
|
||||||
|
default=[0],
|
||||||
|
help='The sleep time when no messages on kafka queue.'),
|
||||||
|
cfg.BoolOpt('drop_data',
|
||||||
|
default=False,
|
||||||
|
help=('Specify if received data should be simply dropped. '
|
||||||
|
'This parameter is only for testing purposes.')),
|
||||||
|
]
|
||||||
|
|
||||||
|
cfg.CONF.register_opts(OPTS, group="kafka")
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class KafkaConnection(object):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
if not cfg.CONF.kafka.uri:
|
||||||
|
raise Exception('Kafka is not configured correctly! '
|
||||||
|
'Use configuration file to specify Kafka '
|
||||||
|
'uri, for example: '
|
||||||
|
'uri=192.168.1.191:9092')
|
||||||
|
|
||||||
|
self.uri = cfg.CONF.kafka.uri
|
||||||
|
self.topic = cfg.CONF.kafka.topic
|
||||||
|
self.group = cfg.CONF.kafka.group
|
||||||
|
self.wait_time = cfg.CONF.kafka.wait_time
|
||||||
|
self.async = cfg.CONF.kafka.async
|
||||||
|
self.ack_time = cfg.CONF.kafka.ack_time
|
||||||
|
self.max_retry = cfg.CONF.kafka.max_retry
|
||||||
|
self.auto_commit = cfg.CONF.kafka.auto_commit
|
||||||
|
self.compact = cfg.CONF.kafka.compact
|
||||||
|
self.partitions = cfg.CONF.kafka.partitions
|
||||||
|
self.drop_data = cfg.CONF.kafka.drop_data
|
||||||
|
|
||||||
|
self._client = None
|
||||||
|
self._consumer = None
|
||||||
|
self._producer = None
|
||||||
|
|
||||||
|
LOG.debug('Kafka Connection initialized successfully!')
|
||||||
|
|
||||||
|
def _init_client(self, wait_time=None):
|
||||||
|
for i in range(self.max_retry):
|
||||||
|
try:
|
||||||
|
# if there is a client instance, but _init_client is called
|
||||||
|
# again, most likely the connection has gone stale, close that
|
||||||
|
# connection and reconnect.
|
||||||
|
if self._client:
|
||||||
|
self._client.close()
|
||||||
|
|
||||||
|
if not wait_time:
|
||||||
|
wait_time = self.wait_time
|
||||||
|
time.sleep(wait_time)
|
||||||
|
|
||||||
|
self._client = client.KafkaClient(self.uri)
|
||||||
|
|
||||||
|
# when a client is re-initialized, existing consumer should be
|
||||||
|
# reset as well.
|
||||||
|
self._consumer = None
|
||||||
|
self._producer = None
|
||||||
|
LOG.debug("Successfully connected to Kafka server at topic: "
|
||||||
|
"\"%s\" partitions %s" % (self.topic,
|
||||||
|
self.partitions))
|
||||||
|
break
|
||||||
|
except common.KafkaUnavailableError:
|
||||||
|
LOG.error('Kafka server at %s is down.' % self.uri)
|
||||||
|
except common.LeaderNotAvailableError:
|
||||||
|
LOG.error('Kafka at %s has no leader available.' % self.uri)
|
||||||
|
except Exception:
|
||||||
|
LOG.error('Kafka at %s initialization failed.' % self.uri)
|
||||||
|
# Wait a bit and try again to get a client
|
||||||
|
time.sleep(self.wait_time)
|
||||||
|
|
||||||
|
def _init_consumer(self):
|
||||||
|
try:
|
||||||
|
if not self._client:
|
||||||
|
self._init_client()
|
||||||
|
self._consumer = consumer.SimpleConsumer(
|
||||||
|
self._client, self.group, self.topic,
|
||||||
|
auto_commit=self.auto_commit,
|
||||||
|
partitions=self.partitions)
|
||||||
|
LOG.debug('Consumer was created successfully.')
|
||||||
|
except Exception:
|
||||||
|
self._consumer = None
|
||||||
|
LOG.exception('Kafka (%s) consumer can not be created.' %
|
||||||
|
self.uri)
|
||||||
|
|
||||||
|
def _init_producer(self):
|
||||||
|
try:
|
||||||
|
if not self._client:
|
||||||
|
self._init_client()
|
||||||
|
self._producer = producer.SimpleProducer(
|
||||||
|
self._client, async=self.async, ack_timeout=self.ack_time)
|
||||||
|
LOG.debug('Producer was created successfully.')
|
||||||
|
except Exception:
|
||||||
|
self._producer = None
|
||||||
|
LOG.exception('Kafka (%s) producer can not be created.' %
|
||||||
|
self.uri)
|
||||||
|
|
||||||
|
def commit(self):
|
||||||
|
if self._consumer and self.auto_commit:
|
||||||
|
self._consumer.commit()
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
if self._client:
|
||||||
|
self._consumer = None
|
||||||
|
self._producer = None
|
||||||
|
self._client.close()
|
||||||
|
|
||||||
|
def get_messages(self):
|
||||||
|
try:
|
||||||
|
if not self._consumer:
|
||||||
|
self._init_consumer()
|
||||||
|
|
||||||
|
for message in self._consumer:
|
||||||
|
LOG.debug(message.message.value)
|
||||||
|
yield message
|
||||||
|
except Exception:
|
||||||
|
LOG.error('Error occurred while handling kafka messages.')
|
||||||
|
self._consumer = None
|
||||||
|
yield None
|
||||||
|
|
||||||
|
def send_messages(self, messages):
|
||||||
|
LOG.debug('Prepare to send messages.')
|
||||||
|
if not messages or self.drop_data:
|
||||||
|
return 204
|
||||||
|
|
||||||
|
code = 400
|
||||||
|
try:
|
||||||
|
if not self._producer:
|
||||||
|
self._init_producer()
|
||||||
|
|
||||||
|
LOG.debug('Start sending messages to kafka.')
|
||||||
|
if self.compact:
|
||||||
|
self._producer.send_messages(self.topic, messages)
|
||||||
|
else:
|
||||||
|
data = json.loads(messages)
|
||||||
|
LOG.debug('Msg parsed successfully.')
|
||||||
|
if isinstance(data, list):
|
||||||
|
for item in data:
|
||||||
|
self._producer.send_messages(
|
||||||
|
self.topic, json.dumps(item))
|
||||||
|
else:
|
||||||
|
self._producer.send_messages(self.topic, messages)
|
||||||
|
LOG.debug('Message posted successfully.')
|
||||||
|
code = 204
|
||||||
|
except (common.KafkaUnavailableError,
|
||||||
|
common.LeaderNotAvailableError):
|
||||||
|
self._client = None
|
||||||
|
code = 503
|
||||||
|
LOG.exception('Error occurred while posting data to '
|
||||||
|
'Kafka.')
|
||||||
|
except ValueError:
|
||||||
|
code = 406
|
||||||
|
LOG.exception('Message %s is not valid json.' % messages)
|
||||||
|
except Exception:
|
||||||
|
code = 500
|
||||||
|
LOG.exception('Unknown error.')
|
||||||
|
|
||||||
|
return code
|
|
@ -0,0 +1,117 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import falcon
|
||||||
|
from falcon import api_helpers
|
||||||
|
|
||||||
|
from monasca.openstack.common import log
|
||||||
|
|
||||||
|
|
||||||
|
RESOURCE_METHOD_FLAG = 'fab05a04-b861-4651-bd0c-9cb3eb9a6088'
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class Restify(object):
|
||||||
|
def __init__(self, path='', method='GET'):
|
||||||
|
if not path:
|
||||||
|
raise Exception('Path has to be specified.')
|
||||||
|
|
||||||
|
if method.upper() not in falcon.HTTP_METHODS:
|
||||||
|
raise Exception('Invalid request method.')
|
||||||
|
self.path = path
|
||||||
|
self.method = method.upper()
|
||||||
|
|
||||||
|
def __call__(self, func):
|
||||||
|
setattr(func, RESOURCE_METHOD_FLAG, self)
|
||||||
|
return func
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceAPI(falcon.API):
|
||||||
|
|
||||||
|
def add_route(self, uri_template, resource):
|
||||||
|
"""Associates uri patterns with resource methods.
|
||||||
|
|
||||||
|
A resource is an instance of a class that defines various methods
|
||||||
|
to handle http requests.
|
||||||
|
|
||||||
|
Use this class to create applications which serve a standard
|
||||||
|
compliant ReSTful API. For example, you may have an API which manage
|
||||||
|
monitoring data, there can be multiple implementations of the API
|
||||||
|
using different technologies. One can use Mongodb, the other can use
|
||||||
|
Cassandra. To make the configuration of the application very easy,
|
||||||
|
each implementation provides a class with set of methods decorated
|
||||||
|
by class Restify, the application can simply using single entry
|
||||||
|
configuration to load different implementations.
|
||||||
|
|
||||||
|
For example::
|
||||||
|
|
||||||
|
class ExampleResource(object):
|
||||||
|
@Restify(path='/path1/', method='post')
|
||||||
|
def func1(self, req, res):
|
||||||
|
pass
|
||||||
|
|
||||||
|
@Restify(path='/path2/{id}/key/', method='get')
|
||||||
|
def func2(self, req, res, id):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def func3(self, req, res, id):
|
||||||
|
pass
|
||||||
|
|
||||||
|
With the above class, the following code will add the class method
|
||||||
|
func1, func2 to handle post and get requests respectively, method
|
||||||
|
func3 won't be added to the routes.::
|
||||||
|
|
||||||
|
app.add_route(None, ExampleResource())
|
||||||
|
|
||||||
|
Args:
|
||||||
|
uri_template (url pattern): the url pattern which a client will
|
||||||
|
post a request against. If none, ResourceAPI will
|
||||||
|
automatically look up the decorated methods.
|
||||||
|
resource (instance): Object which represents an HTTP/REST
|
||||||
|
"resource". Falcon will pass requests to various decorated
|
||||||
|
methods to handle http requests.
|
||||||
|
"""
|
||||||
|
if not resource:
|
||||||
|
raise Exception('Not a valid resource')
|
||||||
|
|
||||||
|
path_maps = {}
|
||||||
|
try:
|
||||||
|
if uri_template:
|
||||||
|
super(ResourceAPI, self).add_route(uri_template, resource)
|
||||||
|
else:
|
||||||
|
for attr in dir(resource):
|
||||||
|
method = getattr(resource, attr)
|
||||||
|
if callable(method) and hasattr(method,
|
||||||
|
RESOURCE_METHOD_FLAG):
|
||||||
|
flag = getattr(method, RESOURCE_METHOD_FLAG)
|
||||||
|
map = path_maps.get(flag.path)
|
||||||
|
if not map:
|
||||||
|
uri_fields, template = (
|
||||||
|
api_helpers.compile_uri_template(flag.path))
|
||||||
|
map = (template, {})
|
||||||
|
path_maps[flag.path] = map
|
||||||
|
|
||||||
|
new_method = api_helpers._wrap_with_hooks(
|
||||||
|
self._before, self._after, method)
|
||||||
|
map[1][flag.method] = new_method
|
||||||
|
|
||||||
|
for item in path_maps:
|
||||||
|
self._routes.insert(0, (path_maps[item][0],
|
||||||
|
path_maps[item][1]))
|
||||||
|
except Exception:
|
||||||
|
LOG.exception('Error occurred while adding the resource')
|
||||||
|
LOG.debug(self._routes)
|
|
@ -0,0 +1,29 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import abc
|
||||||
|
import six
|
||||||
|
|
||||||
|
|
||||||
|
@six.add_metaclass(abc.ABCMeta)
|
||||||
|
class Base(object):
|
||||||
|
|
||||||
|
def __init__(self, conf):
|
||||||
|
self.conf = conf
|
||||||
|
|
||||||
|
@abc.abstractmethod
|
||||||
|
def define_routes(self, app):
|
||||||
|
"""Post metric data interface."""
|
|
@ -0,0 +1,41 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import falcon
|
||||||
|
|
||||||
|
from monasca.api import monasca_api_v2
|
||||||
|
from monasca.common import kafka_conn
|
||||||
|
from monasca.common import resource_api
|
||||||
|
from monasca.openstack.common import log
|
||||||
|
|
||||||
|
|
||||||
|
LOG = log.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class KafkaDispatcher(monasca_api_v2.V2API):
|
||||||
|
def __init__(self, global_conf):
|
||||||
|
LOG.debug('initializing KafkaDispatcher!')
|
||||||
|
super(KafkaDispatcher, self).__init__(global_conf)
|
||||||
|
|
||||||
|
self._kafka_conn = kafka_conn.KafkaConnection()
|
||||||
|
|
||||||
|
@resource_api.Restify('/v2.0/metrics/', method='post')
|
||||||
|
def do_post_metrics(self, req, res):
|
||||||
|
LOG.debug('Getting the call.')
|
||||||
|
msg = req.stream.read()
|
||||||
|
|
||||||
|
code = self._kafka_conn.send_messages(msg)
|
||||||
|
res.status = getattr(falcon, 'HTTP_' + str(code))
|
|
@ -0,0 +1,22 @@
|
||||||
|
# Translations template for monasca.
|
||||||
|
# Copyright (C) 2014 ORGANIZATION
|
||||||
|
# This file is distributed under the same license as the marconi project.
|
||||||
|
# FIRST AUTHOR <EMAIL@ADDRESS>, 2014.
|
||||||
|
#
|
||||||
|
#, fuzzy
|
||||||
|
msgid ""
|
||||||
|
msgstr ""
|
||||||
|
"Project-Id-Version: monasca 2014.2.dev44.g366d9b5\n"
|
||||||
|
"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n"
|
||||||
|
"POT-Creation-Date: 2014-06-04 22:30+0200\n"
|
||||||
|
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
|
||||||
|
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
|
||||||
|
"Language-Team: LANGUAGE <LL@li.org>\n"
|
||||||
|
"MIME-Version: 1.0\n"
|
||||||
|
"Content-Type: text/plain; charset=utf-8\n"
|
||||||
|
"Content-Transfer-Encoding: 8bit\n"
|
||||||
|
"Generated-By: Babel 1.3\n"
|
||||||
|
|
||||||
|
#: monasca/common/utils.py:48
|
||||||
|
msgid "{0} is not a valid operation"
|
||||||
|
msgstr ""
|
|
@ -0,0 +1,51 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
class Inspector(object):
|
||||||
|
"""The middleware that logs the request body and header
|
||||||
|
|
||||||
|
This is a middleware for debug purposes. To enable this middleware, add
|
||||||
|
the following lines into the configuration file, for example:
|
||||||
|
|
||||||
|
[pipeline:main]
|
||||||
|
pipeline = inspector api
|
||||||
|
|
||||||
|
[filter:inspector]
|
||||||
|
use = egg: monasca_api_server#inspector
|
||||||
|
"""
|
||||||
|
def __init__(self, app, conf):
|
||||||
|
"""Inspect each request
|
||||||
|
|
||||||
|
:param app: OptionParser to use. If not sent one will be created.
|
||||||
|
:param conf: Override sys.argv; used in testing
|
||||||
|
"""
|
||||||
|
self.app = app
|
||||||
|
self.conf = conf
|
||||||
|
print('Inspect config:', self.conf)
|
||||||
|
|
||||||
|
def __call__(self, environ, start_response):
|
||||||
|
|
||||||
|
print('environ is ', environ)
|
||||||
|
return self.app(environ, start_response)
|
||||||
|
|
||||||
|
|
||||||
|
def filter_factory(global_conf, **local_conf):
|
||||||
|
|
||||||
|
def login_filter(app):
|
||||||
|
return Inspector(app, local_conf)
|
||||||
|
|
||||||
|
return login_filter
|
|
@ -0,0 +1,47 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
class SimpleLogin(object):
|
||||||
|
"""Example middleware that demostrates how a login middleware should work.
|
||||||
|
|
||||||
|
In this example, the middleware checks if a request path starts with
|
||||||
|
string '/datapoints/', if it does, the request gets pass this
|
||||||
|
middleware, otherwise, a 401 response is returned.
|
||||||
|
"""
|
||||||
|
def __init__(self, app, conf):
|
||||||
|
self.app = app
|
||||||
|
self.conf = conf
|
||||||
|
|
||||||
|
def __call__(self, environ, start_response):
|
||||||
|
# if request starts with /datapoints/, then let it go on.
|
||||||
|
# this login middle
|
||||||
|
if environ.get('PATH_INFO', '').startswith('/datapoints/'):
|
||||||
|
return self.app(environ, start_response)
|
||||||
|
# otherwise, send something else, request stops here.
|
||||||
|
else:
|
||||||
|
status = "401 Unauthorized"
|
||||||
|
response_headers = [("content-type", "text/plain")]
|
||||||
|
start_response(status, response_headers, "please login first")
|
||||||
|
return ['Please log in!']
|
||||||
|
|
||||||
|
|
||||||
|
def filter_factory(global_conf, **local_conf):
|
||||||
|
|
||||||
|
def login_filter(app):
|
||||||
|
return SimpleLogin(app, local_conf)
|
||||||
|
|
||||||
|
return login_filter
|
|
@ -0,0 +1,125 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import StringIO
|
||||||
|
try:
|
||||||
|
import ujson as json
|
||||||
|
except ImportError:
|
||||||
|
import json
|
||||||
|
|
||||||
|
|
||||||
|
class MetricValidator(object):
|
||||||
|
"""middleware that validate the metric input stream.
|
||||||
|
|
||||||
|
This middleware checks if the input stream actually follows metric spec
|
||||||
|
and all the messages in the request has valid metric data. If the body
|
||||||
|
is valid json and compliant with the spec, then the request will forward
|
||||||
|
the request to the next in the pipeline, otherwise, it will reject the
|
||||||
|
request with response code of 400 or 406.
|
||||||
|
"""
|
||||||
|
def __init__(self, app, conf):
|
||||||
|
self.app = app
|
||||||
|
self.conf = conf
|
||||||
|
|
||||||
|
def _is_valid_metric(self, metric):
|
||||||
|
"""Validate a message
|
||||||
|
|
||||||
|
The external message format is
|
||||||
|
{
|
||||||
|
"name":"name1",
|
||||||
|
"dimensions":{
|
||||||
|
"key1":"value1",
|
||||||
|
"key2":"value2"
|
||||||
|
},
|
||||||
|
"timestamp":1405630174,
|
||||||
|
"value":1.0
|
||||||
|
}
|
||||||
|
|
||||||
|
Once this is validated, the message needs to be transformed into
|
||||||
|
the following internal format:
|
||||||
|
|
||||||
|
The current valid message format is as follows (interna):
|
||||||
|
{
|
||||||
|
"metric": {"something": "The metric as a JSON object"},
|
||||||
|
"meta": {
|
||||||
|
"tenantId": "the tenant ID acquired",
|
||||||
|
"region": "the region that the metric was submitted under",
|
||||||
|
},
|
||||||
|
"creation_time": "the time when the API received the metric",
|
||||||
|
}
|
||||||
|
"""
|
||||||
|
if (metric.get('name') and metric.get('dimensions') and
|
||||||
|
metric.get('timestamp') and metric.get('value')):
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def __call__(self, env, start_response):
|
||||||
|
# if request starts with /datapoints/, then let it go on.
|
||||||
|
# this login middle
|
||||||
|
if (env.get('PATH_INFO', '').startswith('/v2.0/metrics') and
|
||||||
|
env.get('REQUEST_METHOD', '') == 'POST'):
|
||||||
|
# We only check the requests which are posting against metrics
|
||||||
|
# endpoint
|
||||||
|
try:
|
||||||
|
body = env['wsgi.input'].read()
|
||||||
|
metrics = json.loads(body)
|
||||||
|
# Do business logic validation here.
|
||||||
|
is_valid = True
|
||||||
|
if isinstance(metrics, list):
|
||||||
|
for metric in metrics:
|
||||||
|
if not self._is_valid_metric(metric):
|
||||||
|
is_valid = False
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
is_valid = self._is_valid_metric(metrics)
|
||||||
|
|
||||||
|
if is_valid:
|
||||||
|
# If the message is valid, then wrap it into this internal
|
||||||
|
# format. The tenantId should be available from the
|
||||||
|
# request since this should have been authenticated.
|
||||||
|
# ideally this transformation should be done somewhere
|
||||||
|
# else. For the sake of simplicity, do the simple one
|
||||||
|
# here to make the life a bit easier.
|
||||||
|
|
||||||
|
# TODO(HP) Add logic to get region id from request header
|
||||||
|
# HTTP_X_SERVICE_CATALOG, then find endpoints, then region
|
||||||
|
region_id = None
|
||||||
|
msg = {'metric': metrics,
|
||||||
|
'meta': {'tenantId': env.get('HTTP_X_PROJECT_ID'),
|
||||||
|
'region': region_id},
|
||||||
|
'creation_time': datetime.datetime.now()}
|
||||||
|
env['wsgi.input'] = StringIO.StringIO(json.dumps(msg))
|
||||||
|
return self.app(env, start_response)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
# It is either invalid or exceptioned out while parsing json
|
||||||
|
# we will send the request back with 400.
|
||||||
|
start_response("400 Bad Request", [], '')
|
||||||
|
return []
|
||||||
|
else:
|
||||||
|
# not a metric post request, move on.
|
||||||
|
return self.app(env, start_response)
|
||||||
|
|
||||||
|
|
||||||
|
def filter_factory(global_conf, **local_conf):
|
||||||
|
|
||||||
|
def validator_filter(app):
|
||||||
|
return MetricValidator(app, local_conf)
|
||||||
|
|
||||||
|
return validator_filter
|
|
@ -0,0 +1,17 @@
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
|
|
||||||
|
six.add_move(six.MovedModule('mox', 'mox', 'mox3.mox'))
|
|
@ -0,0 +1,145 @@
|
||||||
|
# Copyright (c) 2012 OpenStack Foundation.
|
||||||
|
# Administrator of the National Aeronautics and Space Administration.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
|
||||||
|
import errno
|
||||||
|
import gc
|
||||||
|
import os
|
||||||
|
import pprint
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
import eventlet
|
||||||
|
import eventlet.backdoor
|
||||||
|
import greenlet
|
||||||
|
from oslo.config import cfg
|
||||||
|
|
||||||
|
from monasca.openstack.common.gettextutils import _LI
|
||||||
|
from monasca.openstack.common import log as logging
|
||||||
|
|
||||||
|
help_for_backdoor_port = (
|
||||||
|
"Acceptable values are 0, <port>, and <start>:<end>, where 0 results "
|
||||||
|
"in listening on a random tcp port number; <port> results in listening "
|
||||||
|
"on the specified port number (and not enabling backdoor if that port "
|
||||||
|
"is in use); and <start>:<end> results in listening on the smallest "
|
||||||
|
"unused port number within the specified range of port numbers. The "
|
||||||
|
"chosen port is displayed in the service's log file.")
|
||||||
|
eventlet_backdoor_opts = [
|
||||||
|
cfg.StrOpt('backdoor_port',
|
||||||
|
help="Enable eventlet backdoor. %s" % help_for_backdoor_port)
|
||||||
|
]
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
CONF.register_opts(eventlet_backdoor_opts)
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class EventletBackdoorConfigValueError(Exception):
|
||||||
|
def __init__(self, port_range, help_msg, ex):
|
||||||
|
msg = ('Invalid backdoor_port configuration %(range)s: %(ex)s. '
|
||||||
|
'%(help)s' %
|
||||||
|
{'range': port_range, 'ex': ex, 'help': help_msg})
|
||||||
|
super(EventletBackdoorConfigValueError, self).__init__(msg)
|
||||||
|
self.port_range = port_range
|
||||||
|
|
||||||
|
|
||||||
|
def _dont_use_this():
|
||||||
|
print("Don't use this, just disconnect instead")
|
||||||
|
|
||||||
|
|
||||||
|
def _find_objects(t):
|
||||||
|
return [o for o in gc.get_objects() if isinstance(o, t)]
|
||||||
|
|
||||||
|
|
||||||
|
def _print_greenthreads():
|
||||||
|
for i, gt in enumerate(_find_objects(greenlet.greenlet)):
|
||||||
|
print(i, gt)
|
||||||
|
traceback.print_stack(gt.gr_frame)
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
def _print_nativethreads():
|
||||||
|
for threadId, stack in sys._current_frames().items():
|
||||||
|
print(threadId)
|
||||||
|
traceback.print_stack(stack)
|
||||||
|
print()
|
||||||
|
|
||||||
|
|
||||||
|
def _parse_port_range(port_range):
|
||||||
|
if ':' not in port_range:
|
||||||
|
start, end = port_range, port_range
|
||||||
|
else:
|
||||||
|
start, end = port_range.split(':', 1)
|
||||||
|
try:
|
||||||
|
start, end = int(start), int(end)
|
||||||
|
if end < start:
|
||||||
|
raise ValueError
|
||||||
|
return start, end
|
||||||
|
except ValueError as ex:
|
||||||
|
raise EventletBackdoorConfigValueError(port_range, ex,
|
||||||
|
help_for_backdoor_port)
|
||||||
|
|
||||||
|
|
||||||
|
def _listen(host, start_port, end_port, listen_func):
|
||||||
|
try_port = start_port
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
return listen_func((host, try_port))
|
||||||
|
except socket.error as exc:
|
||||||
|
if (exc.errno != errno.EADDRINUSE or
|
||||||
|
try_port >= end_port):
|
||||||
|
raise
|
||||||
|
try_port += 1
|
||||||
|
|
||||||
|
|
||||||
|
def initialize_if_enabled():
|
||||||
|
backdoor_locals = {
|
||||||
|
'exit': _dont_use_this, # So we don't exit the entire process
|
||||||
|
'quit': _dont_use_this, # So we don't exit the entire process
|
||||||
|
'fo': _find_objects,
|
||||||
|
'pgt': _print_greenthreads,
|
||||||
|
'pnt': _print_nativethreads,
|
||||||
|
}
|
||||||
|
|
||||||
|
if CONF.backdoor_port is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
start_port, end_port = _parse_port_range(str(CONF.backdoor_port))
|
||||||
|
|
||||||
|
# NOTE(johannes): The standard sys.displayhook will print the value of
|
||||||
|
# the last expression and set it to __builtin__._, which overwrites
|
||||||
|
# the __builtin__._ that gettext sets. Let's switch to using pprint
|
||||||
|
# since it won't interact poorly with gettext, and it's easier to
|
||||||
|
# read the output too.
|
||||||
|
def displayhook(val):
|
||||||
|
if val is not None:
|
||||||
|
pprint.pprint(val)
|
||||||
|
sys.displayhook = displayhook
|
||||||
|
|
||||||
|
sock = _listen('localhost', start_port, end_port, eventlet.listen)
|
||||||
|
|
||||||
|
# In the case of backdoor port being zero, a port number is assigned by
|
||||||
|
# listen(). In any case, pull the port number out here.
|
||||||
|
port = sock.getsockname()[1]
|
||||||
|
LOG.info(
|
||||||
|
_LI('Eventlet backdoor listening on %(port)s for process %(pid)d') %
|
||||||
|
{'port': port, 'pid': os.getpid()}
|
||||||
|
)
|
||||||
|
eventlet.spawn_n(eventlet.backdoor.backdoor_server, sock,
|
||||||
|
locals=backdoor_locals)
|
||||||
|
return port
|
|
@ -0,0 +1,113 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# Copyright 2012, Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Exception related utilities.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
|
from monasca.openstack.common.gettextutils import _LE
|
||||||
|
|
||||||
|
|
||||||
|
class save_and_reraise_exception(object):
|
||||||
|
"""Save current exception, run some code and then re-raise.
|
||||||
|
|
||||||
|
In some cases the exception context can be cleared, resulting in None
|
||||||
|
being attempted to be re-raised after an exception handler is run. This
|
||||||
|
can happen when eventlet switches greenthreads or when running an
|
||||||
|
exception handler, code raises and catches an exception. In both
|
||||||
|
cases the exception context will be cleared.
|
||||||
|
|
||||||
|
To work around this, we save the exception state, run handler code, and
|
||||||
|
then re-raise the original exception. If another exception occurs, the
|
||||||
|
saved exception is logged and the new exception is re-raised.
|
||||||
|
|
||||||
|
In some cases the caller may not want to re-raise the exception, and
|
||||||
|
for those circumstances this context provides a reraise flag that
|
||||||
|
can be used to suppress the exception. For example::
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
with save_and_reraise_exception() as ctxt:
|
||||||
|
decide_if_need_reraise()
|
||||||
|
if not should_be_reraised:
|
||||||
|
ctxt.reraise = False
|
||||||
|
|
||||||
|
If another exception occurs and reraise flag is False,
|
||||||
|
the saved exception will not be logged.
|
||||||
|
|
||||||
|
If the caller wants to raise new exception during exception handling
|
||||||
|
he/she sets reraise to False initially with an ability to set it back to
|
||||||
|
True if needed::
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
with save_and_reraise_exception(reraise=False) as ctxt:
|
||||||
|
[if statements to determine whether to raise a new exception]
|
||||||
|
# Not raising a new exception, so reraise
|
||||||
|
ctxt.reraise = True
|
||||||
|
"""
|
||||||
|
def __init__(self, reraise=True):
|
||||||
|
self.reraise = reraise
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
self.type_, self.value, self.tb, = sys.exc_info()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
|
if exc_type is not None:
|
||||||
|
if self.reraise:
|
||||||
|
logging.error(_LE('Original exception being dropped: %s'),
|
||||||
|
traceback.format_exception(self.type_,
|
||||||
|
self.value,
|
||||||
|
self.tb))
|
||||||
|
return False
|
||||||
|
if self.reraise:
|
||||||
|
six.reraise(self.type_, self.value, self.tb)
|
||||||
|
|
||||||
|
|
||||||
|
def forever_retry_uncaught_exceptions(infunc):
|
||||||
|
def inner_func(*args, **kwargs):
|
||||||
|
last_log_time = 0
|
||||||
|
last_exc_message = None
|
||||||
|
exc_count = 0
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
return infunc(*args, **kwargs)
|
||||||
|
except Exception as exc:
|
||||||
|
this_exc_message = six.u(str(exc))
|
||||||
|
if this_exc_message == last_exc_message:
|
||||||
|
exc_count += 1
|
||||||
|
else:
|
||||||
|
exc_count = 1
|
||||||
|
# Do not log any more frequently than once a minute unless
|
||||||
|
# the exception message changes
|
||||||
|
cur_time = int(time.time())
|
||||||
|
if (cur_time - last_log_time > 60 or
|
||||||
|
this_exc_message != last_exc_message):
|
||||||
|
logging.exception(
|
||||||
|
_LE('Unexpected exception occurred %d time(s)... '
|
||||||
|
'retrying.') % exc_count)
|
||||||
|
last_log_time = cur_time
|
||||||
|
last_exc_message = this_exc_message
|
||||||
|
exc_count = 0
|
||||||
|
# This should be a very rare event. In case it isn't, do
|
||||||
|
# a sleep.
|
||||||
|
time.sleep(1)
|
||||||
|
return inner_func
|
|
@ -0,0 +1,146 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import errno
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from monasca.openstack.common import excutils
|
||||||
|
from monasca.openstack.common import log as logging
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_FILE_CACHE = {}
|
||||||
|
|
||||||
|
|
||||||
|
def ensure_tree(path):
|
||||||
|
"""Create a directory (and any ancestor directories required)
|
||||||
|
|
||||||
|
:param path: Directory to create
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
os.makedirs(path)
|
||||||
|
except OSError as exc:
|
||||||
|
if exc.errno == errno.EEXIST:
|
||||||
|
if not os.path.isdir(path):
|
||||||
|
raise
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def read_cached_file(filename, force_reload=False):
|
||||||
|
"""Read from a file if it has been modified.
|
||||||
|
|
||||||
|
:param force_reload: Whether to reload the file.
|
||||||
|
:returns: A tuple with a boolean specifying if the data is fresh
|
||||||
|
or not.
|
||||||
|
"""
|
||||||
|
global _FILE_CACHE
|
||||||
|
|
||||||
|
if force_reload:
|
||||||
|
delete_cached_file(filename)
|
||||||
|
|
||||||
|
reloaded = False
|
||||||
|
mtime = os.path.getmtime(filename)
|
||||||
|
cache_info = _FILE_CACHE.setdefault(filename, {})
|
||||||
|
|
||||||
|
if not cache_info or mtime > cache_info.get('mtime', 0):
|
||||||
|
LOG.debug("Reloading cached file %s" % filename)
|
||||||
|
with open(filename) as fap:
|
||||||
|
cache_info['data'] = fap.read()
|
||||||
|
cache_info['mtime'] = mtime
|
||||||
|
reloaded = True
|
||||||
|
return (reloaded, cache_info['data'])
|
||||||
|
|
||||||
|
|
||||||
|
def delete_cached_file(filename):
|
||||||
|
"""Delete cached file if present.
|
||||||
|
|
||||||
|
:param filename: filename to delete
|
||||||
|
"""
|
||||||
|
global _FILE_CACHE
|
||||||
|
|
||||||
|
if filename in _FILE_CACHE:
|
||||||
|
del _FILE_CACHE[filename]
|
||||||
|
|
||||||
|
|
||||||
|
def delete_if_exists(path, remove=os.unlink):
|
||||||
|
"""Delete a file, but ignore file not found error.
|
||||||
|
|
||||||
|
:param path: File to delete
|
||||||
|
:param remove: Optional function to remove passed path
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
remove(path)
|
||||||
|
except OSError as e:
|
||||||
|
if e.errno != errno.ENOENT:
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def remove_path_on_error(path, remove=delete_if_exists):
|
||||||
|
"""Protect code that wants to operate on PATH atomically.
|
||||||
|
Any exception will cause PATH to be removed.
|
||||||
|
|
||||||
|
:param path: File to work with
|
||||||
|
:param remove: Optional function to remove passed path
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
except Exception:
|
||||||
|
with excutils.save_and_reraise_exception():
|
||||||
|
remove(path)
|
||||||
|
|
||||||
|
|
||||||
|
def file_open(*args, **kwargs):
|
||||||
|
"""Open file
|
||||||
|
|
||||||
|
see built-in open() documentation for more details
|
||||||
|
|
||||||
|
Note: The reason this is kept in a separate module is to easily
|
||||||
|
be able to provide a stub module that doesn't alter system
|
||||||
|
state at all (for unit tests)
|
||||||
|
"""
|
||||||
|
return open(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def write_to_tempfile(content, path=None, suffix='', prefix='tmp'):
|
||||||
|
"""Create temporary file or use existing file.
|
||||||
|
|
||||||
|
This util is needed for creating temporary file with
|
||||||
|
specified content, suffix and prefix. If path is not None,
|
||||||
|
it will be used for writing content. If the path doesn't
|
||||||
|
exist it'll be created.
|
||||||
|
|
||||||
|
:param content: content for temporary file.
|
||||||
|
:param path: same as parameter 'dir' for mkstemp
|
||||||
|
:param suffix: same as parameter 'suffix' for mkstemp
|
||||||
|
:param prefix: same as parameter 'prefix' for mkstemp
|
||||||
|
|
||||||
|
For example: it can be used in database tests for creating
|
||||||
|
configuration files.
|
||||||
|
"""
|
||||||
|
if path:
|
||||||
|
ensure_tree(path)
|
||||||
|
|
||||||
|
(fd, path) = tempfile.mkstemp(suffix=suffix, dir=path, prefix=prefix)
|
||||||
|
try:
|
||||||
|
os.write(fd, content)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
return path
|
|
@ -0,0 +1,85 @@
|
||||||
|
#
|
||||||
|
# Copyright 2013 Mirantis, Inc.
|
||||||
|
# Copyright 2013 OpenStack Foundation
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import fixtures
|
||||||
|
from oslo.config import cfg
|
||||||
|
import six
|
||||||
|
|
||||||
|
|
||||||
|
class Config(fixtures.Fixture):
|
||||||
|
"""Allows overriding configuration settings for the test.
|
||||||
|
|
||||||
|
`conf` will be reset on cleanup.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, conf=cfg.CONF):
|
||||||
|
self.conf = conf
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(Config, self).setUp()
|
||||||
|
# NOTE(morganfainberg): unregister must be added to cleanup before
|
||||||
|
# reset is because cleanup works in reverse order of registered items,
|
||||||
|
# and a reset must occur before unregistering options can occur.
|
||||||
|
self.addCleanup(self._unregister_config_opts)
|
||||||
|
self.addCleanup(self.conf.reset)
|
||||||
|
self._registered_config_opts = {}
|
||||||
|
|
||||||
|
def config(self, **kw):
|
||||||
|
"""Override configuration values.
|
||||||
|
|
||||||
|
The keyword arguments are the names of configuration options to
|
||||||
|
override and their values.
|
||||||
|
|
||||||
|
If a `group` argument is supplied, the overrides are applied to
|
||||||
|
the specified configuration option group, otherwise the overrides
|
||||||
|
are applied to the ``default`` group.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
group = kw.pop('group', None)
|
||||||
|
for k, v in six.iteritems(kw):
|
||||||
|
self.conf.set_override(k, v, group)
|
||||||
|
|
||||||
|
def _unregister_config_opts(self):
|
||||||
|
for group in self._registered_config_opts:
|
||||||
|
self.conf.unregister_opts(self._registered_config_opts[group],
|
||||||
|
group=group)
|
||||||
|
|
||||||
|
def register_opt(self, opt, group=None):
|
||||||
|
"""Register a single option for the test run.
|
||||||
|
|
||||||
|
Options registered in this manner will automatically be unregistered
|
||||||
|
during cleanup.
|
||||||
|
|
||||||
|
If a `group` argument is supplied, it will register the new option
|
||||||
|
to that group, otherwise the option is registered to the ``default``
|
||||||
|
group.
|
||||||
|
"""
|
||||||
|
self.conf.register_opt(opt, group=group)
|
||||||
|
self._registered_config_opts.setdefault(group, set()).add(opt)
|
||||||
|
|
||||||
|
def register_opts(self, opts, group=None):
|
||||||
|
"""Register multiple options for the test run.
|
||||||
|
|
||||||
|
This works in the same manner as register_opt() but takes a list of
|
||||||
|
options as the first argument. All arguments will be registered to the
|
||||||
|
same group if the ``group`` argument is supplied, otherwise all options
|
||||||
|
will be registered to the ``default`` group.
|
||||||
|
"""
|
||||||
|
for opt in opts:
|
||||||
|
self.register_opt(opt, group=group)
|
|
@ -0,0 +1,51 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import fixtures
|
||||||
|
|
||||||
|
from monasca.openstack.common import lockutils
|
||||||
|
|
||||||
|
|
||||||
|
class LockFixture(fixtures.Fixture):
|
||||||
|
"""External locking fixture.
|
||||||
|
|
||||||
|
This fixture is basically an alternative to the synchronized decorator with
|
||||||
|
the external flag so that tearDowns and addCleanups will be included in
|
||||||
|
the lock context for locking between tests. The fixture is recommended to
|
||||||
|
be the first line in a test method, like so::
|
||||||
|
|
||||||
|
def test_method(self):
|
||||||
|
self.useFixture(LockFixture)
|
||||||
|
...
|
||||||
|
|
||||||
|
or the first line in setUp if all the test methods in the class are
|
||||||
|
required to be serialized. Something like::
|
||||||
|
|
||||||
|
class TestCase(testtools.testcase):
|
||||||
|
def setUp(self):
|
||||||
|
self.useFixture(LockFixture)
|
||||||
|
super(TestCase, self).setUp()
|
||||||
|
...
|
||||||
|
|
||||||
|
This is because addCleanups are put on a LIFO queue that gets run after the
|
||||||
|
test method exits. (either by completing or raising an exception)
|
||||||
|
"""
|
||||||
|
def __init__(self, name, lock_file_prefix=None):
|
||||||
|
self.mgr = lockutils.lock(name, lock_file_prefix, True)
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(LockFixture, self).setUp()
|
||||||
|
self.addCleanup(self.mgr.__exit__, None, None, None)
|
||||||
|
self.lock = self.mgr.__enter__()
|
|
@ -0,0 +1,34 @@
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import fixtures
|
||||||
|
|
||||||
|
|
||||||
|
def get_logging_handle_error_fixture():
|
||||||
|
"""returns a fixture to make logging raise formatting exceptions.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
self.useFixture(logging.get_logging_handle_error_fixture())
|
||||||
|
"""
|
||||||
|
return fixtures.MonkeyPatch('logging.Handler.handleError',
|
||||||
|
_handleError)
|
||||||
|
|
||||||
|
|
||||||
|
def _handleError(self, record):
|
||||||
|
"""Monkey patch for logging.Handler.handleError.
|
||||||
|
|
||||||
|
The default handleError just logs the error to stderr but we want
|
||||||
|
the option of actually raising an exception.
|
||||||
|
"""
|
||||||
|
raise
|
|
@ -0,0 +1,62 @@
|
||||||
|
# Copyright 2010 United States Government as represented by the
|
||||||
|
# Administrator of the National Aeronautics and Space Administration.
|
||||||
|
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# DO NOT MODIFY THIS FILE
|
||||||
|
#
|
||||||
|
# This file is being graduated to the oslotest library. Please make all
|
||||||
|
# changes there, and only backport critical fixes here. - dhellmann
|
||||||
|
#
|
||||||
|
##############################################################################
|
||||||
|
##############################################################################
|
||||||
|
|
||||||
|
import fixtures
|
||||||
|
import mock
|
||||||
|
|
||||||
|
|
||||||
|
class PatchObject(fixtures.Fixture):
|
||||||
|
"""Deal with code around mock."""
|
||||||
|
|
||||||
|
def __init__(self, obj, attr, new=mock.DEFAULT, **kwargs):
|
||||||
|
self.obj = obj
|
||||||
|
self.attr = attr
|
||||||
|
self.kwargs = kwargs
|
||||||
|
self.new = new
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(PatchObject, self).setUp()
|
||||||
|
_p = mock.patch.object(self.obj, self.attr, self.new, **self.kwargs)
|
||||||
|
self.mock = _p.start()
|
||||||
|
self.addCleanup(_p.stop)
|
||||||
|
|
||||||
|
|
||||||
|
class Patch(fixtures.Fixture):
|
||||||
|
|
||||||
|
"""Deal with code around mock.patch."""
|
||||||
|
|
||||||
|
def __init__(self, obj, new=mock.DEFAULT, **kwargs):
|
||||||
|
self.obj = obj
|
||||||
|
self.kwargs = kwargs
|
||||||
|
self.new = new
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(Patch, self).setUp()
|
||||||
|
_p = mock.patch(self.obj, self.new, **self.kwargs)
|
||||||
|
self.mock = _p.start()
|
||||||
|
self.addCleanup(_p.stop)
|
|
@ -0,0 +1,43 @@
|
||||||
|
# Copyright 2010 United States Government as represented by the
|
||||||
|
# Administrator of the National Aeronautics and Space Administration.
|
||||||
|
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
##############################################################################
|
||||||
|
##############################################################################
|
||||||
|
#
|
||||||
|
# DO NOT MODIFY THIS FILE
|
||||||
|
#
|
||||||
|
# This file is being graduated to the oslotest library. Please make all
|
||||||
|
# changes there, and only backport critical fixes here. - dhellmann
|
||||||
|
#
|
||||||
|
##############################################################################
|
||||||
|
##############################################################################
|
||||||
|
|
||||||
|
import fixtures
|
||||||
|
from six.moves import mox
|
||||||
|
|
||||||
|
|
||||||
|
class MoxStubout(fixtures.Fixture):
|
||||||
|
"""Deal with code around mox and stubout as a fixture."""
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(MoxStubout, self).setUp()
|
||||||
|
# emulate some of the mox stuff, we can't use the metaclass
|
||||||
|
# because it screws with our generators
|
||||||
|
self.mox = mox.Mox()
|
||||||
|
self.stubs = self.mox.stubs
|
||||||
|
self.addCleanup(self.mox.UnsetStubs)
|
||||||
|
self.addCleanup(self.mox.VerifyAll)
|
|
@ -0,0 +1,479 @@
|
||||||
|
# Copyright 2012 Red Hat, Inc.
|
||||||
|
# Copyright 2013 IBM Corp.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
gettext for openstack-common modules.
|
||||||
|
|
||||||
|
Usual usage in an openstack.common module:
|
||||||
|
|
||||||
|
from monasca.openstack.common.gettextutils import _
|
||||||
|
"""
|
||||||
|
|
||||||
|
import copy
|
||||||
|
import gettext
|
||||||
|
import locale
|
||||||
|
from logging import handlers
|
||||||
|
import os
|
||||||
|
|
||||||
|
from babel import localedata
|
||||||
|
import six
|
||||||
|
|
||||||
|
_AVAILABLE_LANGUAGES = {}
|
||||||
|
|
||||||
|
# FIXME(dhellmann): Remove this when moving to oslo.i18n.
|
||||||
|
USE_LAZY = False
|
||||||
|
|
||||||
|
|
||||||
|
class TranslatorFactory(object):
|
||||||
|
"""Create translator functions
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, domain, localedir=None):
|
||||||
|
"""Establish a set of translation functions for the domain.
|
||||||
|
|
||||||
|
:param domain: Name of translation domain,
|
||||||
|
specifying a message catalog.
|
||||||
|
:type domain: str
|
||||||
|
:param lazy: Delays translation until a message is emitted.
|
||||||
|
Defaults to False.
|
||||||
|
:type lazy: Boolean
|
||||||
|
:param localedir: Directory with translation catalogs.
|
||||||
|
:type localedir: str
|
||||||
|
"""
|
||||||
|
self.domain = domain
|
||||||
|
if localedir is None:
|
||||||
|
localedir = os.environ.get(domain.upper() + '_LOCALEDIR')
|
||||||
|
self.localedir = localedir
|
||||||
|
|
||||||
|
def _make_translation_func(self, domain=None):
|
||||||
|
"""Return a new translation function ready for use.
|
||||||
|
|
||||||
|
Takes into account whether or not lazy translation is being
|
||||||
|
done.
|
||||||
|
|
||||||
|
The domain can be specified to override the default from the
|
||||||
|
factory, but the localedir from the factory is always used
|
||||||
|
because we assume the log-level translation catalogs are
|
||||||
|
installed in the same directory as the main application
|
||||||
|
catalog.
|
||||||
|
|
||||||
|
"""
|
||||||
|
if domain is None:
|
||||||
|
domain = self.domain
|
||||||
|
t = gettext.translation(domain,
|
||||||
|
localedir=self.localedir,
|
||||||
|
fallback=True)
|
||||||
|
# Use the appropriate method of the translation object based
|
||||||
|
# on the python version.
|
||||||
|
m = t.gettext if six.PY3 else t.ugettext
|
||||||
|
|
||||||
|
def f(msg):
|
||||||
|
"""oslo.i18n.gettextutils translation function."""
|
||||||
|
if USE_LAZY:
|
||||||
|
return Message(msg, domain=domain)
|
||||||
|
return m(msg)
|
||||||
|
return f
|
||||||
|
|
||||||
|
@property
|
||||||
|
def primary(self):
|
||||||
|
"The default translation function."
|
||||||
|
return self._make_translation_func()
|
||||||
|
|
||||||
|
def _make_log_translation_func(self, level):
|
||||||
|
return self._make_translation_func(self.domain + '-log-' + level)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def log_info(self):
|
||||||
|
"Translate info-level log messages."
|
||||||
|
return self._make_log_translation_func('info')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def log_warning(self):
|
||||||
|
"Translate warning-level log messages."
|
||||||
|
return self._make_log_translation_func('warning')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def log_error(self):
|
||||||
|
"Translate error-level log messages."
|
||||||
|
return self._make_log_translation_func('error')
|
||||||
|
|
||||||
|
@property
|
||||||
|
def log_critical(self):
|
||||||
|
"Translate critical-level log messages."
|
||||||
|
return self._make_log_translation_func('critical')
|
||||||
|
|
||||||
|
|
||||||
|
# NOTE(dhellmann): When this module moves out of the incubator into
|
||||||
|
# oslo.i18n, these global variables can be moved to an integration
|
||||||
|
# module within each application.
|
||||||
|
|
||||||
|
# Create the global translation functions.
|
||||||
|
_translators = TranslatorFactory('monasca')
|
||||||
|
|
||||||
|
# The primary translation function using the well-known name "_"
|
||||||
|
_ = _translators.primary
|
||||||
|
|
||||||
|
# Translators for log levels.
|
||||||
|
#
|
||||||
|
# The abbreviated names are meant to reflect the usual use of a short
|
||||||
|
# name like '_'. The "L" is for "log" and the other letter comes from
|
||||||
|
# the level.
|
||||||
|
_LI = _translators.log_info
|
||||||
|
_LW = _translators.log_warning
|
||||||
|
_LE = _translators.log_error
|
||||||
|
_LC = _translators.log_critical
|
||||||
|
|
||||||
|
# NOTE(dhellmann): End of globals that will move to the application's
|
||||||
|
# integration module.
|
||||||
|
|
||||||
|
|
||||||
|
def enable_lazy():
|
||||||
|
"""Convenience function for configuring _() to use lazy gettext
|
||||||
|
|
||||||
|
Call this at the start of execution to enable the gettextutils._
|
||||||
|
function to use lazy gettext functionality. This is useful if
|
||||||
|
your project is importing _ directly instead of using the
|
||||||
|
gettextutils.install() way of importing the _ function.
|
||||||
|
"""
|
||||||
|
global USE_LAZY
|
||||||
|
USE_LAZY = True
|
||||||
|
|
||||||
|
|
||||||
|
def install(domain):
|
||||||
|
"""Install a _() function using the given translation domain.
|
||||||
|
|
||||||
|
Given a translation domain, install a _() function using gettext's
|
||||||
|
install() function.
|
||||||
|
|
||||||
|
The main difference from gettext.install() is that we allow
|
||||||
|
overriding the default localedir (e.g. /usr/share/locale) using
|
||||||
|
a translation-domain-specific environment variable (e.g.
|
||||||
|
NOVA_LOCALEDIR).
|
||||||
|
|
||||||
|
Note that to enable lazy translation, enable_lazy must be
|
||||||
|
called.
|
||||||
|
|
||||||
|
:param domain: the translation domain
|
||||||
|
"""
|
||||||
|
from six import moves
|
||||||
|
tf = TranslatorFactory(domain)
|
||||||
|
moves.builtins.__dict__['_'] = tf.primary
|
||||||
|
|
||||||
|
|
||||||
|
class Message(six.text_type):
|
||||||
|
"""A Message object is a unicode object that can be translated.
|
||||||
|
|
||||||
|
Translation of Message is done explicitly using the translate() method.
|
||||||
|
For all non-translation intents and purposes, a Message is simply unicode,
|
||||||
|
and can be treated as such.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __new__(cls, msgid, msgtext=None, params=None,
|
||||||
|
domain='monasca', *args):
|
||||||
|
"""Create a new Message object.
|
||||||
|
|
||||||
|
In order for translation to work gettext requires a message ID, this
|
||||||
|
msgid will be used as the base unicode text. It is also possible
|
||||||
|
for the msgid and the base unicode text to be different by passing
|
||||||
|
the msgtext parameter.
|
||||||
|
"""
|
||||||
|
# If the base msgtext is not given, we use the default translation
|
||||||
|
# of the msgid (which is in English) just in case the system locale is
|
||||||
|
# not English, so that the base text will be in that locale by default.
|
||||||
|
if not msgtext:
|
||||||
|
msgtext = Message._translate_msgid(msgid, domain)
|
||||||
|
# We want to initialize the parent unicode with the actual object that
|
||||||
|
# would have been plain unicode if 'Message' was not enabled.
|
||||||
|
msg = super(Message, cls).__new__(cls, msgtext)
|
||||||
|
msg.msgid = msgid
|
||||||
|
msg.domain = domain
|
||||||
|
msg.params = params
|
||||||
|
return msg
|
||||||
|
|
||||||
|
def translate(self, desired_locale=None):
|
||||||
|
"""Translate this message to the desired locale.
|
||||||
|
|
||||||
|
:param desired_locale: The desired locale to translate the message to,
|
||||||
|
if no locale is provided the message will be
|
||||||
|
translated to the system's default locale.
|
||||||
|
|
||||||
|
:returns: the translated message in unicode
|
||||||
|
"""
|
||||||
|
|
||||||
|
translated_message = Message._translate_msgid(self.msgid,
|
||||||
|
self.domain,
|
||||||
|
desired_locale)
|
||||||
|
if self.params is None:
|
||||||
|
# No need for more translation
|
||||||
|
return translated_message
|
||||||
|
|
||||||
|
# This Message object may have been formatted with one or more
|
||||||
|
# Message objects as substitution arguments, given either as a single
|
||||||
|
# argument, part of a tuple, or as one or more values in a dictionary.
|
||||||
|
# When translating this Message we need to translate those Messages too
|
||||||
|
translated_params = _translate_args(self.params, desired_locale)
|
||||||
|
|
||||||
|
translated_message = translated_message % translated_params
|
||||||
|
|
||||||
|
return translated_message
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _translate_msgid(msgid, domain, desired_locale=None):
|
||||||
|
if not desired_locale:
|
||||||
|
system_locale = locale.getdefaultlocale()
|
||||||
|
# If the system locale is not available to the runtime use English
|
||||||
|
if not system_locale[0]:
|
||||||
|
desired_locale = 'en_US'
|
||||||
|
else:
|
||||||
|
desired_locale = system_locale[0]
|
||||||
|
|
||||||
|
locale_dir = os.environ.get(domain.upper() + '_LOCALEDIR')
|
||||||
|
lang = gettext.translation(domain,
|
||||||
|
localedir=locale_dir,
|
||||||
|
languages=[desired_locale],
|
||||||
|
fallback=True)
|
||||||
|
if six.PY3:
|
||||||
|
translator = lang.gettext
|
||||||
|
else:
|
||||||
|
translator = lang.ugettext
|
||||||
|
|
||||||
|
translated_message = translator(msgid)
|
||||||
|
return translated_message
|
||||||
|
|
||||||
|
def __mod__(self, other):
|
||||||
|
# When we mod a Message we want the actual operation to be performed
|
||||||
|
# by the parent class (i.e. unicode()), the only thing we do here is
|
||||||
|
# save the original msgid and the parameters in case of a translation
|
||||||
|
params = self._sanitize_mod_params(other)
|
||||||
|
unicode_mod = super(Message, self).__mod__(params)
|
||||||
|
modded = Message(self.msgid,
|
||||||
|
msgtext=unicode_mod,
|
||||||
|
params=params,
|
||||||
|
domain=self.domain)
|
||||||
|
return modded
|
||||||
|
|
||||||
|
def _sanitize_mod_params(self, other):
|
||||||
|
"""Sanitize the object being modded with this Message.
|
||||||
|
|
||||||
|
- Add support for modding 'None' so translation supports it
|
||||||
|
- Trim the modded object, which can be a large dictionary, to only
|
||||||
|
those keys that would actually be used in a translation
|
||||||
|
- Snapshot the object being modded, in case the message is
|
||||||
|
translated, it will be used as it was when the Message was created
|
||||||
|
"""
|
||||||
|
if other is None:
|
||||||
|
params = (other,)
|
||||||
|
elif isinstance(other, dict):
|
||||||
|
# Merge the dictionaries
|
||||||
|
# Copy each item in case one does not support deep copy.
|
||||||
|
params = {}
|
||||||
|
if isinstance(self.params, dict):
|
||||||
|
for key, val in self.params.items():
|
||||||
|
params[key] = self._copy_param(val)
|
||||||
|
for key, val in other.items():
|
||||||
|
params[key] = self._copy_param(val)
|
||||||
|
else:
|
||||||
|
params = self._copy_param(other)
|
||||||
|
return params
|
||||||
|
|
||||||
|
def _copy_param(self, param):
|
||||||
|
try:
|
||||||
|
return copy.deepcopy(param)
|
||||||
|
except Exception:
|
||||||
|
# Fallback to casting to unicode this will handle the
|
||||||
|
# python code-like objects that can't be deep-copied
|
||||||
|
return six.text_type(param)
|
||||||
|
|
||||||
|
def __add__(self, other):
|
||||||
|
msg = _('Message objects do not support addition.')
|
||||||
|
raise TypeError(msg)
|
||||||
|
|
||||||
|
def __radd__(self, other):
|
||||||
|
return self.__add__(other)
|
||||||
|
|
||||||
|
if six.PY2:
|
||||||
|
def __str__(self):
|
||||||
|
# NOTE(luisg): Logging in python 2.6 tries to str() log records,
|
||||||
|
# and it expects specifically a UnicodeError in order to proceed.
|
||||||
|
msg = _('Message objects do not support str() because they may '
|
||||||
|
'contain non-ascii characters. '
|
||||||
|
'Please use unicode() or translate() instead.')
|
||||||
|
raise UnicodeError(msg)
|
||||||
|
|
||||||
|
|
||||||
|
def get_available_languages(domain):
|
||||||
|
"""Lists the available languages for the given translation domain.
|
||||||
|
|
||||||
|
:param domain: the domain to get languages for
|
||||||
|
"""
|
||||||
|
if domain in _AVAILABLE_LANGUAGES:
|
||||||
|
return copy.copy(_AVAILABLE_LANGUAGES[domain])
|
||||||
|
|
||||||
|
localedir = '%s_LOCALEDIR' % domain.upper()
|
||||||
|
find = lambda x: gettext.find(domain,
|
||||||
|
localedir=os.environ.get(localedir),
|
||||||
|
languages=[x])
|
||||||
|
|
||||||
|
# NOTE(mrodden): en_US should always be available (and first in case
|
||||||
|
# order matters) since our in-line message strings are en_US
|
||||||
|
language_list = ['en_US']
|
||||||
|
# NOTE(luisg): Babel <1.0 used a function called list(), which was
|
||||||
|
# renamed to locale_identifiers() in >=1.0, the requirements master list
|
||||||
|
# requires >=0.9.6, uncapped, so defensively work with both. We can remove
|
||||||
|
# this check when the master list updates to >=1.0, and update all projects
|
||||||
|
list_identifiers = (getattr(localedata, 'list', None) or
|
||||||
|
getattr(localedata, 'locale_identifiers'))
|
||||||
|
locale_identifiers = list_identifiers()
|
||||||
|
|
||||||
|
for i in locale_identifiers:
|
||||||
|
if find(i) is not None:
|
||||||
|
language_list.append(i)
|
||||||
|
|
||||||
|
# NOTE(luisg): Babel>=1.0,<1.3 has a bug where some OpenStack supported
|
||||||
|
# locales (e.g. 'zh_CN', and 'zh_TW') aren't supported even though they
|
||||||
|
# are perfectly legitimate locales:
|
||||||
|
# https://github.com/mitsuhiko/babel/issues/37
|
||||||
|
# In Babel 1.3 they fixed the bug and they support these locales, but
|
||||||
|
# they are still not explicitly "listed" by locale_identifiers().
|
||||||
|
# That is why we add the locales here explicitly if necessary so that
|
||||||
|
# they are listed as supported.
|
||||||
|
aliases = {'zh': 'zh_CN',
|
||||||
|
'zh_Hant_HK': 'zh_HK',
|
||||||
|
'zh_Hant': 'zh_TW',
|
||||||
|
'fil': 'tl_PH'}
|
||||||
|
for (locale_, alias) in six.iteritems(aliases):
|
||||||
|
if locale_ in language_list and alias not in language_list:
|
||||||
|
language_list.append(alias)
|
||||||
|
|
||||||
|
_AVAILABLE_LANGUAGES[domain] = language_list
|
||||||
|
return copy.copy(language_list)
|
||||||
|
|
||||||
|
|
||||||
|
def translate(obj, desired_locale=None):
|
||||||
|
"""Gets the translated unicode representation of the given object.
|
||||||
|
|
||||||
|
If the object is not translatable it is returned as-is.
|
||||||
|
If the locale is None the object is translated to the system locale.
|
||||||
|
|
||||||
|
:param obj: the object to translate
|
||||||
|
:param desired_locale: the locale to translate the message to, if None the
|
||||||
|
default system locale will be used
|
||||||
|
:returns: the translated object in unicode, or the original object if
|
||||||
|
it could not be translated
|
||||||
|
"""
|
||||||
|
message = obj
|
||||||
|
if not isinstance(message, Message):
|
||||||
|
# If the object to translate is not already translatable,
|
||||||
|
# let's first get its unicode representation
|
||||||
|
message = six.text_type(obj)
|
||||||
|
if isinstance(message, Message):
|
||||||
|
# Even after unicoding() we still need to check if we are
|
||||||
|
# running with translatable unicode before translating
|
||||||
|
return message.translate(desired_locale)
|
||||||
|
return obj
|
||||||
|
|
||||||
|
|
||||||
|
def _translate_args(args, desired_locale=None):
|
||||||
|
"""Translates all the translatable elements of the given arguments object.
|
||||||
|
|
||||||
|
This method is used for translating the translatable values in method
|
||||||
|
arguments which include values of tuples or dictionaries.
|
||||||
|
If the object is not a tuple or a dictionary the object itself is
|
||||||
|
translated if it is translatable.
|
||||||
|
|
||||||
|
If the locale is None the object is translated to the system locale.
|
||||||
|
|
||||||
|
:param args: the args to translate
|
||||||
|
:param desired_locale: the locale to translate the args to, if None the
|
||||||
|
default system locale will be used
|
||||||
|
:returns: a new args object with the translated contents of the original
|
||||||
|
"""
|
||||||
|
if isinstance(args, tuple):
|
||||||
|
return tuple(translate(v, desired_locale) for v in args)
|
||||||
|
if isinstance(args, dict):
|
||||||
|
translated_dict = {}
|
||||||
|
for (k, v) in six.iteritems(args):
|
||||||
|
translated_v = translate(v, desired_locale)
|
||||||
|
translated_dict[k] = translated_v
|
||||||
|
return translated_dict
|
||||||
|
return translate(args, desired_locale)
|
||||||
|
|
||||||
|
|
||||||
|
class TranslationHandler(handlers.MemoryHandler):
|
||||||
|
"""Handler that translates records before logging them.
|
||||||
|
|
||||||
|
The TranslationHandler takes a locale and a target logging.Handler object
|
||||||
|
to forward LogRecord objects to after translating them. This handler
|
||||||
|
depends on Message objects being logged, instead of regular strings.
|
||||||
|
|
||||||
|
The handler can be configured declaratively in the logging.conf as follows:
|
||||||
|
|
||||||
|
[handlers]
|
||||||
|
keys = translatedlog, translator
|
||||||
|
|
||||||
|
[handler_translatedlog]
|
||||||
|
class = handlers.WatchedFileHandler
|
||||||
|
args = ('/var/log/api-localized.log',)
|
||||||
|
formatter = context
|
||||||
|
|
||||||
|
[handler_translator]
|
||||||
|
class = openstack.common.log.TranslationHandler
|
||||||
|
target = translatedlog
|
||||||
|
args = ('zh_CN',)
|
||||||
|
|
||||||
|
If the specified locale is not available in the system, the handler will
|
||||||
|
log in the default locale.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, locale=None, target=None):
|
||||||
|
"""Initialize a TranslationHandler
|
||||||
|
|
||||||
|
:param locale: locale to use for translating messages
|
||||||
|
:param target: logging.Handler object to forward
|
||||||
|
LogRecord objects to after translation
|
||||||
|
"""
|
||||||
|
# NOTE(luisg): In order to allow this handler to be a wrapper for
|
||||||
|
# other handlers, such as a FileHandler, and still be able to
|
||||||
|
# configure it using logging.conf, this handler has to extend
|
||||||
|
# MemoryHandler because only the MemoryHandlers' logging.conf
|
||||||
|
# parsing is implemented such that it accepts a target handler.
|
||||||
|
handlers.MemoryHandler.__init__(self, capacity=0, target=target)
|
||||||
|
self.locale = locale
|
||||||
|
|
||||||
|
def setFormatter(self, fmt):
|
||||||
|
self.target.setFormatter(fmt)
|
||||||
|
|
||||||
|
def emit(self, record):
|
||||||
|
# We save the message from the original record to restore it
|
||||||
|
# after translation, so other handlers are not affected by this
|
||||||
|
original_msg = record.msg
|
||||||
|
original_args = record.args
|
||||||
|
|
||||||
|
try:
|
||||||
|
self._translate_and_log_record(record)
|
||||||
|
finally:
|
||||||
|
record.msg = original_msg
|
||||||
|
record.args = original_args
|
||||||
|
|
||||||
|
def _translate_and_log_record(self, record):
|
||||||
|
record.msg = translate(record.msg, self.locale)
|
||||||
|
|
||||||
|
# In addition to translating the message, we also need to translate
|
||||||
|
# arguments that were passed to the log method that were not part
|
||||||
|
# of the main message e.g., log.info(_('Some message %s'), this_one))
|
||||||
|
record.args = _translate_args(record.args, self.locale)
|
||||||
|
|
||||||
|
self.target.emit(record)
|
|
@ -0,0 +1,73 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Import related utilities and helper functions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
|
||||||
|
def import_class(import_str):
|
||||||
|
"""Returns a class from a string including module and class."""
|
||||||
|
mod_str, _sep, class_str = import_str.rpartition('.')
|
||||||
|
__import__(mod_str)
|
||||||
|
try:
|
||||||
|
return getattr(sys.modules[mod_str], class_str)
|
||||||
|
except AttributeError:
|
||||||
|
raise ImportError('Class %s cannot be found (%s)' %
|
||||||
|
(class_str,
|
||||||
|
traceback.format_exception(*sys.exc_info())))
|
||||||
|
|
||||||
|
|
||||||
|
def import_object(import_str, *args, **kwargs):
|
||||||
|
"""Import a class and return an instance of it."""
|
||||||
|
return import_class(import_str)(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def import_object_ns(name_space, import_str, *args, **kwargs):
|
||||||
|
"""Tries to import object from default namespace.
|
||||||
|
|
||||||
|
Imports a class and return an instance of it, first by trying
|
||||||
|
to find the class in a default namespace, then failing back to
|
||||||
|
a full path if not found in the default namespace.
|
||||||
|
"""
|
||||||
|
import_value = "%s.%s" % (name_space, import_str)
|
||||||
|
try:
|
||||||
|
return import_class(import_value)(*args, **kwargs)
|
||||||
|
except ImportError:
|
||||||
|
return import_class(import_str)(*args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def import_module(import_str):
|
||||||
|
"""Import a module."""
|
||||||
|
__import__(import_str)
|
||||||
|
return sys.modules[import_str]
|
||||||
|
|
||||||
|
|
||||||
|
def import_versioned_module(version, submodule=None):
|
||||||
|
module = 'monasca.v%s' % version
|
||||||
|
if submodule:
|
||||||
|
module = '.'.join((module, submodule))
|
||||||
|
return import_module(module)
|
||||||
|
|
||||||
|
|
||||||
|
def try_import(import_str, default=None):
|
||||||
|
"""Try to import a module and if it fails return default."""
|
||||||
|
try:
|
||||||
|
return import_module(import_str)
|
||||||
|
except ImportError:
|
||||||
|
return default
|
|
@ -0,0 +1,202 @@
|
||||||
|
# Copyright 2010 United States Government as represented by the
|
||||||
|
# Administrator of the National Aeronautics and Space Administration.
|
||||||
|
# Copyright 2011 Justin Santa Barbara
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
'''
|
||||||
|
JSON related utilities.
|
||||||
|
|
||||||
|
This module provides a few things:
|
||||||
|
|
||||||
|
1) A handy function for getting an object down to something that can be
|
||||||
|
JSON serialized. See to_primitive().
|
||||||
|
|
||||||
|
2) Wrappers around loads() and dumps(). The dumps() wrapper will
|
||||||
|
automatically use to_primitive() for you if needed.
|
||||||
|
|
||||||
|
3) This sets up anyjson to use the loads() and dumps() wrappers if anyjson
|
||||||
|
is available.
|
||||||
|
'''
|
||||||
|
|
||||||
|
|
||||||
|
import codecs
|
||||||
|
import datetime
|
||||||
|
import functools
|
||||||
|
import inspect
|
||||||
|
import itertools
|
||||||
|
import sys
|
||||||
|
|
||||||
|
is_simplejson = False
|
||||||
|
if sys.version_info < (2, 7):
|
||||||
|
# On Python <= 2.6, json module is not C boosted, so try to use
|
||||||
|
# simplejson module if available
|
||||||
|
try:
|
||||||
|
import simplejson as json
|
||||||
|
# NOTE(mriedem): Make sure we have a new enough version of simplejson
|
||||||
|
# to support the namedobject_as_tuple argument. This can be removed
|
||||||
|
# in the Kilo release when python 2.6 support is dropped.
|
||||||
|
if 'namedtuple_as_object' in inspect.getargspec(json.dumps).args:
|
||||||
|
is_simplejson = True
|
||||||
|
else:
|
||||||
|
import json
|
||||||
|
except ImportError:
|
||||||
|
import json
|
||||||
|
else:
|
||||||
|
import json
|
||||||
|
|
||||||
|
import six
|
||||||
|
import six.moves.xmlrpc_client as xmlrpclib
|
||||||
|
|
||||||
|
from monasca.openstack.common import gettextutils
|
||||||
|
from monasca.openstack.common import importutils
|
||||||
|
from monasca.openstack.common import strutils
|
||||||
|
from monasca.openstack.common import timeutils
|
||||||
|
|
||||||
|
netaddr = importutils.try_import("netaddr")
|
||||||
|
|
||||||
|
_nasty_type_tests = [inspect.ismodule, inspect.isclass, inspect.ismethod,
|
||||||
|
inspect.isfunction, inspect.isgeneratorfunction,
|
||||||
|
inspect.isgenerator, inspect.istraceback, inspect.isframe,
|
||||||
|
inspect.iscode, inspect.isbuiltin, inspect.isroutine,
|
||||||
|
inspect.isabstract]
|
||||||
|
|
||||||
|
_simple_types = (six.string_types + six.integer_types
|
||||||
|
+ (type(None), bool, float))
|
||||||
|
|
||||||
|
|
||||||
|
def to_primitive(value, convert_instances=False, convert_datetime=True,
|
||||||
|
level=0, max_depth=3):
|
||||||
|
"""Convert a complex object into primitives.
|
||||||
|
|
||||||
|
Handy for JSON serialization. We can optionally handle instances,
|
||||||
|
but since this is a recursive function, we could have cyclical
|
||||||
|
data structures.
|
||||||
|
|
||||||
|
To handle cyclical data structures we could track the actual objects
|
||||||
|
visited in a set, but not all objects are hashable. Instead we just
|
||||||
|
track the depth of the object inspections and don't go too deep.
|
||||||
|
|
||||||
|
Therefore, convert_instances=True is lossy ... be aware.
|
||||||
|
|
||||||
|
"""
|
||||||
|
# handle obvious types first - order of basic types determined by running
|
||||||
|
# full tests on nova project, resulting in the following counts:
|
||||||
|
# 572754 <type 'NoneType'>
|
||||||
|
# 460353 <type 'int'>
|
||||||
|
# 379632 <type 'unicode'>
|
||||||
|
# 274610 <type 'str'>
|
||||||
|
# 199918 <type 'dict'>
|
||||||
|
# 114200 <type 'datetime.datetime'>
|
||||||
|
# 51817 <type 'bool'>
|
||||||
|
# 26164 <type 'list'>
|
||||||
|
# 6491 <type 'float'>
|
||||||
|
# 283 <type 'tuple'>
|
||||||
|
# 19 <type 'long'>
|
||||||
|
if isinstance(value, _simple_types):
|
||||||
|
return value
|
||||||
|
|
||||||
|
if isinstance(value, datetime.datetime):
|
||||||
|
if convert_datetime:
|
||||||
|
return timeutils.strtime(value)
|
||||||
|
else:
|
||||||
|
return value
|
||||||
|
|
||||||
|
# value of itertools.count doesn't get caught by nasty_type_tests
|
||||||
|
# and results in infinite loop when list(value) is called.
|
||||||
|
if type(value) == itertools.count:
|
||||||
|
return six.text_type(value)
|
||||||
|
|
||||||
|
# FIXME(vish): Workaround for LP bug 852095. Without this workaround,
|
||||||
|
# tests that raise an exception in a mocked method that
|
||||||
|
# has a @wrap_exception with a notifier will fail. If
|
||||||
|
# we up the dependency to 0.5.4 (when it is released) we
|
||||||
|
# can remove this workaround.
|
||||||
|
if getattr(value, '__module__', None) == 'mox':
|
||||||
|
return 'mock'
|
||||||
|
|
||||||
|
if level > max_depth:
|
||||||
|
return '?'
|
||||||
|
|
||||||
|
# The try block may not be necessary after the class check above,
|
||||||
|
# but just in case ...
|
||||||
|
try:
|
||||||
|
recursive = functools.partial(to_primitive,
|
||||||
|
convert_instances=convert_instances,
|
||||||
|
convert_datetime=convert_datetime,
|
||||||
|
level=level,
|
||||||
|
max_depth=max_depth)
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return dict((k, recursive(v)) for k, v in six.iteritems(value))
|
||||||
|
elif isinstance(value, (list, tuple)):
|
||||||
|
return [recursive(lv) for lv in value]
|
||||||
|
|
||||||
|
# It's not clear why xmlrpclib created their own DateTime type, but
|
||||||
|
# for our purposes, make it a datetime type which is explicitly
|
||||||
|
# handled
|
||||||
|
if isinstance(value, xmlrpclib.DateTime):
|
||||||
|
value = datetime.datetime(*tuple(value.timetuple())[:6])
|
||||||
|
|
||||||
|
if convert_datetime and isinstance(value, datetime.datetime):
|
||||||
|
return timeutils.strtime(value)
|
||||||
|
elif isinstance(value, gettextutils.Message):
|
||||||
|
return value.data
|
||||||
|
elif hasattr(value, 'iteritems'):
|
||||||
|
return recursive(dict(value.iteritems()), level=level + 1)
|
||||||
|
elif hasattr(value, '__iter__'):
|
||||||
|
return recursive(list(value))
|
||||||
|
elif convert_instances and hasattr(value, '__dict__'):
|
||||||
|
# Likely an instance of something. Watch for cycles.
|
||||||
|
# Ignore class member vars.
|
||||||
|
return recursive(value.__dict__, level=level + 1)
|
||||||
|
elif netaddr and isinstance(value, netaddr.IPAddress):
|
||||||
|
return six.text_type(value)
|
||||||
|
else:
|
||||||
|
if any(test(value) for test in _nasty_type_tests):
|
||||||
|
return six.text_type(value)
|
||||||
|
return value
|
||||||
|
except TypeError:
|
||||||
|
# Class objects are tricky since they may define something like
|
||||||
|
# __iter__ defined but it isn't callable as list().
|
||||||
|
return six.text_type(value)
|
||||||
|
|
||||||
|
|
||||||
|
def dumps(value, default=to_primitive, **kwargs):
|
||||||
|
if is_simplejson:
|
||||||
|
kwargs['namedtuple_as_object'] = False
|
||||||
|
return json.dumps(value, default=default, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def dump(obj, fp, *args, **kwargs):
|
||||||
|
if is_simplejson:
|
||||||
|
kwargs['namedtuple_as_object'] = False
|
||||||
|
return json.dump(obj, fp, *args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def loads(s, encoding='utf-8', **kwargs):
|
||||||
|
return json.loads(strutils.safe_decode(s, encoding), **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
def load(fp, encoding='utf-8', **kwargs):
|
||||||
|
return json.load(codecs.getreader(encoding)(fp), **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
import anyjson
|
||||||
|
except ImportError:
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
anyjson._modules.append((__name__, 'dumps', TypeError,
|
||||||
|
'loads', ValueError, 'load'))
|
||||||
|
anyjson.force_implementation(__name__)
|
|
@ -0,0 +1,45 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Local storage of variables using weak references"""
|
||||||
|
|
||||||
|
import threading
|
||||||
|
import weakref
|
||||||
|
|
||||||
|
|
||||||
|
class WeakLocal(threading.local):
|
||||||
|
def __getattribute__(self, attr):
|
||||||
|
rval = super(WeakLocal, self).__getattribute__(attr)
|
||||||
|
if rval:
|
||||||
|
# NOTE(mikal): this bit is confusing. What is stored is a weak
|
||||||
|
# reference, not the value itself. We therefore need to lookup
|
||||||
|
# the weak reference and return the inner value here.
|
||||||
|
rval = rval()
|
||||||
|
return rval
|
||||||
|
|
||||||
|
def __setattr__(self, attr, value):
|
||||||
|
value = weakref.ref(value)
|
||||||
|
return super(WeakLocal, self).__setattr__(attr, value)
|
||||||
|
|
||||||
|
|
||||||
|
# NOTE(mikal): the name "store" should be deprecated in the future
|
||||||
|
store = WeakLocal()
|
||||||
|
|
||||||
|
# A "weak" store uses weak references and allows an object to fall out of scope
|
||||||
|
# when it falls out of scope in the code that uses the thread local storage. A
|
||||||
|
# "strong" store will hold a reference to the object so that it never falls out
|
||||||
|
# of scope.
|
||||||
|
weak_store = WeakLocal()
|
||||||
|
strong_store = threading.local()
|
|
@ -0,0 +1,322 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import contextlib
|
||||||
|
import errno
|
||||||
|
import functools
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
import weakref
|
||||||
|
|
||||||
|
from oslo.config import cfg
|
||||||
|
|
||||||
|
from monasca.openstack.common import fileutils
|
||||||
|
from monasca.openstack.common.gettextutils import _, _LE, _LI
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
util_opts = [
|
||||||
|
cfg.BoolOpt('disable_process_locking', default=False,
|
||||||
|
help='Enables or disables inter-process locks.'),
|
||||||
|
cfg.StrOpt('lock_path',
|
||||||
|
default=os.environ.get("MONASCA_LOCK_PATH"),
|
||||||
|
help='Directory to use for lock files.')
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
CONF.register_opts(util_opts)
|
||||||
|
|
||||||
|
|
||||||
|
def set_defaults(lock_path):
|
||||||
|
cfg.set_defaults(util_opts, lock_path=lock_path)
|
||||||
|
|
||||||
|
|
||||||
|
class _FileLock(object):
|
||||||
|
"""Lock implementation which allows multiple locks, working around
|
||||||
|
issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does
|
||||||
|
not require any cleanup. Since the lock is always held on a file
|
||||||
|
descriptor rather than outside of the process, the lock gets dropped
|
||||||
|
automatically if the process crashes, even if __exit__ is not executed.
|
||||||
|
|
||||||
|
There are no guarantees regarding usage by multiple green threads in a
|
||||||
|
single process here. This lock works only between processes. Exclusive
|
||||||
|
access between local threads should be achieved using the semaphores
|
||||||
|
in the @synchronized decorator.
|
||||||
|
|
||||||
|
Note these locks are released when the descriptor is closed, so it's not
|
||||||
|
safe to close the file descriptor while another green thread holds the
|
||||||
|
lock. Just opening and closing the lock file can break synchronisation,
|
||||||
|
so lock files must be accessed only using this abstraction.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, name):
|
||||||
|
self.lockfile = None
|
||||||
|
self.fname = name
|
||||||
|
|
||||||
|
def acquire(self):
|
||||||
|
basedir = os.path.dirname(self.fname)
|
||||||
|
|
||||||
|
if not os.path.exists(basedir):
|
||||||
|
fileutils.ensure_tree(basedir)
|
||||||
|
LOG.info(_LI('Created lock path: %s'), basedir)
|
||||||
|
|
||||||
|
self.lockfile = open(self.fname, 'w')
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
# Using non-blocking locks since green threads are not
|
||||||
|
# patched to deal with blocking locking calls.
|
||||||
|
# Also upon reading the MSDN docs for locking(), it seems
|
||||||
|
# to have a laughable 10 attempts "blocking" mechanism.
|
||||||
|
self.trylock()
|
||||||
|
LOG.debug('Got file lock "%s"', self.fname)
|
||||||
|
return True
|
||||||
|
except IOError as e:
|
||||||
|
if e.errno in (errno.EACCES, errno.EAGAIN):
|
||||||
|
# external locks synchronise things like iptables
|
||||||
|
# updates - give it some time to prevent busy spinning
|
||||||
|
time.sleep(0.01)
|
||||||
|
else:
|
||||||
|
raise threading.ThreadError(_("Unable to acquire lock on"
|
||||||
|
" `%(filename)s` due to"
|
||||||
|
" %(exception)s") %
|
||||||
|
{'filename': self.fname,
|
||||||
|
'exception': e})
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
self.acquire()
|
||||||
|
return self
|
||||||
|
|
||||||
|
def release(self):
|
||||||
|
try:
|
||||||
|
self.unlock()
|
||||||
|
self.lockfile.close()
|
||||||
|
LOG.debug('Released file lock "%s"', self.fname)
|
||||||
|
except IOError:
|
||||||
|
LOG.exception(_LE("Could not release the acquired lock `%s`"),
|
||||||
|
self.fname)
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||||
|
self.release()
|
||||||
|
|
||||||
|
def exists(self):
|
||||||
|
return os.path.exists(self.fname)
|
||||||
|
|
||||||
|
def trylock(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def unlock(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
|
||||||
|
class _WindowsLock(_FileLock):
|
||||||
|
def trylock(self):
|
||||||
|
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1)
|
||||||
|
|
||||||
|
def unlock(self):
|
||||||
|
msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1)
|
||||||
|
|
||||||
|
|
||||||
|
class _FcntlLock(_FileLock):
|
||||||
|
def trylock(self):
|
||||||
|
fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
|
||||||
|
def unlock(self):
|
||||||
|
fcntl.lockf(self.lockfile, fcntl.LOCK_UN)
|
||||||
|
|
||||||
|
|
||||||
|
if os.name == 'nt':
|
||||||
|
import msvcrt
|
||||||
|
InterProcessLock = _WindowsLock
|
||||||
|
else:
|
||||||
|
import fcntl
|
||||||
|
InterProcessLock = _FcntlLock
|
||||||
|
|
||||||
|
_semaphores = weakref.WeakValueDictionary()
|
||||||
|
_semaphores_lock = threading.Lock()
|
||||||
|
|
||||||
|
|
||||||
|
def _get_lock_path(name, lock_file_prefix, lock_path=None):
|
||||||
|
# NOTE(mikal): the lock name cannot contain directory
|
||||||
|
# separators
|
||||||
|
name = name.replace(os.sep, '_')
|
||||||
|
if lock_file_prefix:
|
||||||
|
sep = '' if lock_file_prefix.endswith('-') else '-'
|
||||||
|
name = '%s%s%s' % (lock_file_prefix, sep, name)
|
||||||
|
|
||||||
|
local_lock_path = lock_path or CONF.lock_path
|
||||||
|
|
||||||
|
if not local_lock_path:
|
||||||
|
raise cfg.RequiredOptError('lock_path')
|
||||||
|
|
||||||
|
return os.path.join(local_lock_path, name)
|
||||||
|
|
||||||
|
|
||||||
|
def external_lock(name, lock_file_prefix=None, lock_path=None):
|
||||||
|
LOG.debug('Attempting to grab external lock "%(lock)s"',
|
||||||
|
{'lock': name})
|
||||||
|
|
||||||
|
lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path)
|
||||||
|
|
||||||
|
return InterProcessLock(lock_file_path)
|
||||||
|
|
||||||
|
|
||||||
|
def remove_external_lock_file(name, lock_file_prefix=None):
|
||||||
|
"""Remove an external lock file when it's not used anymore
|
||||||
|
This will be helpful when we have a lot of lock files
|
||||||
|
"""
|
||||||
|
with internal_lock(name):
|
||||||
|
lock_file_path = _get_lock_path(name, lock_file_prefix)
|
||||||
|
try:
|
||||||
|
os.remove(lock_file_path)
|
||||||
|
except OSError:
|
||||||
|
LOG.info(_LI('Failed to remove file %(file)s'),
|
||||||
|
{'file': lock_file_path})
|
||||||
|
|
||||||
|
|
||||||
|
def internal_lock(name):
|
||||||
|
with _semaphores_lock:
|
||||||
|
try:
|
||||||
|
sem = _semaphores[name]
|
||||||
|
except KeyError:
|
||||||
|
sem = threading.Semaphore()
|
||||||
|
_semaphores[name] = sem
|
||||||
|
|
||||||
|
LOG.debug('Got semaphore "%(lock)s"', {'lock': name})
|
||||||
|
return sem
|
||||||
|
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def lock(name, lock_file_prefix=None, external=False, lock_path=None):
|
||||||
|
"""Context based lock
|
||||||
|
|
||||||
|
This function yields a `threading.Semaphore` instance (if we don't use
|
||||||
|
eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is
|
||||||
|
True, in which case, it'll yield an InterProcessLock instance.
|
||||||
|
|
||||||
|
:param lock_file_prefix: The lock_file_prefix argument is used to provide
|
||||||
|
lock files on disk with a meaningful prefix.
|
||||||
|
|
||||||
|
:param external: The external keyword argument denotes whether this lock
|
||||||
|
should work across multiple processes. This means that if two different
|
||||||
|
workers both run a method decorated with @synchronized('mylock',
|
||||||
|
external=True), only one of them will execute at a time.
|
||||||
|
"""
|
||||||
|
int_lock = internal_lock(name)
|
||||||
|
with int_lock:
|
||||||
|
if external and not CONF.disable_process_locking:
|
||||||
|
ext_lock = external_lock(name, lock_file_prefix, lock_path)
|
||||||
|
with ext_lock:
|
||||||
|
yield ext_lock
|
||||||
|
else:
|
||||||
|
yield int_lock
|
||||||
|
LOG.debug('Released semaphore "%(lock)s"', {'lock': name})
|
||||||
|
|
||||||
|
|
||||||
|
def synchronized(name, lock_file_prefix=None, external=False, lock_path=None):
|
||||||
|
"""Synchronization decorator.
|
||||||
|
|
||||||
|
Decorating a method like so::
|
||||||
|
|
||||||
|
@synchronized('mylock')
|
||||||
|
def foo(self, *args):
|
||||||
|
...
|
||||||
|
|
||||||
|
ensures that only one thread will execute the foo method at a time.
|
||||||
|
|
||||||
|
Different methods can share the same lock::
|
||||||
|
|
||||||
|
@synchronized('mylock')
|
||||||
|
def foo(self, *args):
|
||||||
|
...
|
||||||
|
|
||||||
|
@synchronized('mylock')
|
||||||
|
def bar(self, *args):
|
||||||
|
...
|
||||||
|
|
||||||
|
This way only one of either foo or bar can be executing at a time.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def wrap(f):
|
||||||
|
@functools.wraps(f)
|
||||||
|
def inner(*args, **kwargs):
|
||||||
|
try:
|
||||||
|
with lock(name, lock_file_prefix, external, lock_path):
|
||||||
|
LOG.debug('Got semaphore / lock "%(function)s"',
|
||||||
|
{'function': f.__name__})
|
||||||
|
return f(*args, **kwargs)
|
||||||
|
finally:
|
||||||
|
LOG.debug('Semaphore / lock released "%(function)s"',
|
||||||
|
{'function': f.__name__})
|
||||||
|
return inner
|
||||||
|
return wrap
|
||||||
|
|
||||||
|
|
||||||
|
def synchronized_with_prefix(lock_file_prefix):
|
||||||
|
"""Partial object generator for the synchronization decorator.
|
||||||
|
|
||||||
|
Redefine @synchronized in each project like so::
|
||||||
|
|
||||||
|
(in nova/utils.py)
|
||||||
|
from nova.openstack.common import lockutils
|
||||||
|
|
||||||
|
synchronized = lockutils.synchronized_with_prefix('nova-')
|
||||||
|
|
||||||
|
|
||||||
|
(in nova/foo.py)
|
||||||
|
from nova import utils
|
||||||
|
|
||||||
|
@utils.synchronized('mylock')
|
||||||
|
def bar(self, *args):
|
||||||
|
...
|
||||||
|
|
||||||
|
The lock_file_prefix argument is used to provide lock files on disk with a
|
||||||
|
meaningful prefix.
|
||||||
|
"""
|
||||||
|
|
||||||
|
return functools.partial(synchronized, lock_file_prefix=lock_file_prefix)
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv):
|
||||||
|
"""Create a dir for locks and pass it to command from arguments
|
||||||
|
|
||||||
|
If you run this:
|
||||||
|
python -m openstack.common.lockutils python setup.py testr <etc>
|
||||||
|
|
||||||
|
a temporary directory will be created for all your locks and passed to all
|
||||||
|
your tests in an environment variable. The temporary dir will be deleted
|
||||||
|
afterwards and the return value will be preserved.
|
||||||
|
"""
|
||||||
|
|
||||||
|
lock_dir = tempfile.mkdtemp()
|
||||||
|
os.environ["MONASCA_LOCK_PATH"] = lock_dir
|
||||||
|
try:
|
||||||
|
ret_val = subprocess.call(argv[1:])
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(lock_dir, ignore_errors=True)
|
||||||
|
return ret_val
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
sys.exit(main(sys.argv))
|
|
@ -0,0 +1,713 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# Copyright 2010 United States Government as represented by the
|
||||||
|
# Administrator of the National Aeronautics and Space Administration.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""OpenStack logging handler.
|
||||||
|
|
||||||
|
This module adds to logging functionality by adding the option to specify
|
||||||
|
a context object when calling the various log methods. If the context object
|
||||||
|
is not specified, default formatting is used. Additionally, an instance uuid
|
||||||
|
may be passed as part of the log message, which is intended to make it easier
|
||||||
|
for admins to find messages related to a specific instance.
|
||||||
|
|
||||||
|
It also allows setting of formatting information through conf.
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
import inspect
|
||||||
|
import itertools
|
||||||
|
import logging
|
||||||
|
import logging.config
|
||||||
|
import logging.handlers
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
import traceback
|
||||||
|
|
||||||
|
from oslo.config import cfg
|
||||||
|
import six
|
||||||
|
from six import moves
|
||||||
|
|
||||||
|
_PY26 = sys.version_info[0:2] == (2, 6)
|
||||||
|
|
||||||
|
from monasca.openstack.common.gettextutils import _
|
||||||
|
from monasca.openstack.common import importutils
|
||||||
|
from monasca.openstack.common import jsonutils
|
||||||
|
from monasca.openstack.common import local
|
||||||
|
# NOTE(flaper87): Pls, remove when graduating this module
|
||||||
|
# from the incubator.
|
||||||
|
from monasca.openstack.common.strutils import mask_password # noqa
|
||||||
|
|
||||||
|
|
||||||
|
_DEFAULT_LOG_DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||||
|
|
||||||
|
|
||||||
|
common_cli_opts = [
|
||||||
|
cfg.BoolOpt('debug',
|
||||||
|
short='d',
|
||||||
|
default=False,
|
||||||
|
help='Print debugging output (set logging level to '
|
||||||
|
'DEBUG instead of default WARNING level).'),
|
||||||
|
cfg.BoolOpt('verbose',
|
||||||
|
short='v',
|
||||||
|
default=False,
|
||||||
|
help='Print more verbose output (set logging level to '
|
||||||
|
'INFO instead of default WARNING level).'),
|
||||||
|
]
|
||||||
|
|
||||||
|
logging_cli_opts = [
|
||||||
|
cfg.StrOpt('log-config-append',
|
||||||
|
metavar='PATH',
|
||||||
|
deprecated_name='log-config',
|
||||||
|
help='The name of a logging configuration file. This file '
|
||||||
|
'is appended to any existing logging configuration '
|
||||||
|
'files. For details about logging configuration files, '
|
||||||
|
'see the Python logging module documentation.'),
|
||||||
|
cfg.StrOpt('log-format',
|
||||||
|
metavar='FORMAT',
|
||||||
|
help='DEPRECATED. '
|
||||||
|
'A logging.Formatter log message format string which may '
|
||||||
|
'use any of the available logging.LogRecord attributes. '
|
||||||
|
'This option is deprecated. Please use '
|
||||||
|
'logging_context_format_string and '
|
||||||
|
'logging_default_format_string instead.'),
|
||||||
|
cfg.StrOpt('log-date-format',
|
||||||
|
default=_DEFAULT_LOG_DATE_FORMAT,
|
||||||
|
metavar='DATE_FORMAT',
|
||||||
|
help='Format string for %%(asctime)s in log records. '
|
||||||
|
'Default: %(default)s .'),
|
||||||
|
cfg.StrOpt('log-file',
|
||||||
|
metavar='PATH',
|
||||||
|
deprecated_name='logfile',
|
||||||
|
help='(Optional) Name of log file to output to. '
|
||||||
|
'If no default is set, logging will go to stdout.'),
|
||||||
|
cfg.StrOpt('log-dir',
|
||||||
|
deprecated_name='logdir',
|
||||||
|
help='(Optional) The base directory used for relative '
|
||||||
|
'--log-file paths.'),
|
||||||
|
cfg.BoolOpt('use-syslog',
|
||||||
|
default=False,
|
||||||
|
help='Use syslog for logging. '
|
||||||
|
'Existing syslog format is DEPRECATED during I, '
|
||||||
|
'and will change in J to honor RFC5424.'),
|
||||||
|
cfg.BoolOpt('use-syslog-rfc-format',
|
||||||
|
# TODO(bogdando) remove or use True after existing
|
||||||
|
# syslog format deprecation in J
|
||||||
|
default=False,
|
||||||
|
help='(Optional) Enables or disables syslog rfc5424 format '
|
||||||
|
'for logging. If enabled, prefixes the MSG part of the '
|
||||||
|
'syslog message with APP-NAME (RFC5424). The '
|
||||||
|
'format without the APP-NAME is deprecated in I, '
|
||||||
|
'and will be removed in J.'),
|
||||||
|
cfg.StrOpt('syslog-log-facility',
|
||||||
|
default='LOG_USER',
|
||||||
|
help='Syslog facility to receive log lines.')
|
||||||
|
]
|
||||||
|
|
||||||
|
generic_log_opts = [
|
||||||
|
cfg.BoolOpt('use_stderr',
|
||||||
|
default=True,
|
||||||
|
help='Log output to standard error.')
|
||||||
|
]
|
||||||
|
|
||||||
|
DEFAULT_LOG_LEVELS = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN',
|
||||||
|
'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO',
|
||||||
|
'oslo.messaging=INFO', 'iso8601=WARN',
|
||||||
|
'requests.packages.urllib3.connectionpool=WARN',
|
||||||
|
'urllib3.connectionpool=WARN', 'websocket=WARN',
|
||||||
|
"keystonemiddleware=WARN", "routes.middleware=WARN",
|
||||||
|
"stevedore=WARN"]
|
||||||
|
|
||||||
|
log_opts = [
|
||||||
|
cfg.StrOpt('logging_context_format_string',
|
||||||
|
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
||||||
|
'%(name)s [%(request_id)s %(user_identity)s] '
|
||||||
|
'%(instance)s%(message)s',
|
||||||
|
help='Format string to use for log messages with context.'),
|
||||||
|
cfg.StrOpt('logging_default_format_string',
|
||||||
|
default='%(asctime)s.%(msecs)03d %(process)d %(levelname)s '
|
||||||
|
'%(name)s [-] %(instance)s%(message)s',
|
||||||
|
help='Format string to use for log messages without context.'),
|
||||||
|
cfg.StrOpt('logging_debug_format_suffix',
|
||||||
|
default='%(funcName)s %(pathname)s:%(lineno)d',
|
||||||
|
help='Data to append to log format when level is DEBUG.'),
|
||||||
|
cfg.StrOpt('logging_exception_prefix',
|
||||||
|
default='%(asctime)s.%(msecs)03d %(process)d TRACE %(name)s '
|
||||||
|
'%(instance)s',
|
||||||
|
help='Prefix each line of exception output with this format.'),
|
||||||
|
cfg.ListOpt('default_log_levels',
|
||||||
|
default=DEFAULT_LOG_LEVELS,
|
||||||
|
help='List of logger=LEVEL pairs.'),
|
||||||
|
cfg.BoolOpt('publish_errors',
|
||||||
|
default=False,
|
||||||
|
help='Enables or disables publication of error events.'),
|
||||||
|
cfg.BoolOpt('fatal_deprecations',
|
||||||
|
default=False,
|
||||||
|
help='Enables or disables fatal status of deprecations.'),
|
||||||
|
|
||||||
|
# NOTE(mikal): there are two options here because sometimes we are handed
|
||||||
|
# a full instance (and could include more information), and other times we
|
||||||
|
# are just handed a UUID for the instance.
|
||||||
|
cfg.StrOpt('instance_format',
|
||||||
|
default='[instance: %(uuid)s] ',
|
||||||
|
help='The format for an instance that is passed with the log '
|
||||||
|
'message.'),
|
||||||
|
cfg.StrOpt('instance_uuid_format',
|
||||||
|
default='[instance: %(uuid)s] ',
|
||||||
|
help='The format for an instance UUID that is passed with the '
|
||||||
|
'log message.'),
|
||||||
|
]
|
||||||
|
|
||||||
|
CONF = cfg.CONF
|
||||||
|
CONF.register_cli_opts(common_cli_opts)
|
||||||
|
CONF.register_cli_opts(logging_cli_opts)
|
||||||
|
CONF.register_opts(generic_log_opts)
|
||||||
|
CONF.register_opts(log_opts)
|
||||||
|
|
||||||
|
# our new audit level
|
||||||
|
# NOTE(jkoelker) Since we synthesized an audit level, make the logging
|
||||||
|
# module aware of it so it acts like other levels.
|
||||||
|
logging.AUDIT = logging.INFO + 1
|
||||||
|
logging.addLevelName(logging.AUDIT, 'AUDIT')
|
||||||
|
|
||||||
|
|
||||||
|
try:
|
||||||
|
NullHandler = logging.NullHandler
|
||||||
|
except AttributeError: # NOTE(jkoelker) NullHandler added in Python 2.7
|
||||||
|
class NullHandler(logging.Handler):
|
||||||
|
def handle(self, record):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def emit(self, record):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def createLock(self):
|
||||||
|
self.lock = None
|
||||||
|
|
||||||
|
|
||||||
|
def _dictify_context(context):
|
||||||
|
if context is None:
|
||||||
|
return None
|
||||||
|
if not isinstance(context, dict) and getattr(context, 'to_dict', None):
|
||||||
|
context = context.to_dict()
|
||||||
|
return context
|
||||||
|
|
||||||
|
|
||||||
|
def _get_binary_name():
|
||||||
|
return os.path.basename(inspect.stack()[-1][1])
|
||||||
|
|
||||||
|
|
||||||
|
def _get_log_file_path(binary=None):
|
||||||
|
logfile = CONF.log_file
|
||||||
|
logdir = CONF.log_dir
|
||||||
|
|
||||||
|
if logfile and not logdir:
|
||||||
|
return logfile
|
||||||
|
|
||||||
|
if logfile and logdir:
|
||||||
|
return os.path.join(logdir, logfile)
|
||||||
|
|
||||||
|
if logdir:
|
||||||
|
binary = binary or _get_binary_name()
|
||||||
|
return '%s.log' % (os.path.join(logdir, binary),)
|
||||||
|
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class BaseLoggerAdapter(logging.LoggerAdapter):
|
||||||
|
|
||||||
|
def audit(self, msg, *args, **kwargs):
|
||||||
|
self.log(logging.AUDIT, msg, *args, **kwargs)
|
||||||
|
|
||||||
|
def isEnabledFor(self, level):
|
||||||
|
if _PY26:
|
||||||
|
# This method was added in python 2.7 (and it does the exact
|
||||||
|
# same logic, so we need to do the exact same logic so that
|
||||||
|
# python 2.6 has this capability as well).
|
||||||
|
return self.logger.isEnabledFor(level)
|
||||||
|
else:
|
||||||
|
return super(BaseLoggerAdapter, self).isEnabledFor(level)
|
||||||
|
|
||||||
|
|
||||||
|
class LazyAdapter(BaseLoggerAdapter):
|
||||||
|
def __init__(self, name='unknown', version='unknown'):
|
||||||
|
self._logger = None
|
||||||
|
self.extra = {}
|
||||||
|
self.name = name
|
||||||
|
self.version = version
|
||||||
|
|
||||||
|
@property
|
||||||
|
def logger(self):
|
||||||
|
if not self._logger:
|
||||||
|
self._logger = getLogger(self.name, self.version)
|
||||||
|
if six.PY3:
|
||||||
|
# In Python 3, the code fails because the 'manager' attribute
|
||||||
|
# cannot be found when using a LoggerAdapter as the
|
||||||
|
# underlying logger. Work around this issue.
|
||||||
|
self._logger.manager = self._logger.logger.manager
|
||||||
|
return self._logger
|
||||||
|
|
||||||
|
|
||||||
|
class ContextAdapter(BaseLoggerAdapter):
|
||||||
|
warn = logging.LoggerAdapter.warning
|
||||||
|
|
||||||
|
def __init__(self, logger, project_name, version_string):
|
||||||
|
self.logger = logger
|
||||||
|
self.project = project_name
|
||||||
|
self.version = version_string
|
||||||
|
self._deprecated_messages_sent = dict()
|
||||||
|
|
||||||
|
@property
|
||||||
|
def handlers(self):
|
||||||
|
return self.logger.handlers
|
||||||
|
|
||||||
|
def deprecated(self, msg, *args, **kwargs):
|
||||||
|
"""Call this method when a deprecated feature is used.
|
||||||
|
|
||||||
|
If the system is configured for fatal deprecations then the message
|
||||||
|
is logged at the 'critical' level and :class:`DeprecatedConfig` will
|
||||||
|
be raised.
|
||||||
|
|
||||||
|
Otherwise, the message will be logged (once) at the 'warn' level.
|
||||||
|
|
||||||
|
:raises: :class:`DeprecatedConfig` if the system is configured for
|
||||||
|
fatal deprecations.
|
||||||
|
|
||||||
|
"""
|
||||||
|
stdmsg = _("Deprecated: %s") % msg
|
||||||
|
if CONF.fatal_deprecations:
|
||||||
|
self.critical(stdmsg, *args, **kwargs)
|
||||||
|
raise DeprecatedConfig(msg=stdmsg)
|
||||||
|
|
||||||
|
# Using a list because a tuple with dict can't be stored in a set.
|
||||||
|
sent_args = self._deprecated_messages_sent.setdefault(msg, list())
|
||||||
|
|
||||||
|
if args in sent_args:
|
||||||
|
# Already logged this message, so don't log it again.
|
||||||
|
return
|
||||||
|
|
||||||
|
sent_args.append(args)
|
||||||
|
self.warn(stdmsg, *args, **kwargs)
|
||||||
|
|
||||||
|
def process(self, msg, kwargs):
|
||||||
|
# NOTE(jecarey): If msg is not unicode, coerce it into unicode
|
||||||
|
# before it can get to the python logging and
|
||||||
|
# possibly cause string encoding trouble
|
||||||
|
if not isinstance(msg, six.text_type):
|
||||||
|
msg = six.text_type(msg)
|
||||||
|
|
||||||
|
if 'extra' not in kwargs:
|
||||||
|
kwargs['extra'] = {}
|
||||||
|
extra = kwargs['extra']
|
||||||
|
|
||||||
|
context = kwargs.pop('context', None)
|
||||||
|
if not context:
|
||||||
|
context = getattr(local.store, 'context', None)
|
||||||
|
if context:
|
||||||
|
extra.update(_dictify_context(context))
|
||||||
|
|
||||||
|
instance = kwargs.pop('instance', None)
|
||||||
|
instance_uuid = (extra.get('instance_uuid') or
|
||||||
|
kwargs.pop('instance_uuid', None))
|
||||||
|
instance_extra = ''
|
||||||
|
if instance:
|
||||||
|
instance_extra = CONF.instance_format % instance
|
||||||
|
elif instance_uuid:
|
||||||
|
instance_extra = (CONF.instance_uuid_format
|
||||||
|
% {'uuid': instance_uuid})
|
||||||
|
extra['instance'] = instance_extra
|
||||||
|
|
||||||
|
extra.setdefault('user_identity', kwargs.pop('user_identity', None))
|
||||||
|
|
||||||
|
extra['project'] = self.project
|
||||||
|
extra['version'] = self.version
|
||||||
|
extra['extra'] = extra.copy()
|
||||||
|
return msg, kwargs
|
||||||
|
|
||||||
|
|
||||||
|
class JSONFormatter(logging.Formatter):
|
||||||
|
def __init__(self, fmt=None, datefmt=None):
|
||||||
|
# NOTE(jkoelker) we ignore the fmt argument, but its still there
|
||||||
|
# since logging.config.fileConfig passes it.
|
||||||
|
self.datefmt = datefmt
|
||||||
|
|
||||||
|
def formatException(self, ei, strip_newlines=True):
|
||||||
|
lines = traceback.format_exception(*ei)
|
||||||
|
if strip_newlines:
|
||||||
|
lines = [moves.filter(
|
||||||
|
lambda x: x,
|
||||||
|
line.rstrip().splitlines()) for line in lines]
|
||||||
|
lines = list(itertools.chain(*lines))
|
||||||
|
return lines
|
||||||
|
|
||||||
|
def format(self, record):
|
||||||
|
message = {'message': record.getMessage(),
|
||||||
|
'asctime': self.formatTime(record, self.datefmt),
|
||||||
|
'name': record.name,
|
||||||
|
'msg': record.msg,
|
||||||
|
'args': record.args,
|
||||||
|
'levelname': record.levelname,
|
||||||
|
'levelno': record.levelno,
|
||||||
|
'pathname': record.pathname,
|
||||||
|
'filename': record.filename,
|
||||||
|
'module': record.module,
|
||||||
|
'lineno': record.lineno,
|
||||||
|
'funcname': record.funcName,
|
||||||
|
'created': record.created,
|
||||||
|
'msecs': record.msecs,
|
||||||
|
'relative_created': record.relativeCreated,
|
||||||
|
'thread': record.thread,
|
||||||
|
'thread_name': record.threadName,
|
||||||
|
'process_name': record.processName,
|
||||||
|
'process': record.process,
|
||||||
|
'traceback': None}
|
||||||
|
|
||||||
|
if hasattr(record, 'extra'):
|
||||||
|
message['extra'] = record.extra
|
||||||
|
|
||||||
|
if record.exc_info:
|
||||||
|
message['traceback'] = self.formatException(record.exc_info)
|
||||||
|
|
||||||
|
return jsonutils.dumps(message)
|
||||||
|
|
||||||
|
|
||||||
|
def _create_logging_excepthook(product_name):
|
||||||
|
def logging_excepthook(exc_type, value, tb):
|
||||||
|
extra = {'exc_info': (exc_type, value, tb)}
|
||||||
|
getLogger(product_name).critical(
|
||||||
|
"".join(traceback.format_exception_only(exc_type, value)),
|
||||||
|
**extra)
|
||||||
|
return logging_excepthook
|
||||||
|
|
||||||
|
|
||||||
|
class LogConfigError(Exception):
|
||||||
|
|
||||||
|
message = _('Error loading logging config %(log_config)s: %(err_msg)s')
|
||||||
|
|
||||||
|
def __init__(self, log_config, err_msg):
|
||||||
|
self.log_config = log_config
|
||||||
|
self.err_msg = err_msg
|
||||||
|
|
||||||
|
def __str__(self):
|
||||||
|
return self.message % dict(log_config=self.log_config,
|
||||||
|
err_msg=self.err_msg)
|
||||||
|
|
||||||
|
|
||||||
|
def _load_log_config(log_config_append):
|
||||||
|
try:
|
||||||
|
logging.config.fileConfig(log_config_append,
|
||||||
|
disable_existing_loggers=False)
|
||||||
|
except (moves.configparser.Error, KeyError) as exc:
|
||||||
|
raise LogConfigError(log_config_append, six.text_type(exc))
|
||||||
|
|
||||||
|
|
||||||
|
def setup(product_name, version='unknown'):
|
||||||
|
"""Setup logging."""
|
||||||
|
if CONF.log_config_append:
|
||||||
|
_load_log_config(CONF.log_config_append)
|
||||||
|
else:
|
||||||
|
_setup_logging_from_conf(product_name, version)
|
||||||
|
sys.excepthook = _create_logging_excepthook(product_name)
|
||||||
|
|
||||||
|
|
||||||
|
def set_defaults(logging_context_format_string=None,
|
||||||
|
default_log_levels=None):
|
||||||
|
# Just in case the caller is not setting the
|
||||||
|
# default_log_level. This is insurance because
|
||||||
|
# we introduced the default_log_level parameter
|
||||||
|
# later in a backwards in-compatible change
|
||||||
|
if default_log_levels is not None:
|
||||||
|
cfg.set_defaults(
|
||||||
|
log_opts,
|
||||||
|
default_log_levels=default_log_levels)
|
||||||
|
if logging_context_format_string is not None:
|
||||||
|
cfg.set_defaults(
|
||||||
|
log_opts,
|
||||||
|
logging_context_format_string=logging_context_format_string)
|
||||||
|
|
||||||
|
|
||||||
|
def _find_facility_from_conf():
|
||||||
|
facility_names = logging.handlers.SysLogHandler.facility_names
|
||||||
|
facility = getattr(logging.handlers.SysLogHandler,
|
||||||
|
CONF.syslog_log_facility,
|
||||||
|
None)
|
||||||
|
|
||||||
|
if facility is None and CONF.syslog_log_facility in facility_names:
|
||||||
|
facility = facility_names.get(CONF.syslog_log_facility)
|
||||||
|
|
||||||
|
if facility is None:
|
||||||
|
valid_facilities = facility_names.keys()
|
||||||
|
consts = ['LOG_AUTH', 'LOG_AUTHPRIV', 'LOG_CRON', 'LOG_DAEMON',
|
||||||
|
'LOG_FTP', 'LOG_KERN', 'LOG_LPR', 'LOG_MAIL', 'LOG_NEWS',
|
||||||
|
'LOG_AUTH', 'LOG_SYSLOG', 'LOG_USER', 'LOG_UUCP',
|
||||||
|
'LOG_LOCAL0', 'LOG_LOCAL1', 'LOG_LOCAL2', 'LOG_LOCAL3',
|
||||||
|
'LOG_LOCAL4', 'LOG_LOCAL5', 'LOG_LOCAL6', 'LOG_LOCAL7']
|
||||||
|
valid_facilities.extend(consts)
|
||||||
|
raise TypeError(_('syslog facility must be one of: %s') %
|
||||||
|
', '.join("'%s'" % fac
|
||||||
|
for fac in valid_facilities))
|
||||||
|
|
||||||
|
return facility
|
||||||
|
|
||||||
|
|
||||||
|
class RFCSysLogHandler(logging.handlers.SysLogHandler):
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
self.binary_name = _get_binary_name()
|
||||||
|
# Do not use super() unless type(logging.handlers.SysLogHandler)
|
||||||
|
# is 'type' (Python 2.7).
|
||||||
|
# Use old style calls, if the type is 'classobj' (Python 2.6)
|
||||||
|
logging.handlers.SysLogHandler.__init__(self, *args, **kwargs)
|
||||||
|
|
||||||
|
def format(self, record):
|
||||||
|
# Do not use super() unless type(logging.handlers.SysLogHandler)
|
||||||
|
# is 'type' (Python 2.7).
|
||||||
|
# Use old style calls, if the type is 'classobj' (Python 2.6)
|
||||||
|
msg = logging.handlers.SysLogHandler.format(self, record)
|
||||||
|
msg = self.binary_name + ' ' + msg
|
||||||
|
return msg
|
||||||
|
|
||||||
|
|
||||||
|
def _setup_logging_from_conf(project, version):
|
||||||
|
log_root = getLogger(None).logger
|
||||||
|
for handler in log_root.handlers:
|
||||||
|
log_root.removeHandler(handler)
|
||||||
|
|
||||||
|
logpath = _get_log_file_path()
|
||||||
|
if logpath:
|
||||||
|
filelog = logging.handlers.WatchedFileHandler(logpath)
|
||||||
|
log_root.addHandler(filelog)
|
||||||
|
|
||||||
|
if CONF.use_stderr:
|
||||||
|
streamlog = ColorHandler()
|
||||||
|
log_root.addHandler(streamlog)
|
||||||
|
|
||||||
|
elif not logpath:
|
||||||
|
# pass sys.stdout as a positional argument
|
||||||
|
# python2.6 calls the argument strm, in 2.7 it's stream
|
||||||
|
streamlog = logging.StreamHandler(sys.stdout)
|
||||||
|
log_root.addHandler(streamlog)
|
||||||
|
|
||||||
|
if CONF.publish_errors:
|
||||||
|
try:
|
||||||
|
handler = importutils.import_object(
|
||||||
|
"monasca.openstack.common.log_handler.PublishErrorsHandler",
|
||||||
|
logging.ERROR)
|
||||||
|
except ImportError:
|
||||||
|
handler = importutils.import_object(
|
||||||
|
"oslo.messaging.notify.log_handler.PublishErrorsHandler",
|
||||||
|
logging.ERROR)
|
||||||
|
log_root.addHandler(handler)
|
||||||
|
|
||||||
|
datefmt = CONF.log_date_format
|
||||||
|
for handler in log_root.handlers:
|
||||||
|
# NOTE(alaski): CONF.log_format overrides everything currently. This
|
||||||
|
# should be deprecated in favor of context aware formatting.
|
||||||
|
if CONF.log_format:
|
||||||
|
handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
|
||||||
|
datefmt=datefmt))
|
||||||
|
log_root.info('Deprecated: log_format is now deprecated and will '
|
||||||
|
'be removed in the next release')
|
||||||
|
else:
|
||||||
|
handler.setFormatter(ContextFormatter(project=project,
|
||||||
|
version=version,
|
||||||
|
datefmt=datefmt))
|
||||||
|
|
||||||
|
if CONF.debug:
|
||||||
|
log_root.setLevel(logging.DEBUG)
|
||||||
|
elif CONF.verbose:
|
||||||
|
log_root.setLevel(logging.INFO)
|
||||||
|
else:
|
||||||
|
log_root.setLevel(logging.WARNING)
|
||||||
|
|
||||||
|
for pair in CONF.default_log_levels:
|
||||||
|
mod, _sep, level_name = pair.partition('=')
|
||||||
|
logger = logging.getLogger(mod)
|
||||||
|
# NOTE(AAzza) in python2.6 Logger.setLevel doesn't convert string name
|
||||||
|
# to integer code.
|
||||||
|
if sys.version_info < (2, 7):
|
||||||
|
level = logging.getLevelName(level_name)
|
||||||
|
logger.setLevel(level)
|
||||||
|
else:
|
||||||
|
logger.setLevel(level_name)
|
||||||
|
|
||||||
|
if CONF.use_syslog:
|
||||||
|
try:
|
||||||
|
facility = _find_facility_from_conf()
|
||||||
|
# TODO(bogdando) use the format provided by RFCSysLogHandler
|
||||||
|
# after existing syslog format deprecation in J
|
||||||
|
if CONF.use_syslog_rfc_format:
|
||||||
|
syslog = RFCSysLogHandler(facility=facility)
|
||||||
|
else:
|
||||||
|
syslog = logging.handlers.SysLogHandler(facility=facility)
|
||||||
|
log_root.addHandler(syslog)
|
||||||
|
except socket.error:
|
||||||
|
log_root.error('Unable to add syslog handler. Verify that syslog'
|
||||||
|
'is running.')
|
||||||
|
|
||||||
|
|
||||||
|
_loggers = {}
|
||||||
|
|
||||||
|
|
||||||
|
def getLogger(name='unknown', version='unknown'):
|
||||||
|
if name not in _loggers:
|
||||||
|
_loggers[name] = ContextAdapter(logging.getLogger(name),
|
||||||
|
name,
|
||||||
|
version)
|
||||||
|
return _loggers[name]
|
||||||
|
|
||||||
|
|
||||||
|
def getLazyLogger(name='unknown', version='unknown'):
|
||||||
|
"""Returns lazy logger.
|
||||||
|
|
||||||
|
Creates a pass-through logger that does not create the real logger
|
||||||
|
until it is really needed and delegates all calls to the real logger
|
||||||
|
once it is created.
|
||||||
|
"""
|
||||||
|
return LazyAdapter(name, version)
|
||||||
|
|
||||||
|
|
||||||
|
class WritableLogger(object):
|
||||||
|
"""A thin wrapper that responds to `write` and logs."""
|
||||||
|
|
||||||
|
def __init__(self, logger, level=logging.INFO):
|
||||||
|
self.logger = logger
|
||||||
|
self.level = level
|
||||||
|
|
||||||
|
def write(self, msg):
|
||||||
|
self.logger.log(self.level, msg.rstrip())
|
||||||
|
|
||||||
|
|
||||||
|
class ContextFormatter(logging.Formatter):
|
||||||
|
"""A context.RequestContext aware formatter configured through flags.
|
||||||
|
|
||||||
|
The flags used to set format strings are: logging_context_format_string
|
||||||
|
and logging_default_format_string. You can also specify
|
||||||
|
logging_debug_format_suffix to append extra formatting if the log level is
|
||||||
|
debug.
|
||||||
|
|
||||||
|
For information about what variables are available for the formatter see:
|
||||||
|
http://docs.python.org/library/logging.html#formatter
|
||||||
|
|
||||||
|
If available, uses the context value stored in TLS - local.store.context
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, *args, **kwargs):
|
||||||
|
"""Initialize ContextFormatter instance
|
||||||
|
|
||||||
|
Takes additional keyword arguments which can be used in the message
|
||||||
|
format string.
|
||||||
|
|
||||||
|
:keyword project: project name
|
||||||
|
:type project: string
|
||||||
|
:keyword version: project version
|
||||||
|
:type version: string
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
self.project = kwargs.pop('project', 'unknown')
|
||||||
|
self.version = kwargs.pop('version', 'unknown')
|
||||||
|
|
||||||
|
logging.Formatter.__init__(self, *args, **kwargs)
|
||||||
|
|
||||||
|
def format(self, record):
|
||||||
|
"""Uses contextstring if request_id is set, otherwise default."""
|
||||||
|
|
||||||
|
# NOTE(jecarey): If msg is not unicode, coerce it into unicode
|
||||||
|
# before it can get to the python logging and
|
||||||
|
# possibly cause string encoding trouble
|
||||||
|
if not isinstance(record.msg, six.text_type):
|
||||||
|
record.msg = six.text_type(record.msg)
|
||||||
|
|
||||||
|
# store project info
|
||||||
|
record.project = self.project
|
||||||
|
record.version = self.version
|
||||||
|
|
||||||
|
# store request info
|
||||||
|
context = getattr(local.store, 'context', None)
|
||||||
|
if context:
|
||||||
|
d = _dictify_context(context)
|
||||||
|
for k, v in d.items():
|
||||||
|
setattr(record, k, v)
|
||||||
|
|
||||||
|
# NOTE(sdague): default the fancier formatting params
|
||||||
|
# to an empty string so we don't throw an exception if
|
||||||
|
# they get used
|
||||||
|
for key in ('instance', 'color', 'user_identity'):
|
||||||
|
if key not in record.__dict__:
|
||||||
|
record.__dict__[key] = ''
|
||||||
|
|
||||||
|
if record.__dict__.get('request_id'):
|
||||||
|
fmt = CONF.logging_context_format_string
|
||||||
|
else:
|
||||||
|
fmt = CONF.logging_default_format_string
|
||||||
|
|
||||||
|
if (record.levelno == logging.DEBUG and
|
||||||
|
CONF.logging_debug_format_suffix):
|
||||||
|
fmt += " " + CONF.logging_debug_format_suffix
|
||||||
|
|
||||||
|
if sys.version_info < (3, 2):
|
||||||
|
self._fmt = fmt
|
||||||
|
else:
|
||||||
|
self._style = logging.PercentStyle(fmt)
|
||||||
|
self._fmt = self._style._fmt
|
||||||
|
# Cache this on the record, Logger will respect our formatted copy
|
||||||
|
if record.exc_info:
|
||||||
|
record.exc_text = self.formatException(record.exc_info, record)
|
||||||
|
return logging.Formatter.format(self, record)
|
||||||
|
|
||||||
|
def formatException(self, exc_info, record=None):
|
||||||
|
"""Format exception output with CONF.logging_exception_prefix."""
|
||||||
|
if not record:
|
||||||
|
return logging.Formatter.formatException(self, exc_info)
|
||||||
|
|
||||||
|
stringbuffer = moves.StringIO()
|
||||||
|
traceback.print_exception(exc_info[0], exc_info[1], exc_info[2],
|
||||||
|
None, stringbuffer)
|
||||||
|
lines = stringbuffer.getvalue().split('\n')
|
||||||
|
stringbuffer.close()
|
||||||
|
|
||||||
|
if CONF.logging_exception_prefix.find('%(asctime)') != -1:
|
||||||
|
record.asctime = self.formatTime(record, self.datefmt)
|
||||||
|
|
||||||
|
formatted_lines = []
|
||||||
|
for line in lines:
|
||||||
|
pl = CONF.logging_exception_prefix % record.__dict__
|
||||||
|
fl = '%s%s' % (pl, line)
|
||||||
|
formatted_lines.append(fl)
|
||||||
|
return '\n'.join(formatted_lines)
|
||||||
|
|
||||||
|
|
||||||
|
class ColorHandler(logging.StreamHandler):
|
||||||
|
LEVEL_COLORS = {
|
||||||
|
logging.DEBUG: '\033[00;32m', # GREEN
|
||||||
|
logging.INFO: '\033[00;36m', # CYAN
|
||||||
|
logging.AUDIT: '\033[01;36m', # BOLD CYAN
|
||||||
|
logging.WARN: '\033[01;33m', # BOLD YELLOW
|
||||||
|
logging.ERROR: '\033[01;31m', # BOLD RED
|
||||||
|
logging.CRITICAL: '\033[01;31m', # BOLD RED
|
||||||
|
}
|
||||||
|
|
||||||
|
def format(self, record):
|
||||||
|
record.color = self.LEVEL_COLORS[record.levelno]
|
||||||
|
return logging.StreamHandler.format(self, record)
|
||||||
|
|
||||||
|
|
||||||
|
class DeprecatedConfig(Exception):
|
||||||
|
message = _("Fatal call to deprecated config: %(msg)s")
|
||||||
|
|
||||||
|
def __init__(self, msg):
|
||||||
|
super(Exception, self).__init__(self.message % dict(msg=msg))
|
|
@ -0,0 +1,147 @@
|
||||||
|
# Copyright 2010 United States Government as represented by the
|
||||||
|
# Administrator of the National Aeronautics and Space Administration.
|
||||||
|
# Copyright 2011 Justin Santa Barbara
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
from eventlet import event
|
||||||
|
from eventlet import greenthread
|
||||||
|
|
||||||
|
from monasca.openstack.common.gettextutils import _LE, _LW
|
||||||
|
from monasca.openstack.common import log as logging
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# NOTE(zyluo): This lambda function was declared to avoid mocking collisions
|
||||||
|
# with time.time() called in the standard logging module
|
||||||
|
# during unittests.
|
||||||
|
_ts = lambda: time.time()
|
||||||
|
|
||||||
|
|
||||||
|
class LoopingCallDone(Exception):
|
||||||
|
"""Exception to break out and stop a LoopingCallBase.
|
||||||
|
|
||||||
|
The poll-function passed to LoopingCallBase can raise this exception to
|
||||||
|
break out of the loop normally. This is somewhat analogous to
|
||||||
|
StopIteration.
|
||||||
|
|
||||||
|
An optional return-value can be included as the argument to the exception;
|
||||||
|
this return-value will be returned by LoopingCallBase.wait()
|
||||||
|
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, retvalue=True):
|
||||||
|
""":param retvalue: Value that LoopingCallBase.wait() should return."""
|
||||||
|
self.retvalue = retvalue
|
||||||
|
|
||||||
|
|
||||||
|
class LoopingCallBase(object):
|
||||||
|
def __init__(self, f=None, *args, **kw):
|
||||||
|
self.args = args
|
||||||
|
self.kw = kw
|
||||||
|
self.f = f
|
||||||
|
self._running = False
|
||||||
|
self.done = None
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self._running = False
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
return self.done.wait()
|
||||||
|
|
||||||
|
|
||||||
|
class FixedIntervalLoopingCall(LoopingCallBase):
|
||||||
|
"""A fixed interval looping call."""
|
||||||
|
|
||||||
|
def start(self, interval, initial_delay=None):
|
||||||
|
self._running = True
|
||||||
|
done = event.Event()
|
||||||
|
|
||||||
|
def _inner():
|
||||||
|
if initial_delay:
|
||||||
|
greenthread.sleep(initial_delay)
|
||||||
|
|
||||||
|
try:
|
||||||
|
while self._running:
|
||||||
|
start = _ts()
|
||||||
|
self.f(*self.args, **self.kw)
|
||||||
|
end = _ts()
|
||||||
|
if not self._running:
|
||||||
|
break
|
||||||
|
delay = end - start - interval
|
||||||
|
if delay > 0:
|
||||||
|
LOG.warn(_LW('task %(func_name)s run outlasted '
|
||||||
|
'interval by %(delay).2f sec'),
|
||||||
|
{'func_name': repr(self.f), 'delay': delay})
|
||||||
|
greenthread.sleep(-delay if delay < 0 else 0)
|
||||||
|
except LoopingCallDone as e:
|
||||||
|
self.stop()
|
||||||
|
done.send(e.retvalue)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception(_LE('in fixed duration looping call'))
|
||||||
|
done.send_exception(*sys.exc_info())
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
done.send(True)
|
||||||
|
|
||||||
|
self.done = done
|
||||||
|
|
||||||
|
greenthread.spawn_n(_inner)
|
||||||
|
return self.done
|
||||||
|
|
||||||
|
|
||||||
|
class DynamicLoopingCall(LoopingCallBase):
|
||||||
|
"""A looping call which sleeps until the next known event.
|
||||||
|
|
||||||
|
The function called should return how long to sleep for before being
|
||||||
|
called again.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def start(self, initial_delay=None, periodic_interval_max=None):
|
||||||
|
self._running = True
|
||||||
|
done = event.Event()
|
||||||
|
|
||||||
|
def _inner():
|
||||||
|
if initial_delay:
|
||||||
|
greenthread.sleep(initial_delay)
|
||||||
|
|
||||||
|
try:
|
||||||
|
while self._running:
|
||||||
|
idle = self.f(*self.args, **self.kw)
|
||||||
|
if not self._running:
|
||||||
|
break
|
||||||
|
|
||||||
|
if periodic_interval_max is not None:
|
||||||
|
idle = min(idle, periodic_interval_max)
|
||||||
|
LOG.debug('Dynamic looping call %(func_name)s sleeping '
|
||||||
|
'for %(idle).02f seconds',
|
||||||
|
{'func_name': repr(self.f), 'idle': idle})
|
||||||
|
greenthread.sleep(idle)
|
||||||
|
except LoopingCallDone as e:
|
||||||
|
self.stop()
|
||||||
|
done.send(e.retvalue)
|
||||||
|
except Exception:
|
||||||
|
LOG.exception(_LE('in dynamic looping call'))
|
||||||
|
done.send_exception(*sys.exc_info())
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
done.send(True)
|
||||||
|
|
||||||
|
self.done = done
|
||||||
|
|
||||||
|
greenthread.spawn(_inner)
|
||||||
|
return self.done
|
|
@ -0,0 +1,512 @@
|
||||||
|
# Copyright 2010 United States Government as represented by the
|
||||||
|
# Administrator of the National Aeronautics and Space Administration.
|
||||||
|
# Copyright 2011 Justin Santa Barbara
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""Generic Node base class for all workers that run on hosts."""
|
||||||
|
|
||||||
|
import errno
|
||||||
|
import logging as std_logging
|
||||||
|
import os
|
||||||
|
import random
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
try:
|
||||||
|
# Importing just the symbol here because the io module does not
|
||||||
|
# exist in Python 2.6.
|
||||||
|
from io import UnsupportedOperation # noqa
|
||||||
|
except ImportError:
|
||||||
|
# Python 2.6
|
||||||
|
UnsupportedOperation = None
|
||||||
|
|
||||||
|
import eventlet
|
||||||
|
from eventlet import event
|
||||||
|
from oslo.config import cfg
|
||||||
|
|
||||||
|
from monasca.openstack.common import eventlet_backdoor
|
||||||
|
from monasca.openstack.common.gettextutils import _LE, _LI, _LW
|
||||||
|
from monasca.openstack.common import importutils
|
||||||
|
from monasca.openstack.common import log as logging
|
||||||
|
from monasca.openstack.common import systemd
|
||||||
|
from monasca.openstack.common import threadgroup
|
||||||
|
|
||||||
|
|
||||||
|
rpc = importutils.try_import('monasca.openstack.common.rpc')
|
||||||
|
CONF = cfg.CONF
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _sighup_supported():
|
||||||
|
return hasattr(signal, 'SIGHUP')
|
||||||
|
|
||||||
|
|
||||||
|
def _is_daemon():
|
||||||
|
# The process group for a foreground process will match the
|
||||||
|
# process group of the controlling terminal. If those values do
|
||||||
|
# not match, or ioctl() fails on the stdout file handle, we assume
|
||||||
|
# the process is running in the background as a daemon.
|
||||||
|
# http://www.gnu.org/software/bash/manual/bashref.html#Job-Control-Basics
|
||||||
|
try:
|
||||||
|
is_daemon = os.getpgrp() != os.tcgetpgrp(sys.stdout.fileno())
|
||||||
|
except OSError as err:
|
||||||
|
if err.errno == errno.ENOTTY:
|
||||||
|
# Assume we are a daemon because there is no terminal.
|
||||||
|
is_daemon = True
|
||||||
|
else:
|
||||||
|
raise
|
||||||
|
except UnsupportedOperation:
|
||||||
|
# Could not get the fileno for stdout, so we must be a daemon.
|
||||||
|
is_daemon = True
|
||||||
|
return is_daemon
|
||||||
|
|
||||||
|
|
||||||
|
def _is_sighup_and_daemon(signo):
|
||||||
|
if not (_sighup_supported() and signo == signal.SIGHUP):
|
||||||
|
# Avoid checking if we are a daemon, because the signal isn't
|
||||||
|
# SIGHUP.
|
||||||
|
return False
|
||||||
|
return _is_daemon()
|
||||||
|
|
||||||
|
|
||||||
|
def _signo_to_signame(signo):
|
||||||
|
signals = {signal.SIGTERM: 'SIGTERM',
|
||||||
|
signal.SIGINT: 'SIGINT'}
|
||||||
|
if _sighup_supported():
|
||||||
|
signals[signal.SIGHUP] = 'SIGHUP'
|
||||||
|
return signals[signo]
|
||||||
|
|
||||||
|
|
||||||
|
def _set_signals_handler(handler):
|
||||||
|
signal.signal(signal.SIGTERM, handler)
|
||||||
|
signal.signal(signal.SIGINT, handler)
|
||||||
|
if _sighup_supported():
|
||||||
|
signal.signal(signal.SIGHUP, handler)
|
||||||
|
|
||||||
|
|
||||||
|
class Launcher(object):
|
||||||
|
"""Launch one or more services and wait for them to complete."""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
"""Initialize the service launcher.
|
||||||
|
|
||||||
|
:returns: None
|
||||||
|
|
||||||
|
"""
|
||||||
|
self.services = Services()
|
||||||
|
self.backdoor_port = eventlet_backdoor.initialize_if_enabled()
|
||||||
|
|
||||||
|
def launch_service(self, service):
|
||||||
|
"""Load and start the given service.
|
||||||
|
|
||||||
|
:param service: The service you would like to start.
|
||||||
|
:returns: None
|
||||||
|
|
||||||
|
"""
|
||||||
|
service.backdoor_port = self.backdoor_port
|
||||||
|
self.services.add(service)
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""Stop all services which are currently running.
|
||||||
|
|
||||||
|
:returns: None
|
||||||
|
|
||||||
|
"""
|
||||||
|
self.services.stop()
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
"""Waits until all services have been stopped, and then returns.
|
||||||
|
|
||||||
|
:returns: None
|
||||||
|
|
||||||
|
"""
|
||||||
|
self.services.wait()
|
||||||
|
|
||||||
|
def restart(self):
|
||||||
|
"""Reload config files and restart service.
|
||||||
|
|
||||||
|
:returns: None
|
||||||
|
|
||||||
|
"""
|
||||||
|
cfg.CONF.reload_config_files()
|
||||||
|
self.services.restart()
|
||||||
|
|
||||||
|
|
||||||
|
class SignalExit(SystemExit):
|
||||||
|
def __init__(self, signo, exccode=1):
|
||||||
|
super(SignalExit, self).__init__(exccode)
|
||||||
|
self.signo = signo
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceLauncher(Launcher):
|
||||||
|
def _handle_signal(self, signo, frame):
|
||||||
|
# Allow the process to be killed again and die from natural causes
|
||||||
|
_set_signals_handler(signal.SIG_DFL)
|
||||||
|
raise SignalExit(signo)
|
||||||
|
|
||||||
|
def handle_signal(self):
|
||||||
|
_set_signals_handler(self._handle_signal)
|
||||||
|
|
||||||
|
def _wait_for_exit_or_signal(self, ready_callback=None):
|
||||||
|
status = None
|
||||||
|
signo = 0
|
||||||
|
|
||||||
|
LOG.debug('Full set of CONF:')
|
||||||
|
CONF.log_opt_values(LOG, std_logging.DEBUG)
|
||||||
|
|
||||||
|
try:
|
||||||
|
if ready_callback:
|
||||||
|
ready_callback()
|
||||||
|
super(ServiceLauncher, self).wait()
|
||||||
|
except SignalExit as exc:
|
||||||
|
signame = _signo_to_signame(exc.signo)
|
||||||
|
LOG.info(_LI('Caught %s, exiting'), signame)
|
||||||
|
status = exc.code
|
||||||
|
signo = exc.signo
|
||||||
|
except SystemExit as exc:
|
||||||
|
status = exc.code
|
||||||
|
finally:
|
||||||
|
self.stop()
|
||||||
|
if rpc:
|
||||||
|
try:
|
||||||
|
rpc.cleanup()
|
||||||
|
except Exception:
|
||||||
|
# We're shutting down, so it doesn't matter at this point.
|
||||||
|
LOG.exception(_LE('Exception during rpc cleanup.'))
|
||||||
|
|
||||||
|
return status, signo
|
||||||
|
|
||||||
|
def wait(self, ready_callback=None):
|
||||||
|
systemd.notify_once()
|
||||||
|
while True:
|
||||||
|
self.handle_signal()
|
||||||
|
status, signo = self._wait_for_exit_or_signal(ready_callback)
|
||||||
|
if not _is_sighup_and_daemon(signo):
|
||||||
|
return status
|
||||||
|
self.restart()
|
||||||
|
|
||||||
|
|
||||||
|
class ServiceWrapper(object):
|
||||||
|
def __init__(self, service, workers):
|
||||||
|
self.service = service
|
||||||
|
self.workers = workers
|
||||||
|
self.children = set()
|
||||||
|
self.forktimes = []
|
||||||
|
|
||||||
|
|
||||||
|
class ProcessLauncher(object):
|
||||||
|
def __init__(self, wait_interval=0.01):
|
||||||
|
"""Constructor.
|
||||||
|
|
||||||
|
:param wait_interval: The interval to sleep for between checks
|
||||||
|
of child process exit.
|
||||||
|
"""
|
||||||
|
self.children = {}
|
||||||
|
self.sigcaught = None
|
||||||
|
self.running = True
|
||||||
|
self.wait_interval = wait_interval
|
||||||
|
rfd, self.writepipe = os.pipe()
|
||||||
|
self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')
|
||||||
|
self.handle_signal()
|
||||||
|
|
||||||
|
def handle_signal(self):
|
||||||
|
_set_signals_handler(self._handle_signal)
|
||||||
|
|
||||||
|
def _handle_signal(self, signo, frame):
|
||||||
|
self.sigcaught = signo
|
||||||
|
self.running = False
|
||||||
|
|
||||||
|
# Allow the process to be killed again and die from natural causes
|
||||||
|
_set_signals_handler(signal.SIG_DFL)
|
||||||
|
|
||||||
|
def _pipe_watcher(self):
|
||||||
|
# This will block until the write end is closed when the parent
|
||||||
|
# dies unexpectedly
|
||||||
|
self.readpipe.read()
|
||||||
|
|
||||||
|
LOG.info(_LI('Parent process has died unexpectedly, exiting'))
|
||||||
|
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
def _child_process_handle_signal(self):
|
||||||
|
# Setup child signal handlers differently
|
||||||
|
def _sigterm(*args):
|
||||||
|
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||||
|
raise SignalExit(signal.SIGTERM)
|
||||||
|
|
||||||
|
def _sighup(*args):
|
||||||
|
signal.signal(signal.SIGHUP, signal.SIG_DFL)
|
||||||
|
raise SignalExit(signal.SIGHUP)
|
||||||
|
|
||||||
|
signal.signal(signal.SIGTERM, _sigterm)
|
||||||
|
if _sighup_supported():
|
||||||
|
signal.signal(signal.SIGHUP, _sighup)
|
||||||
|
# Block SIGINT and let the parent send us a SIGTERM
|
||||||
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||||
|
|
||||||
|
def _child_wait_for_exit_or_signal(self, launcher):
|
||||||
|
status = 0
|
||||||
|
signo = 0
|
||||||
|
|
||||||
|
# NOTE(johannes): All exceptions are caught to ensure this
|
||||||
|
# doesn't fallback into the loop spawning children. It would
|
||||||
|
# be bad for a child to spawn more children.
|
||||||
|
try:
|
||||||
|
launcher.wait()
|
||||||
|
except SignalExit as exc:
|
||||||
|
signame = _signo_to_signame(exc.signo)
|
||||||
|
LOG.info(_LI('Child caught %s, exiting'), signame)
|
||||||
|
status = exc.code
|
||||||
|
signo = exc.signo
|
||||||
|
except SystemExit as exc:
|
||||||
|
status = exc.code
|
||||||
|
except BaseException:
|
||||||
|
LOG.exception(_LE('Unhandled exception'))
|
||||||
|
status = 2
|
||||||
|
finally:
|
||||||
|
launcher.stop()
|
||||||
|
|
||||||
|
return status, signo
|
||||||
|
|
||||||
|
def _child_process(self, service):
|
||||||
|
self._child_process_handle_signal()
|
||||||
|
|
||||||
|
# Reopen the eventlet hub to make sure we don't share an epoll
|
||||||
|
# fd with parent and/or siblings, which would be bad
|
||||||
|
eventlet.hubs.use_hub()
|
||||||
|
|
||||||
|
# Close write to ensure only parent has it open
|
||||||
|
os.close(self.writepipe)
|
||||||
|
# Create greenthread to watch for parent to close pipe
|
||||||
|
eventlet.spawn_n(self._pipe_watcher)
|
||||||
|
|
||||||
|
# Reseed random number generator
|
||||||
|
random.seed()
|
||||||
|
|
||||||
|
launcher = Launcher()
|
||||||
|
launcher.launch_service(service)
|
||||||
|
return launcher
|
||||||
|
|
||||||
|
def _start_child(self, wrap):
|
||||||
|
if len(wrap.forktimes) > wrap.workers:
|
||||||
|
# Limit ourselves to one process a second (over the period of
|
||||||
|
# number of workers * 1 second). This will allow workers to
|
||||||
|
# start up quickly but ensure we don't fork off children that
|
||||||
|
# die instantly too quickly.
|
||||||
|
if time.time() - wrap.forktimes[0] < wrap.workers:
|
||||||
|
LOG.info(_LI('Forking too fast, sleeping'))
|
||||||
|
time.sleep(1)
|
||||||
|
|
||||||
|
wrap.forktimes.pop(0)
|
||||||
|
|
||||||
|
wrap.forktimes.append(time.time())
|
||||||
|
|
||||||
|
pid = os.fork()
|
||||||
|
if pid == 0:
|
||||||
|
launcher = self._child_process(wrap.service)
|
||||||
|
while True:
|
||||||
|
self._child_process_handle_signal()
|
||||||
|
status, signo = self._child_wait_for_exit_or_signal(launcher)
|
||||||
|
if not _is_sighup_and_daemon(signo):
|
||||||
|
break
|
||||||
|
launcher.restart()
|
||||||
|
|
||||||
|
os._exit(status)
|
||||||
|
|
||||||
|
LOG.info(_LI('Started child %d'), pid)
|
||||||
|
|
||||||
|
wrap.children.add(pid)
|
||||||
|
self.children[pid] = wrap
|
||||||
|
|
||||||
|
return pid
|
||||||
|
|
||||||
|
def launch_service(self, service, workers=1):
|
||||||
|
wrap = ServiceWrapper(service, workers)
|
||||||
|
|
||||||
|
LOG.info(_LI('Starting %d workers'), wrap.workers)
|
||||||
|
while self.running and len(wrap.children) < wrap.workers:
|
||||||
|
self._start_child(wrap)
|
||||||
|
|
||||||
|
def _wait_child(self):
|
||||||
|
try:
|
||||||
|
# Don't block if no child processes have exited
|
||||||
|
pid, status = os.waitpid(0, os.WNOHANG)
|
||||||
|
if not pid:
|
||||||
|
return None
|
||||||
|
except OSError as exc:
|
||||||
|
if exc.errno not in (errno.EINTR, errno.ECHILD):
|
||||||
|
raise
|
||||||
|
return None
|
||||||
|
|
||||||
|
if os.WIFSIGNALED(status):
|
||||||
|
sig = os.WTERMSIG(status)
|
||||||
|
LOG.info(_LI('Child %(pid)d killed by signal %(sig)d'),
|
||||||
|
dict(pid=pid, sig=sig))
|
||||||
|
else:
|
||||||
|
code = os.WEXITSTATUS(status)
|
||||||
|
LOG.info(_LI('Child %(pid)s exited with status %(code)d'),
|
||||||
|
dict(pid=pid, code=code))
|
||||||
|
|
||||||
|
if pid not in self.children:
|
||||||
|
LOG.warning(_LW('pid %d not in child list'), pid)
|
||||||
|
return None
|
||||||
|
|
||||||
|
wrap = self.children.pop(pid)
|
||||||
|
wrap.children.remove(pid)
|
||||||
|
return wrap
|
||||||
|
|
||||||
|
def _respawn_children(self):
|
||||||
|
while self.running:
|
||||||
|
wrap = self._wait_child()
|
||||||
|
if not wrap:
|
||||||
|
# Yield to other threads if no children have exited
|
||||||
|
# Sleep for a short time to avoid excessive CPU usage
|
||||||
|
# (see bug #1095346)
|
||||||
|
eventlet.greenthread.sleep(self.wait_interval)
|
||||||
|
continue
|
||||||
|
while self.running and len(wrap.children) < wrap.workers:
|
||||||
|
self._start_child(wrap)
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
"""Loop waiting on children to die and respawning as necessary."""
|
||||||
|
|
||||||
|
systemd.notify_once()
|
||||||
|
LOG.debug('Full set of CONF:')
|
||||||
|
CONF.log_opt_values(LOG, std_logging.DEBUG)
|
||||||
|
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
self.handle_signal()
|
||||||
|
self._respawn_children()
|
||||||
|
# No signal means that stop was called. Don't clean up here.
|
||||||
|
if not self.sigcaught:
|
||||||
|
return
|
||||||
|
|
||||||
|
signame = _signo_to_signame(self.sigcaught)
|
||||||
|
LOG.info(_LI('Caught %s, stopping children'), signame)
|
||||||
|
if not _is_sighup_and_daemon(self.sigcaught):
|
||||||
|
break
|
||||||
|
|
||||||
|
for pid in self.children:
|
||||||
|
os.kill(pid, signal.SIGHUP)
|
||||||
|
self.running = True
|
||||||
|
self.sigcaught = None
|
||||||
|
except eventlet.greenlet.GreenletExit:
|
||||||
|
LOG.info(_LI("Wait called after thread killed. Cleaning up."))
|
||||||
|
|
||||||
|
self.stop()
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
"""Terminate child processes and wait on each."""
|
||||||
|
self.running = False
|
||||||
|
for pid in self.children:
|
||||||
|
try:
|
||||||
|
os.kill(pid, signal.SIGTERM)
|
||||||
|
except OSError as exc:
|
||||||
|
if exc.errno != errno.ESRCH:
|
||||||
|
raise
|
||||||
|
|
||||||
|
# Wait for children to die
|
||||||
|
if self.children:
|
||||||
|
LOG.info(_LI('Waiting on %d children to exit'), len(self.children))
|
||||||
|
while self.children:
|
||||||
|
self._wait_child()
|
||||||
|
|
||||||
|
|
||||||
|
class Service(object):
|
||||||
|
"""Service object for binaries running on hosts."""
|
||||||
|
|
||||||
|
def __init__(self, threads=1000):
|
||||||
|
self.tg = threadgroup.ThreadGroup(threads)
|
||||||
|
|
||||||
|
# signal that the service is done shutting itself down:
|
||||||
|
self._done = event.Event()
|
||||||
|
|
||||||
|
def reset(self):
|
||||||
|
# NOTE(Fengqian): docs for Event.reset() recommend against using it
|
||||||
|
self._done = event.Event()
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.tg.stop()
|
||||||
|
self.tg.wait()
|
||||||
|
# Signal that service cleanup is done:
|
||||||
|
if not self._done.ready():
|
||||||
|
self._done.send()
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
self._done.wait()
|
||||||
|
|
||||||
|
|
||||||
|
class Services(object):
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.services = []
|
||||||
|
self.tg = threadgroup.ThreadGroup()
|
||||||
|
self.done = event.Event()
|
||||||
|
|
||||||
|
def add(self, service):
|
||||||
|
self.services.append(service)
|
||||||
|
self.tg.add_thread(self.run_service, service, self.done)
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
# wait for graceful shutdown of services:
|
||||||
|
for service in self.services:
|
||||||
|
service.stop()
|
||||||
|
service.wait()
|
||||||
|
|
||||||
|
# Each service has performed cleanup, now signal that the run_service
|
||||||
|
# wrapper threads can now die:
|
||||||
|
if not self.done.ready():
|
||||||
|
self.done.send()
|
||||||
|
|
||||||
|
# reap threads:
|
||||||
|
self.tg.stop()
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
self.tg.wait()
|
||||||
|
|
||||||
|
def restart(self):
|
||||||
|
self.stop()
|
||||||
|
self.done = event.Event()
|
||||||
|
for restart_service in self.services:
|
||||||
|
restart_service.reset()
|
||||||
|
self.tg.add_thread(self.run_service, restart_service, self.done)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def run_service(service, done):
|
||||||
|
"""Service start wrapper.
|
||||||
|
|
||||||
|
:param service: service to run
|
||||||
|
:param done: event to wait on until a shutdown is triggered
|
||||||
|
:returns: None
|
||||||
|
|
||||||
|
"""
|
||||||
|
service.start()
|
||||||
|
done.wait()
|
||||||
|
|
||||||
|
|
||||||
|
def launch(service, workers=1):
|
||||||
|
if workers is None or workers == 1:
|
||||||
|
launcher = ServiceLauncher()
|
||||||
|
launcher.launch_service(service)
|
||||||
|
else:
|
||||||
|
launcher = ProcessLauncher()
|
||||||
|
launcher.launch_service(service, workers=workers)
|
||||||
|
|
||||||
|
return launcher
|
|
@ -0,0 +1,311 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
System-level utilities and helper functions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import math
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
import six
|
||||||
|
|
||||||
|
from monasca.openstack.common.gettextutils import _
|
||||||
|
|
||||||
|
|
||||||
|
UNIT_PREFIX_EXPONENT = {
|
||||||
|
'k': 1,
|
||||||
|
'K': 1,
|
||||||
|
'Ki': 1,
|
||||||
|
'M': 2,
|
||||||
|
'Mi': 2,
|
||||||
|
'G': 3,
|
||||||
|
'Gi': 3,
|
||||||
|
'T': 4,
|
||||||
|
'Ti': 4,
|
||||||
|
}
|
||||||
|
UNIT_SYSTEM_INFO = {
|
||||||
|
'IEC': (1024, re.compile(r'(^[-+]?\d*\.?\d+)([KMGT]i?)?(b|bit|B)$')),
|
||||||
|
'SI': (1000, re.compile(r'(^[-+]?\d*\.?\d+)([kMGT])?(b|bit|B)$')),
|
||||||
|
}
|
||||||
|
|
||||||
|
TRUE_STRINGS = ('1', 't', 'true', 'on', 'y', 'yes')
|
||||||
|
FALSE_STRINGS = ('0', 'f', 'false', 'off', 'n', 'no')
|
||||||
|
|
||||||
|
SLUGIFY_STRIP_RE = re.compile(r"[^\w\s-]")
|
||||||
|
SLUGIFY_HYPHENATE_RE = re.compile(r"[-\s]+")
|
||||||
|
|
||||||
|
|
||||||
|
# NOTE(flaper87): The following globals are used by `mask_password`
|
||||||
|
_SANITIZE_KEYS = ['adminPass', 'admin_pass', 'password', 'admin_password']
|
||||||
|
|
||||||
|
# NOTE(ldbragst): Let's build a list of regex objects using the list of
|
||||||
|
# _SANITIZE_KEYS we already have. This way, we only have to add the new key
|
||||||
|
# to the list of _SANITIZE_KEYS and we can generate regular expressions
|
||||||
|
# for XML and JSON automatically.
|
||||||
|
_SANITIZE_PATTERNS_2 = []
|
||||||
|
_SANITIZE_PATTERNS_1 = []
|
||||||
|
|
||||||
|
# NOTE(amrith): Some regular expressions have only one parameter, some
|
||||||
|
# have two parameters. Use different lists of patterns here.
|
||||||
|
_FORMAT_PATTERNS_1 = [r'(%(key)s\s*[=]\s*)[^\s^\'^\"]+']
|
||||||
|
_FORMAT_PATTERNS_2 = [r'(%(key)s\s*[=]\s*[\"\']).*?([\"\'])',
|
||||||
|
r'(%(key)s\s+[\"\']).*?([\"\'])',
|
||||||
|
r'([-]{2}%(key)s\s+)[^\'^\"^=^\s]+([\s]*)',
|
||||||
|
r'(<%(key)s>).*?(</%(key)s>)',
|
||||||
|
r'([\"\']%(key)s[\"\']\s*:\s*[\"\']).*?([\"\'])',
|
||||||
|
r'([\'"].*?%(key)s[\'"]\s*:\s*u?[\'"]).*?([\'"])',
|
||||||
|
r'([\'"].*?%(key)s[\'"]\s*,\s*\'--?[A-z]+\'\s*,\s*u?'
|
||||||
|
'[\'"]).*?([\'"])',
|
||||||
|
r'(%(key)s\s*--?[A-z]+\s*)\S+(\s*)']
|
||||||
|
|
||||||
|
for key in _SANITIZE_KEYS:
|
||||||
|
for pattern in _FORMAT_PATTERNS_2:
|
||||||
|
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
|
||||||
|
_SANITIZE_PATTERNS_2.append(reg_ex)
|
||||||
|
|
||||||
|
for pattern in _FORMAT_PATTERNS_1:
|
||||||
|
reg_ex = re.compile(pattern % {'key': key}, re.DOTALL)
|
||||||
|
_SANITIZE_PATTERNS_1.append(reg_ex)
|
||||||
|
|
||||||
|
|
||||||
|
def int_from_bool_as_string(subject):
|
||||||
|
"""Interpret a string as a boolean and return either 1 or 0.
|
||||||
|
|
||||||
|
Any string value in:
|
||||||
|
|
||||||
|
('True', 'true', 'On', 'on', '1')
|
||||||
|
|
||||||
|
is interpreted as a boolean True.
|
||||||
|
|
||||||
|
Useful for JSON-decoded stuff and config file parsing
|
||||||
|
"""
|
||||||
|
return bool_from_string(subject) and 1 or 0
|
||||||
|
|
||||||
|
|
||||||
|
def bool_from_string(subject, strict=False, default=False):
|
||||||
|
"""Interpret a string as a boolean.
|
||||||
|
|
||||||
|
A case-insensitive match is performed such that strings matching 't',
|
||||||
|
'true', 'on', 'y', 'yes', or '1' are considered True and, when
|
||||||
|
`strict=False`, anything else returns the value specified by 'default'.
|
||||||
|
|
||||||
|
Useful for JSON-decoded stuff and config file parsing.
|
||||||
|
|
||||||
|
If `strict=True`, unrecognized values, including None, will raise a
|
||||||
|
ValueError which is useful when parsing values passed in from an API call.
|
||||||
|
Strings yielding False are 'f', 'false', 'off', 'n', 'no', or '0'.
|
||||||
|
"""
|
||||||
|
if not isinstance(subject, six.string_types):
|
||||||
|
subject = six.text_type(subject)
|
||||||
|
|
||||||
|
lowered = subject.strip().lower()
|
||||||
|
|
||||||
|
if lowered in TRUE_STRINGS:
|
||||||
|
return True
|
||||||
|
elif lowered in FALSE_STRINGS:
|
||||||
|
return False
|
||||||
|
elif strict:
|
||||||
|
acceptable = ', '.join(
|
||||||
|
"'%s'" % s for s in sorted(TRUE_STRINGS + FALSE_STRINGS))
|
||||||
|
msg = _("Unrecognized value '%(val)s', acceptable values are:"
|
||||||
|
" %(acceptable)s") % {'val': subject,
|
||||||
|
'acceptable': acceptable}
|
||||||
|
raise ValueError(msg)
|
||||||
|
else:
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def safe_decode(text, incoming=None, errors='strict'):
|
||||||
|
"""Decodes incoming text/bytes string using `incoming` if they're not
|
||||||
|
already unicode.
|
||||||
|
|
||||||
|
:param incoming: Text's current encoding
|
||||||
|
:param errors: Errors handling policy. See here for valid
|
||||||
|
values http://docs.python.org/2/library/codecs.html
|
||||||
|
:returns: text or a unicode `incoming` encoded
|
||||||
|
representation of it.
|
||||||
|
:raises TypeError: If text is not an instance of str
|
||||||
|
"""
|
||||||
|
if not isinstance(text, (six.string_types, six.binary_type)):
|
||||||
|
raise TypeError("%s can't be decoded" % type(text))
|
||||||
|
|
||||||
|
if isinstance(text, six.text_type):
|
||||||
|
return text
|
||||||
|
|
||||||
|
if not incoming:
|
||||||
|
incoming = (sys.stdin.encoding or
|
||||||
|
sys.getdefaultencoding())
|
||||||
|
|
||||||
|
try:
|
||||||
|
return text.decode(incoming, errors)
|
||||||
|
except UnicodeDecodeError:
|
||||||
|
# Note(flaper87) If we get here, it means that
|
||||||
|
# sys.stdin.encoding / sys.getdefaultencoding
|
||||||
|
# didn't return a suitable encoding to decode
|
||||||
|
# text. This happens mostly when global LANG
|
||||||
|
# var is not set correctly and there's no
|
||||||
|
# default encoding. In this case, most likely
|
||||||
|
# python will use ASCII or ANSI encoders as
|
||||||
|
# default encodings but they won't be capable
|
||||||
|
# of decoding non-ASCII characters.
|
||||||
|
#
|
||||||
|
# Also, UTF-8 is being used since it's an ASCII
|
||||||
|
# extension.
|
||||||
|
return text.decode('utf-8', errors)
|
||||||
|
|
||||||
|
|
||||||
|
def safe_encode(text, incoming=None,
|
||||||
|
encoding='utf-8', errors='strict'):
|
||||||
|
"""Encodes incoming text/bytes string using `encoding`.
|
||||||
|
|
||||||
|
If incoming is not specified, text is expected to be encoded with
|
||||||
|
current python's default encoding. (`sys.getdefaultencoding`)
|
||||||
|
|
||||||
|
:param incoming: Text's current encoding
|
||||||
|
:param encoding: Expected encoding for text (Default UTF-8)
|
||||||
|
:param errors: Errors handling policy. See here for valid
|
||||||
|
values http://docs.python.org/2/library/codecs.html
|
||||||
|
:returns: text or a bytestring `encoding` encoded
|
||||||
|
representation of it.
|
||||||
|
:raises TypeError: If text is not an instance of str
|
||||||
|
"""
|
||||||
|
if not isinstance(text, (six.string_types, six.binary_type)):
|
||||||
|
raise TypeError("%s can't be encoded" % type(text))
|
||||||
|
|
||||||
|
if not incoming:
|
||||||
|
incoming = (sys.stdin.encoding or
|
||||||
|
sys.getdefaultencoding())
|
||||||
|
|
||||||
|
if isinstance(text, six.text_type):
|
||||||
|
return text.encode(encoding, errors)
|
||||||
|
elif text and encoding != incoming:
|
||||||
|
# Decode text before encoding it with `encoding`
|
||||||
|
text = safe_decode(text, incoming, errors)
|
||||||
|
return text.encode(encoding, errors)
|
||||||
|
else:
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def string_to_bytes(text, unit_system='IEC', return_int=False):
|
||||||
|
"""Converts a string into an float representation of bytes.
|
||||||
|
|
||||||
|
The units supported for IEC ::
|
||||||
|
|
||||||
|
Kb(it), Kib(it), Mb(it), Mib(it), Gb(it), Gib(it), Tb(it), Tib(it)
|
||||||
|
KB, KiB, MB, MiB, GB, GiB, TB, TiB
|
||||||
|
|
||||||
|
The units supported for SI ::
|
||||||
|
|
||||||
|
kb(it), Mb(it), Gb(it), Tb(it)
|
||||||
|
kB, MB, GB, TB
|
||||||
|
|
||||||
|
Note that the SI unit system does not support capital letter 'K'
|
||||||
|
|
||||||
|
:param text: String input for bytes size conversion.
|
||||||
|
:param unit_system: Unit system for byte size conversion.
|
||||||
|
:param return_int: If True, returns integer representation of text
|
||||||
|
in bytes. (default: decimal)
|
||||||
|
:returns: Numerical representation of text in bytes.
|
||||||
|
:raises ValueError: If text has an invalid value.
|
||||||
|
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
base, reg_ex = UNIT_SYSTEM_INFO[unit_system]
|
||||||
|
except KeyError:
|
||||||
|
msg = _('Invalid unit system: "%s"') % unit_system
|
||||||
|
raise ValueError(msg)
|
||||||
|
match = reg_ex.match(text)
|
||||||
|
if match:
|
||||||
|
magnitude = float(match.group(1))
|
||||||
|
unit_prefix = match.group(2)
|
||||||
|
if match.group(3) in ['b', 'bit']:
|
||||||
|
magnitude /= 8
|
||||||
|
else:
|
||||||
|
msg = _('Invalid string format: %s') % text
|
||||||
|
raise ValueError(msg)
|
||||||
|
if not unit_prefix:
|
||||||
|
res = magnitude
|
||||||
|
else:
|
||||||
|
res = magnitude * pow(base, UNIT_PREFIX_EXPONENT[unit_prefix])
|
||||||
|
if return_int:
|
||||||
|
return int(math.ceil(res))
|
||||||
|
return res
|
||||||
|
|
||||||
|
|
||||||
|
def to_slug(value, incoming=None, errors="strict"):
|
||||||
|
"""Normalize string.
|
||||||
|
|
||||||
|
Convert to lowercase, remove non-word characters, and convert spaces
|
||||||
|
to hyphens.
|
||||||
|
|
||||||
|
Inspired by Django's `slugify` filter.
|
||||||
|
|
||||||
|
:param value: Text to slugify
|
||||||
|
:param incoming: Text's current encoding
|
||||||
|
:param errors: Errors handling policy. See here for valid
|
||||||
|
values http://docs.python.org/2/library/codecs.html
|
||||||
|
:returns: slugified unicode representation of `value`
|
||||||
|
:raises TypeError: If text is not an instance of str
|
||||||
|
"""
|
||||||
|
value = safe_decode(value, incoming, errors)
|
||||||
|
# NOTE(aababilov): no need to use safe_(encode|decode) here:
|
||||||
|
# encodings are always "ascii", error handling is always "ignore"
|
||||||
|
# and types are always known (first: unicode; second: str)
|
||||||
|
value = unicodedata.normalize("NFKD", value).encode(
|
||||||
|
"ascii", "ignore").decode("ascii")
|
||||||
|
value = SLUGIFY_STRIP_RE.sub("", value).strip().lower()
|
||||||
|
return SLUGIFY_HYPHENATE_RE.sub("-", value)
|
||||||
|
|
||||||
|
|
||||||
|
def mask_password(message, secret="***"):
|
||||||
|
"""Replace password with 'secret' in message.
|
||||||
|
|
||||||
|
:param message: The string which includes security information.
|
||||||
|
:param secret: value with which to replace passwords.
|
||||||
|
:returns: The unicode value of message with the password fields masked.
|
||||||
|
|
||||||
|
For example:
|
||||||
|
|
||||||
|
>>> mask_password("'adminPass' : 'aaaaa'")
|
||||||
|
"'adminPass' : '***'"
|
||||||
|
>>> mask_password("'admin_pass' : 'aaaaa'")
|
||||||
|
"'admin_pass' : '***'"
|
||||||
|
>>> mask_password('"password" : "aaaaa"')
|
||||||
|
'"password" : "***"'
|
||||||
|
>>> mask_password("'original_password' : 'aaaaa'")
|
||||||
|
"'original_password' : '***'"
|
||||||
|
>>> mask_password("u'original_password' : u'aaaaa'")
|
||||||
|
"u'original_password' : u'***'"
|
||||||
|
"""
|
||||||
|
message = six.text_type(message)
|
||||||
|
|
||||||
|
# NOTE(ldbragst): Check to see if anything in message contains any key
|
||||||
|
# specified in _SANITIZE_KEYS, if not then just return the message since
|
||||||
|
# we don't have to mask any passwords.
|
||||||
|
if not any(key in message for key in _SANITIZE_KEYS):
|
||||||
|
return message
|
||||||
|
|
||||||
|
substitute = r'\g<1>' + secret + r'\g<2>'
|
||||||
|
for pattern in _SANITIZE_PATTERNS_2:
|
||||||
|
message = re.sub(pattern, substitute, message)
|
||||||
|
|
||||||
|
substitute = r'\g<1>' + secret
|
||||||
|
for pattern in _SANITIZE_PATTERNS_1:
|
||||||
|
message = re.sub(pattern, substitute, message)
|
||||||
|
|
||||||
|
return message
|
|
@ -0,0 +1,106 @@
|
||||||
|
# Copyright 2012-2014 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Helper module for systemd service readiness notification.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import socket
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from monasca.openstack.common import log as logging
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _abstractify(socket_name):
|
||||||
|
if socket_name.startswith('@'):
|
||||||
|
# abstract namespace socket
|
||||||
|
socket_name = '\0%s' % socket_name[1:]
|
||||||
|
return socket_name
|
||||||
|
|
||||||
|
|
||||||
|
def _sd_notify(unset_env, msg):
|
||||||
|
notify_socket = os.getenv('NOTIFY_SOCKET')
|
||||||
|
if notify_socket:
|
||||||
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||||
|
try:
|
||||||
|
sock.connect(_abstractify(notify_socket))
|
||||||
|
sock.sendall(msg)
|
||||||
|
if unset_env:
|
||||||
|
del os.environ['NOTIFY_SOCKET']
|
||||||
|
except EnvironmentError:
|
||||||
|
LOG.debug("Systemd notification failed", exc_info=True)
|
||||||
|
finally:
|
||||||
|
sock.close()
|
||||||
|
|
||||||
|
|
||||||
|
def notify():
|
||||||
|
"""Send notification to Systemd that service is ready.
|
||||||
|
|
||||||
|
For details see
|
||||||
|
http://www.freedesktop.org/software/systemd/man/sd_notify.html
|
||||||
|
"""
|
||||||
|
_sd_notify(False, 'READY=1')
|
||||||
|
|
||||||
|
|
||||||
|
def notify_once():
|
||||||
|
"""Send notification once to Systemd that service is ready.
|
||||||
|
|
||||||
|
Systemd sets NOTIFY_SOCKET environment variable with the name of the
|
||||||
|
socket listening for notifications from services.
|
||||||
|
This method removes the NOTIFY_SOCKET environment variable to ensure
|
||||||
|
notification is sent only once.
|
||||||
|
"""
|
||||||
|
_sd_notify(True, 'READY=1')
|
||||||
|
|
||||||
|
|
||||||
|
def onready(notify_socket, timeout):
|
||||||
|
"""Wait for systemd style notification on the socket.
|
||||||
|
|
||||||
|
:param notify_socket: local socket address
|
||||||
|
:type notify_socket: string
|
||||||
|
:param timeout: socket timeout
|
||||||
|
:type timeout: float
|
||||||
|
:returns: 0 service ready
|
||||||
|
1 service not ready
|
||||||
|
2 timeout occurred
|
||||||
|
"""
|
||||||
|
sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||||
|
sock.settimeout(timeout)
|
||||||
|
sock.bind(_abstractify(notify_socket))
|
||||||
|
try:
|
||||||
|
msg = sock.recv(512)
|
||||||
|
except socket.timeout:
|
||||||
|
return 2
|
||||||
|
finally:
|
||||||
|
sock.close()
|
||||||
|
if 'READY=1' in msg:
|
||||||
|
return 0
|
||||||
|
else:
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# simple CLI for testing
|
||||||
|
if len(sys.argv) == 1:
|
||||||
|
notify()
|
||||||
|
elif len(sys.argv) >= 2:
|
||||||
|
timeout = float(sys.argv[1])
|
||||||
|
notify_socket = os.getenv('NOTIFY_SOCKET')
|
||||||
|
if notify_socket:
|
||||||
|
retval = onready(notify_socket, timeout)
|
||||||
|
sys.exit(retval)
|
|
@ -0,0 +1,147 @@
|
||||||
|
# Copyright 2012 Red Hat, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
import threading
|
||||||
|
|
||||||
|
import eventlet
|
||||||
|
from eventlet import greenpool
|
||||||
|
|
||||||
|
from monasca.openstack.common import log as logging
|
||||||
|
from monasca.openstack.common import loopingcall
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
def _thread_done(gt, *args, **kwargs):
|
||||||
|
"""Callback function to be passed to GreenThread.link() when we spawn()
|
||||||
|
Calls the :class:`ThreadGroup` to notify if.
|
||||||
|
|
||||||
|
"""
|
||||||
|
kwargs['group'].thread_done(kwargs['thread'])
|
||||||
|
|
||||||
|
|
||||||
|
class Thread(object):
|
||||||
|
"""Wrapper around a greenthread, that holds a reference to the
|
||||||
|
:class:`ThreadGroup`. The Thread will notify the :class:`ThreadGroup` when
|
||||||
|
it has done so it can be removed from the threads list.
|
||||||
|
"""
|
||||||
|
def __init__(self, thread, group):
|
||||||
|
self.thread = thread
|
||||||
|
self.thread.link(_thread_done, group=group, thread=self)
|
||||||
|
|
||||||
|
def stop(self):
|
||||||
|
self.thread.kill()
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
return self.thread.wait()
|
||||||
|
|
||||||
|
def link(self, func, *args, **kwargs):
|
||||||
|
self.thread.link(func, *args, **kwargs)
|
||||||
|
|
||||||
|
|
||||||
|
class ThreadGroup(object):
|
||||||
|
"""The point of the ThreadGroup class is to:
|
||||||
|
|
||||||
|
* keep track of timers and greenthreads (making it easier to stop them
|
||||||
|
when need be).
|
||||||
|
* provide an easy API to add timers.
|
||||||
|
"""
|
||||||
|
def __init__(self, thread_pool_size=10):
|
||||||
|
self.pool = greenpool.GreenPool(thread_pool_size)
|
||||||
|
self.threads = []
|
||||||
|
self.timers = []
|
||||||
|
|
||||||
|
def add_dynamic_timer(self, callback, initial_delay=None,
|
||||||
|
periodic_interval_max=None, *args, **kwargs):
|
||||||
|
timer = loopingcall.DynamicLoopingCall(callback, *args, **kwargs)
|
||||||
|
timer.start(initial_delay=initial_delay,
|
||||||
|
periodic_interval_max=periodic_interval_max)
|
||||||
|
self.timers.append(timer)
|
||||||
|
|
||||||
|
def add_timer(self, interval, callback, initial_delay=None,
|
||||||
|
*args, **kwargs):
|
||||||
|
pulse = loopingcall.FixedIntervalLoopingCall(callback, *args, **kwargs)
|
||||||
|
pulse.start(interval=interval,
|
||||||
|
initial_delay=initial_delay)
|
||||||
|
self.timers.append(pulse)
|
||||||
|
|
||||||
|
def add_thread(self, callback, *args, **kwargs):
|
||||||
|
gt = self.pool.spawn(callback, *args, **kwargs)
|
||||||
|
th = Thread(gt, self)
|
||||||
|
self.threads.append(th)
|
||||||
|
return th
|
||||||
|
|
||||||
|
def thread_done(self, thread):
|
||||||
|
self.threads.remove(thread)
|
||||||
|
|
||||||
|
def _stop_threads(self):
|
||||||
|
current = threading.current_thread()
|
||||||
|
|
||||||
|
# Iterate over a copy of self.threads so thread_done doesn't
|
||||||
|
# modify the list while we're iterating
|
||||||
|
for x in self.threads[:]:
|
||||||
|
if x is current:
|
||||||
|
# don't kill the current thread.
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
x.stop()
|
||||||
|
except Exception as ex:
|
||||||
|
LOG.exception(ex)
|
||||||
|
|
||||||
|
def stop_timers(self):
|
||||||
|
for x in self.timers:
|
||||||
|
try:
|
||||||
|
x.stop()
|
||||||
|
except Exception as ex:
|
||||||
|
LOG.exception(ex)
|
||||||
|
self.timers = []
|
||||||
|
|
||||||
|
def stop(self, graceful=False):
|
||||||
|
"""stop function has the option of graceful=True/False.
|
||||||
|
|
||||||
|
* In case of graceful=True, wait for all threads to be finished.
|
||||||
|
Never kill threads.
|
||||||
|
* In case of graceful=False, kill threads immediately.
|
||||||
|
"""
|
||||||
|
self.stop_timers()
|
||||||
|
if graceful:
|
||||||
|
# In case of graceful=True, wait for all threads to be
|
||||||
|
# finished, never kill threads
|
||||||
|
self.wait()
|
||||||
|
else:
|
||||||
|
# In case of graceful=False(Default), kill threads
|
||||||
|
# immediately
|
||||||
|
self._stop_threads()
|
||||||
|
|
||||||
|
def wait(self):
|
||||||
|
for x in self.timers:
|
||||||
|
try:
|
||||||
|
x.wait()
|
||||||
|
except eventlet.greenlet.GreenletExit:
|
||||||
|
pass
|
||||||
|
except Exception as ex:
|
||||||
|
LOG.exception(ex)
|
||||||
|
current = threading.current_thread()
|
||||||
|
|
||||||
|
# Iterate over a copy of self.threads so thread_done doesn't
|
||||||
|
# modify the list while we're iterating
|
||||||
|
for x in self.threads[:]:
|
||||||
|
if x is current:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
x.wait()
|
||||||
|
except eventlet.greenlet.GreenletExit:
|
||||||
|
pass
|
||||||
|
except Exception as ex:
|
||||||
|
LOG.exception(ex)
|
|
@ -0,0 +1,210 @@
|
||||||
|
# Copyright 2011 OpenStack Foundation.
|
||||||
|
# All Rights Reserved.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
"""
|
||||||
|
Time related utilities and helper functions.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import calendar
|
||||||
|
import datetime
|
||||||
|
import time
|
||||||
|
|
||||||
|
import iso8601
|
||||||
|
import six
|
||||||
|
|
||||||
|
|
||||||
|
# ISO 8601 extended time format with microseconds
|
||||||
|
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
|
||||||
|
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
|
||||||
|
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
|
||||||
|
|
||||||
|
|
||||||
|
def isotime(at=None, subsecond=False):
|
||||||
|
"""Stringify time in ISO 8601 format."""
|
||||||
|
if not at:
|
||||||
|
at = utcnow()
|
||||||
|
st = at.strftime(_ISO8601_TIME_FORMAT
|
||||||
|
if not subsecond
|
||||||
|
else _ISO8601_TIME_FORMAT_SUBSECOND)
|
||||||
|
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
|
||||||
|
st += ('Z' if tz == 'UTC' else tz)
|
||||||
|
return st
|
||||||
|
|
||||||
|
|
||||||
|
def parse_isotime(timestr):
|
||||||
|
"""Parse time from ISO 8601 format."""
|
||||||
|
try:
|
||||||
|
return iso8601.parse_date(timestr)
|
||||||
|
except iso8601.ParseError as e:
|
||||||
|
raise ValueError(six.text_type(e))
|
||||||
|
except TypeError as e:
|
||||||
|
raise ValueError(six.text_type(e))
|
||||||
|
|
||||||
|
|
||||||
|
def strtime(at=None, fmt=PERFECT_TIME_FORMAT):
|
||||||
|
"""Returns formatted utcnow."""
|
||||||
|
if not at:
|
||||||
|
at = utcnow()
|
||||||
|
return at.strftime(fmt)
|
||||||
|
|
||||||
|
|
||||||
|
def parse_strtime(timestr, fmt=PERFECT_TIME_FORMAT):
|
||||||
|
"""Turn a formatted time back into a datetime."""
|
||||||
|
return datetime.datetime.strptime(timestr, fmt)
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_time(timestamp):
|
||||||
|
"""Normalize time in arbitrary timezone to UTC naive object."""
|
||||||
|
offset = timestamp.utcoffset()
|
||||||
|
if offset is None:
|
||||||
|
return timestamp
|
||||||
|
return timestamp.replace(tzinfo=None) - offset
|
||||||
|
|
||||||
|
|
||||||
|
def is_older_than(before, seconds):
|
||||||
|
"""Return True if before is older than seconds."""
|
||||||
|
if isinstance(before, six.string_types):
|
||||||
|
before = parse_strtime(before).replace(tzinfo=None)
|
||||||
|
else:
|
||||||
|
before = before.replace(tzinfo=None)
|
||||||
|
|
||||||
|
return utcnow() - before > datetime.timedelta(seconds=seconds)
|
||||||
|
|
||||||
|
|
||||||
|
def is_newer_than(after, seconds):
|
||||||
|
"""Return True if after is newer than seconds."""
|
||||||
|
if isinstance(after, six.string_types):
|
||||||
|
after = parse_strtime(after).replace(tzinfo=None)
|
||||||
|
else:
|
||||||
|
after = after.replace(tzinfo=None)
|
||||||
|
|
||||||
|
return after - utcnow() > datetime.timedelta(seconds=seconds)
|
||||||
|
|
||||||
|
|
||||||
|
def utcnow_ts():
|
||||||
|
"""Timestamp version of our utcnow function."""
|
||||||
|
if utcnow.override_time is None:
|
||||||
|
# NOTE(kgriffs): This is several times faster
|
||||||
|
# than going through calendar.timegm(...)
|
||||||
|
return int(time.time())
|
||||||
|
|
||||||
|
return calendar.timegm(utcnow().timetuple())
|
||||||
|
|
||||||
|
|
||||||
|
def utcnow():
|
||||||
|
"""Overridable version of utils.utcnow."""
|
||||||
|
if utcnow.override_time:
|
||||||
|
try:
|
||||||
|
return utcnow.override_time.pop(0)
|
||||||
|
except AttributeError:
|
||||||
|
return utcnow.override_time
|
||||||
|
return datetime.datetime.utcnow()
|
||||||
|
|
||||||
|
|
||||||
|
def iso8601_from_timestamp(timestamp):
|
||||||
|
"""Returns an iso8601 formatted date from timestamp."""
|
||||||
|
return isotime(datetime.datetime.utcfromtimestamp(timestamp))
|
||||||
|
|
||||||
|
|
||||||
|
utcnow.override_time = None
|
||||||
|
|
||||||
|
|
||||||
|
def set_time_override(override_time=None):
|
||||||
|
"""Overrides utils.utcnow.
|
||||||
|
|
||||||
|
Make it return a constant time or a list thereof, one at a time.
|
||||||
|
|
||||||
|
:param override_time: datetime instance or list thereof. If not
|
||||||
|
given, defaults to the current UTC time.
|
||||||
|
"""
|
||||||
|
utcnow.override_time = override_time or datetime.datetime.utcnow()
|
||||||
|
|
||||||
|
|
||||||
|
def advance_time_delta(timedelta):
|
||||||
|
"""Advance overridden time using a datetime.timedelta."""
|
||||||
|
assert utcnow.override_time is not None
|
||||||
|
try:
|
||||||
|
for dt in utcnow.override_time:
|
||||||
|
dt += timedelta
|
||||||
|
except TypeError:
|
||||||
|
utcnow.override_time += timedelta
|
||||||
|
|
||||||
|
|
||||||
|
def advance_time_seconds(seconds):
|
||||||
|
"""Advance overridden time by seconds."""
|
||||||
|
advance_time_delta(datetime.timedelta(0, seconds))
|
||||||
|
|
||||||
|
|
||||||
|
def clear_time_override():
|
||||||
|
"""Remove the overridden time."""
|
||||||
|
utcnow.override_time = None
|
||||||
|
|
||||||
|
|
||||||
|
def marshall_now(now=None):
|
||||||
|
"""Make an rpc-safe datetime with microseconds.
|
||||||
|
|
||||||
|
Note: tzinfo is stripped, but not required for relative times.
|
||||||
|
"""
|
||||||
|
if not now:
|
||||||
|
now = utcnow()
|
||||||
|
return dict(day=now.day, month=now.month, year=now.year, hour=now.hour,
|
||||||
|
minute=now.minute, second=now.second,
|
||||||
|
microsecond=now.microsecond)
|
||||||
|
|
||||||
|
|
||||||
|
def unmarshall_time(tyme):
|
||||||
|
"""Unmarshall a datetime dict."""
|
||||||
|
return datetime.datetime(day=tyme['day'],
|
||||||
|
month=tyme['month'],
|
||||||
|
year=tyme['year'],
|
||||||
|
hour=tyme['hour'],
|
||||||
|
minute=tyme['minute'],
|
||||||
|
second=tyme['second'],
|
||||||
|
microsecond=tyme['microsecond'])
|
||||||
|
|
||||||
|
|
||||||
|
def delta_seconds(before, after):
|
||||||
|
"""Return the difference between two timing objects.
|
||||||
|
|
||||||
|
Compute the difference in seconds between two date, time, or
|
||||||
|
datetime objects (as a float, to microsecond resolution).
|
||||||
|
"""
|
||||||
|
delta = after - before
|
||||||
|
return total_seconds(delta)
|
||||||
|
|
||||||
|
|
||||||
|
def total_seconds(delta):
|
||||||
|
"""Return the total seconds of datetime.timedelta object.
|
||||||
|
|
||||||
|
Compute total seconds of datetime.timedelta, datetime.timedelta
|
||||||
|
doesn't have method total_seconds in Python2.6, calculate it manually.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return delta.total_seconds()
|
||||||
|
except AttributeError:
|
||||||
|
return ((delta.days * 24 * 3600) + delta.seconds +
|
||||||
|
float(delta.microseconds) / (10 ** 6))
|
||||||
|
|
||||||
|
|
||||||
|
def is_soon(dt, window):
|
||||||
|
"""Determines if time is going to happen in the next window seconds.
|
||||||
|
|
||||||
|
:param dt: the time
|
||||||
|
:param window: minimum seconds to remain to consider the time not soon
|
||||||
|
|
||||||
|
:return: True if expiration is within the given duration
|
||||||
|
"""
|
||||||
|
soon = (utcnow() + datetime.timedelta(seconds=window))
|
||||||
|
return normalize_time(dt) <= soon
|
|
@ -0,0 +1,87 @@
|
||||||
|
# Copyright 2013 IBM Corp
|
||||||
|
#
|
||||||
|
# Author: Tong Li <litong01@us.ibm.com>
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
import fixtures
|
||||||
|
import testtools
|
||||||
|
|
||||||
|
_TRUE_VALUES = ('True', 'true', '1', 'yes')
|
||||||
|
_LOG_FORMAT = "%(levelname)8s [%(name)s] %(message)s"
|
||||||
|
|
||||||
|
|
||||||
|
class BaseTestCase(testtools.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
super(BaseTestCase, self).setUp()
|
||||||
|
self._set_timeout()
|
||||||
|
self._fake_output()
|
||||||
|
self._fake_logs()
|
||||||
|
self.useFixture(fixtures.NestedTempfile())
|
||||||
|
self.useFixture(fixtures.TempHomeDir())
|
||||||
|
self.tempdirs = []
|
||||||
|
|
||||||
|
def _set_timeout(self):
|
||||||
|
test_timeout = os.environ.get('OS_TEST_TIMEOUT', 0)
|
||||||
|
try:
|
||||||
|
test_timeout = int(test_timeout)
|
||||||
|
except ValueError:
|
||||||
|
# If timeout value is invalid do not set a timeout.
|
||||||
|
test_timeout = 0
|
||||||
|
if test_timeout > 0:
|
||||||
|
self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
|
||||||
|
|
||||||
|
def _fake_output(self):
|
||||||
|
if os.environ.get('OS_STDOUT_CAPTURE') in _TRUE_VALUES:
|
||||||
|
stdout = self.useFixture(fixtures.StringStream('stdout')).stream
|
||||||
|
self.useFixture(fixtures.MonkeyPatch('sys.stdout', stdout))
|
||||||
|
if os.environ.get('OS_STDERR_CAPTURE') in _TRUE_VALUES:
|
||||||
|
stderr = self.useFixture(fixtures.StringStream('stderr')).stream
|
||||||
|
self.useFixture(fixtures.MonkeyPatch('sys.stderr', stderr))
|
||||||
|
|
||||||
|
def _fake_logs(self):
|
||||||
|
if os.environ.get('OS_DEBUG') in _TRUE_VALUES:
|
||||||
|
level = logging.DEBUG
|
||||||
|
else:
|
||||||
|
level = logging.INFO
|
||||||
|
capture_logs = os.environ.get('OS_LOG_CAPTURE') in _TRUE_VALUES
|
||||||
|
if capture_logs:
|
||||||
|
self.useFixture(
|
||||||
|
fixtures.FakeLogger(
|
||||||
|
format=_LOG_FORMAT,
|
||||||
|
level=level,
|
||||||
|
nuke_handlers=capture_logs,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logging.basicConfig(format=_LOG_FORMAT, level=level)
|
||||||
|
|
||||||
|
def create_tempfiles(self, files, ext='.conf'):
|
||||||
|
tempfiles = []
|
||||||
|
for (basename, contents) in files:
|
||||||
|
if not os.path.isabs(basename):
|
||||||
|
(fd, path) = tempfile.mkstemp(prefix=basename, suffix=ext)
|
||||||
|
else:
|
||||||
|
path = basename + ext
|
||||||
|
fd = os.open(path, os.O_CREAT | os.O_WRONLY)
|
||||||
|
tempfiles.append(path)
|
||||||
|
try:
|
||||||
|
os.write(fd, contents)
|
||||||
|
finally:
|
||||||
|
os.close(fd)
|
||||||
|
return tempfiles
|
|
@ -0,0 +1,9 @@
|
||||||
|
falcon>=0.1.9
|
||||||
|
pastedeploy>=1.3.3
|
||||||
|
simplejson>=2.0.9
|
||||||
|
pbr>=0.6,!=0.7,<1.0
|
||||||
|
six>=1.7.0
|
||||||
|
stevedore>=0.14
|
||||||
|
gunicorn>=19.1.0
|
||||||
|
oslo.config>=1.2.1
|
||||||
|
ujson>=1.33
|
|
@ -0,0 +1,45 @@
|
||||||
|
[metadata]
|
||||||
|
name = monasca-api
|
||||||
|
|
||||||
|
# Juno
|
||||||
|
version = 2014.7
|
||||||
|
|
||||||
|
summary = OpenStack Monitoring Service
|
||||||
|
description-file =
|
||||||
|
README.md
|
||||||
|
author = OpenStack
|
||||||
|
author-email = openstack-dev@lists.openstack.org
|
||||||
|
home-page = http://www.openstack.org/
|
||||||
|
classifier =
|
||||||
|
Environment :: OpenStack
|
||||||
|
Intended Audience :: Information Technology
|
||||||
|
Intended Audience :: System Administrators
|
||||||
|
License :: OSI Approved :: Apache Software License
|
||||||
|
Operating System :: POSIX :: Linux
|
||||||
|
Programming Language :: Python
|
||||||
|
Programming Language :: Python :: 2
|
||||||
|
Programming Language :: Python :: 2.7
|
||||||
|
|
||||||
|
[files]
|
||||||
|
packages =
|
||||||
|
monasca
|
||||||
|
|
||||||
|
data_files =
|
||||||
|
/etc/monasca =
|
||||||
|
etc/monasca.conf
|
||||||
|
etc/monasca.ini
|
||||||
|
|
||||||
|
[entry_points]
|
||||||
|
console_scripts =
|
||||||
|
monasca-api = monasca.api.server:run
|
||||||
|
|
||||||
|
monasca.dispatcher =
|
||||||
|
kafka = monasca.dispatcher.kafka_dispatcher:KafkaDispatcher
|
||||||
|
|
||||||
|
paste.filter_factory =
|
||||||
|
login = monasca.middleware.login:filter_factory
|
||||||
|
inspector = monasca.middleware.inspector:filter_factory
|
||||||
|
metric_validator = monasca.middleware.metric_validator:filter_factory
|
||||||
|
|
||||||
|
[pbr]
|
||||||
|
warnerrors = True
|
|
@ -0,0 +1,23 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
# Copyright (c) 2013 Hewlett-Packard Development Company, L.P.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||||
|
# implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
|
||||||
|
# THIS FILE IS MANAGED BY THE GLOBAL REQUIREMENTS REPO - DO NOT EDIT
|
||||||
|
from setuptools import setup
|
||||||
|
|
||||||
|
setup(
|
||||||
|
setup_requires=['pbr'],
|
||||||
|
pbr=True,
|
||||||
|
)
|
|
@ -0,0 +1,22 @@
|
||||||
|
# Hacking already pins down pep8, pyflakes and flake8
|
||||||
|
hacking>=0.9.2,<0.10
|
||||||
|
Babel>=1.3
|
||||||
|
coverage>=3.6
|
||||||
|
discover
|
||||||
|
fixtures>=0.3.14
|
||||||
|
flake8==2.1.0
|
||||||
|
pep8<=1.5.6
|
||||||
|
httplib2>=0.7.5
|
||||||
|
mock>=1.0
|
||||||
|
mox>=0.5.3
|
||||||
|
# Docs Requirements
|
||||||
|
oslosphinx
|
||||||
|
oslotest
|
||||||
|
python-subunit>=0.0.18
|
||||||
|
sphinx>=1.1.2,!=1.2.0,<1.3
|
||||||
|
sphinxcontrib-docbookrestapi
|
||||||
|
sphinxcontrib-httpdomain
|
||||||
|
sphinxcontrib-pecanwsme>=0.8
|
||||||
|
testrepository>=0.0.18
|
||||||
|
testscenarios>=0.4
|
||||||
|
testtools>=0.9.34
|
|
@ -0,0 +1,42 @@
|
||||||
|
[tox]
|
||||||
|
minversion = 1.6
|
||||||
|
skipsdist = True
|
||||||
|
envlist = py27,py33,pep8
|
||||||
|
|
||||||
|
[testenv]
|
||||||
|
deps = -r{toxinidir}/requirements.txt
|
||||||
|
-r{toxinidir}/test-requirements.txt
|
||||||
|
install_command = pip install -U --allow-external pytidylib --allow-insecure pytidylib --allow-external netifaces --allow-insecure netifaces {opts} {packages}
|
||||||
|
usedevelop = True
|
||||||
|
setenv = VIRTUAL_ENV={envdir}
|
||||||
|
commands = python setup.py testr --slowest --testr-args='--concurrency 1 {posargs}'
|
||||||
|
downloadcache = {toxworkdir}/_download
|
||||||
|
whitelist_externals = bash
|
||||||
|
|
||||||
|
[testenv:cover]
|
||||||
|
setenv = NOSE_WITH_COVERAGE=1
|
||||||
|
commands =
|
||||||
|
python setup.py testr --coverage \
|
||||||
|
--testr-args='^(?!.*test.*coverage).*$'
|
||||||
|
|
||||||
|
[testenv:pep8]
|
||||||
|
commands =
|
||||||
|
flake8 monasca
|
||||||
|
|
||||||
|
[testenv:genconfig]
|
||||||
|
|
||||||
|
[testenv:docs]
|
||||||
|
commands = python setup.py build_sphinx
|
||||||
|
|
||||||
|
[testenv:venv]
|
||||||
|
commands = {posargs}
|
||||||
|
|
||||||
|
[flake8]
|
||||||
|
# H305 imports not grouped correctly
|
||||||
|
ignore = H305
|
||||||
|
builtins = _
|
||||||
|
exclude=.venv,.git,.tox,dist,doc,./monasca/openstack/common,*lib/python*,*egg,tools,build
|
||||||
|
show-source = True
|
||||||
|
|
||||||
|
[hacking]
|
||||||
|
|
Loading…
Reference in New Issue