From 5acc00c179f109eacb174856e58f75dba1ce73cc Mon Sep 17 00:00:00 2001 From: ZhiQiang Fan Date: Thu, 26 Nov 2015 04:45:04 +0800 Subject: [PATCH] re-implement thread safe fnmatch fnmatch is not thread safe for versions <= 2.7.9. We have used it in some places without any lock for concurrency scenario. This patch re-implements a thread safe match() which is very similar to fnmatch by using its translate function. Change-Id: I4b6c2ea72841201519144eb09ecf8c82b16b6143 ref: https://hg.python.org/cpython/rev/fe12c34c39eb Closes-Bug: #1519767 --- ceilometer/agent/manager.py | 3 +-- ceilometer/dispatcher/gnocchi.py | 4 ++-- ceilometer/event/converter.py | 7 +++---- ceilometer/meter/notifications.py | 4 ++-- ceilometer/pipeline.py | 6 +++--- ceilometer/utils.py | 29 +++++++++++++++++++++++++++++ 6 files changed, 40 insertions(+), 13 deletions(-) diff --git a/ceilometer/agent/manager.py b/ceilometer/agent/manager.py index 0fc153589b..57a90a5955 100644 --- a/ceilometer/agent/manager.py +++ b/ceilometer/agent/manager.py @@ -19,7 +19,6 @@ # under the License. import collections -import fnmatch import itertools import random @@ -241,7 +240,7 @@ class AgentManager(service_base.BaseService): def _match(pollster): """Find out if pollster name matches to one of the list.""" - return any(fnmatch.fnmatch(pollster.name, pattern) for + return any(utils.match(pollster.name, pattern) for pattern in pollster_list) if type(namespaces) is not list: diff --git a/ceilometer/dispatcher/gnocchi.py b/ceilometer/dispatcher/gnocchi.py index 1fd73188f0..26652079d4 100644 --- a/ceilometer/dispatcher/gnocchi.py +++ b/ceilometer/dispatcher/gnocchi.py @@ -13,7 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. import copy -import fnmatch import itertools import operator import os @@ -30,6 +29,7 @@ from ceilometer import dispatcher from ceilometer.dispatcher import gnocchi_client from ceilometer.i18n import _, _LE, _LW from ceilometer import keystone_client +from ceilometer import utils CACHE_NAMESPACE = uuid.uuid4() LOG = log.getLogger(__name__) @@ -110,7 +110,7 @@ class ResourcesDefinition(object): def match(self, metric_name): for t in self.cfg['metrics']: - if fnmatch.fnmatch(metric_name, t): + if utils.match(metric_name, t): return True return False diff --git a/ceilometer/event/converter.py b/ceilometer/event/converter.py index 165632d510..8a66f35aee 100644 --- a/ceilometer/event/converter.py +++ b/ceilometer/event/converter.py @@ -13,8 +13,6 @@ # License for the specific language governing permissions and limitations # under the License. -import fnmatch - from debtcollector import moves from oslo_config import cfg from oslo_log import log @@ -24,6 +22,7 @@ import six from ceilometer import declarative from ceilometer.event.storage import models from ceilometer.i18n import _ +from ceilometer import utils OPTS = [ cfg.StrOpt('definitions_cfg_file', @@ -131,13 +130,13 @@ class EventDefinition(object): def included_type(self, event_type): for t in self._included_types: - if fnmatch.fnmatch(event_type, t): + if utils.match(event_type, t): return True return False def excluded_type(self, event_type): for t in self._excluded_types: - if fnmatch.fnmatch(event_type, t): + if utils.match(event_type, t): return True return False diff --git a/ceilometer/meter/notifications.py b/ceilometer/meter/notifications.py index 083686833d..b593f43884 100644 --- a/ceilometer/meter/notifications.py +++ b/ceilometer/meter/notifications.py @@ -11,7 +11,6 @@ # License for the specific language governing permissions and limitations # under the License. -import fnmatch import itertools import pkg_resources import six @@ -26,6 +25,7 @@ from ceilometer.agent import plugin_base from ceilometer import declarative from ceilometer.i18n import _LE from ceilometer import sample +from ceilometer import utils OPTS = [ cfg.StrOpt('meter_definitions_cfg_file', @@ -97,7 +97,7 @@ class MeterDefinition(object): def match_type(self, meter_name): for t in self._event_type: - if fnmatch.fnmatch(meter_name, t): + if utils.match(meter_name, t): return True def to_samples(self, message, all_values=False): diff --git a/ceilometer/pipeline.py b/ceilometer/pipeline.py index c07299c403..cf69d986e5 100644 --- a/ceilometer/pipeline.py +++ b/ceilometer/pipeline.py @@ -18,7 +18,6 @@ # under the License. import abc -import fnmatch import hashlib import os @@ -36,6 +35,7 @@ from ceilometer.i18n import _, _LI, _LW from ceilometer import publisher from ceilometer.publisher import utils as publisher_utils from ceilometer import sample as sample_util +from ceilometer import utils OPTS = [ @@ -272,11 +272,11 @@ class Source(object): def is_supported(dataset, data_name): # Support wildcard like storage.* and !disk.* # Start with negation, we consider that the order is deny, allow - if any(fnmatch.fnmatch(data_name, datapoint[1:]) + if any(utils.match(data_name, datapoint[1:]) for datapoint in dataset if datapoint[0] == '!'): return False - if any(fnmatch.fnmatch(data_name, datapoint) + if any(utils.match(data_name, datapoint) for datapoint in dataset if datapoint[0] != '!'): return True diff --git a/ceilometer/utils.py b/ceilometer/utils.py index 77a5b94ec2..7c01798782 100644 --- a/ceilometer/utils.py +++ b/ceilometer/utils.py @@ -23,8 +23,11 @@ import calendar import copy import datetime import decimal +import fnmatch import hashlib +import re import struct +import sys from oslo_concurrency import processutils from oslo_config import cfg @@ -254,3 +257,29 @@ def kill_listeners(listeners): for listener in listeners: listener.stop() listener.wait() + + +if sys.version_info > (2, 7, 9): + match = fnmatch.fnmatch +else: + _MATCH_CACHE = {} + _MATCH_CACHE_MAX = 100 + + def match(string, pattern): + """Thread safe fnmatch re-implementation. + + Standard library fnmatch in Python versions <= 2.7.9 has thread safe + issue, this helper function is created for such case. see: + https://bugs.python.org/issue23191 + """ + string = string.lower() + pattern = pattern.lower() + + cached_pattern = _MATCH_CACHE.get(pattern) + if cached_pattern is None: + translated_pattern = fnmatch.translate(pattern) + cached_pattern = re.compile(translated_pattern) + if len(_MATCH_CACHE) >= _MATCH_CACHE_MAX: + _MATCH_CACHE.clear() + _MATCH_CACHE[pattern] = cached_pattern + return cached_pattern.match(string) is not None