Removed embeded collections library.
This commit is contained in:
Tim Kuhlman
parent
6eafc3607c
commit
e4bb04ef38
|
|
@ -7,10 +7,7 @@ from util import json, headers
|
|||
|
||||
import time
|
||||
|
||||
try:
|
||||
from collections import defaultdict
|
||||
except ImportError:
|
||||
from compat.defaultdict import defaultdict
|
||||
from collections import defaultdict
|
||||
|
||||
STATS_URL = "/;csv;norefresh"
|
||||
EVENT_TYPE = SOURCE_TYPE_NAME = 'haproxy'
|
||||
|
|
|
|||
|
|
@ -1,12 +1,8 @@
|
|||
import os
|
||||
import time
|
||||
|
||||
try:
|
||||
from collections import defaultdict
|
||||
except ImportError:
|
||||
from compat.defaultdict import defaultdict
|
||||
from collections import defaultdict
|
||||
|
||||
from datetime import datetime
|
||||
from glob import glob
|
||||
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ if sys.version_info < (2, 6):
|
|||
# be too much work to rewrite, so raise an exception here.
|
||||
raise Exception('kafka_consumer check requires at least Python 2.6')
|
||||
|
||||
from compat.defaultdict import defaultdict
|
||||
from collections import defaultdict
|
||||
from checks import AgentCheck
|
||||
try:
|
||||
from kafka.client import KafkaClient
|
||||
|
|
|
|||
|
|
@ -1,9 +1,6 @@
|
|||
import re
|
||||
import urllib2
|
||||
try:
|
||||
from collections import defaultdict
|
||||
except ImportError:
|
||||
from compat.defaultdict import defaultdict
|
||||
from collections import defaultdict
|
||||
|
||||
from checks import AgentCheck
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ import time
|
|||
|
||||
# project
|
||||
import config
|
||||
from compat.defaultdict import defaultdict
|
||||
from collections import defaultdict
|
||||
from util import get_os, yaml, yLoader
|
||||
|
||||
# 3rd party
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ import time
|
|||
# project
|
||||
from checks import Check, UnknownValue
|
||||
from util import get_hostname, Platform
|
||||
from compat import any
|
||||
|
||||
|
||||
# locale-resilient float converter
|
||||
|
|
|
|||
|
|
@ -1,5 +0,0 @@
|
|||
def any(iterable):
|
||||
for element in iterable:
|
||||
if element:
|
||||
return True
|
||||
return False
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
|
||||
class defaultdict(dict):
|
||||
def __init__(self, default_factory=None, *a, **kw):
|
||||
if (default_factory is not None and
|
||||
not hasattr(default_factory, '__call__')):
|
||||
raise TypeError('first argument must be callable')
|
||||
dict.__init__(self, *a, **kw)
|
||||
self.default_factory = default_factory
|
||||
def __getitem__(self, key):
|
||||
try:
|
||||
return dict.__getitem__(self, key)
|
||||
except KeyError:
|
||||
return self.__missing__(key)
|
||||
def __missing__(self, key):
|
||||
if self.default_factory is None:
|
||||
raise KeyError(key)
|
||||
self[key] = value = self.default_factory()
|
||||
return value
|
||||
def __reduce__(self):
|
||||
if self.default_factory is None:
|
||||
args = tuple()
|
||||
else:
|
||||
args = self.default_factory,
|
||||
return type(self), args, None, None, self.iteritems()
|
||||
def copy(self):
|
||||
return self.__copy__()
|
||||
def __copy__(self):
|
||||
return type(self)(self.default_factory, self)
|
||||
def __deepcopy__(self, memo):
|
||||
import copy
|
||||
return type(self)(self.default_factory,
|
||||
copy.deepcopy(self.items()))
|
||||
def __repr__(self):
|
||||
return 'defaultdict(%s, %s)' % (self.default_factory,
|
||||
dict.__repr__(self))
|
||||
|
||||
|
|
@ -1,112 +0,0 @@
|
|||
from operator import itemgetter as _itemgetter
|
||||
from keyword import iskeyword as _iskeyword
|
||||
import sys as _sys
|
||||
|
||||
def namedtuple(typename, field_names, verbose=False, rename=False):
|
||||
"""Returns a new subclass of tuple with named fields.
|
||||
|
||||
>>> Point = namedtuple('Point', 'x y')
|
||||
>>> Point.__doc__ # docstring for the new class
|
||||
'Point(x, y)'
|
||||
>>> p = Point(11, y=22) # instantiate with positional args or keywords
|
||||
>>> p[0] + p[1] # indexable like a plain tuple
|
||||
33
|
||||
>>> x, y = p # unpack like a regular tuple
|
||||
>>> x, y
|
||||
(11, 22)
|
||||
>>> p.x + p.y # fields also accessable by name
|
||||
33
|
||||
>>> d = p._asdict() # convert to a dictionary
|
||||
>>> d['x']
|
||||
11
|
||||
>>> Point(**d) # convert from a dictionary
|
||||
Point(x=11, y=22)
|
||||
>>> p._replace(x=100) # _replace() is like str.replace() but targets named fields
|
||||
Point(x=100, y=22)
|
||||
|
||||
"""
|
||||
|
||||
# Parse and validate the field names. Validation serves two purposes,
|
||||
# generating informative error messages and preventing template injection attacks.
|
||||
if isinstance(field_names, basestring):
|
||||
field_names = field_names.replace(',', ' ').split() # names separated by whitespace and/or commas
|
||||
field_names = tuple(map(str, field_names))
|
||||
if rename:
|
||||
names = list(field_names)
|
||||
seen = set()
|
||||
for i, name in enumerate(names):
|
||||
if (not min(c.isalnum() or c=='_' for c in name) or _iskeyword(name)
|
||||
or not name or name[0].isdigit() or name.startswith('_')
|
||||
or name in seen):
|
||||
names[i] = '_%d' % i
|
||||
seen.add(name)
|
||||
field_names = tuple(names)
|
||||
for name in (typename,) + field_names:
|
||||
if not min(c.isalnum() or c=='_' for c in name):
|
||||
raise ValueError('Type names and field names can only contain alphanumeric characters and underscores: %r' % name)
|
||||
if _iskeyword(name):
|
||||
raise ValueError('Type names and field names cannot be a keyword: %r' % name)
|
||||
if name[0].isdigit():
|
||||
raise ValueError('Type names and field names cannot start with a number: %r' % name)
|
||||
seen_names = set()
|
||||
for name in field_names:
|
||||
if name.startswith('_') and not rename:
|
||||
raise ValueError('Field names cannot start with an underscore: %r' % name)
|
||||
if name in seen_names:
|
||||
raise ValueError('Encountered duplicate field name: %r' % name)
|
||||
seen_names.add(name)
|
||||
|
||||
# Create and fill-in the class template
|
||||
numfields = len(field_names)
|
||||
argtxt = repr(field_names).replace("'", "")[1:-1] # tuple repr without parens or quotes
|
||||
reprtxt = ', '.join('%s=%%r' % name for name in field_names)
|
||||
template = '''class %(typename)s(tuple):
|
||||
'%(typename)s(%(argtxt)s)' \n
|
||||
__slots__ = () \n
|
||||
_fields = %(field_names)r \n
|
||||
def __new__(_cls, %(argtxt)s):
|
||||
return _tuple.__new__(_cls, (%(argtxt)s)) \n
|
||||
@classmethod
|
||||
def _make(cls, iterable, new=tuple.__new__, len=len):
|
||||
'Make a new %(typename)s object from a sequence or iterable'
|
||||
result = new(cls, iterable)
|
||||
if len(result) != %(numfields)d:
|
||||
raise TypeError('Expected %(numfields)d arguments, got %%d' %% len(result))
|
||||
return result \n
|
||||
def __repr__(self):
|
||||
return '%(typename)s(%(reprtxt)s)' %% self \n
|
||||
def _asdict(self):
|
||||
'Return a new dict which maps field names to their values'
|
||||
return dict(zip(self._fields, self)) \n
|
||||
def _replace(_self, **kwds):
|
||||
'Return a new %(typename)s object replacing specified fields with new values'
|
||||
result = _self._make(map(kwds.pop, %(field_names)r, _self))
|
||||
if kwds:
|
||||
raise ValueError('Got unexpected field names: %%r' %% kwds.keys())
|
||||
return result \n
|
||||
def __getnewargs__(self):
|
||||
return tuple(self) \n\n''' % locals()
|
||||
for i, name in enumerate(field_names):
|
||||
template += ' %s = _property(_itemgetter(%d))\n' % (name, i)
|
||||
if verbose:
|
||||
print template
|
||||
|
||||
# Execute the template string in a temporary namespace
|
||||
namespace = dict(_itemgetter=_itemgetter, __name__='namedtuple_%s' % typename,
|
||||
_property=property, _tuple=tuple)
|
||||
try:
|
||||
exec template in namespace
|
||||
except SyntaxError, e:
|
||||
raise SyntaxError(e.message + ':\n' + template)
|
||||
result = namespace[typename]
|
||||
|
||||
# For pickling to work, the __module__ variable needs to be set to the frame
|
||||
# where the named tuple is created. Bypass this step in enviroments where
|
||||
# sys._getframe is not defined (Jython for example) or sys._getframe is not
|
||||
# defined for arguments greater than 0 (IronPython).
|
||||
try:
|
||||
result.__module__ = _sys._getframe(1).f_globals.get('__name__', '__main__')
|
||||
except (AttributeError, ValueError):
|
||||
pass
|
||||
|
||||
return result
|
||||
|
|
@ -1,309 +0,0 @@
|
|||
"""Miscellaneous network utility code."""
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import stat
|
||||
import struct
|
||||
|
||||
from tornado.ioloop import IOLoop
|
||||
from tornado.iostream import IOStream, SSLIOStream
|
||||
|
||||
import fcntl
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
def set_close_exec(fd):
|
||||
flags = fcntl.fcntl(fd, fcntl.F_GETFD)
|
||||
fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC)
|
||||
|
||||
try:
|
||||
import ssl # Python 2.6+
|
||||
except ImportError:
|
||||
ssl = None
|
||||
|
||||
class TCPServer(object):
|
||||
r"""A non-blocking, single-threaded TCP server.
|
||||
|
||||
To use `TCPServer`, define a subclass which overrides the `handle_stream`
|
||||
method.
|
||||
|
||||
`TCPServer` can serve SSL traffic with Python 2.6+ and OpenSSL.
|
||||
To make this server serve SSL traffic, send the ssl_options dictionary
|
||||
argument with the arguments required for the `ssl.wrap_socket` method,
|
||||
including "certfile" and "keyfile"::
|
||||
|
||||
TCPServer(ssl_options={
|
||||
"certfile": os.path.join(data_dir, "mydomain.crt"),
|
||||
"keyfile": os.path.join(data_dir, "mydomain.key"),
|
||||
})
|
||||
|
||||
`TCPServer` initialization follows one of three patterns:
|
||||
|
||||
1. `listen`: simple single-process::
|
||||
|
||||
server = TCPServer()
|
||||
server.listen(8888)
|
||||
IOLoop.instance().start()
|
||||
|
||||
2. `bind`/`start`: simple multi-process::
|
||||
|
||||
server = TCPServer()
|
||||
server.bind(8888)
|
||||
server.start(0) # Forks multiple sub-processes
|
||||
IOLoop.instance().start()
|
||||
|
||||
When using this interface, an `IOLoop` must *not* be passed
|
||||
to the `TCPServer` constructor. `start` will always start
|
||||
the server on the default singleton `IOLoop`.
|
||||
|
||||
3. `add_sockets`: advanced multi-process::
|
||||
|
||||
sockets = bind_sockets(8888)
|
||||
tornado.process.fork_processes(0)
|
||||
server = TCPServer()
|
||||
server.add_sockets(sockets)
|
||||
IOLoop.instance().start()
|
||||
|
||||
The `add_sockets` interface is more complicated, but it can be
|
||||
used with `tornado.process.fork_processes` to give you more
|
||||
flexibility in when the fork happens. `add_sockets` can
|
||||
also be used in single-process servers if you want to create
|
||||
your listening sockets in some way other than
|
||||
`bind_sockets`.
|
||||
"""
|
||||
def __init__(self, io_loop=None, ssl_options=None):
|
||||
self.io_loop = io_loop
|
||||
self.ssl_options = ssl_options
|
||||
self._sockets = {} # fd -> socket object
|
||||
self._pending_sockets = []
|
||||
self._started = False
|
||||
|
||||
def listen(self, port, address=""):
|
||||
"""Starts accepting connections on the given port.
|
||||
|
||||
This method may be called more than once to listen on multiple ports.
|
||||
`listen` takes effect immediately; it is not necessary to call
|
||||
`TCPServer.start` afterwards. It is, however, necessary to start
|
||||
the `IOLoop`.
|
||||
"""
|
||||
sockets = bind_sockets(port, address=address)
|
||||
self.add_sockets(sockets)
|
||||
|
||||
def add_sockets(self, sockets):
|
||||
"""Makes this server start accepting connections on the given sockets.
|
||||
|
||||
The ``sockets`` parameter is a list of socket objects such as
|
||||
those returned by `bind_sockets`.
|
||||
`add_sockets` is typically used in combination with that
|
||||
method and `tornado.process.fork_processes` to provide greater
|
||||
control over the initialization of a multi-process server.
|
||||
"""
|
||||
if self.io_loop is None:
|
||||
self.io_loop = IOLoop.instance()
|
||||
|
||||
for sock in sockets:
|
||||
self._sockets[sock.fileno()] = sock
|
||||
add_accept_handler(sock, self._handle_connection,
|
||||
io_loop=self.io_loop)
|
||||
|
||||
def add_socket(self, socket):
|
||||
"""Singular version of `add_sockets`. Takes a single socket object."""
|
||||
self.add_sockets([socket])
|
||||
|
||||
def bind(self, port, address=None, family=socket.AF_UNSPEC, backlog=128):
|
||||
"""Binds this server to the given port on the given address.
|
||||
|
||||
To start the server, call `start`. If you want to run this server
|
||||
in a single process, you can call `listen` as a shortcut to the
|
||||
sequence of `bind` and `start` calls.
|
||||
|
||||
Address may be either an IP address or hostname. If it's a hostname,
|
||||
the server will listen on all IP addresses associated with the
|
||||
name. Address may be an empty string or None to listen on all
|
||||
available interfaces. Family may be set to either ``socket.AF_INET``
|
||||
or ``socket.AF_INET6`` to restrict to ipv4 or ipv6 addresses, otherwise
|
||||
both will be used if available.
|
||||
|
||||
The ``backlog`` argument has the same meaning as for
|
||||
`socket.listen`.
|
||||
|
||||
This method may be called multiple times prior to `start` to listen
|
||||
on multiple ports or interfaces.
|
||||
"""
|
||||
sockets = bind_sockets(port, address=address, family=family,
|
||||
backlog=backlog)
|
||||
if self._started:
|
||||
self.add_sockets(sockets)
|
||||
else:
|
||||
self._pending_sockets.extend(sockets)
|
||||
|
||||
def start(self, num_processes=1):
|
||||
"""Starts this server in the IOLoop.
|
||||
|
||||
By default, we run the server in this process and do not fork any
|
||||
additional child process.
|
||||
|
||||
If num_processes is ``None`` or <= 0, we detect the number of cores
|
||||
available on this machine and fork that number of child
|
||||
processes. If num_processes is given and > 1, we fork that
|
||||
specific number of sub-processes.
|
||||
|
||||
Since we use processes and not threads, there is no shared memory
|
||||
between any server code.
|
||||
|
||||
Note that multiple processes are not compatible with the autoreload
|
||||
module (or the ``debug=True`` option to `tornado.web.Application`).
|
||||
When using multiple processes, no IOLoops can be created or
|
||||
referenced until after the call to ``TCPServer.start(n)``.
|
||||
"""
|
||||
assert not self._started
|
||||
self._started = True
|
||||
sockets = self._pending_sockets
|
||||
self._pending_sockets = []
|
||||
self.add_sockets(sockets)
|
||||
|
||||
def stop(self):
|
||||
"""Stops listening for new connections.
|
||||
|
||||
Requests currently in progress may still continue after the
|
||||
server is stopped.
|
||||
"""
|
||||
for fd, sock in self._sockets.iteritems():
|
||||
self.io_loop.remove_handler(fd)
|
||||
sock.close()
|
||||
|
||||
def handle_stream(self, stream, address):
|
||||
"""Override to handle a new `IOStream` from an incoming connection."""
|
||||
raise NotImplementedError()
|
||||
|
||||
def _handle_connection(self, connection, address):
|
||||
if self.ssl_options is not None:
|
||||
assert ssl, "Python 2.6+ and OpenSSL required for SSL"
|
||||
try:
|
||||
connection = ssl.wrap_socket(connection,
|
||||
server_side=True,
|
||||
do_handshake_on_connect=False,
|
||||
**self.ssl_options)
|
||||
except ssl.SSLError, err:
|
||||
if err.args[0] == ssl.SSL_ERROR_EOF:
|
||||
return connection.close()
|
||||
else:
|
||||
raise
|
||||
except socket.error, err:
|
||||
if err.args[0] == errno.ECONNABORTED:
|
||||
return connection.close()
|
||||
else:
|
||||
raise
|
||||
try:
|
||||
if self.ssl_options is not None:
|
||||
stream = SSLIOStream(connection, io_loop=self.io_loop)
|
||||
else:
|
||||
stream = IOStream(connection, io_loop=self.io_loop)
|
||||
self.handle_stream(stream, address)
|
||||
except Exception:
|
||||
log.error("Error in connection callback", exc_info=True)
|
||||
|
||||
|
||||
def bind_sockets(port, address=None, family=socket.AF_UNSPEC, backlog=128):
|
||||
"""Creates listening sockets bound to the given port and address.
|
||||
|
||||
Returns a list of socket objects (multiple sockets are returned if
|
||||
the given address maps to multiple IP addresses, which is most common
|
||||
for mixed IPv4 and IPv6 use).
|
||||
|
||||
Address may be either an IP address or hostname. If it's a hostname,
|
||||
the server will listen on all IP addresses associated with the
|
||||
name. Address may be an empty string or None to listen on all
|
||||
available interfaces. Family may be set to either socket.AF_INET
|
||||
or socket.AF_INET6 to restrict to ipv4 or ipv6 addresses, otherwise
|
||||
both will be used if available.
|
||||
|
||||
The ``backlog`` argument has the same meaning as for
|
||||
``socket.listen()``.
|
||||
"""
|
||||
sockets = []
|
||||
if address == "":
|
||||
address = None
|
||||
flags = socket.AI_PASSIVE
|
||||
if hasattr(socket, "AI_ADDRCONFIG"):
|
||||
# AI_ADDRCONFIG ensures that we only try to bind on ipv6
|
||||
# if the system is configured for it, but the flag doesn't
|
||||
# exist on some platforms (specifically WinXP, although
|
||||
# newer versions of windows have it)
|
||||
flags |= socket.AI_ADDRCONFIG
|
||||
for res in socket.getaddrinfo(address, port, family, socket.SOCK_STREAM,
|
||||
0, flags):
|
||||
af, socktype, proto, canonname, sockaddr = res
|
||||
sock = socket.socket(af, socktype, proto)
|
||||
set_close_exec(sock.fileno())
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
if af == socket.AF_INET6:
|
||||
# On linux, ipv6 sockets accept ipv4 too by default,
|
||||
# but this makes it impossible to bind to both
|
||||
# 0.0.0.0 in ipv4 and :: in ipv6. On other systems,
|
||||
# separate sockets *must* be used to listen for both ipv4
|
||||
# and ipv6. For consistency, always disable ipv4 on our
|
||||
# ipv6 sockets and use a separate ipv4 socket when needed.
|
||||
#
|
||||
# Python 2.x on windows doesn't have IPPROTO_IPV6.
|
||||
if hasattr(socket, "IPPROTO_IPV6"):
|
||||
sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 1)
|
||||
sock.setblocking(0)
|
||||
sock.bind(sockaddr)
|
||||
sock.listen(backlog)
|
||||
sockets.append(sock)
|
||||
return sockets
|
||||
|
||||
if hasattr(socket, 'AF_UNIX'):
|
||||
def bind_unix_socket(file, mode=0600, backlog=128):
|
||||
"""Creates a listening unix socket.
|
||||
|
||||
If a socket with the given name already exists, it will be deleted.
|
||||
If any other file with that name exists, an exception will be
|
||||
raised.
|
||||
|
||||
Returns a socket object (not a list of socket objects like
|
||||
`bind_sockets`)
|
||||
"""
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
set_close_exec(sock.fileno())
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
sock.setblocking(0)
|
||||
try:
|
||||
st = os.stat(file)
|
||||
except OSError, err:
|
||||
if err.errno != errno.ENOENT:
|
||||
raise
|
||||
else:
|
||||
if stat.S_ISSOCK(st.st_mode):
|
||||
os.remove(file)
|
||||
else:
|
||||
raise ValueError("File %s exists and is not a socket", file)
|
||||
sock.bind(file)
|
||||
os.chmod(file, mode)
|
||||
sock.listen(backlog)
|
||||
return sock
|
||||
|
||||
def add_accept_handler(sock, callback, io_loop=None):
|
||||
"""Adds an ``IOLoop`` event handler to accept new connections on ``sock``.
|
||||
|
||||
When a connection is accepted, ``callback(connection, address)`` will
|
||||
be run (``connection`` is a socket object, and ``address`` is the
|
||||
address of the other end of the connection). Note that this signature
|
||||
is different from the ``callback(fd, events)`` signature used for
|
||||
``IOLoop`` handlers.
|
||||
"""
|
||||
if io_loop is None:
|
||||
io_loop = IOLoop.instance()
|
||||
def accept_handler(fd, events):
|
||||
while True:
|
||||
try:
|
||||
connection, address = sock.accept()
|
||||
except socket.error, e:
|
||||
if e.args[0] in (errno.EWOULDBLOCK, errno.EAGAIN):
|
||||
return
|
||||
raise
|
||||
callback(connection, address)
|
||||
io_loop.add_handler(sock.fileno(), accept_handler, IOLoop.READ)
|
||||
|
|
@ -39,10 +39,7 @@ try:
|
|||
except ImportError:
|
||||
from yaml import Loader as yLoader
|
||||
|
||||
try:
|
||||
from collections import namedtuple
|
||||
except ImportError:
|
||||
from compat.namedtuple import namedtuple
|
||||
from collections import namedtuple
|
||||
|
||||
import logging
|
||||
log = logging.getLogger(__name__)
|
||||
|
|
|
|||
|
|
@ -37,9 +37,7 @@ source:
|
|||
cp -r $(SRC)/monagent/checks $(ROOT)/
|
||||
cp -r $(SRC)/monagent/dogstream $(ROOT)/
|
||||
cp -r $(SRC)/monagent/pup $(ROOT)/
|
||||
cp -r $(SRC)/monagent/yaml $(ROOT)/
|
||||
cp -r $(SRC)/monagent/checks.d $(ROOT)/
|
||||
cp -r $(SRC)/monagent/compat $(ROOT)/
|
||||
cp -r $(SRC)/conf.d $(ROOT)/
|
||||
cp -r $(SRC)/monagent/resources $(ROOT)/
|
||||
cp -r $(SRC)/monagent/mon_lib $(ROOT)/
|
||||
|
|
|
|||
Loading…
Reference in New Issue