Fixes for running/building

This commit is contained in:
Tim Kuhlman 2014-05-06 17:01:46 -06:00
parent a516ce6c7b
commit 3f92626c2e
7 changed files with 86 additions and 83 deletions

View File

@ -56,9 +56,9 @@ class Collector(object):
w32.Cpu(log)]
else:
system_checks = [u.Disk(log, agent_config),
u.IO(log, agent_config),
u.IO(log),
u.Load(log, agent_config),
u.Memory(log, agent_config),
u.Memory(log),
u.Cpu(log, agent_config)]
self._checks.extend(system_checks)
@ -135,7 +135,7 @@ class Collector(object):
for check_type in self._checks:
try:
for name, value in check_type.check():
for name, value in check_type.check().iteritems():
metrics_list.append(Measurement(name, timestamp, value, {}))
except Exception:
log.exception('Error running check.')
@ -149,7 +149,8 @@ class Collector(object):
collect_duration = timer.step()
# Add in metrics on the collector run, emit_duration is from the previous run
for name, value in self.collector_stats(len(metrics_list), len(events), collect_duration, self.emit_duration):
for name, value in self.collector_stats(len(metrics_list), len(events),
collect_duration, self.emit_duration).iteritems():
metrics_list.append(Measurement(name, timestamp, value, {}))
emitter_statuses = self._emit(metrics_list)

View File

@ -26,8 +26,12 @@ class Disk(Check):
def check(self):
"""Get disk space/inode stats"""
# First get the configuration.
use_mount = self.agent_config.get("use_mount", False)
blacklist_re = self.agent_config.get('device_blacklist_re', None)
if self.agent_config is not None:
use_mount = self.agent_config.get("use_mount", False)
blacklist_re = self.agent_config.get('device_blacklist_re', None)
else:
use_mount = False
blacklist_re = None
platform_name = sys.platform
try:
@ -52,7 +56,7 @@ class Disk(Check):
except Exception:
self.logger.exception('Error collecting disk stats')
return False
return {}
def parse_df_output(self, df_output, platform_name, inodes=False, use_mount=False, blacklist_re=None):
"""
@ -355,10 +359,13 @@ class IO(Check):
# 6.67 3 0.02 0.00 0 0.00 <-- line of interest
io = self._parse_darwin(iostat)
else:
return False
return {}
# If we filter devices, do it know.
device_blacklist_re = self.agent_config.get('device_blacklist_re', None)
if self.agent_config is not None:
device_blacklist_re = self.agent_config.get('device_blacklist_re', None)
else:
device_blacklist_re = None
if device_blacklist_re:
filtered_io = {}
for device, stats in io.iteritems():
@ -368,7 +375,7 @@ class IO(Check):
filtered_io = io
filtered = {}
for dev_name, stats in filtered_io:
for dev_name, stats in filtered_io.iteritems():
filtered_stats = {stat: stats[stat] for stat in stats.iterkeys() if stat not in self.stat_blacklist}
filtered[dev_name] = filtered_stats
@ -376,7 +383,7 @@ class IO(Check):
except Exception:
self.logger.exception("Cannot extract IO statistics")
return False
return {}
class Load(Check):
@ -389,7 +396,7 @@ class Load(Check):
loadAvrgProc.close()
except Exception:
self.logger.exception('Cannot extract load')
return False
return {}
uptime = uptime[0] # readlines() provides a list but we want a string
@ -401,7 +408,7 @@ class Load(Check):
close_fds=True).communicate()[0]
except Exception:
self.logger.exception('Cannot extract load')
return False
return {}
# Split out the 3 load average values
load = [res.replace(',', '.') for res in re.findall(r'([0-9]+[\.,]\d+)', uptime)]
@ -443,7 +450,7 @@ class Memory(Check):
meminfoProc.close()
except Exception:
self.logger.exception('Cannot get memory metrics from /proc/meminfo')
return False
return {}
# $ cat /proc/meminfo
# MemTotal: 7995360 kB
@ -543,7 +550,7 @@ class Memory(Check):
sysctl = sp.Popen(['sysctl', 'vm.swapusage'], stdout=sp.PIPE, close_fds=True).communicate()[0]
except StandardError:
self.logger.exception('getMemoryUsage')
return False
return {}
# Deal with top
lines = top.split('\n')
@ -569,7 +576,7 @@ class Memory(Check):
sysctl = sp.Popen(['sysctl', 'vm.stats.vm'], stdout=sp.PIPE, close_fds=True).communicate()[0]
except Exception:
self.logger.exception('getMemoryUsage')
return False
return {}
lines = sysctl.split('\n')
@ -625,7 +632,7 @@ class Memory(Check):
sysctl = sp.Popen(['swapinfo', '-m'], stdout=sp.PIPE, close_fds=True).communicate()[0]
except Exception:
self.logger.exception('getMemoryUsage')
return False
return {}
lines = sysctl.split('\n')
@ -692,9 +699,9 @@ class Memory(Check):
return memData
except Exception:
self.logger.exception("Cannot compute mem stats from kstat -c zone_memory_cap")
return False
return {}
else:
return False
return {}
class Cpu(Check):
@ -780,7 +787,7 @@ class Cpu(Check):
cpu_idle,
cpu_stolen)
else:
return False
return {}
elif sys.platform == 'darwin':
# generate 3 seconds of data
@ -800,7 +807,7 @@ class Cpu(Check):
else:
self.logger.warn("Expected to get at least 4 lines of data from iostat instead of just " +
str(iostats[: max(80, len(iostats))]))
return False
return {}
elif sys.platform.startswith("freebsd"):
# generate 3 seconds of data
@ -826,7 +833,7 @@ class Cpu(Check):
else:
self.logger.warn("Expected to get at least 4 lines of data from iostat instead of just " +
str(iostats[:max(80, len(iostats))]))
return False
return {}
elif sys.platform == 'sunos5':
# mpstat -aq 1 2
@ -866,10 +873,10 @@ class Cpu(Check):
0.0)
except Exception:
self.logger.exception("Cannot compute CPU stats")
return False
return {}
else:
self.logger.warn("CPUStats: unsupported platform")
return False
return {}
def _get_subprocess_output(command):

View File

@ -4,9 +4,7 @@ import itertools
import logging
import logging.config
import logging.handlers
import platform
import string
import subprocess
import sys
import glob
import inspect
@ -188,7 +186,7 @@ def get_config(parse_args=True, cfg_path=None, options=None):
options, _ = get_parsed_args()
# General config
agentConfig = {
agent_config = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY,
'dogstatsd_agregator_bucket_size': DEFAULT_STATSD_BUCKET_SIZE,
@ -217,7 +215,7 @@ def get_config(parse_args=True, cfg_path=None, options=None):
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
agent_config[option] = config.get('Main', option)
#
# Core config
@ -228,33 +226,33 @@ def get_config(parse_args=True, cfg_path=None, options=None):
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get('Main', 'additional_checksd')
agent_config['additional_checksd'] = config.get('Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d')
agent_config['additional_checksd'] = os.path.join(common_path, 'Datadog', 'checks.d')
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get('Main', 'use_web_info_page').lower() in ("yes", "true")
agent_config['use_web_info_page'] = config.get('Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
agent_config['use_web_info_page'] = True
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
agent_config['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get('Main', 'non_local_traffic').lower() in ("yes", "true")
agent_config['non_local_traffic'] = config.get('Main', 'non_local_traffic').lower() in ("yes", "true")
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(config.get('Main', 'check_freq'))
agent_config['check_freq'] = int(config.get('Main', 'check_freq'))
except Exception:
pass
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
agent_config['watchdog'] = False
# Dogstatsd config
dogstatsd_defaults = {
@ -265,15 +263,15 @@ def get_config(parse_args=True, cfg_path=None, options=None):
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
agent_config[key] = config.get('Main', key)
else:
agentConfig[key] = value
agent_config[key] = value
#Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get('Main', 'statsd_forward_host')
agent_config['statsd_forward_host'] = config.get('Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port'))
agent_config['statsd_forward_port'] = int(config.get('Main', 'statsd_forward_port'))
# normalize 'yes'/'no' to boolean
dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative(dogstatsd_defaults['dogstatsd_normalize'])
@ -281,50 +279,50 @@ def get_config(parse_args=True, cfg_path=None, options=None):
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(config.get('Main', 'use_mount'))
agent_config['use_mount'] = _is_affirmative(config.get('Main', 'use_mount'))
if config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(config.get('Main', 'autorestart'))
agent_config['autorestart'] = _is_affirmative(config.get('Main', 'autorestart'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
agent_config['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
agent_config['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")])
agent_config["dogstreams"] = ':'.join([log_path, config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
agent_config["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
agent_config["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
agent_config["nagios_perf_cfg"] = config.get("Main", "nagios_perf_cfg")
if config.has_section('WMI'):
agentConfig['WMI'] = {}
agent_config['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
agent_config['WMI'][key] = value
if config.has_option("Main", "limit_memory_consumption") and \
config.get("Main", "limit_memory_consumption") is not None:
agentConfig["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption"))
agent_config["limit_memory_consumption"] = int(config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
agent_config["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation"))
agent_config["skip_ssl_validation"] = _is_affirmative(config.get("Main", "skip_ssl_validation"))
agentConfig['Api'] = get_mon_api_config(config)
agent_config['Api'] = get_mon_api_config(config)
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
@ -338,9 +336,9 @@ def get_config(parse_args=True, cfg_path=None, options=None):
sys.stderr.write('There are some items missing from your config file, but nothing fatal [%s]' % e)
# Storing proxy settings in the agent_config
agentConfig['proxy_settings'] = get_proxy(agentConfig)
agent_config['proxy_settings'] = get_proxy(agent_config)
return agentConfig
return agent_config
def set_win32_cert_path():
@ -364,21 +362,21 @@ def set_win32_cert_path():
log.info("Windows certificate path: %s" % crt_path)
tornado.simple_httpclient._DEFAULT_CA_CERTS = crt_path
def get_proxy(agentConfig, use_system_settings=False):
def get_proxy(agent_config, use_system_settings=False):
proxy_settings = {}
# First we read the proxy configuration from datadog.conf
proxy_host = agentConfig.get('proxy_host', None)
proxy_host = agent_config.get('proxy_host', None)
if proxy_host is not None and not use_system_settings:
proxy_settings['host'] = proxy_host
try:
proxy_settings['port'] = int(agentConfig.get('proxy_port', 3128))
proxy_settings['port'] = int(agent_config.get('proxy_port', 3128))
except ValueError:
log.error('Proxy port must be an Integer. Defaulting it to 3128')
proxy_settings['port'] = 3128
proxy_settings['user'] = agentConfig.get('proxy_user', None)
proxy_settings['password'] = agentConfig.get('proxy_password', None)
proxy_settings['user'] = agent_config.get('proxy_user', None)
proxy_settings['password'] = agent_config.get('proxy_password', None)
proxy_settings['system_settings'] = False
log.debug("Proxy Settings: %s:%s@%s:%s" % (proxy_settings['user'], "*****", proxy_settings['host'], proxy_settings['port']))
return proxy_settings
@ -494,7 +492,7 @@ def check_yaml(conf_path):
finally:
f.close()
def load_check_directory(agentConfig):
def load_check_directory(agent_config):
''' Return the initialized checks from checks.d, and a mapping of checks that failed to
initialize. Only checks that have a configuration
file in conf.d will be returned. '''
@ -504,7 +502,7 @@ def load_check_directory(agentConfig):
init_failed_checks = {}
osname = get_os()
checks_paths = [glob.glob(os.path.join(agentConfig['additional_checksd'], '*.py'))]
checks_paths = [glob.glob(os.path.join(agent_config['additional_checksd'], '*.py'))]
try:
checksd_path = get_checksd_path(osname)
@ -520,7 +518,7 @@ def load_check_directory(agentConfig):
sys.exit(3)
# Start JMXFetch if needed
JMXFetch.init(confd_path, agentConfig, get_logging_config(), DEFAULT_CHECK_FREQUENCY, JMX_COLLECT_COMMAND)
JMXFetch.init(confd_path, agent_config, get_logging_config(), DEFAULT_CHECK_FREQUENCY, JMX_COLLECT_COMMAND)
# For backwards-compatability with old style checks, we have to load every
# checks.d module and check for a corresponding config OR check if the old
@ -578,7 +576,7 @@ def load_check_directory(agentConfig):
elif hasattr(check_class, 'parse_agent_config'):
# FIXME: Remove this check once all old-style checks are gone
try:
check_config = check_class.parse_agent_config(agentConfig)
check_config = check_class.parse_agent_config(agent_config)
except Exception, e:
continue
if not check_config:
@ -610,12 +608,12 @@ def load_check_directory(agentConfig):
try:
try:
c = check_class(check_name, init_config=init_config,
agentConfig=agentConfig, instances=instances)
agent_config=agent_config, instances=instances)
except TypeError, e:
# Backwards compatibility for checks which don't support the
# instances argument in the constructor.
c = check_class(check_name, init_config=init_config,
agentConfig=agentConfig)
agent_config=agent_config)
c.instances = instances
except Exception, e:
log.exception('Unable to initialize check %s' % check_name)

View File

@ -21,15 +21,16 @@ def http_emitter(message, log, agentConfig):
log.debug('http_emitter: attempting postback to ' + agentConfig['forwarder_url'])
# Post back the data
payload = []
partial_payload = []
for measurement in message:
if isinstance(Measurement, measurement):
if isinstance(measurement, Measurement):
# Measurements need their __dict__ encoded to avoid being expressed as a tuple
payload.append(json.dumps(measurement.__dict__))
partial_payload.append(measurement.__dict__)
else:
# Not all measurements are of the Measurement type yet
payload.append(json.dumps(measurement))
partial_payload.append(measurement)
payload = json.dumps(partial_payload)
url = "%s/intake" % agentConfig['forwarder_url']
headers = post_headers(agentConfig, payload)

View File

@ -137,6 +137,7 @@ class AgentInputHandler(tornado.web.RequestHandler):
# read the message it should be a list of Measurements - from monagent.common.metrics import Measurement
msg = tornado.escape.json_decode(self.request.body)
try:
log.debug(msg)
measurements = [Measurement(**m) for m in msg]
except Exception:
log.exception('Error parsing body of Agent Input')

View File

@ -10,7 +10,7 @@ SRC=../
ROOT=root
BUILD_NUMBER ?= 0
VERSION=`PYTHONPATH=$(SRC)/monagent python -c "from config import get_version; print get_version()"`
VERSION=`PYTHONPATH=$(SRC) python -c "from monagent.common.config import get_version; print get_version()"`
FPM_BUILD=fpm --epoch 1 -s dir -e -C $(BUILD) \
-a all -m "HP Cloud Monitoring <hpcs-mon@hp.com>" \
@ -34,13 +34,8 @@ clean:
# root directory.
source:
mkdir -p $(ROOT)
cp -r $(SRC)/monagent/checks $(ROOT)/
cp -r $(SRC)/monagent/dogstream $(ROOT)/
cp -r $(SRC)/monagent/checks.d $(ROOT)/
cp -r $(SRC)/monagent $(ROOT)/
cp -r $(SRC)/conf.d $(ROOT)/
cp -r $(SRC)/monagent/resources $(ROOT)/
cp -r $(SRC)/monagent/api $(ROOT)/
cp -r $(SRC)/monagent/*.py $(ROOT)/
cp -r $(SRC)/LICENSE* $(ROOT)/
cp -r $(SRC)/agent.conf.example $(ROOT)/
find $(ROOT) -name "*.pyc" -exec rm {} \;
@ -58,9 +53,9 @@ install_full: source
# Install the source to usr/share
cp -r $(ROOT)/* $(BUILD)/usr/share/mon/agent/
# Install the common executables.
ln -sf ../share/mon/agent/monstatsd/dogstatsd.py $(BUILD)/usr/bin/dogstatsd
ln -sf ../share/mon/agent/forwarder/__init__.py $(BUILD)/usr/bin/mon-forwarder
ln -sf ../share/mon/agent/collector/daemon.py $(BUILD)/usr/bin/mon-collector
ln -sf ../share/mon/agent/monagent/monstatsd/dogstatsd.py $(BUILD)/usr/bin/dogstatsd
ln -sf ../share/mon/agent/monagent/forwarder/__init__.py $(BUILD)/usr/bin/mon-forwarder
ln -sf ../share/mon/agent/monagent/collector/daemon.py $(BUILD)/usr/bin/mon-collector
chmod 755 $(BUILD)/usr/bin/dogstatsd
chmod 755 $(BUILD)/usr/bin/mon-forwarder
chmod 755 $(BUILD)/usr/bin/mon-collector

View File

@ -1,7 +1,7 @@
import sys
from config import *
from collector.jmxfetch import JMX_FETCH_JAR_NAME
from monagent.common.config import *
from monagent.collector.jmxfetch import JMX_FETCH_JAR_NAME
try: