opendev
/
puppet-log_processor
Code Issues Proposed changes

Better logstash field data. 

We are currently using a lot of wildcard searches in elasticsearch which
are slow. Provide better field data so that we can replace those
wildcard searches with filters. In particular add a short uuid field and
make the filename tag field the basename of the filepath so that grenade
and non grenade files all end up with the same tags.

Change-Id: If558017fceae96bcf197e611ab5cac1cfe7ae9bf
This commit is contained in:
Clark Boylan 2014-03-10 13:54:39 -07:00
parent c24a8a75e7
commit 5a3ff67db4
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
files/log-gearman-client.py
View File

@ -20,6 +20,7 @@ import gear
import json
import logging
import os
import os.path
import re
import signal
import threading
@ -94,7 +95,9 @@ class EventProcessor(threading.Thread):
fields["build_master"] = event["build"].get("host_name", "UNKNOWN")
parameters = event["build"].get("parameters", {})
fields["project"] = parameters.get("ZUUL_PROJECT", "UNKNOWN")
# TODO(clarkb) can we do better without duplicated data here?
fields["build_uuid"] = parameters.get("ZUUL_UUID", "UNKNOWN")
fields["build_short_uuid"] = fields["build_uuid"][:7]
fields["build_queue"] = parameters.get("ZUUL_PIPELINE", "UNKNOWN")
fields["build_ref"] = parameters.get("ZUUL_REF", "UNKNOWN")
fields["build_branch"] = parameters.get("ZUUL_BRANCH", "UNKNOWN")
@ -115,7 +118,8 @@ class EventProcessor(threading.Thread):
fields["log_url"] = source_url
out_event = {}
out_event["fields"] = fields
out_event["tags"] = [fileopts['name']] + fileopts.get('tags', [])
out_event["tags"] = [os.path.basename(fileopts['name'])] + \
fileopts.get('tags', [])
return source_url, out_event