openstack
/
puppet-monasca
Code Issues Proposed changes

Support pruning a time range in vertica 

Change-Id: I66b95e15804603b7b9f2f4fee9e79609f04cd849
This commit is contained in:
Brad Klein 2016-06-27 14:42:45 -06:00
parent 57cf143e29
commit f3810193bf
1 changed files with 61 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

94
templates/vertica/prune_vertica.py.erb
View File

@ -25,19 +25,20 @@ MON_TABLES = ["Definitions",
def main():
parser = argparse.ArgumentParser()
group = parser.add_mutually_exclusive_group()
limit_ignore_group = parser.add_mutually_exclusive_group()
date_group = parser.add_mutually_exclusive_group(required=True)
help_text = """
Limit pruning to just this list of tenant name(s).
"""
group.add_argument('-l', '--limit', required=False,
nargs='+', help=help_text)
limit_ignore_group.add_argument('-l', '--limit', required=False,
nargs='+', help=help_text)
help_text = """
Ignore pruning for this list of tenant name(s).
"""
group.add_argument('-i', '--ignore', required=False,
nargs='+', help=help_text)
limit_ignore_group.add_argument('-i', '--ignore', required=False,
nargs='+', help=help_text)
help_text = """
The name of the admin tenant. This tenant will not be
pruned, and libvirt metrics that are cross-posted here will
@ -49,25 +50,52 @@ def main():
help_text = """
Number of days of measurements to retain.
"""
parser.add_argument('-r', '--retain_days', required=True,
date_group.add_argument('-r', '--retain_days', required=False,
help=help_text)
help_text = """
Start time of the pruning period.
"""
date_group.add_argument('-s', '--start_time', required=False,
help=help_text)
help_text = """
End time of the pruning period.
"""
parser.add_argument('-e', '--end_time', required=False,
help=help_text)
help_text = """
Metric name(s) to delete, only works with -l (limit) argument.
Metric name(s) separated by spaces to delete, only works with
-l (limit) argument. As in '-m mem.total_mb cpu.idle_perc'.
"""
parser.add_argument('-m', '--metric_name', required=False,
nargs='+', help=help_text)
help_text = """
Dimension name(s) to delete, only works with -m (name) argument.
Dimension key/value pairs to delete, only works with -m (name)
argument. Syntax is the same as the monasca command:
-d 'region=dev02,hostname=dev02-keystone-001'
"""
parser.add_argument('-d', '--dimensions', required=False,
nargs='+', help=help_text)
help=help_text)
args = parser.parse_args()
if args.limit and args.admin_tenant in args.limit:
print "Pruning the admin project is not allowed!"
print "Error: Pruning the admin project is not allowed!"
sys.exit(1)
if args.start_time and not args.end_time:
print "Error: Please provide both start and end time."
sys.exit(1)
if args.start_time and args.end_time and not args.start_time < args.end_time:
print "Error: Start time must preceed end time."
sys.exit(1)
if args.dimensions and not args.metric_name:
print "Error: Please provide metric name with dimensions."
sys.exit(1)
before_counts = get_table_counts(MON_TABLES)
@ -77,18 +105,23 @@ def main():
def prune_tables(args):
now = time.time()
start_time = get_isotime(now)
retain_time = get_isotime(now - (int(args.retain_days) * 60 * 60 * 24))
run_time = get_isotime(now)
tenants = get_tenant_list()
admin_id = get_tenant_id(tenants, args.admin_tenant)
print "Database pruning executing at %s" % start_time
print "Keeping records later than %s" % retain_time
print "Database pruning executing at %s" % run_time
print "Admin project '%s' (%s)" % (args.admin_tenant, admin_id)
if args.retain_days:
retain_time = get_isotime(now - (int(args.retain_days) * 60 * 60 * 24))
time_clause = "time_stamp < '%s'" % retain_time
print "Keeping records more recent than %s" % retain_time
else:
print "Pruning records between %s and %s" % (args.start_time, args.end_time)
time_clause = "time_stamp >= '%s' and time_stamp <= '%s'" % (args.start_time, args.end_time)
delete_sql = """
DELETE
FROM MonMetrics.Measurements
WHERE time_stamp < '%s'
WHERE %s
AND definition_dimensions_id IN (SELECT
defdims.id
FROM MonMetrics.Definitions def,
@ -121,7 +154,7 @@ def prune_tables(args):
"AND def.tenant_id != '%s' AND def.tenant_id IN (%s)" \
% (admin_id, limited_in_list)
vsql(delete_sql % (retain_time, limit_clause))
vsql(delete_sql % (time_clause, limit_clause))
elif args.ignore:
@ -130,13 +163,13 @@ def prune_tables(args):
non_admin_with_ignore_clause = \
"AND def.tenant_id != '%s' AND def.tenant_id NOT IN (%s)" \
% (admin_id, ignore_in_list)
vsql(delete_sql % (retain_time, non_admin_with_ignore_clause))
vsql(delete_sql % (retain_time, admin_clause))
vsql(delete_sql % (time_clause, non_admin_with_ignore_clause))
vsql(delete_sql % (time_clause, admin_clause))
else:
non_admin_clause = "AND def.tenant_id != '%s'" % admin_id
vsql(delete_sql % (retain_time, non_admin_clause))
vsql(delete_sql % (retain_time, admin_clause))
vsql(delete_sql % (time_clause, non_admin_clause))
vsql(delete_sql % (time_clause, admin_clause))
delete_defdims_sql = """
DELETE FROM MonMetrics.DefinitionDimensions
@ -185,23 +218,18 @@ def get_dimensions_sql(dimensions):
where
"""
key_val_dict = dict(s.split('=') for s in dimensions)
first_time = True
for key, value in key_val_dict.iteritems():
if not first_time:
pairs = dimensions.split(',')
for pair in pairs:
kv = pair.split('=')
if first_time:
sql += " ( "
else:
sql += " or "
sql += " name = '%s' and value = '%s'" % (key, value)
sql += " (name = '%s' and value = '%s')" % (kv[0], kv[1])
first_time = False
sql += """
group by
dimension_set_id
having
count(*) = %d
order by
dimension_set_id)
""" % len(key_val_dict)
sql += ") group by dimension_set_id)"
return sql