Better vertica partition defaults, new drop script
Change-Id: Iebc96b72a62b8650e4cafa82f759b2847efa64f5
This commit is contained in:
parent
4d642836b5
commit
098d07d7c3
|
@ -11,4 +11,4 @@ CREATE TABLE MonAlarms.StateHistory(
|
||||||
reason VARCHAR(65000),
|
reason VARCHAR(65000),
|
||||||
reason_data VARCHAR(65000),
|
reason_data VARCHAR(65000),
|
||||||
time_stamp TIMESTAMP NOT NULL
|
time_stamp TIMESTAMP NOT NULL
|
||||||
) PARTITION BY EXTRACT('year' FROM time_stamp)*10000 + EXTRACT('month' FROM time_stamp)*100 + EXTRACT('day' FROM time_stamp);
|
) PARTITION BY EXTRACT('year' FROM time_stamp)*100 + EXTRACT('month' FROM time_stamp);
|
||||||
|
|
|
@ -7,7 +7,7 @@ CREATE TABLE MonMetrics.Measurements (
|
||||||
time_stamp TIMESTAMP NOT NULL,
|
time_stamp TIMESTAMP NOT NULL,
|
||||||
value FLOAT NOT NULL,
|
value FLOAT NOT NULL,
|
||||||
value_meta VARCHAR(8192)
|
value_meta VARCHAR(8192)
|
||||||
) PARTITION BY EXTRACT('year' FROM time_stamp)*10000 + EXTRACT('month' FROM time_stamp)*100 + EXTRACT('day' FROM time_stamp);
|
) PARTITION BY EXTRACT('year' FROM time_stamp)*100 + EXTRACT('month' FROM time_stamp);
|
||||||
|
|
||||||
CREATE TABLE MonMetrics.Definitions(
|
CREATE TABLE MonMetrics.Definitions(
|
||||||
id BINARY(20) NOT NULL,
|
id BINARY(20) NOT NULL,
|
||||||
|
|
|
@ -125,6 +125,8 @@ class monasca::vertica::config (
|
||||||
$single_node_script = 'create_mon_db.sh'
|
$single_node_script = 'create_mon_db.sh'
|
||||||
$prune_script_name = 'prune_vertica.py'
|
$prune_script_name = 'prune_vertica.py'
|
||||||
$prune_script = "${virtual_env}/bin/${prune_script_name}"
|
$prune_script = "${virtual_env}/bin/${prune_script_name}"
|
||||||
|
$partition_drop_script_name = 'drop_vertica_partitions.py'
|
||||||
|
$partition_drop_script = "${virtual_env}/bin/${partition_drop_script_name}"
|
||||||
|
|
||||||
file { $install_dir:
|
file { $install_dir:
|
||||||
ensure => directory,
|
ensure => directory,
|
||||||
|
@ -154,7 +156,7 @@ class monasca::vertica::config (
|
||||||
python::virtualenv { $virtual_env :
|
python::virtualenv { $virtual_env :
|
||||||
owner => 'root',
|
owner => 'root',
|
||||||
group => 'root',
|
group => 'root',
|
||||||
before => [File[$prune_script]],
|
before => [File[$prune_script], File[$partition_drop_script]],
|
||||||
require => [Package['python-virtualenv'],Package['python-dev']],
|
require => [Package['python-virtualenv'],Package['python-dev']],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -167,6 +169,15 @@ class monasca::vertica::config (
|
||||||
require => File[$install_dir],
|
require => File[$install_dir],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
file { $partition_drop_script:
|
||||||
|
ensure => file,
|
||||||
|
content => template("${templates}/${partition_drop_script_name}.erb"),
|
||||||
|
mode => '0755',
|
||||||
|
owner => $db_user,
|
||||||
|
group => $db_group,
|
||||||
|
require => File[$install_dir],
|
||||||
|
}
|
||||||
|
|
||||||
file { '/usr/sbin/update_vertica_stats.sh':
|
file { '/usr/sbin/update_vertica_stats.sh':
|
||||||
ensure => file,
|
ensure => file,
|
||||||
source => "${files}/update_vertica_stats.sh",
|
source => "${files}/update_vertica_stats.sh",
|
||||||
|
|
|
@ -0,0 +1,80 @@
|
||||||
|
#!<%= @virtual_env %>/bin/python
|
||||||
|
#
|
||||||
|
from datetime import datetime
|
||||||
|
from dateutil.relativedelta import relativedelta
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
VSQL = "/usr/sbin/vsql"
|
||||||
|
|
||||||
|
GET_KEYS_SQL = """
|
||||||
|
SELECT DISTINCT
|
||||||
|
partition_key
|
||||||
|
FROM
|
||||||
|
partitions
|
||||||
|
WHERE
|
||||||
|
partition_key <= %s AND table_schema = '%s';
|
||||||
|
"""
|
||||||
|
|
||||||
|
DROP_SQL = "SELECT DROP_PARTITION('%s', %s);"
|
||||||
|
|
||||||
|
SCHEMAS_DICT = {"MonMetrics": "MonMetrics.Measurements",
|
||||||
|
"MonAlarms": "MonAlarms.StateHistory"}
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
|
||||||
|
help_text = """
|
||||||
|
Number of months of partitions to retain.
|
||||||
|
"""
|
||||||
|
parser.add_argument('-m', '--months', required=True,
|
||||||
|
help=help_text)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
drop_partitions(args)
|
||||||
|
|
||||||
|
|
||||||
|
def get_drop_key(months_ago):
|
||||||
|
the_past = datetime.utcnow() - relativedelta(months=+int(months_ago))
|
||||||
|
partition_key = str(the_past.year) + str(the_past.month).zfill(2)
|
||||||
|
return partition_key
|
||||||
|
|
||||||
|
|
||||||
|
def drop_partitions(args):
|
||||||
|
drop_key = get_drop_key(args.months)
|
||||||
|
print "Deleting partitions '%s' and older" % drop_key
|
||||||
|
for schema, table in SCHEMAS_DICT.iteritems():
|
||||||
|
keys = vsql(GET_KEYS_SQL % (drop_key, schema))
|
||||||
|
if len(keys) == 0:
|
||||||
|
msg = " No partitions older than or equal to '%s' found for '%s'"
|
||||||
|
print msg % (drop_key, schema)
|
||||||
|
continue
|
||||||
|
for key in keys:
|
||||||
|
print " Dropping '%s' partition for '%s'" % (key, table)
|
||||||
|
vsql(DROP_SQL % (table, drop_key))
|
||||||
|
|
||||||
|
|
||||||
|
def vsql(sql):
|
||||||
|
results = []
|
||||||
|
p = subprocess.Popen([VSQL, "-c", sql],
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE)
|
||||||
|
|
||||||
|
out, err = p.communicate()
|
||||||
|
if p.returncode != 0:
|
||||||
|
print("vsql failed: %s %s" % (out, err))
|
||||||
|
return ""
|
||||||
|
for l in out.splitlines():
|
||||||
|
if '--' in l or 'partition_key' in l:
|
||||||
|
# header row, skip
|
||||||
|
continue
|
||||||
|
if 'row' in l:
|
||||||
|
# we're done
|
||||||
|
return results
|
||||||
|
results.append(l.strip())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
Loading…
Reference in New Issue