Better vertica partition defaults, new drop script

Change-Id: Iebc96b72a62b8650e4cafa82f759b2847efa64f5

files/vertica/mon_alarms_schema.sql

@@ -11,4 +11,4 @@ CREATE TABLE MonAlarms.StateHistory(
     reason VARCHAR(65000),
     reason_data VARCHAR(65000),
     time_stamp TIMESTAMP NOT NULL
-) PARTITION BY EXTRACT('year' FROM time_stamp)*10000 + EXTRACT('month' FROM time_stamp)*100 + EXTRACT('day' FROM time_stamp);
+) PARTITION BY EXTRACT('year' FROM time_stamp)*100 + EXTRACT('month' FROM time_stamp);

files/vertica/mon_metrics_schema.sql

@@ -7,7 +7,7 @@ CREATE TABLE MonMetrics.Measurements (
     time_stamp TIMESTAMP NOT NULL,
     value FLOAT NOT NULL,
     value_meta VARCHAR(8192)
-) PARTITION BY EXTRACT('year' FROM time_stamp)*10000 + EXTRACT('month' FROM time_stamp)*100 + EXTRACT('day' FROM time_stamp);
+) PARTITION BY EXTRACT('year' FROM time_stamp)*100 + EXTRACT('month' FROM time_stamp);
 
 CREATE TABLE MonMetrics.Definitions(
     id BINARY(20) NOT NULL,

manifests/vertica/config.pp

@@ -125,6 +125,8 @@ class monasca::vertica::config (
   $single_node_script = 'create_mon_db.sh'
   $prune_script_name = 'prune_vertica.py'
   $prune_script = "${virtual_env}/bin/${prune_script_name}"
+  $partition_drop_script_name = 'drop_vertica_partitions.py'
+  $partition_drop_script = "${virtual_env}/bin/${partition_drop_script_name}"
 
   file { $install_dir:
     ensure => directory,
@@ -154,7 +156,7 @@ class monasca::vertica::config (
   python::virtualenv { $virtual_env :
     owner   => 'root',
     group   => 'root',
-    before  => [File[$prune_script]],
+    before  => [File[$prune_script], File[$partition_drop_script]],
     require => [Package['python-virtualenv'],Package['python-dev']],
   }
 
@@ -167,6 +169,15 @@ class monasca::vertica::config (
     require => File[$install_dir],
   }
 
+  file { $partition_drop_script:
+    ensure  => file,
+    content => template("${templates}/${partition_drop_script_name}.erb"),
+    mode    => '0755',
+    owner   => $db_user,
+    group   => $db_group,
+    require => File[$install_dir],
+  }
+
   file { '/usr/sbin/update_vertica_stats.sh':
     ensure  => file,
     source  => "${files}/update_vertica_stats.sh",

templates/vertica/drop_vertica_partitions.py.erb

@@ -0,0 +1,80 @@
+#!<%= @virtual_env %>/bin/python
+#
+from datetime import datetime
+from dateutil.relativedelta import relativedelta
+
+import argparse
+import subprocess
+
+VSQL = "/usr/sbin/vsql"
+
+GET_KEYS_SQL = """
+  SELECT DISTINCT
+     partition_key
+  FROM
+     partitions
+  WHERE
+     partition_key <= %s AND table_schema = '%s';
+"""
+
+DROP_SQL = "SELECT DROP_PARTITION('%s', %s);"
+
+SCHEMAS_DICT = {"MonMetrics": "MonMetrics.Measurements",
+                "MonAlarms": "MonAlarms.StateHistory"}
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    help_text = """
+    Number of months of partitions to retain.
+    """
+    parser.add_argument('-m', '--months', required=True,
+                        help=help_text)
+
+    args = parser.parse_args()
+    drop_partitions(args)
+
+
+def get_drop_key(months_ago):
+    the_past = datetime.utcnow() - relativedelta(months=+int(months_ago))
+    partition_key = str(the_past.year) + str(the_past.month).zfill(2)
+    return partition_key
+
+
+def drop_partitions(args):
+    drop_key = get_drop_key(args.months)
+    print "Deleting partitions '%s' and older" % drop_key
+    for schema, table in SCHEMAS_DICT.iteritems():
+        keys = vsql(GET_KEYS_SQL % (drop_key, schema))
+        if len(keys) == 0:
+            msg = "  No partitions older than or equal to '%s' found for '%s'"
+            print msg % (drop_key, schema)
+            continue
+        for key in keys:
+            print "  Dropping '%s' partition for '%s'" % (key, table)
+            vsql(DROP_SQL % (table, drop_key))
+
+
+def vsql(sql):
+    results = []
+    p = subprocess.Popen([VSQL, "-c", sql],
+                         stdout=subprocess.PIPE,
+                         stderr=subprocess.PIPE)
+
+    out, err = p.communicate()
+    if p.returncode != 0:
+        print("vsql failed: %s %s" % (out, err))
+        return ""
+    for l in out.splitlines():
+        if '--' in l or 'partition_key' in l:
+            # header row, skip
+            continue
+        if 'row' in l:
+            # we're done
+            return results
+        results.append(l.strip())
+
+
+if __name__ == '__main__':
+    main()