From 88b756cf645c1d2b61ad1bff04c4f6cc0dc79f80 Mon Sep 17 00:00:00 2001 From: Ryan Brandt Date: Tue, 19 Apr 2016 12:10:41 -0600 Subject: [PATCH] Fix metric-list limits Change-Id: I5ad17fa686cd00ecaff7ced34821e6cb05528a05 --- .../vertica/MeasurementVerticaRepoImpl.java | 44 ++---- .../MetricDefinitionVerticaRepoImpl.java | 75 ++++++---- .../persistence/vertica/MetricQueries.java | 128 +++++++++++------- .../vertica/StatisticVerticaRepoImpl.java | 23 +--- .../vertica/MetricQueriesTest.java | 30 ++-- 5 files changed, 155 insertions(+), 145 deletions(-) diff --git a/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MeasurementVerticaRepoImpl.java b/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MeasurementVerticaRepoImpl.java index 8d2f5e9cd..679aea54b 100644 --- a/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MeasurementVerticaRepoImpl.java +++ b/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MeasurementVerticaRepoImpl.java @@ -29,7 +29,6 @@ import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Iterator; import java.util.Set; import javax.annotation.Nullable; @@ -53,6 +52,8 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo { public static final DateTimeFormatter DATETIME_FORMATTER = ISODateTimeFormat.dateTime().withZoneUTC(); + public static final ByteBuffer EMPTY_DEF_ID = ByteBuffer.wrap(new byte[0]); + private static final String FIND_BY_METRIC_DEF_SQL = "select mes.definition_dimensions_id, " + "mes.time_stamp, mes.value, mes.value_meta " @@ -66,14 +67,9 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo { private static final String DEFDIM_IDS_SELECT = - "SELECT defDims.id, defDims.dimension_set_id, defDims.definition_id " - + "FROM MonMetrics.Definitions def, MonMetrics.DefinitionDimensions defDims " - + "WHERE defDims.definition_id = def.id " - + "AND def.tenant_id = :tenantId " - + "%s " // Name clause here - + "%s;"; // Dimensions and clause goes here - - private static final String TABLE_TO_JOIN_DIMENSIONS_ON = "defDims"; + "SELECT defDims.id " + + "FROM MonMetrics.DefinitionDimensions defDims " + + "WHERE defDims.id IN (%s)"; private final DBI db; @@ -104,45 +100,29 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo { Map results = new LinkedHashMap<>(); Set defDimIdSet = new HashSet<>(); - Set dimSetIdSet = new HashSet<>(); - - String namePart = ""; - - if (name != null && !name.isEmpty()) { - namePart = "AND def.name = :name "; - } String defDimSql = String.format( DEFDIM_IDS_SELECT, - namePart, - MetricQueries.buildDimensionAndClause(dimensions, "defDims", 0)); + MetricQueries.buildMetricDefinitionSubSql(name, dimensions)); Query> query = h.createQuery(defDimSql).bind("tenantId", tenantId); - MetricQueries.bindDimensionsToQuery(query, dimensions); - if (name != null && !name.isEmpty()) { query.bind("name", name); } - List> rows = query.list(); + MetricQueries.bindDimensionsToQuery(query, dimensions); - ByteBuffer defId = ByteBuffer.wrap(new byte[0]); + List> rows = query.list(); for (Map row : rows) { byte[] defDimId = (byte[]) row.get("id"); defDimIdSet.add(defDimId); - byte[] dimSetIdBytes = (byte[]) row.get("dimension_set_id"); - dimSetIdSet.add(dimSetIdBytes); - - byte[] defIdBytes = (byte[]) row.get("definition_id"); - defId = ByteBuffer.wrap(defIdBytes); - } - if (!Boolean.TRUE.equals(mergeMetricsFlag) && (dimSetIdSet.size() > 1)) { + if (!Boolean.TRUE.equals(mergeMetricsFlag) && (defDimIdSet.size() > 1)) { throw new MultipleMetricsException(name, dimensions); } @@ -211,7 +191,7 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo { } - Measurements measurements = (Boolean.TRUE.equals(mergeMetricsFlag)) ? results.get(defId) : results.get(defdimsId); + Measurements measurements = (Boolean.TRUE.equals(mergeMetricsFlag)) ? results.get(EMPTY_DEF_ID) : results.get(defdimsId); if (measurements == null) { if (Boolean.TRUE.equals(mergeMetricsFlag)) { @@ -219,10 +199,10 @@ public class MeasurementVerticaRepoImpl implements MeasurementRepo { new Measurements(name, new HashMap(), new ArrayList()); - results.put(defId, measurements); + results.put(EMPTY_DEF_ID, measurements); } else { measurements = - new Measurements(name, MetricQueries.dimensionsFor(h, (byte[]) dimSetIdSet.toArray()[0]), + new Measurements(name, MetricQueries.dimensionsFor(h, (byte[]) defDimIdSet.toArray()[0]), new ArrayList()); results.put(defdimsId, measurements); } diff --git a/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MetricDefinitionVerticaRepoImpl.java b/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MetricDefinitionVerticaRepoImpl.java index bcd0f5d60..7dd3a6981 100644 --- a/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MetricDefinitionVerticaRepoImpl.java +++ b/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MetricDefinitionVerticaRepoImpl.java @@ -44,40 +44,46 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo { logger = LoggerFactory.getLogger(MetricDefinitionVerticaRepoImpl.class); - private static final String - FIND_METRIC_DEFS_SQL = + private static final String FIND_METRIC_DEFS_SQL = "SELECT defDims.id as defDimsId, def.name, dims.name as dName, dims.value AS dValue " - + "FROM MonMetrics.Definitions def, MonMetrics.DefinitionDimensions defDims " + + "FROM MonMetrics.Definitions def " + + "JOIN MonMetrics.DefinitionDimensions defDims ON def.id = defDims.definition_id " // Outer join needed in case there are no dimensions for a definition. + "LEFT OUTER JOIN MonMetrics.Dimensions dims ON dims.dimension_set_id = defDims" - + ".dimension_set_id WHERE def.id = defDims.definition_id " - + "and def.tenant_id = :tenantId " - + "%s " // Name goes here. - + "%s " // Offset goes here. - + "%s " // Dimensions and clause goes here - + "%s " // Optional timestamp qualifier goes here + + ".dimension_set_id " + + "WHERE defDims.id in (%s) " + "ORDER BY defDims.id ASC"; - private static final String - FIND_METRIC_NAMES_SQL = + private static final String METRIC_DEF_SUB_QUERY = + "SELECT defDimsSub.id " + + "FROM MonMetrics.Definitions defSub " + + "JOIN MonMetrics.DefinitionDimensions defDimsSub ON defSub.id = defDimsSub.definition_id " + + "WHERE defSub.tenant_id = :tenantId " + + "%s " // Name goes here + + "%s " // Offset goes here + + "%s " // Dimensions and clause goes here + + "%s " // Time qualifier goes here + + "GROUP BY defDimsSub.id " + + "ORDER BY defDimsSub.id ASC " + + "%s "; // limit goes here + + private static final String FIND_METRIC_NAMES_SQL = "SELECT distinct def.id, def.name " + "FROM MonMetrics.Definitions def " + "WHERE def.id IN (%s) " // Subselect goes here + "ORDER BY def.id ASC "; - private static final String - METRIC_NAMES_SUB_SELECT = + private static final String METRIC_NAMES_SUB_SELECT = "SELECT distinct MAX(defSub.id) as max_id " // The aggregation function gives us one id per name - + "FROM MonMetrics.Definitions defSub, MonMetrics.DefinitionDimensions defDimsSub " - + "WHERE defDimsSub.definition_id = defSub.id " - + "AND defSub.tenant_id = :tenantId " + + "FROM MonMetrics.Definitions defSub " + + "JOIN MonMetrics.DefinitionDimensions defDimsSub ON defDimsSub.definition_id = defSub.id " + + "WHERE defSub.tenant_id = :tenantId " + "%s " // Offset goes here. + "%s " // Dimensions and clause goes here + "GROUP BY defSub.name " // This is to reduce the (id, name) sets to only include unique names + "ORDER BY max_id ASC %s"; // Limit goes here. - private static final String - DEFDIM_IDS_SELECT = + private static final String DEFDIM_IDS_SELECT = "SELECT defDims.id " + "FROM MonMetrics.Definitions def, MonMetrics.DefinitionDimensions defDims " + "WHERE defDims.definition_id = def.id " @@ -85,9 +91,8 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo { + "%s " // Name clause here + "%s;"; // Dimensions and clause goes here - private static final String - MEASUREMENT_AND_CLAUSE = - "AND defDims.id IN (" + private static final String MEASUREMENT_AND_CLAUSE = + "AND defDimsSub.id IN (" + "SELECT definition_dimensions_id FROM MonMetrics.Measurements " + "WHERE to_hex(definition_dimensions_id) " + "%s " // List of definition dimension ids here @@ -152,8 +157,7 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo { String.format(METRIC_NAMES_SUB_SELECT, offsetPart, MetricQueries.buildDimensionAndClause(dimensions, - TABLE_TO_JOIN_DIMENSIONS_ON, - 0), // No limit on dim ids + TABLE_TO_JOIN_DIMENSIONS_ON), limitPart); String sql = String.format(FIND_METRIC_NAMES_SQL, defSubSelect); @@ -254,7 +258,7 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo { if (name != null && !name.isEmpty()) { - namePart = " and def.name = :name "; + namePart = " and defSub.name = :name "; } @@ -262,7 +266,15 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo { if (offset != null && !offset.isEmpty()) { - offsetPart = " and defDims.id > :offset "; + offsetPart = " and defDimsSub.id > :offset "; + + } + + String limitPart = ""; + + if (limit > 0) { + + limitPart = "limit " + Integer.toString(limit + 1); } @@ -273,9 +285,14 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo { String sql = String.format(FIND_METRIC_DEFS_SQL, - namePart, offsetPart, - MetricQueries.buildDimensionAndClause(dimensions, "defDims", limit), - timeInClause); + String.format(METRIC_DEF_SUB_QUERY, + namePart, + offsetPart, + MetricQueries.buildDimensionAndClause(dimensions, + TABLE_TO_JOIN_DIMENSIONS_ON), + timeInClause, + limitPart) + ); Query> query = h.createQuery(sql).bind("tenantId", tenantId); @@ -338,7 +355,7 @@ public class MetricDefinitionVerticaRepoImpl implements MetricDefinitionRepo { String defDimSql = String.format( DEFDIM_IDS_SELECT, namePart, - MetricQueries.buildDimensionAndClause(dimensions, "defDims", 0)); + MetricQueries.buildDimensionAndClause(dimensions, "defDims")); Query> query = dbHandle.createQuery(defDimSql).bind("tenantId", tenantId); diff --git a/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MetricQueries.java b/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MetricQueries.java index d991ab905..9a17acc88 100644 --- a/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MetricQueries.java +++ b/java/src/main/java/monasca/api/infrastructure/persistence/vertica/MetricQueries.java @@ -32,61 +32,84 @@ import monasca.common.persistence.SqlQueries; * Vertica utilities for building metric queries. */ final class MetricQueries { + private static Splitter BAR_SPLITTER = Splitter.on('|').omitEmptyStrings().trimResults(); + + static final String METRIC_DEF_SUB_SQL = + "SELECT defDimsSub.id " + + "FROM MonMetrics.Definitions as defSub " + + "JOIN MonMetrics.DefinitionDimensions as defDimsSub ON defDimsSub.definition_id = defSub.id " + + "WHERE defSub.tenant_id = :tenantId " + + "%s " // metric name here + + "%s " // dimension and clause here + + "GROUP BY defDimsSub.id"; + + private static final String TABLE_TO_JOIN_DIMENSIONS_ON = "defDimsSub"; + private MetricQueries() {} - static String buildDimensionAndClause(Map dimensions, - String tableToJoinName, - int limit) { + static String buildMetricDefinitionSubSql(String name, Map dimensions) { - StringBuilder sb = null; + String namePart = ""; - if (dimensions != null && dimensions.size() > 0) { - - int numDims = dimensions.size(); - sb = new StringBuilder(); - sb.append(" and " + tableToJoinName + ".dimension_set_id in ") - .append("(select dimension_set_id from MonMetrics.Dimensions where "); - - int i = 0; - for (Iterator> it = dimensions.entrySet().iterator(); it.hasNext(); i++) { - Map.Entry entry = it.next(); - sb.append("name = :dname").append(i); - String dim_value = entry.getValue(); - if (!Strings.isNullOrEmpty(dim_value)) { - List values = Splitter.on('|').splitToList(dim_value); - if (values.size() > 1) { - sb.append(" and ( "); - for (int j = 0; j < values.size(); j++) { - sb.append("value = :dvalue").append(i).append('_').append(j); - if (j < values.size() - 1) { - sb.append(" or "); - } - } - sb.append(" )"); - } else { - sb.append(" and value = :dvalue").append(i); - } - } - if (it.hasNext()) { - sb.append(" or "); - } - } - sb.append(" group by dimension_set_id") - .append(" having count(*) = " + numDims +" "); - - // - // Limit is non-deterministic here unless we also - // order by. - // - if (limit > 0) { - sb.append("order by dimension_set_id ") - .append("limit " + Integer.toString(limit + 1)); - } - - sb.append(")"); + if (name != null && !name.isEmpty()) { + namePart = "AND defSub.name = :name "; } - return sb == null ? "" : sb.toString(); + return String.format(METRIC_DEF_SUB_SQL, + namePart, + buildDimensionAndClause(dimensions, + TABLE_TO_JOIN_DIMENSIONS_ON)); + } + + static String buildDimensionAndClause(Map dimensions, + String tableToJoinName) { + + if (dimensions == null || dimensions.isEmpty()) { + return ""; + } + + StringBuilder sb = new StringBuilder(); + sb.append(" and ").append(tableToJoinName).append( + ".dimension_set_id in ( " + + "SELECT dimension_set_id FROM MonMetrics.Dimensions WHERE ("); + + int i = 0; + for (Iterator> it = dimensions.entrySet().iterator(); it.hasNext(); i++) { + Map.Entry entry = it.next(); + + sb.append("(name = :dname").append(i); + + String dim_value = entry.getValue(); + if (!Strings.isNullOrEmpty(dim_value)) { + List values = BAR_SPLITTER.splitToList(dim_value); + + if (values.size() > 1) { + sb.append(" and ( "); + + for (int j = 0; j < values.size(); j++) { + sb.append("value = :dvalue").append(i).append('_').append(j); + + if (j < values.size() - 1) { + sb.append(" or "); + } + } + sb.append(")"); + + } else { + sb.append(" and value = :dvalue").append(i); + } + } + sb.append(")"); + + if (it.hasNext()) { + sb.append(" or "); + } + } + + sb.append(") GROUP BY dimension_set_id HAVING count(*) = ").append(dimensions.size()).append(") "); + + + return sb.toString(); } static void bindDimensionsToQuery(Query query, Map dimensions) { @@ -113,8 +136,11 @@ final class MetricQueries { static Map dimensionsFor(Handle handle, byte[] dimensionSetId) { - return SqlQueries.keyValuesFor(handle, "select name, value from MonMetrics.Dimensions " - + "where" + " dimension_set_id = ?", dimensionSetId); + return SqlQueries.keyValuesFor(handle, + "select name, value from MonMetrics.Dimensions as d " + + "join MonMetrics.DefinitionDimensions as dd " + + "on d.dimension_set_id = dd.dimension_set_id " + + "where" + " dd.id = ?", dimensionSetId); } static String createDefDimIdInClause(Set defDimIdSet) { diff --git a/java/src/main/java/monasca/api/infrastructure/persistence/vertica/StatisticVerticaRepoImpl.java b/java/src/main/java/monasca/api/infrastructure/persistence/vertica/StatisticVerticaRepoImpl.java index 04c6bac7a..b00fc6d69 100644 --- a/java/src/main/java/monasca/api/infrastructure/persistence/vertica/StatisticVerticaRepoImpl.java +++ b/java/src/main/java/monasca/api/infrastructure/persistence/vertica/StatisticVerticaRepoImpl.java @@ -50,13 +50,10 @@ public class StatisticVerticaRepoImpl implements StatisticRepo { "select defdims.id, def.name, d.name as dname, d.value as dvalue " + "from MonMetrics.Definitions def, MonMetrics.DefinitionDimensions defdims " + "left outer join MonMetrics.Dimensions d on d.dimension_set_id = defdims.dimension_set_id " - + "where def.id = defdims.definition_id and def.tenant_id = :tenantId " - + "%s " // metric name here - + "%s " // dimension and clause here + + "where def.id = defdims.definition_id " + + "and defdims.id in (%s) " + "order by defdims.id ASC"; - private static final String TABLE_TO_JOIN_DIMENSIONS_ON = "defdims"; - private final DBI db; @Inject @@ -195,19 +192,9 @@ public class StatisticVerticaRepoImpl implements StatisticRepo { List bytes = new ArrayList<>(); - StringBuilder sb = new StringBuilder(); - - if (name != null && !name.isEmpty()) { - - sb.append(" and def.name = :name"); - - } - - String sql = - String - .format(FIND_BY_METRIC_DEF_SQL, - sb, - MetricQueries.buildDimensionAndClause(dimensions, TABLE_TO_JOIN_DIMENSIONS_ON, 0)); + String sql = String.format( + FIND_BY_METRIC_DEF_SQL, + MetricQueries.buildMetricDefinitionSubSql(name, dimensions)); Query> query = h.createQuery(sql) diff --git a/java/src/test/java/monasca/api/infrastructure/persistence/vertica/MetricQueriesTest.java b/java/src/test/java/monasca/api/infrastructure/persistence/vertica/MetricQueriesTest.java index 786867b7e..3943bfa71 100644 --- a/java/src/test/java/monasca/api/infrastructure/persistence/vertica/MetricQueriesTest.java +++ b/java/src/test/java/monasca/api/infrastructure/persistence/vertica/MetricQueriesTest.java @@ -28,55 +28,55 @@ public class MetricQueriesTest { public void metricQueriesBuildDimensionAndClauseTest1() { String expectedResult = - " and defdims.dimension_set_id in (select dimension_set_id from MonMetrics.Dimensions " - + "where name = :dname0 and value = :dvalue0 or name = :dname1 and value = :dvalue1 " - + "group by dimension_set_id having count(*) = 2 order by dimension_set_id limit 2)"; + " and defdims.dimension_set_id in ( SELECT dimension_set_id FROM MonMetrics.Dimensions WHERE" + + " ((name = :dname0 and value = :dvalue0) or (name = :dname1 and value = :dvalue1))" + + " GROUP BY dimension_set_id HAVING count(*) = 2) "; Map dimsMap = new HashMap<>(); dimsMap.put("foo", "bar"); dimsMap.put("biz", "baz"); - String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON, 1); + String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON); assertEquals(expectedResult, s); } public void metricQueriesBuildDimensionAndClauseTest2() { String expectedResult = ""; Map dimsMap = new HashMap<>(); - assertEquals(expectedResult, MetricQueries.buildDimensionAndClause(dimsMap,TABLE_TO_JOIN_DIMENSIONS_ON, 0)); + assertEquals(expectedResult, MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON)); } public void metricQueriesBuildDimensionAndClauseForTest3() { String expectedResult = ""; Map dimsMap = null; - assertEquals(expectedResult, MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON, 0)); + assertEquals(expectedResult, MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON)); } public void metricQueriesBuildDimensionAndClauseTest4() { String expectedResult = - " and defdims.dimension_set_id in (select dimension_set_id from MonMetrics.Dimensions " - + "where name = :dname0 and ( value = :dvalue0_0 or value = :dvalue0_1 ) " - + "group by dimension_set_id having count(*) = 1 order by dimension_set_id limit 2)"; + " and defdims.dimension_set_id in ( SELECT dimension_set_id FROM MonMetrics.Dimensions WHERE" + + " ((name = :dname0 and ( value = :dvalue0_0 or value = :dvalue0_1)))" + + " GROUP BY dimension_set_id HAVING count(*) = 1) "; Map dimsMap = new HashMap<>(); dimsMap.put("foo", "bar|baz"); - String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON, 1); + String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON); assertEquals(expectedResult, s); } public void metricQueriesBuildDimensionAndClauseTest5() { String expectedResult = - " and defdims.dimension_set_id in (select dimension_set_id from MonMetrics.Dimensions " - + "where name = :dname0 and ( value = :dvalue0_0 or value = :dvalue0_1 ) " - + "or name = :dname1 and ( value = :dvalue1_0 or value = :dvalue1_1 ) " - + "group by dimension_set_id having count(*) = 2 order by dimension_set_id limit 2)"; + " and defdims.dimension_set_id in ( SELECT dimension_set_id FROM MonMetrics.Dimensions WHERE" + + " ((name = :dname0 and ( value = :dvalue0_0 or value = :dvalue0_1))" + + " or (name = :dname1 and ( value = :dvalue1_0 or value = :dvalue1_1)))" + + " GROUP BY dimension_set_id HAVING count(*) = 2) "; Map dimsMap = new HashMap<>(); dimsMap.put("foo", "bar|baz"); dimsMap.put("biz", "baz|baf"); - String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON, 1); + String s = MetricQueries.buildDimensionAndClause(dimsMap, TABLE_TO_JOIN_DIMENSIONS_ON); assertEquals(expectedResult, s); } }