From 836cc38c5d147808c6d71dfd369834c1177ff1f3 Mon Sep 17 00:00:00 2001 From: Jonathan Halterman Date: Thu, 10 Apr 2014 15:11:34 -0700 Subject: [PATCH] Updated metric and measurement queries to support new normalized db schema --- pom.xml | 4 +- .../java/com/hpcloud/mon/MonApiModule.java | 2 +- .../com/hpcloud/mon/app/AlarmService.java | 3 +- .../MeasurementRepositoryImpl.java | 35 ++++++--------- .../MetricDefinitionRepositoryImpl.java | 32 ++++++------- .../persistence/MetricQueries.java | 34 ++++++++++---- .../MeasurementRepositoryImplTest.java | 45 +++++++++++++------ .../MetricDefinitionRepositoryImplTest.java | 42 +++++++++++------ 8 files changed, 117 insertions(+), 80 deletions(-) diff --git a/pom.xml b/pom.xml index dcc90ab65..53c32e496 100644 --- a/pom.xml +++ b/pom.xml @@ -15,8 +15,8 @@ ${project.version}-${timestamp}-${buildNumber} ${project.artifactId}-${computedVersion} - 1.0.0.23 - 0.7.0-rc3 + 1.0.0.29 + 0.7.0 true UTF-8 diff --git a/src/main/java/com/hpcloud/mon/MonApiModule.java b/src/main/java/com/hpcloud/mon/MonApiModule.java index 4cc9e136e..b76cfc8d8 100644 --- a/src/main/java/com/hpcloud/mon/MonApiModule.java +++ b/src/main/java/com/hpcloud/mon/MonApiModule.java @@ -41,7 +41,7 @@ public class MonApiModule extends AbstractModule { @Override protected void configure() { bind(MonApiConfiguration.class).toInstance(config); - bind(MetricRegistry.class).in(Singleton.class); + bind(MetricRegistry.class).toInstance(environment.metrics()); bind(DataSourceFactory.class).annotatedWith(Names.named("mysql")).toInstance(config.mysql); bind(DataSourceFactory.class).annotatedWith(Names.named("vertiva")).toInstance(config.vertica); diff --git a/src/main/java/com/hpcloud/mon/app/AlarmService.java b/src/main/java/com/hpcloud/mon/app/AlarmService.java index 682c45b60..9aa84d839 100644 --- a/src/main/java/com/hpcloud/mon/app/AlarmService.java +++ b/src/main/java/com/hpcloud/mon/app/AlarmService.java @@ -179,8 +179,9 @@ public class AlarmService { undeterminedActions); // Notify interested parties of new alarm + // TODO pass changed sub alarms String event = Serialization.toJson(new AlarmUpdatedEvent(tenantId, alarmId, name, - expression, state, enabled, subAlarms.getKey(), subAlarms.getValue())); + expression, state, enabled, subAlarms.getKey(), null, subAlarms.getValue())); producer.send(new KeyedMessage<>(config.eventsTopic, tenantId, event)); } catch (Exception e) { throw Exceptions.uncheck(e, "Error updating alarm for project / tenant %s", tenantId); diff --git a/src/main/java/com/hpcloud/mon/infrastructure/persistence/MeasurementRepositoryImpl.java b/src/main/java/com/hpcloud/mon/infrastructure/persistence/MeasurementRepositoryImpl.java index ebdc502f1..22968426b 100644 --- a/src/main/java/com/hpcloud/mon/infrastructure/persistence/MeasurementRepositoryImpl.java +++ b/src/main/java/com/hpcloud/mon/infrastructure/persistence/MeasurementRepositoryImpl.java @@ -4,7 +4,6 @@ import java.nio.ByteBuffer; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collection; -import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -28,10 +27,10 @@ import com.hpcloud.mon.domain.model.measurement.Measurements; * @author Jonathan Halterman */ public class MeasurementRepositoryImpl implements MeasurementRepository { - private static final String FIND_BY_METRIC_DEF_SQL = "select m.definition_id, m.time_stamp, m.value " - + "from MonMetrics.Measurements m, MonMetrics.Definitions def%s " - + "where def.tenant_id = :tenantId and m.definition_id = def.id and m.time_stamp >= :startTime%s " - + "order by m.definition_id"; + private static final String FIND_BY_METRIC_DEF_SQL = "select m.definition_dimensions_id, dd.dimension_set_id, m.time_stamp, m.value " + + "from MonMetrics.Measurements m, MonMetrics.Definitions def, MonMetrics.DefinitionDimensions dd%s " + + "where m.definition_dimensions_id = dd.id and def.id = dd.definition_id " + + "and def.tenant_id = :tenantId and m.time_stamp >= :startTime%s order by dd.id"; private final DBI db; @@ -46,16 +45,16 @@ public class MeasurementRepositoryImpl implements MeasurementRepository { Handle h = db.open(); try { - // Build query - StringBuilder sbFrom = new StringBuilder(); + // Build sql StringBuilder sbWhere = new StringBuilder(); - MetricQueries.buildClausesForDimensions(sbFrom, sbWhere, dimensions); - if (name != null) sbWhere.append(" and def.name = :name"); if (endTime != null) sbWhere.append(" and m.time_stamp <= :endTime"); - String sql = String.format(FIND_BY_METRIC_DEF_SQL, sbFrom.toString(), sbWhere.toString()); + String sql = String.format(FIND_BY_METRIC_DEF_SQL, + MetricQueries.buildJoinClauseFor(dimensions), sbWhere); + + // Build query Query> query = h.createQuery(sql) .bind("tenantId", tenantId) .bind("startTime", new Timestamp(startTime.getMillis())); @@ -63,29 +62,23 @@ public class MeasurementRepositoryImpl implements MeasurementRepository { query.bind("name", name); if (endTime != null) query.bind("endTime", new Timestamp(endTime.getMillis())); - if (dimensions != null) { - int i = 0; - for (Iterator> it = dimensions.entrySet().iterator(); it.hasNext(); i++) { - Map.Entry entry = it.next(); - query.bind("dname" + i, entry.getKey()); - query.bind("dvalue" + i, entry.getValue()); - } - } + MetricQueries.bindDimensionsToQuery(query, dimensions); - // Execute + // Execute query List> rows = query.list(); // Build results Map results = new LinkedHashMap<>(); for (Map row : rows) { - byte[] defIdBytes = (byte[]) row.get("definition_id"); + byte[] defIdBytes = (byte[]) row.get("definition_dimensions_id"); + byte[] dimSetIdBytes = (byte[]) row.get("dimension_set_id"); ByteBuffer defId = ByteBuffer.wrap(defIdBytes); long timestamp = ((Timestamp) row.get("time_stamp")).getTime() / 1000; double value = (double) row.get("value"); Measurements measurements = results.get(defId); if (measurements == null) { - measurements = new Measurements(name, MetricQueries.dimensionsFor(h, defIdBytes), + measurements = new Measurements(name, MetricQueries.dimensionsFor(h, dimSetIdBytes), new ArrayList()); results.put(defId, measurements); } diff --git a/src/main/java/com/hpcloud/mon/infrastructure/persistence/MetricDefinitionRepositoryImpl.java b/src/main/java/com/hpcloud/mon/infrastructure/persistence/MetricDefinitionRepositoryImpl.java index d5f15041e..282fc09a7 100644 --- a/src/main/java/com/hpcloud/mon/infrastructure/persistence/MetricDefinitionRepositoryImpl.java +++ b/src/main/java/com/hpcloud/mon/infrastructure/persistence/MetricDefinitionRepositoryImpl.java @@ -3,7 +3,6 @@ package com.hpcloud.mon.infrastructure.persistence; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; -import java.util.Iterator; import java.util.List; import java.util.Map; @@ -23,9 +22,10 @@ import com.hpcloud.mon.domain.model.metric.MetricDefinitionRepository; * @author Jonathan Halterman */ public class MetricDefinitionRepositoryImpl implements MetricDefinitionRepository { - private static final String FIND_BY_METRIC_DEF_SQL = "select def.id, def.name, d.name as dname, d.value as dvalue " - + "from MonMetrics.Definitions def, MonMetrics.Dimensions d%s " - + "where def.tenant_id = :tenantId and d.definition_id = def.id%s order by def.id"; + private static final String FIND_BY_METRIC_DEF_SQL = "select dd.id, def.name, d.name as dname, d.value as dvalue " + + "from MonMetrics.Definitions def, MonMetrics.DefinitionDimensions dd " + + "left outer join MonMetrics.Dimensions d on d.dimension_set_id = dd.dimension_set_id%s " + + "where def.id = dd.definition_id and def.tenant_id = :tenantId%s order by dd.id"; private final DBI db; @@ -39,27 +39,20 @@ public class MetricDefinitionRepositoryImpl implements MetricDefinitionRepositor Handle h = db.open(); try { - // Build query - StringBuilder sbFrom = new StringBuilder(); + // Build sql StringBuilder sbWhere = new StringBuilder(); - MetricQueries.buildClausesForDimensions(sbFrom, sbWhere, dimensions); - if (name != null) sbWhere.append(" and def.name = :name"); - String sql = String.format(FIND_BY_METRIC_DEF_SQL, sbFrom.toString(), sbWhere.toString()); + String sql = String.format(FIND_BY_METRIC_DEF_SQL, + MetricQueries.buildJoinClauseFor(dimensions), sbWhere); + + // Build query Query> query = h.createQuery(sql).bind("tenantId", tenantId); if (name != null) query.bind("name", name); - if (dimensions != null) { - int i = 0; - for (Iterator> it = dimensions.entrySet().iterator(); it.hasNext(); i++) { - Map.Entry entry = it.next(); - query.bind("dname" + i, entry.getKey()); - query.bind("dvalue" + i, entry.getValue()); - } - } + MetricQueries.bindDimensionsToQuery(query, dimensions); - // Execute + // Execute query List> rows = query.list(); // Build results @@ -75,7 +68,8 @@ public class MetricDefinitionRepositoryImpl implements MetricDefinitionRepositor if (defId == null || !Arrays.equals(currentId, defId)) { currentId = defId; dims = new HashMap<>(); - dims.put(dName, dValue); + if (dName != null && dValue != null) + dims.put(dName, dValue); metricDefs.add(new MetricDefinition(metricName, dims)); } else dims.put(dName, dValue); diff --git a/src/main/java/com/hpcloud/mon/infrastructure/persistence/MetricQueries.java b/src/main/java/com/hpcloud/mon/infrastructure/persistence/MetricQueries.java index 7230278f2..abcf0c5ae 100644 --- a/src/main/java/com/hpcloud/mon/infrastructure/persistence/MetricQueries.java +++ b/src/main/java/com/hpcloud/mon/infrastructure/persistence/MetricQueries.java @@ -1,8 +1,10 @@ package com.hpcloud.mon.infrastructure.persistence; +import java.util.Iterator; import java.util.Map; import org.skife.jdbi.v2.Handle; +import org.skife.jdbi.v2.Query; import com.hpcloud.persistence.SqlQueries; @@ -10,12 +12,14 @@ final class MetricQueries { private MetricQueries() { } - static void buildClausesForDimensions(StringBuilder sbFrom, StringBuilder sbWhere, - Map dimensions) { + static String buildJoinClauseFor(Map dimensions) { + StringBuilder sbJoin = null; if (dimensions != null) { - for (int i = 0; i < dimensions.size(); i++) { - sbFrom.append(", MonMetrics.Dimensions d").append(i); - sbWhere.append(" and d") + sbJoin = new StringBuilder(); + for (int i = 0; i < dimensions.size(); i++) + sbJoin.append(" inner join MonMetrics.Dimensions d") + .append(i) + .append(" on d") .append(i) .append(".name = :dname") .append(i) @@ -23,15 +27,27 @@ final class MetricQueries { .append(i) .append(".value = :dvalue") .append(i) - .append(" and def.id = d") + .append(" and dd.dimension_set_id = d") .append(i) - .append(".definition_id"); + .append(".dimension_set_id"); + } + + return sbJoin == null ? "" : sbJoin.toString(); + } + + static void bindDimensionsToQuery(Query query, Map dimensions) { + if (dimensions != null) { + int i = 0; + for (Iterator> it = dimensions.entrySet().iterator(); it.hasNext(); i++) { + Map.Entry entry = it.next(); + query.bind("dname" + i, entry.getKey()); + query.bind("dvalue" + i, entry.getValue()); } } } - static Map dimensionsFor(Handle handle, byte[] definitionId) { + static Map dimensionsFor(Handle handle, byte[] dimensionSetId) { return SqlQueries.keyValuesFor(handle, - "select name, value from MonMetrics.Dimensions where definition_id = ?", definitionId); + "select name, value from MonMetrics.Dimensions where dimension_set_id = ?", dimensionSetId); } } diff --git a/src/test/java/com/hpcloud/mon/infrastructure/persistence/MeasurementRepositoryImplTest.java b/src/test/java/com/hpcloud/mon/infrastructure/persistence/MeasurementRepositoryImplTest.java index 1de79aa90..bb4ddcc7c 100644 --- a/src/test/java/com/hpcloud/mon/infrastructure/persistence/MeasurementRepositoryImplTest.java +++ b/src/test/java/com/hpcloud/mon/infrastructure/persistence/MeasurementRepositoryImplTest.java @@ -44,29 +44,46 @@ public class MeasurementRepositoryImplTest { handle.execute("truncate table MonMetrics.Definitions"); handle.execute("truncate table MonMetrics.Dimensions"); handle.execute("truncate table MonMetrics.Measurements"); + handle.execute("truncate table MonMetrics.DefinitionDimensions"); handle.execute("insert into MonMetrics.Definitions values ('/1', 'cpu_utilization', 'bob', '1')"); - handle.execute("insert into MonMetrics.Dimensions values ('/1', 'service', 'compute')"); - handle.execute("insert into MonMetrics.Dimensions values ('/1', 'instance_id', '123')"); - handle.execute("insert into MonMetrics.Dimensions values ('/1', 'flavor_id', '1')"); - handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)"); - handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)"); - handle.execute("insert into MonMetrics.Definitions values ('/2', 'cpu_utilization', 'bob', '1')"); - handle.execute("insert into MonMetrics.Dimensions values ('/2', 'service', 'compute')"); - handle.execute("insert into MonMetrics.Dimensions values ('/2', 'instance_id', '123')"); - handle.execute("insert into MonMetrics.Dimensions values ('/2', 'flavor_id', '2')"); - handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)"); - handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)"); + handle.execute("insert into MonMetrics.Dimensions values ('/5', 'service', 'compute')"); + handle.execute("insert into MonMetrics.Dimensions values ('/5', 'instance_id', '123')"); + handle.execute("insert into MonMetrics.Dimensions values ('/5', 'flavor_id', '1')"); + handle.execute("insert into MonMetrics.DefinitionDimensions values ('/1', '/1', '/5')"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)"); + + handle.execute("insert into MonMetrics.Dimensions values ('/8', 'service', 'compute')"); + handle.execute("insert into MonMetrics.Dimensions values ('/8', 'instance_id', '123')"); + handle.execute("insert into MonMetrics.Dimensions values ('/8', 'flavor_id', '2')"); + handle.execute("insert into MonMetrics.DefinitionDimensions values ('/2', '/1', '/8')"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)"); + + handle.execute("insert into MonMetrics.DefinitionDimensions values ('/3', '/1', '')"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/3', '2014-01-01 00:00:00', 4)"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/3', '2014-01-01 00:01:00', 8)"); } - public void shouldFind() { + public void shouldFindWithoutDimensions() { + Collection measurements = repo.find("bob", "cpu_utilization", null, new DateTime( + 2014, 1, 1, 0, 0, 0), null); + assertEquals(measurements.size(), 3); + } + + public void shouldFindWithDimensions() { Map dims = new HashMap<>(); dims.put("service", "compute"); dims.put("instance_id", "123"); - Collection measurements = repo.find("1234", "cpu_utilization", dims, new DateTime( - 2014, 1, 1, 0, 0, 0), null); + Collection measurements = repo.find("bob", "cpu_utilization", dims, new DateTime( + 2014, 1, 1, 0, 0), null); assertEquals(measurements.size(), 2); + + dims.put("flavor_id", "2"); + measurements = repo.find("bob", "cpu_utilization", dims, new DateTime(2014, 1, 1, 0, 0), null); + assertEquals(measurements.size(), 1); } } diff --git a/src/test/java/com/hpcloud/mon/infrastructure/persistence/MetricDefinitionRepositoryImplTest.java b/src/test/java/com/hpcloud/mon/infrastructure/persistence/MetricDefinitionRepositoryImplTest.java index 496b19ee6..88890d33e 100644 --- a/src/test/java/com/hpcloud/mon/infrastructure/persistence/MetricDefinitionRepositoryImplTest.java +++ b/src/test/java/com/hpcloud/mon/infrastructure/persistence/MetricDefinitionRepositoryImplTest.java @@ -43,28 +43,44 @@ public class MetricDefinitionRepositoryImplTest { handle.execute("truncate table MonMetrics.Definitions"); handle.execute("truncate table MonMetrics.Dimensions"); handle.execute("truncate table MonMetrics.Measurements"); + handle.execute("truncate table MonMetrics.DefinitionDimensions"); handle.execute("insert into MonMetrics.Definitions values ('/1', 'cpu_utilization', 'bob', '1')"); - handle.execute("insert into MonMetrics.Dimensions values ('/1', 'service', 'compute')"); - handle.execute("insert into MonMetrics.Dimensions values ('/1', 'instance_id', '123')"); - handle.execute("insert into MonMetrics.Dimensions values ('/1', 'flavor_id', '1')"); - handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)"); - handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)"); - handle.execute("insert into MonMetrics.Definitions values ('/2', 'cpu_utilization', 'bob', '1')"); - handle.execute("insert into MonMetrics.Dimensions values ('/2', 'service', 'compute')"); - handle.execute("insert into MonMetrics.Dimensions values ('/2', 'instance_id', '123')"); - handle.execute("insert into MonMetrics.Dimensions values ('/2', 'flavor_id', '2')"); - handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)"); - handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)"); + handle.execute("insert into MonMetrics.Dimensions values ('/5', 'service', 'compute')"); + handle.execute("insert into MonMetrics.Dimensions values ('/5', 'instance_id', '123')"); + handle.execute("insert into MonMetrics.Dimensions values ('/5', 'flavor_id', '1')"); + handle.execute("insert into MonMetrics.DefinitionDimensions values ('/1', '/1', '/5')"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)"); + + handle.execute("insert into MonMetrics.Dimensions values ('/8', 'service', 'compute')"); + handle.execute("insert into MonMetrics.Dimensions values ('/8', 'instance_id', '123')"); + handle.execute("insert into MonMetrics.Dimensions values ('/8', 'flavor_id', '2')"); + handle.execute("insert into MonMetrics.DefinitionDimensions values ('/2', '/1', '/8')"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)"); + + handle.execute("insert into MonMetrics.DefinitionDimensions values ('/3', '/1', '')"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/3', '2014-01-01 00:00:00', 4)"); + handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/3', '2014-01-01 00:01:00', 8)"); } - public void shouldFind() { + public void shouldFindWithoutDimensions() { + List defs = repo.find("bob", "cpu_utilization", null); + assertEquals(defs.size(), 3); + } + + public void shouldFindWithDimensions() { Map dims = new HashMap<>(); dims.put("service", "compute"); dims.put("instance_id", "123"); - List defs = repo.find("1234", "cpu_utilization", dims); + List defs = repo.find("bob", "cpu_utilization", dims); assertEquals(defs.size(), 2); + + dims.put("flavor_id", "2"); + defs = repo.find("bob", "cpu_utilization", dims); + assertEquals(defs.size(), 1); } }