Updated metric and measurement queries to support new normalized db schema

This commit is contained in:
Jonathan Halterman 2014-04-10 15:11:34 -07:00
parent 1b6d5653fc
commit 836cc38c5d
8 changed files with 117 additions and 80 deletions

View File

@ -15,8 +15,8 @@
<properties>
<computedVersion>${project.version}-${timestamp}-${buildNumber}</computedVersion>
<computedName>${project.artifactId}-${computedVersion}</computedName>
<mon.common.version>1.0.0.23</mon.common.version>
<dropwizard.version>0.7.0-rc3</dropwizard.version>
<mon.common.version>1.0.0.29</mon.common.version>
<dropwizard.version>0.7.0</dropwizard.version>
<skipITs>true</skipITs>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

View File

@ -41,7 +41,7 @@ public class MonApiModule extends AbstractModule {
@Override
protected void configure() {
bind(MonApiConfiguration.class).toInstance(config);
bind(MetricRegistry.class).in(Singleton.class);
bind(MetricRegistry.class).toInstance(environment.metrics());
bind(DataSourceFactory.class).annotatedWith(Names.named("mysql")).toInstance(config.mysql);
bind(DataSourceFactory.class).annotatedWith(Names.named("vertiva")).toInstance(config.vertica);

View File

@ -179,8 +179,9 @@ public class AlarmService {
undeterminedActions);
// Notify interested parties of new alarm
// TODO pass changed sub alarms
String event = Serialization.toJson(new AlarmUpdatedEvent(tenantId, alarmId, name,
expression, state, enabled, subAlarms.getKey(), subAlarms.getValue()));
expression, state, enabled, subAlarms.getKey(), null, subAlarms.getValue()));
producer.send(new KeyedMessage<>(config.eventsTopic, tenantId, event));
} catch (Exception e) {
throw Exceptions.uncheck(e, "Error updating alarm for project / tenant %s", tenantId);

View File

@ -4,7 +4,6 @@ import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@ -28,10 +27,10 @@ import com.hpcloud.mon.domain.model.measurement.Measurements;
* @author Jonathan Halterman
*/
public class MeasurementRepositoryImpl implements MeasurementRepository {
private static final String FIND_BY_METRIC_DEF_SQL = "select m.definition_id, m.time_stamp, m.value "
+ "from MonMetrics.Measurements m, MonMetrics.Definitions def%s "
+ "where def.tenant_id = :tenantId and m.definition_id = def.id and m.time_stamp >= :startTime%s "
+ "order by m.definition_id";
private static final String FIND_BY_METRIC_DEF_SQL = "select m.definition_dimensions_id, dd.dimension_set_id, m.time_stamp, m.value "
+ "from MonMetrics.Measurements m, MonMetrics.Definitions def, MonMetrics.DefinitionDimensions dd%s "
+ "where m.definition_dimensions_id = dd.id and def.id = dd.definition_id "
+ "and def.tenant_id = :tenantId and m.time_stamp >= :startTime%s order by dd.id";
private final DBI db;
@ -46,16 +45,16 @@ public class MeasurementRepositoryImpl implements MeasurementRepository {
Handle h = db.open();
try {
// Build query
StringBuilder sbFrom = new StringBuilder();
// Build sql
StringBuilder sbWhere = new StringBuilder();
MetricQueries.buildClausesForDimensions(sbFrom, sbWhere, dimensions);
if (name != null)
sbWhere.append(" and def.name = :name");
if (endTime != null)
sbWhere.append(" and m.time_stamp <= :endTime");
String sql = String.format(FIND_BY_METRIC_DEF_SQL, sbFrom.toString(), sbWhere.toString());
String sql = String.format(FIND_BY_METRIC_DEF_SQL,
MetricQueries.buildJoinClauseFor(dimensions), sbWhere);
// Build query
Query<Map<String, Object>> query = h.createQuery(sql)
.bind("tenantId", tenantId)
.bind("startTime", new Timestamp(startTime.getMillis()));
@ -63,29 +62,23 @@ public class MeasurementRepositoryImpl implements MeasurementRepository {
query.bind("name", name);
if (endTime != null)
query.bind("endTime", new Timestamp(endTime.getMillis()));
if (dimensions != null) {
int i = 0;
for (Iterator<Map.Entry<String, String>> it = dimensions.entrySet().iterator(); it.hasNext(); i++) {
Map.Entry<String, String> entry = it.next();
query.bind("dname" + i, entry.getKey());
query.bind("dvalue" + i, entry.getValue());
}
}
MetricQueries.bindDimensionsToQuery(query, dimensions);
// Execute
// Execute query
List<Map<String, Object>> rows = query.list();
// Build results
Map<ByteBuffer, Measurements> results = new LinkedHashMap<>();
for (Map<String, Object> row : rows) {
byte[] defIdBytes = (byte[]) row.get("definition_id");
byte[] defIdBytes = (byte[]) row.get("definition_dimensions_id");
byte[] dimSetIdBytes = (byte[]) row.get("dimension_set_id");
ByteBuffer defId = ByteBuffer.wrap(defIdBytes);
long timestamp = ((Timestamp) row.get("time_stamp")).getTime() / 1000;
double value = (double) row.get("value");
Measurements measurements = results.get(defId);
if (measurements == null) {
measurements = new Measurements(name, MetricQueries.dimensionsFor(h, defIdBytes),
measurements = new Measurements(name, MetricQueries.dimensionsFor(h, dimSetIdBytes),
new ArrayList<Measurement>());
results.put(defId, measurements);
}

View File

@ -3,7 +3,6 @@ package com.hpcloud.mon.infrastructure.persistence;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -23,9 +22,10 @@ import com.hpcloud.mon.domain.model.metric.MetricDefinitionRepository;
* @author Jonathan Halterman
*/
public class MetricDefinitionRepositoryImpl implements MetricDefinitionRepository {
private static final String FIND_BY_METRIC_DEF_SQL = "select def.id, def.name, d.name as dname, d.value as dvalue "
+ "from MonMetrics.Definitions def, MonMetrics.Dimensions d%s "
+ "where def.tenant_id = :tenantId and d.definition_id = def.id%s order by def.id";
private static final String FIND_BY_METRIC_DEF_SQL = "select dd.id, def.name, d.name as dname, d.value as dvalue "
+ "from MonMetrics.Definitions def, MonMetrics.DefinitionDimensions dd "
+ "left outer join MonMetrics.Dimensions d on d.dimension_set_id = dd.dimension_set_id%s "
+ "where def.id = dd.definition_id and def.tenant_id = :tenantId%s order by dd.id";
private final DBI db;
@ -39,27 +39,20 @@ public class MetricDefinitionRepositoryImpl implements MetricDefinitionRepositor
Handle h = db.open();
try {
// Build query
StringBuilder sbFrom = new StringBuilder();
// Build sql
StringBuilder sbWhere = new StringBuilder();
MetricQueries.buildClausesForDimensions(sbFrom, sbWhere, dimensions);
if (name != null)
sbWhere.append(" and def.name = :name");
String sql = String.format(FIND_BY_METRIC_DEF_SQL, sbFrom.toString(), sbWhere.toString());
String sql = String.format(FIND_BY_METRIC_DEF_SQL,
MetricQueries.buildJoinClauseFor(dimensions), sbWhere);
// Build query
Query<Map<String, Object>> query = h.createQuery(sql).bind("tenantId", tenantId);
if (name != null)
query.bind("name", name);
if (dimensions != null) {
int i = 0;
for (Iterator<Map.Entry<String, String>> it = dimensions.entrySet().iterator(); it.hasNext(); i++) {
Map.Entry<String, String> entry = it.next();
query.bind("dname" + i, entry.getKey());
query.bind("dvalue" + i, entry.getValue());
}
}
MetricQueries.bindDimensionsToQuery(query, dimensions);
// Execute
// Execute query
List<Map<String, Object>> rows = query.list();
// Build results
@ -75,7 +68,8 @@ public class MetricDefinitionRepositoryImpl implements MetricDefinitionRepositor
if (defId == null || !Arrays.equals(currentId, defId)) {
currentId = defId;
dims = new HashMap<>();
dims.put(dName, dValue);
if (dName != null && dValue != null)
dims.put(dName, dValue);
metricDefs.add(new MetricDefinition(metricName, dims));
} else
dims.put(dName, dValue);

View File

@ -1,8 +1,10 @@
package com.hpcloud.mon.infrastructure.persistence;
import java.util.Iterator;
import java.util.Map;
import org.skife.jdbi.v2.Handle;
import org.skife.jdbi.v2.Query;
import com.hpcloud.persistence.SqlQueries;
@ -10,12 +12,14 @@ final class MetricQueries {
private MetricQueries() {
}
static void buildClausesForDimensions(StringBuilder sbFrom, StringBuilder sbWhere,
Map<String, String> dimensions) {
static String buildJoinClauseFor(Map<String, String> dimensions) {
StringBuilder sbJoin = null;
if (dimensions != null) {
for (int i = 0; i < dimensions.size(); i++) {
sbFrom.append(", MonMetrics.Dimensions d").append(i);
sbWhere.append(" and d")
sbJoin = new StringBuilder();
for (int i = 0; i < dimensions.size(); i++)
sbJoin.append(" inner join MonMetrics.Dimensions d")
.append(i)
.append(" on d")
.append(i)
.append(".name = :dname")
.append(i)
@ -23,15 +27,27 @@ final class MetricQueries {
.append(i)
.append(".value = :dvalue")
.append(i)
.append(" and def.id = d")
.append(" and dd.dimension_set_id = d")
.append(i)
.append(".definition_id");
.append(".dimension_set_id");
}
return sbJoin == null ? "" : sbJoin.toString();
}
static void bindDimensionsToQuery(Query<?> query, Map<String, String> dimensions) {
if (dimensions != null) {
int i = 0;
for (Iterator<Map.Entry<String, String>> it = dimensions.entrySet().iterator(); it.hasNext(); i++) {
Map.Entry<String, String> entry = it.next();
query.bind("dname" + i, entry.getKey());
query.bind("dvalue" + i, entry.getValue());
}
}
}
static Map<String, String> dimensionsFor(Handle handle, byte[] definitionId) {
static Map<String, String> dimensionsFor(Handle handle, byte[] dimensionSetId) {
return SqlQueries.keyValuesFor(handle,
"select name, value from MonMetrics.Dimensions where definition_id = ?", definitionId);
"select name, value from MonMetrics.Dimensions where dimension_set_id = ?", dimensionSetId);
}
}

View File

@ -44,29 +44,46 @@ public class MeasurementRepositoryImplTest {
handle.execute("truncate table MonMetrics.Definitions");
handle.execute("truncate table MonMetrics.Dimensions");
handle.execute("truncate table MonMetrics.Measurements");
handle.execute("truncate table MonMetrics.DefinitionDimensions");
handle.execute("insert into MonMetrics.Definitions values ('/1', 'cpu_utilization', 'bob', '1')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'flavor_id', '1')");
handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)");
handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)");
handle.execute("insert into MonMetrics.Definitions values ('/2', 'cpu_utilization', 'bob', '1')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'flavor_id', '2')");
handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)");
handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)");
handle.execute("insert into MonMetrics.Dimensions values ('/5', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/5', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/5', 'flavor_id', '1')");
handle.execute("insert into MonMetrics.DefinitionDimensions values ('/1', '/1', '/5')");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)");
handle.execute("insert into MonMetrics.Dimensions values ('/8', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/8', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/8', 'flavor_id', '2')");
handle.execute("insert into MonMetrics.DefinitionDimensions values ('/2', '/1', '/8')");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)");
handle.execute("insert into MonMetrics.DefinitionDimensions values ('/3', '/1', '')");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/3', '2014-01-01 00:00:00', 4)");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/3', '2014-01-01 00:01:00', 8)");
}
public void shouldFind() {
public void shouldFindWithoutDimensions() {
Collection<Measurements> measurements = repo.find("bob", "cpu_utilization", null, new DateTime(
2014, 1, 1, 0, 0, 0), null);
assertEquals(measurements.size(), 3);
}
public void shouldFindWithDimensions() {
Map<String, String> dims = new HashMap<>();
dims.put("service", "compute");
dims.put("instance_id", "123");
Collection<Measurements> measurements = repo.find("1234", "cpu_utilization", dims, new DateTime(
2014, 1, 1, 0, 0, 0), null);
Collection<Measurements> measurements = repo.find("bob", "cpu_utilization", dims, new DateTime(
2014, 1, 1, 0, 0), null);
assertEquals(measurements.size(), 2);
dims.put("flavor_id", "2");
measurements = repo.find("bob", "cpu_utilization", dims, new DateTime(2014, 1, 1, 0, 0), null);
assertEquals(measurements.size(), 1);
}
}

View File

@ -43,28 +43,44 @@ public class MetricDefinitionRepositoryImplTest {
handle.execute("truncate table MonMetrics.Definitions");
handle.execute("truncate table MonMetrics.Dimensions");
handle.execute("truncate table MonMetrics.Measurements");
handle.execute("truncate table MonMetrics.DefinitionDimensions");
handle.execute("insert into MonMetrics.Definitions values ('/1', 'cpu_utilization', 'bob', '1')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'flavor_id', '1')");
handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)");
handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)");
handle.execute("insert into MonMetrics.Definitions values ('/2', 'cpu_utilization', 'bob', '1')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'flavor_id', '2')");
handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)");
handle.execute("insert into MonMetrics.Measurements (definition_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)");
handle.execute("insert into MonMetrics.Dimensions values ('/5', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/5', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/5', 'flavor_id', '1')");
handle.execute("insert into MonMetrics.DefinitionDimensions values ('/1', '/1', '/5')");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)");
handle.execute("insert into MonMetrics.Dimensions values ('/8', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/8', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/8', 'flavor_id', '2')");
handle.execute("insert into MonMetrics.DefinitionDimensions values ('/2', '/1', '/8')");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)");
handle.execute("insert into MonMetrics.DefinitionDimensions values ('/3', '/1', '')");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/3', '2014-01-01 00:00:00', 4)");
handle.execute("insert into MonMetrics.Measurements (definition_dimensions_id, time_stamp, value) values ('/3', '2014-01-01 00:01:00', 8)");
}
public void shouldFind() {
public void shouldFindWithoutDimensions() {
List<MetricDefinition> defs = repo.find("bob", "cpu_utilization", null);
assertEquals(defs.size(), 3);
}
public void shouldFindWithDimensions() {
Map<String, String> dims = new HashMap<>();
dims.put("service", "compute");
dims.put("instance_id", "123");
List<MetricDefinition> defs = repo.find("1234", "cpu_utilization", dims);
List<MetricDefinition> defs = repo.find("bob", "cpu_utilization", dims);
assertEquals(defs.size(), 2);
dims.put("flavor_id", "2");
defs = repo.find("bob", "cpu_utilization", dims);
assertEquals(defs.size(), 1);
}
}