Added initial support for querying measurements

This commit is contained in:
Jonathan Halterman 2014-03-28 16:44:25 -07:00
parent bae312f3cd
commit 4120f6deb0
15 changed files with 466 additions and 65 deletions

10
pom.xml
View File

@ -104,6 +104,11 @@
<artifactId>dropwizard-client</artifactId>
<version>${dropwizard.version}</version>
</dependency>
<dependency>
<groupId>com.vertica</groupId>
<artifactId>vertica-jdbc</artifactId>
<version>6.1.0</version>
</dependency>
<dependency>
<groupId>com.palominolabs.metrics</groupId>
<artifactId>metrics-guice</artifactId>
@ -119,11 +124,6 @@
<artifactId>jsr305</artifactId>
<version>2.0.0</version>
</dependency>
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
<version>2.3</version>
</dependency>
<dependency>
<groupId>com.hp.csbu.cc</groupId>
<artifactId>CsMiddleware</artifactId>

View File

@ -19,7 +19,8 @@ public class MonApiConfiguration extends Configuration {
@NotEmpty public String metricsTopic = "metrics";
@NotEmpty public String eventsTopic = "events";
@Valid @NotNull public DataSourceFactory database;
@Valid @NotNull public DataSourceFactory mysql;
@Valid @NotNull public DataSourceFactory vertica;
@Valid @NotNull public KafkaConfiguration kafka;
@Valid @NotNull public MiddlewareConfiguration middleware;
}

View File

@ -6,6 +6,7 @@ import io.dropwizard.setup.Environment;
import java.util.Properties;
import javax.inject.Named;
import javax.inject.Singleton;
import kafka.javaapi.producer.Producer;
@ -18,6 +19,7 @@ import com.google.common.base.Joiner;
import com.google.inject.AbstractModule;
import com.google.inject.Provides;
import com.google.inject.ProvisionException;
import com.google.inject.name.Names;
import com.hpcloud.mon.app.ApplicationModule;
import com.hpcloud.mon.domain.DomainModule;
import com.hpcloud.mon.infrastructure.InfrastructureModule;
@ -40,7 +42,8 @@ public class MonApiModule extends AbstractModule {
protected void configure() {
bind(MonApiConfiguration.class).toInstance(config);
bind(MetricRegistry.class).in(Singleton.class);
bind(DataSourceFactory.class).toInstance(config.database);
bind(DataSourceFactory.class).annotatedWith(Names.named("mysql")).toInstance(config.mysql);
bind(DataSourceFactory.class).annotatedWith(Names.named("vertiva")).toInstance(config.vertica);
install(new ApplicationModule());
install(new DomainModule());
@ -49,11 +52,23 @@ public class MonApiModule extends AbstractModule {
@Provides
@Singleton
public DBI getDBI() {
@Named("mysql")
public DBI getMySqlDBI() {
try {
return new DBIFactory().build(environment, config.database, "mysql");
return new DBIFactory().build(environment, config.mysql, "mysql");
} catch (ClassNotFoundException e) {
throw new ProvisionException("Failed to provision DBI", e);
throw new ProvisionException("Failed to provision MySQL DBI", e);
}
}
@Provides
@Singleton
@Named("vertica")
public DBI getVerticaDBI() {
try {
return new DBIFactory().build(environment, config.vertica, "vertica");
} catch (ClassNotFoundException e) {
throw new ProvisionException("Failed to provision Vertica DBI", e);
}
}

View File

@ -1,60 +1,67 @@
package com.hpcloud.mon.domain.model.measurement;
import java.util.Map;
/**
* Encapsulates a metric measurements.
*
* @author Jonathan Halterman
*/
public class Measurement {
private Map<String, String> dimensions;
private Map<String, Object> measurements;
private long timestamp;
private double value;
public Measurement() {
}
public Measurement(long timestamp, double value) {
this.timestamp = timestamp;
this.value = value;
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (!super.equals(obj))
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
Measurement other = (Measurement) obj;
if (measurements == null) {
if (other.measurements != null)
return false;
} else if (!measurements.equals(other.measurements))
if (timestamp != other.timestamp)
return false;
if (dimensions == null) {
if (other.dimensions != null)
return false;
} else if (!dimensions.equals(other.dimensions))
if (Double.doubleToLongBits(value) != Double.doubleToLongBits(other.value))
return false;
return true;
}
public Map<String, Object> getMeasurements() {
return measurements;
public long getTimestamp() {
return timestamp;
}
public Map<String, String> getDimensions() {
return dimensions;
public double getValue() {
return value;
}
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
result = prime * result + ((measurements == null) ? 0 : measurements.hashCode());
result = prime * result + ((dimensions == null) ? 0 : dimensions.hashCode());
int result = 1;
result = prime * result + (int) (timestamp ^ (timestamp >>> 32));
long temp;
temp = Double.doubleToLongBits(value);
result = prime * result + (int) (temp ^ (temp >>> 32));
return result;
}
public void setMeasurements(Map<String, Object> measurements) {
this.measurements = measurements;
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
public void setDimensions(Map<String, String> dimensions) {
this.dimensions = dimensions;
public void setValue(double value) {
this.value = value;
}
@Override
public String toString() {
return String.format("Measurement [timestamp=%s, value=%s]", timestamp, value);
}
}

View File

@ -1,6 +1,6 @@
package com.hpcloud.mon.domain.model.measurement;
import java.util.List;
import java.util.Collection;
import java.util.Map;
import javax.annotation.Nullable;
@ -16,6 +16,6 @@ public interface MeasurementRepository {
/**
* Finds measurements for the given criteria.
*/
List<Measurement> find(String tenantId, String name, Map<String, String> dimensions,
Collection<Measurements> find(String tenantId, String name, Map<String, String> dimensions,
DateTime startTime, @Nullable DateTime endTime);
}

View File

@ -0,0 +1,95 @@
package com.hpcloud.mon.domain.model.measurement;
import java.util.List;
import java.util.Map;
/**
* Encapsulates a metric measurements.
*
* @author Jonathan Halterman
*/
public class Measurements {
private String name;
private Map<String, String> dimensions;
private List<Measurement> measurements;
public Measurements() {
}
public Measurements(String name, Map<String, String> dimensions, List<Measurement> measurements) {
this.name = name;
this.dimensions = dimensions;
this.measurements = measurements;
}
public void addMeasurement(Measurement measurement) {
measurements.add(measurement);
}
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
Measurements other = (Measurements) obj;
if (dimensions == null) {
if (other.dimensions != null)
return false;
} else if (!dimensions.equals(other.dimensions))
return false;
if (measurements == null) {
if (other.measurements != null)
return false;
} else if (!measurements.equals(other.measurements))
return false;
if (name == null) {
if (other.name != null)
return false;
} else if (!name.equals(other.name))
return false;
return true;
}
public Map<String, String> getDimensions() {
return dimensions;
}
public List<Measurement> getMeasurements() {
return measurements;
}
public String getName() {
return name;
}
@Override
public int hashCode() {
final int prime = 31;
int result = 1;
result = prime * result + ((dimensions == null) ? 0 : dimensions.hashCode());
result = prime * result + ((measurements == null) ? 0 : measurements.hashCode());
result = prime * result + ((name == null) ? 0 : name.hashCode());
return result;
}
public void setDimensions(Map<String, String> dimensions) {
this.dimensions = dimensions;
}
public void setMeasurements(List<Measurement> measurements) {
this.measurements = measurements;
}
public void setName(String name) {
this.name = name;
}
@Override
public String toString() {
return String.format("Measurement [name=%s, dimensions=%s, measurements=%s]", name, dimensions,
measurements);
}
}

View File

@ -8,53 +8,74 @@ import java.util.Map;
* @author Jonathan Halterman
*/
public class Statistic {
private String name;
private Map<String, String> dimensions;
private Map<String, Object> measurements;
private Map<String, Object> statistics;
@Override
public boolean equals(Object obj) {
if (this == obj)
return true;
if (!super.equals(obj))
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
Statistic other = (Statistic) obj;
if (measurements == null) {
if (other.measurements != null)
return false;
} else if (!measurements.equals(other.measurements))
return false;
if (dimensions == null) {
if (other.dimensions != null)
return false;
} else if (!dimensions.equals(other.dimensions))
return false;
if (name == null) {
if (other.name != null)
return false;
} else if (!name.equals(other.name))
return false;
if (statistics == null) {
if (other.statistics != null)
return false;
} else if (!statistics.equals(other.statistics))
return false;
return true;
}
public Map<String, Object> getMeasurements() {
return measurements;
}
public Map<String, String> getDimensions() {
return dimensions;
}
public String getName() {
return name;
}
public Map<String, Object> getStatistics() {
return statistics;
}
@Override
public int hashCode() {
final int prime = 31;
int result = super.hashCode();
result = prime * result + ((measurements == null) ? 0 : measurements.hashCode());
int result = 1;
result = prime * result + ((dimensions == null) ? 0 : dimensions.hashCode());
result = prime * result + ((name == null) ? 0 : name.hashCode());
result = prime * result + ((statistics == null) ? 0 : statistics.hashCode());
return result;
}
public void setMeasurements(Map<String, Object> measurements) {
this.measurements = measurements;
}
public void setDimensions(Map<String, String> dimensions) {
this.dimensions = dimensions;
}
public void setName(String name) {
this.name = name;
}
public void setStatistics(Map<String, Object> statistics) {
this.statistics = statistics;
}
@Override
public String toString() {
return String.format("Statistic [name=%s, dimensions=%s, statistics=%s]", name, dimensions,
statistics);
}
}

View File

@ -3,8 +3,10 @@ package com.hpcloud.mon.infrastructure.persistence;
import java.util.List;
import javax.inject.Inject;
import javax.inject.Named;
import org.skife.jdbi.v2.DBI;
import org.skife.jdbi.v2.Handle;
import com.hpcloud.mon.domain.model.alarmhistory.AlarmHistory;
import com.hpcloud.mon.domain.model.alarmhistory.AlarmHistoryRepository;
@ -18,12 +20,21 @@ public class AlarmHistoryRepositoryImpl implements AlarmHistoryRepository {
private final DBI db;
@Inject
public AlarmHistoryRepositoryImpl(DBI db) {
public AlarmHistoryRepositoryImpl(@Named("vertica") DBI db) {
this.db = db;
}
@Override
public List<AlarmHistory> findById(String tenantId, String alarmId) {
return null;
Handle h = db.open();
try {
return null;
} catch (RuntimeException e) {
h.rollback();
throw e;
} finally {
h.close();
}
}
}

View File

@ -7,6 +7,7 @@ import java.util.List;
import java.util.Map;
import javax.inject.Inject;
import javax.inject.Named;
import org.skife.jdbi.v2.DBI;
import org.skife.jdbi.v2.Handle;
@ -35,7 +36,7 @@ public class AlarmRepositoryImpl implements AlarmRepository {
private final DBI db;
@Inject
public AlarmRepositoryImpl(DBI db) {
public AlarmRepositoryImpl(@Named("mysql") DBI db) {
this.db = db;
}

View File

@ -1,22 +1,111 @@
package com.hpcloud.mon.infrastructure.persistence;
import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.inject.Named;
import org.joda.time.DateTime;
import org.skife.jdbi.v2.DBI;
import org.skife.jdbi.v2.Handle;
import org.skife.jdbi.v2.Query;
import com.hpcloud.mon.domain.model.measurement.Measurement;
import com.hpcloud.mon.domain.model.measurement.MeasurementRepository;
import com.hpcloud.mon.domain.model.measurement.Measurements;
import com.hpcloud.persistence.SqlQueries;
/**
* Vertica measurement repository implementation.
*/
public class MeasurementRepositoryImpl implements MeasurementRepository {
@Override
public List<Measurement> find(String tenantId, String name, Map<String, String> dimensions,
DateTime startTime, @Nullable DateTime endTime) {
return null;
private static final String FIND_BY_METRIC_DEF_SQL = "select m.metric_definition_id, m.time_stamp, m.value "
+ "from MonMetrics.Measurements m, MonMetrics.Definitions def%s "
+ "where m.metric_definition_id = def.id%s order by m.metric_definition_id";
private final DBI db;
@Inject
public MeasurementRepositoryImpl(@Named("vertica") DBI db) {
this.db = db;
}
@Override
public Collection<Measurements> find(String tenantId, String name,
Map<String, String> dimensions, DateTime startTime, @Nullable DateTime endTime) {
Handle h = db.open();
try {
// Build query
StringBuilder sbFrom = new StringBuilder();
StringBuilder sbWhere = new StringBuilder();
if (dimensions != null) {
for (int i = 0; i < dimensions.size(); i++) {
sbFrom.append(", MonMetrics.Dimensions d").append(i);
sbWhere.append(" and d")
.append(i)
.append(".name = :dname")
.append(i)
.append(" and d")
.append(i)
.append(".value = :dvalue")
.append(i)
.append(" and def.id = d")
.append(i)
.append(".metric_definition_id");
}
}
if (name != null)
sbWhere.append(" and def.name = :name");
String sql = String.format(FIND_BY_METRIC_DEF_SQL, sbFrom.toString(), sbWhere.toString());
Query<Map<String, Object>> query = h.createQuery(sql);
if (name != null)
query.bind("name", name);
if (dimensions != null) {
int i = 0;
for (Iterator<Map.Entry<String, String>> it = dimensions.entrySet().iterator(); it.hasNext(); i++) {
Map.Entry<String, String> entry = it.next();
query.bind("dname" + i, entry.getKey());
query.bind("dvalue" + i, entry.getValue());
}
}
// Execute
List<Map<String, Object>> rows = query.list();
// Build results
Map<ByteBuffer, Measurements> results = new LinkedHashMap<>();
for (Map<String, Object> row : rows) {
byte[] defIdBytes = (byte[]) row.get("metric_definition_id");
ByteBuffer defId = ByteBuffer.wrap(defIdBytes);
long timestamp = ((Timestamp) row.get("time_stamp")).getTime();
double value = (double) row.get("value");
Measurements measurements = results.get(defId);
if (measurements == null) {
Map<String, String> dims = SqlQueries.keyValuesFor(h,
"select name, value from MonMetrics.Dimensions where metric_definition_id = ?",
defIdBytes);
measurements = new Measurements(name, dims, new ArrayList<Measurement>());
results.put(defId, measurements);
}
measurements.addMeasurement(new Measurement(timestamp, value));
}
return results.values();
} finally {
h.close();
}
}
}

View File

@ -4,6 +4,7 @@ import java.util.List;
import java.util.UUID;
import javax.inject.Inject;
import javax.inject.Named;
import org.skife.jdbi.v2.DBI;
import org.skife.jdbi.v2.Handle;
@ -27,7 +28,7 @@ public class NotificationMethodRepositoryImpl implements NotificationMethodRepos
private final DBI db;
@Inject
public NotificationMethodRepositoryImpl(DBI db) {
public NotificationMethodRepositoryImpl(@Named("mysql") DBI db) {
this.db = db;
}

View File

@ -1,6 +1,6 @@
package com.hpcloud.mon.resource;
import java.util.List;
import java.util.Collection;
import java.util.Map;
import javax.inject.Inject;
@ -15,8 +15,8 @@ import org.joda.time.DateTime;
import com.codahale.metrics.annotation.Timed;
import com.hpcloud.mon.app.validation.Validation;
import com.hpcloud.mon.domain.model.measurement.Measurement;
import com.hpcloud.mon.domain.model.measurement.MeasurementRepository;
import com.hpcloud.mon.domain.model.measurement.Measurements;
/**
* Measurement resource implementation.
@ -35,7 +35,7 @@ public class MeasurementResource {
@GET
@Timed
@Produces(MediaType.APPLICATION_JSON)
public List<Measurement> get(@HeaderParam("X-Tenant-Id") String tenantId,
public Collection<Measurements> get(@HeaderParam("X-Tenant-Id") String tenantId,
@QueryParam("name") String name, @QueryParam("dimensions") String dimensionsStr,
@QueryParam("start_time") String startTimeStr, @QueryParam("end_time") String endTimeStr) {

View File

@ -0,0 +1,65 @@
package com.hpcloud.mon.infrastructure.persistence;
import java.util.HashMap;
import java.util.Map;
import org.skife.jdbi.v2.DBI;
import org.skife.jdbi.v2.Handle;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import com.hpcloud.mon.domain.model.measurement.MeasurementRepository;
/**
* @author Jonathan Halterman
*/
@Test(groups = "database")
public class MeasurementRepositoryImplTest {
private DBI db;
private Handle handle;
private MeasurementRepository repo;
@BeforeClass
protected void setupClass() throws Exception {
Class.forName("com.vertica.jdbc.Driver");
db = new DBI("jdbc:vertica://192.168.10.8/mon", "dbadmin", "password");
handle = db.open();
repo = new MeasurementRepositoryImpl(db);
}
@AfterClass
protected void afterClass() {
handle.close();
}
@BeforeMethod
protected void beforeMethod() {
handle.execute("truncate table MonMetrics.Definitions");
handle.execute("truncate table MonMetrics.Dimensions");
handle.execute("truncate table MonMetrics.Measurements");
handle.execute("insert into MonMetrics.Definitions values ('/1', 'cpu_utilization', 'bob', '1')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/1', 'flavor_id', '1')");
handle.execute("insert into MonMetrics.Measurements (metric_definition_id, time_stamp, value) values ('/1', '2014-01-01 00:00:00', 10)");
handle.execute("insert into MonMetrics.Measurements (metric_definition_id, time_stamp, value) values ('/1', '2014-01-01 00:01:00', 15)");
handle.execute("insert into MonMetrics.Definitions values ('/2', 'cpu_utilization', 'bob', '1')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'service', 'compute')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'instance_id', '123')");
handle.execute("insert into MonMetrics.Dimensions values ('/2', 'flavor_id', '2')");
handle.execute("insert into MonMetrics.Measurements (metric_definition_id, time_stamp, value) values ('/2', '2014-01-01 00:00:00', 12)");
handle.execute("insert into MonMetrics.Measurements (metric_definition_id, time_stamp, value) values ('/2', '2014-01-01 00:01:00', 13)");
}
public void shouldFind() {
Map<String, String> dims = new HashMap<>();
dims.put("service", "compute");
dims.put("instance_id", "123");
repo.find("1234", "cpu_utilization", dims, null, null);
}
}

View File

@ -2,8 +2,12 @@ package com.hpcloud.mon.resource;
import static org.testng.Assert.assertEquals;
import java.nio.charset.Charset;
import org.testng.annotations.Test;
import com.google.common.hash.Hashing;
/**
* @author Jonathan Halterman
*/
@ -11,7 +15,8 @@ import org.testng.annotations.Test;
public class LinksTest {
public void shouldPrefixForHttps() {
Links.accessedViaHttps = true;
assertEquals(Links.prefixForHttps("http://abc123blah/blah/blah"), "https://abc123blah/blah/blah");
assertEquals(Links.prefixForHttps("http://abc123blah/blah/blah"),
"https://abc123blah/blah/blah");
assertEquals(Links.prefixForHttps("https://abc123blah/blah/blah"),
"https://abc123blah/blah/blah");
@ -21,4 +26,15 @@ public class LinksTest {
assertEquals(Links.prefixForHttps("https://abc123blah/blah/blah"),
"https://abc123blah/blah/blah");
}
public void foo() throws Exception {
byte[] foo = Hashing.sha1()
.newHasher()
.putString("foo", Charset.defaultCharset())
.putString("bar", Charset.defaultCharset())
.hash()
.asBytes();
String text1 = new String(foo, Charset.forName("US-ASCII"));
int i = 1;
}
}

View File

@ -0,0 +1,79 @@
CREATE SCHEMA MonMetrics;
CREATE TABLE MonMetrics.Measurements (
id AUTO_INCREMENT,
metric_definition_id BINARY(20) NOT NULL,
time_stamp TIMESTAMP NOT NULL,
value FLOAT NOT NULL,
PRIMARY KEY(id)
) PARTITION BY EXTRACT('year' FROM time_stamp)*10000 + EXTRACT('month' FROM time_stamp)*100 + EXTRACT('day' FROM time_stamp);
CREATE TABLE MonMetrics.Definitions (
id BINARY(20) NOT NULL,
name VARCHAR NOT NULL,
tenant_id VARCHAR(14) NOT NULL,
region VARCHAR NOT NULL,
PRIMARY KEY(id),
CONSTRAINT MetricsDefinitionsConstraint UNIQUE(id, name, tenant_id, region)
);
CREATE TABLE MonMetrics.Dimensions (
metric_definition_id BINARY(20) NOT NULL,
name VARCHAR NOT NULL,
value VARCHAR NOT NULL,
CONSTRAINT MetricsDimensionsConstraint UNIQUE(metric_definition_id, name, value)
);
CREATE PROJECTION Measurements_DBD_1_rep_MonMetrics /*+createtype(D)*/
(
id ENCODING AUTO,
metric_definition_id ENCODING RLE,
time_stamp ENCODING DELTAVAL,
value ENCODING AUTO
)
AS
SELECT id,
metric_definition_id,
time_stamp,
value
FROM MonMetrics.Measurements
ORDER BY metric_definition_id,
time_stamp,
id
UNSEGMENTED ALL NODES;
CREATE PROJECTION Definitions_DBD_2_rep_MonMetrics /*+createtype(D)*/
(
id ENCODING RLE,
name ENCODING AUTO,
tenant_id ENCODING RLE,
region ENCODING RLE
)
AS
SELECT id,
name,
tenant_id,
region
FROM MonMetrics.Definitions
ORDER BY id,
tenant_id,
region,
name
UNSEGMENTED ALL NODES;
CREATE PROJECTION Dimensions_DBD_4_rep_MonMetrics /*+createtype(D)*/
(
metric_definition_id ENCODING RLE,
name ENCODING AUTO,
value ENCODING AUTO
)
AS
SELECT metric_definition_id,
name,
value
FROM MonMetrics.Dimensions
ORDER BY metric_definition_id,
name
UNSEGMENTED ALL NODES;
select refresh('MonMetrics.Measurements, MonMetrics.Definitions, MonMetrics.Dimensions');