Changes to handle AlarmUpdatedEvent properly. Now reuses measurements if possible (when only operator or threshold have changed). Added test not run by normal Unit tests.
Go to mon-common version 30. Gives alarmDescription in AlarmUpdatedEvent and new name alarmActionsEnabled
This commit is contained in:
Craig Bryant
parent
feb5433432
commit
e02b6f8483
2
pom.xml
2
pom.xml
|
|
@ -14,7 +14,7 @@
|
|||
</prerequisites>
|
||||
|
||||
<properties>
|
||||
<mon.common.version>1.0.0.30</mon.common.version>
|
||||
<mon.common.version>1.0.0.33</mon.common.version>
|
||||
|
||||
<skipITs>true</skipITs>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
|
|
|
|||
|
|
@ -227,4 +227,8 @@ public class Alarm extends AbstractEntity {
|
|||
public void updateSubAlarm(SubAlarm subAlarm) {
|
||||
subAlarms.put(subAlarm.getId(), subAlarm);
|
||||
}
|
||||
|
||||
public boolean removeSubAlarmById(String toDeleteId) {
|
||||
return subAlarms.remove(toDeleteId) != null;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -111,4 +111,23 @@ public class SubAlarm extends AbstractEntity implements Serializable {
|
|||
return String.format("SubAlarm [id=%s, alarmId=%s, expression=%s, state=%s noState=%s]", id, alarmId,
|
||||
expression, state, noState);
|
||||
}
|
||||
|
||||
/**
|
||||
* Determine if this SubAlarm and 'other' could reuse saved measurements. Only possible
|
||||
* only operator and/or threshold are the only properties from the expression that are different
|
||||
* @param other SubAlarm to compare to
|
||||
* @return true if 'other' is "compatible", false otherwise
|
||||
*/
|
||||
public boolean isCompatible(final SubAlarm other) {
|
||||
if (!this.expression.getMetricDefinition().equals(other.expression.getMetricDefinition()))
|
||||
return false;
|
||||
if (!this.expression.getFunction().equals(other.expression.getFunction()))
|
||||
return false;
|
||||
if (this.expression.getPeriod() != other.expression.getPeriod())
|
||||
return false;
|
||||
if (this.expression.getPeriods() != other.expression.getPeriods())
|
||||
return false;
|
||||
// Operator and Threshold can vary
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ public class SubAlarmStats {
|
|||
private static final int UNDETERMINED_COEFFICIENT = 2;
|
||||
|
||||
private final int slotWidth;
|
||||
private final SubAlarm subAlarm;
|
||||
private SubAlarm subAlarm;
|
||||
private final SlidingWindowStats stats;
|
||||
/** The number of times we can observe an empty window before transitioning to UNDETERMINED state. */
|
||||
final int emptyWindowObservationThreshold;
|
||||
|
|
@ -125,4 +125,13 @@ public class SubAlarmStats {
|
|||
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* This MUST only be used for compatible SubAlarms, i.e. where
|
||||
* this.subAlarm.isCompatible(subAlarm) is true
|
||||
* @param subAlarm
|
||||
*/
|
||||
public void updateSubAlarm(final SubAlarm subAlarm) {
|
||||
this.subAlarm = subAlarm;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,9 @@ import backtype.storm.tuple.Tuple;
|
|||
|
||||
import com.hpcloud.mon.ThresholdingConfiguration;
|
||||
import com.hpcloud.mon.common.event.AlarmStateTransitionedEvent;
|
||||
import com.hpcloud.mon.common.event.AlarmUpdatedEvent;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmState;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmSubExpression;
|
||||
import com.hpcloud.mon.domain.model.Alarm;
|
||||
import com.hpcloud.mon.domain.model.SubAlarm;
|
||||
import com.hpcloud.mon.domain.service.AlarmDAO;
|
||||
|
|
@ -42,7 +44,7 @@ public class AlarmThresholdingBolt extends BaseRichBolt {
|
|||
|
||||
private transient Logger LOG;
|
||||
private DataSourceFactory dbConfig;
|
||||
private final Map<String, Alarm> alarms = new HashMap<String, Alarm>();
|
||||
final Map<String, Alarm> alarms = new HashMap<String, Alarm>();
|
||||
private String alertExchange;
|
||||
private String alertRoutingKey;
|
||||
private transient AlarmDAO alarmDAO;
|
||||
|
|
@ -83,7 +85,7 @@ public class AlarmThresholdingBolt extends BaseRichBolt {
|
|||
if (EventProcessingBolt.DELETED.equals(eventType))
|
||||
handleAlarmDeleted(alarmId);
|
||||
else if (EventProcessingBolt.UPDATED.equals(eventType))
|
||||
handleAlarmUpdated(alarmId);
|
||||
handleAlarmUpdated(alarmId, (AlarmUpdatedEvent) tuple.getValue(2));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
LOG.error("Error processing tuple {}", tuple, e);
|
||||
|
|
@ -155,23 +157,52 @@ public class AlarmThresholdingBolt extends BaseRichBolt {
|
|||
alarms.remove(alarmId);
|
||||
}
|
||||
|
||||
void handleAlarmUpdated(String alarmId) {
|
||||
// Just flush the Alarm from our cache so it gets read fresh from the database
|
||||
// when a sub alarm is received
|
||||
LOG.debug("Received AlarmUpdatedEvent for alarm id {}", alarmId);
|
||||
alarms.remove(alarmId);
|
||||
final Alarm alarm = alarmDAO.findById(alarmId);
|
||||
if (alarm == null)
|
||||
LOG.error("Failed to locate alarm for id {}", alarmId);
|
||||
else {
|
||||
// TODO - Should the API be doing this? If so, does it also do the kafka event?
|
||||
if (alarm.getState() != AlarmState.UNDETERMINED) {
|
||||
final AlarmState initialState = alarm.getState();
|
||||
alarm.setState(AlarmState.UNDETERMINED);
|
||||
changeAlarmState(alarm, initialState, "Alarm updated by User");
|
||||
}
|
||||
alarms.put(alarmId, alarm);
|
||||
void handleAlarmUpdated(String alarmId, AlarmUpdatedEvent alarmUpdatedEvent) {
|
||||
final Alarm oldAlarm = alarms.get(alarmId);
|
||||
if (oldAlarm == null) {
|
||||
LOG.debug("Updated Alarm {} not loaded, ignoring");
|
||||
return;
|
||||
}
|
||||
|
||||
oldAlarm.setName(alarmUpdatedEvent.alarmName);
|
||||
oldAlarm.setDescription(alarmUpdatedEvent.alarmDescription);
|
||||
oldAlarm.setExpression(alarmUpdatedEvent.alarmExpression);
|
||||
oldAlarm.setState(alarmUpdatedEvent.alarmState);
|
||||
oldAlarm.setActionsEnabled(alarmUpdatedEvent.alarmActionsEnabled);
|
||||
|
||||
// Now handle the SubAlarms
|
||||
// First remove the deleted SubAlarms so we don't have to consider them later
|
||||
for (Map.Entry<String, AlarmSubExpression> entry : alarmUpdatedEvent.oldAlarmSubExpressions.entrySet()) {
|
||||
LOG.debug("Removing deleted SubAlarm {}", entry.getValue());
|
||||
if (!oldAlarm.removeSubAlarmById(entry.getKey()))
|
||||
LOG.error("Did not find removed SubAlarm {}", entry.getValue());
|
||||
}
|
||||
|
||||
// Reuse what we can from the changed SubAlarms
|
||||
for (Map.Entry<String, AlarmSubExpression> entry : alarmUpdatedEvent.changedSubExpressions.entrySet()) {
|
||||
final SubAlarm oldSubAlarm = oldAlarm.getSubAlarm(entry.getKey());
|
||||
if (oldSubAlarm == null) {
|
||||
LOG.error("Did not find changed SubAlarm {}", entry.getValue());
|
||||
continue;
|
||||
}
|
||||
final SubAlarm newSubAlarm = new SubAlarm(entry.getKey(), oldAlarm.getId(), entry.getValue());
|
||||
newSubAlarm.setState(oldSubAlarm.getState());
|
||||
if (!oldSubAlarm.isCompatible(newSubAlarm)) {
|
||||
newSubAlarm.setNoState(true);
|
||||
}
|
||||
LOG.debug("Changing SubAlarm from {} to {}", oldSubAlarm, newSubAlarm);
|
||||
oldAlarm.updateSubAlarm(newSubAlarm);
|
||||
}
|
||||
|
||||
// Add the new SubAlarms
|
||||
for (Map.Entry<String, AlarmSubExpression> entry : alarmUpdatedEvent.newAlarmSubExpressions.entrySet()) {
|
||||
final SubAlarm newSubAlarm = new SubAlarm(entry.getKey(), oldAlarm.getId(), entry.getValue());
|
||||
newSubAlarm.setNoState(true);
|
||||
LOG.debug("Adding SubAlarm {}", newSubAlarm);
|
||||
oldAlarm.updateSubAlarm(newSubAlarm);
|
||||
}
|
||||
|
||||
alarms.put(alarmId, oldAlarm);
|
||||
}
|
||||
|
||||
String buildStateChangeReason() {
|
||||
|
|
|
|||
|
|
@ -46,7 +46,7 @@ public class EventProcessingBolt extends BaseRichBolt {
|
|||
/** Stream for metric and sub-alarm specific events. */
|
||||
public static final String METRIC_SUB_ALARM_EVENT_STREAM_ID = "metric-sub-alarm-events";
|
||||
|
||||
public static final String[] ALARM_EVENT_STREAM_FIELDS = new String[] {"eventType", "alarmId"};
|
||||
public static final String[] ALARM_EVENT_STREAM_FIELDS = new String[] {"eventType", "alarmId", "alarm"};
|
||||
public static final String[] METRIC_ALARM_EVENT_STREAM_FIELDS = new String[] {"eventType",
|
||||
"metricDefinitionAndTenantId", "subAlarmId"};
|
||||
public static final String[] METRIC_SUB_ALARM_EVENT_STREAM_FIELDS = new String[] {"eventType",
|
||||
|
|
@ -99,8 +99,17 @@ public class EventProcessingBolt extends BaseRichBolt {
|
|||
}
|
||||
|
||||
private void sendAddSubAlarm(String alarmId, String subAlarmId, String tenantId, AlarmSubExpression alarmSubExpression) {
|
||||
sendSubAlarm(CREATED, alarmId, subAlarmId, tenantId, alarmSubExpression);
|
||||
}
|
||||
|
||||
private void sendUpdateSubAlarm(String alarmId, String subAlarmId, String tenantId, AlarmSubExpression alarmSubExpression) {
|
||||
sendSubAlarm(UPDATED, alarmId, subAlarmId, tenantId, alarmSubExpression);
|
||||
}
|
||||
|
||||
private void sendSubAlarm(String eventType, String alarmId, String subAlarmId, String tenantId,
|
||||
AlarmSubExpression alarmSubExpression) {
|
||||
MetricDefinition metricDef = alarmSubExpression.getMetricDefinition();
|
||||
collector.emit(METRIC_SUB_ALARM_EVENT_STREAM_ID, new Values(CREATED, new MetricDefinitionAndTenantId(metricDef, tenantId),
|
||||
collector.emit(METRIC_SUB_ALARM_EVENT_STREAM_ID, new Values(eventType, new MetricDefinitionAndTenantId(metricDef, tenantId),
|
||||
new SubAlarm(subAlarmId, alarmId, alarmSubExpression)));
|
||||
}
|
||||
|
||||
|
|
@ -109,7 +118,7 @@ public class EventProcessingBolt extends BaseRichBolt {
|
|||
sendDeletedSubAlarm(entry.getKey(), event.tenantId, entry.getValue());
|
||||
}
|
||||
|
||||
collector.emit(ALARM_EVENT_STREAM_ID, new Values(DELETED, event.alarmId));
|
||||
collector.emit(ALARM_EVENT_STREAM_ID, new Values(DELETED, event.alarmId, event));
|
||||
}
|
||||
|
||||
private void sendDeletedSubAlarm(String subAlarmId, String tenantId, MetricDefinition metricDef) {
|
||||
|
|
@ -121,9 +130,12 @@ public class EventProcessingBolt extends BaseRichBolt {
|
|||
for (Map.Entry<String, AlarmSubExpression> entry : event.oldAlarmSubExpressions.entrySet()) {
|
||||
sendDeletedSubAlarm(entry.getKey(), event.tenantId, entry.getValue().getMetricDefinition());
|
||||
}
|
||||
for (Map.Entry<String, AlarmSubExpression> entry : event.changedSubExpressions.entrySet()) {
|
||||
sendUpdateSubAlarm(event.alarmId, entry.getKey(), event.tenantId, entry.getValue());
|
||||
}
|
||||
for (Map.Entry<String, AlarmSubExpression> entry : event.newAlarmSubExpressions.entrySet()) {
|
||||
sendAddSubAlarm(event.alarmId, entry.getKey(), event.tenantId, entry.getValue());
|
||||
}
|
||||
collector.emit(ALARM_EVENT_STREAM_ID, new Values(UPDATED, event.alarmId));
|
||||
collector.emit(ALARM_EVENT_STREAM_ID, new Values(UPDATED, event.alarmId, event));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,6 +100,8 @@ public class MetricAggregationBolt extends BaseRichBolt {
|
|||
SubAlarm subAlarm = (SubAlarm) tuple.getValue(2);
|
||||
if (EventProcessingBolt.CREATED.equals(eventType))
|
||||
handleAlarmCreated(metricDefinitionAndTenantId, subAlarm);
|
||||
else if (EventProcessingBolt.UPDATED.equals(eventType))
|
||||
handleAlarmUpdated(metricDefinitionAndTenantId, subAlarm);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -198,6 +200,10 @@ public class MetricAggregationBolt extends BaseRichBolt {
|
|||
*/
|
||||
void handleAlarmCreated(MetricDefinitionAndTenantId metricDefinitionAndTenantId, SubAlarm subAlarm) {
|
||||
LOG.debug("Received AlarmCreatedEvent for {}", subAlarm);
|
||||
addSubAlarm(metricDefinitionAndTenantId, subAlarm);
|
||||
}
|
||||
|
||||
private void addSubAlarm(MetricDefinitionAndTenantId metricDefinitionAndTenantId, SubAlarm subAlarm) {
|
||||
SubAlarmStatsRepository subAlarmStatsRepo = getOrCreateSubAlarmStatsRepo(metricDefinitionAndTenantId);
|
||||
if (subAlarmStatsRepo == null)
|
||||
return;
|
||||
|
|
@ -207,6 +213,39 @@ public class MetricAggregationBolt extends BaseRichBolt {
|
|||
}
|
||||
|
||||
/**
|
||||
* Adds the {@code subAlarm} subAlarmStatsRepo for the {@code metricDefinition}.
|
||||
*
|
||||
* MetricDefinition can't have changed, just how it is evaluated
|
||||
*/
|
||||
void handleAlarmUpdated(MetricDefinitionAndTenantId metricDefinitionAndTenantId, SubAlarm subAlarm) {
|
||||
LOG.debug("Received AlarmUpdatedEvent for {}", subAlarm);
|
||||
// Clear the old SubAlarm, but save the SubAlarm state
|
||||
final SubAlarmStatsRepository oldSubAlarmStatsRepo = subAlarmStatsRepos.get(metricDefinitionAndTenantId);
|
||||
if (oldSubAlarmStatsRepo == null) {
|
||||
LOG.error("Did not find SubAlarmStatsRepository for MetricDefinition {}", metricDefinitionAndTenantId);
|
||||
}
|
||||
else {
|
||||
final SubAlarmStats oldSubAlarmStats = oldSubAlarmStatsRepo.get(subAlarm.getId());
|
||||
if (oldSubAlarmStats == null)
|
||||
LOG.error("Did not find existing SubAlarm {} in SubAlarmStatsRepository", subAlarm);
|
||||
else {
|
||||
final SubAlarm oldSubAlarm = oldSubAlarmStats.getSubAlarm();
|
||||
subAlarm.setState(oldSubAlarm.getState());
|
||||
subAlarm.setNoState(true); // Doesn't hurt to send too many state changes, just too few
|
||||
if (oldSubAlarm.isCompatible(subAlarm)) {
|
||||
LOG.debug("Changing SubAlarm {} to SubAlarm {} and keeping measurements", oldSubAlarm, subAlarm);
|
||||
oldSubAlarmStats.updateSubAlarm(subAlarm);
|
||||
return;
|
||||
}
|
||||
// Have to completely change the SubAlarmStats
|
||||
LOG.debug("Changing SubAlarm {} to SubAlarm {} and flushing measurements", oldSubAlarm, subAlarm);
|
||||
oldSubAlarmStatsRepo.remove(subAlarm.getId());
|
||||
}
|
||||
}
|
||||
addSubAlarm(metricDefinitionAndTenantId, subAlarm);
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes the sub-alarm for the {@code subAlarmId} from the subAlarmStatsRepo for the
|
||||
* {@code metricDefinitionAndTenantId}.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -104,6 +104,7 @@ public class MetricFilteringBolt extends BaseRichBolt {
|
|||
MetricDefinitionAndTenantId metricDefinitionAndTenantId = (MetricDefinitionAndTenantId) tuple.getValue(1);
|
||||
|
||||
LOG.debug("Received {} for {}", eventType, metricDefinitionAndTenantId);
|
||||
// UPDATED events can be ignored because the MetricDefinitionAndTenantId doesn't change
|
||||
if (EventProcessingBolt.METRIC_ALARM_EVENT_STREAM_ID.equals(tuple.getSourceStreamId())) {
|
||||
if (EventProcessingBolt.DELETED.equals(eventType))
|
||||
removeSubAlarm(metricDefinitionAndTenantId, tuple.getString(2));
|
||||
|
|
|
|||
|
|
@ -0,0 +1,273 @@
|
|||
package com.hpcloud.mon;
|
||||
|
||||
import static org.mockito.Matchers.any;
|
||||
import static org.mockito.Matchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.stubbing.Answer;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
import static org.testng.Assert.assertEquals;
|
||||
import static org.testng.Assert.assertNotEquals;
|
||||
import backtype.storm.Config;
|
||||
import backtype.storm.testing.FeederSpout;
|
||||
import backtype.storm.tuple.Fields;
|
||||
import backtype.storm.tuple.Values;
|
||||
|
||||
import com.google.inject.AbstractModule;
|
||||
import com.hpcloud.configuration.KafkaProducerConfiguration;
|
||||
import com.hpcloud.mon.common.event.AlarmCreatedEvent;
|
||||
import com.hpcloud.mon.common.event.AlarmStateTransitionedEvent;
|
||||
import com.hpcloud.mon.common.event.AlarmUpdatedEvent;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmExpression;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmState;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmSubExpression;
|
||||
import com.hpcloud.mon.common.model.metric.Metric;
|
||||
import com.hpcloud.mon.domain.model.Alarm;
|
||||
import com.hpcloud.mon.domain.model.MetricDefinitionAndTenantId;
|
||||
import com.hpcloud.mon.domain.model.SubAlarm;
|
||||
import com.hpcloud.mon.domain.service.AlarmDAO;
|
||||
import com.hpcloud.mon.domain.service.MetricDefinitionDAO;
|
||||
import com.hpcloud.mon.domain.service.SubAlarmDAO;
|
||||
import com.hpcloud.mon.domain.service.SubAlarmMetricDefinition;
|
||||
import com.hpcloud.mon.infrastructure.thresholding.AlarmEventForwarder;
|
||||
import com.hpcloud.mon.infrastructure.thresholding.EventProcessingBoltTest;
|
||||
import com.hpcloud.mon.infrastructure.thresholding.MetricAggregationBolt;
|
||||
import com.hpcloud.mon.infrastructure.thresholding.MetricSpout;
|
||||
import com.hpcloud.streaming.storm.TopologyTestCase;
|
||||
import com.hpcloud.util.Injector;
|
||||
import com.hpcloud.util.Serialization;
|
||||
|
||||
/**
|
||||
* Simulates a real'ish run of the thresholding engine with alarms being created, updated and deleted
|
||||
*/
|
||||
@Test(groups = "integration")
|
||||
public class ThresholdingEngineAlarmTest extends TopologyTestCase {
|
||||
private static final String TEST_ALARM_TENANT_ID = "bob";
|
||||
private static final String TEST_ALARM_ID = "1";
|
||||
private static final String TEST_ALARM_NAME = "test-alarm";
|
||||
private static final String TEST_ALARM_DESCRIPTION = "Description of test-alarm";
|
||||
private FeederSpout metricSpout;
|
||||
private FeederSpout eventSpout;
|
||||
private AlarmDAO alarmDAO;
|
||||
private SubAlarmDAO subAlarmDAO;
|
||||
private MetricDefinitionDAO metricDefinitionDAO;
|
||||
private final AlarmEventForwarder alarmEventForwarder;
|
||||
private int nextSubAlarmId = 4242;
|
||||
private List<SubAlarm> subAlarms;
|
||||
private AlarmExpression expression = new AlarmExpression(
|
||||
"max(hpcs.compute.cpu{id=5}) >= 3 or max(hpcs.compute.mem{id=5}) >= 557");
|
||||
|
||||
private AlarmState currentState = AlarmState.OK;
|
||||
private volatile int alarmsSent = 0;
|
||||
|
||||
public ThresholdingEngineAlarmTest() {
|
||||
// Fixtures
|
||||
|
||||
subAlarms = subAlarmsFor(TEST_ALARM_ID, expression);
|
||||
|
||||
// Mocks
|
||||
alarmDAO = mock(AlarmDAO.class);
|
||||
when(alarmDAO.findById(anyString())).thenAnswer(new Answer<Alarm>() {
|
||||
@Override
|
||||
public Alarm answer(InvocationOnMock invocation) throws Throwable {
|
||||
return new Alarm(TEST_ALARM_ID, TEST_ALARM_TENANT_ID, TEST_ALARM_NAME,
|
||||
TEST_ALARM_DESCRIPTION, expression, subAlarms, currentState, Boolean.TRUE);
|
||||
}
|
||||
});
|
||||
|
||||
subAlarmDAO = mock(SubAlarmDAO.class);
|
||||
when(subAlarmDAO.find(any(MetricDefinitionAndTenantId.class))).thenAnswer(new Answer<List<SubAlarm>>() {
|
||||
@Override
|
||||
public List<SubAlarm> answer(InvocationOnMock invocation) throws Throwable {
|
||||
MetricDefinitionAndTenantId metricDefinitionAndTenantId = (MetricDefinitionAndTenantId) invocation.getArguments()[0];
|
||||
for (final SubAlarm subAlarm : subAlarms) {
|
||||
if (metricDefinitionAndTenantId.metricDefinition.equals(subAlarm.getExpression().getMetricDefinition())) {
|
||||
return Arrays.asList(subAlarm);
|
||||
}
|
||||
}
|
||||
return Collections.emptyList();
|
||||
}
|
||||
});
|
||||
|
||||
metricDefinitionDAO = mock(MetricDefinitionDAO.class);
|
||||
List<SubAlarmMetricDefinition> metricDefs = new ArrayList<>(0);
|
||||
when(metricDefinitionDAO.findForAlarms()).thenReturn(metricDefs);
|
||||
|
||||
// Bindings
|
||||
Injector.reset();
|
||||
Injector.registerModules(new AbstractModule() {
|
||||
protected void configure() {
|
||||
bind(AlarmDAO.class).toInstance(alarmDAO);
|
||||
bind(SubAlarmDAO.class).toInstance(subAlarmDAO);
|
||||
bind(MetricDefinitionDAO.class).toInstance(metricDefinitionDAO);
|
||||
}
|
||||
});
|
||||
|
||||
// Config
|
||||
ThresholdingConfiguration threshConfig = new ThresholdingConfiguration();
|
||||
threshConfig.sporadicMetricNamespaces = new HashSet<String>();
|
||||
Serialization.registerTarget(KafkaProducerConfiguration.class);
|
||||
|
||||
threshConfig.kafkaProducerConfig = Serialization.fromJson("{\"KafkaProducerConfiguration\":{\"topic\":\"alarm-state-transitions\",\"metadataBrokerList\":\"192.168.10.10:9092\",\"requestRequiredAcks\":1,\"requestTimeoutMs\":10000,\"producerType\":\"sync\",\"serializerClass\":\"kafka.serializer.StringEncoder\",\"keySerializerClass\":\"\",\"partitionerClass\":\"\",\"compressionCodec\":\"none\",\"compressedTopics\":\"\",\"messageSendMaxRetries\":3,\"retryBackoffMs\":100,\"topicMetadataRefreshIntervalMs\":600000,\"queueBufferingMaxMs\":5000,\"queueBufferingMaxMessages\":10000,\"queueEnqueueTimeoutMs\":-1,\"batchNumMessages\":200,\"sendBufferBytes\":102400,\"clientId\":\"Threshold_Engine\"}}");
|
||||
Config stormConfig = new Config();
|
||||
stormConfig.setMaxTaskParallelism(1);
|
||||
metricSpout = new FeederSpout(new Fields(MetricSpout.FIELDS));
|
||||
eventSpout = new FeederSpout(new Fields("event"));
|
||||
alarmEventForwarder = mock(AlarmEventForwarder.class);
|
||||
Injector.registerModules(new TopologyModule(threshConfig, stormConfig,
|
||||
metricSpout, eventSpout, alarmEventForwarder));
|
||||
|
||||
// Evaluate alarm stats every 1 seconds
|
||||
System.setProperty(MetricAggregationBolt.TICK_TUPLE_SECONDS_KEY, "5");
|
||||
}
|
||||
|
||||
private List<SubAlarm> subAlarmsFor(final String alarmId,
|
||||
final AlarmExpression expression,
|
||||
final String ... ids) {
|
||||
final List<SubAlarm> result = new ArrayList<SubAlarm>(expression.getSubExpressions().size());
|
||||
int index = 0;
|
||||
for (final AlarmSubExpression expr : expression.getSubExpressions()) {
|
||||
final String id;
|
||||
if ((index >= ids.length) || (ids[index] == null)) {
|
||||
id = String.valueOf(nextSubAlarmId++);
|
||||
}
|
||||
else {
|
||||
id = ids[index];
|
||||
}
|
||||
index++;
|
||||
result.add(new SubAlarm(id, TEST_ALARM_ID, expr));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
final AlarmState[] expectedStates = { AlarmState.ALARM, AlarmState.OK, AlarmState.ALARM };
|
||||
public void shouldThreshold() throws Exception {
|
||||
doAnswer(new Answer<Object>() {
|
||||
public Object answer(InvocationOnMock invocation) {
|
||||
final Object[] args = invocation.getArguments();
|
||||
AlarmStateTransitionedEvent event = Serialization.fromJson((String)args[2]);
|
||||
System.out.printf("Alarm transitioned from %s to %s%n", event.oldState, event.newState);
|
||||
assertEquals(event.alarmName, TEST_ALARM_NAME);
|
||||
assertEquals(event.alarmId, TEST_ALARM_ID);
|
||||
assertEquals(event.tenantId, TEST_ALARM_TENANT_ID);
|
||||
assertEquals(event.oldState, currentState);
|
||||
currentState = event.newState;
|
||||
assertEquals(event.newState, expectedStates[alarmsSent++]);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
)
|
||||
.when(alarmEventForwarder).send(anyString(), anyString(), anyString());
|
||||
int waitCount = 0;
|
||||
int feedCount = 5;
|
||||
int goodValueCount = 0;
|
||||
boolean firstUpdate = true;
|
||||
boolean secondUpdate = true;
|
||||
final Alarm initialAlarm = new Alarm(TEST_ALARM_ID, TEST_ALARM_TENANT_ID, TEST_ALARM_NAME,
|
||||
TEST_ALARM_DESCRIPTION, expression, subAlarms, AlarmState.UNDETERMINED, Boolean.TRUE);
|
||||
final int expectedAlarms = expectedStates.length;
|
||||
for (int i = 1; alarmsSent != expectedAlarms && i < 150; i++) {
|
||||
if (i == 5) {
|
||||
final Map<String, AlarmSubExpression> exprs = createSubExpressionMap();
|
||||
final AlarmCreatedEvent event = new AlarmCreatedEvent(TEST_ALARM_TENANT_ID, TEST_ALARM_ID, TEST_ALARM_NAME,
|
||||
expression.getExpression(), exprs);
|
||||
eventSpout.feed(new Values(event));
|
||||
System.out.printf("Send AlarmCreatedEvent for expression %s%n", expression.getExpression());
|
||||
}
|
||||
else if (alarmsSent == 1 && firstUpdate) {
|
||||
firstUpdate = false;
|
||||
final String originalExpression = expression.getExpression();
|
||||
expression = new AlarmExpression(originalExpression.replace(">= 3", ">= 556"));
|
||||
assertNotEquals(expression.getExpression(), originalExpression);
|
||||
final List<SubAlarm> updatedSubAlarms = new ArrayList<>();
|
||||
updatedSubAlarms.add(new SubAlarm(subAlarms.get(0).getId(), initialAlarm.getId(), expression.getSubExpressions().get(0)));
|
||||
for (int index = 1; index < subAlarms.size(); index++) {
|
||||
final SubAlarm subAlarm = subAlarms.get(index);
|
||||
updatedSubAlarms.add(new SubAlarm(subAlarm.getId(), initialAlarm.getId(), subAlarm.getExpression()));
|
||||
}
|
||||
|
||||
final AlarmUpdatedEvent event = EventProcessingBoltTest.createAlarmUpdatedEvent(initialAlarm, expression, updatedSubAlarms);
|
||||
event.alarmState = currentState;
|
||||
subAlarms = updatedSubAlarms;
|
||||
eventSpout.feed(new Values(event));
|
||||
|
||||
System.out.printf("Send AlarmUpdatedEvent for expression %s%n", expression.getExpression());
|
||||
}
|
||||
else if (alarmsSent == 2 && secondUpdate) {
|
||||
secondUpdate = false;
|
||||
expression = new AlarmExpression("max(hpcs.compute.load{id=5}) > 551 and (" + expression.getExpression().replace("556", "554") + ")");
|
||||
final List<SubAlarm> updatedSubAlarms = new ArrayList<>();
|
||||
updatedSubAlarms.add(new SubAlarm(UUID.randomUUID().toString(), initialAlarm.getId(), expression.getSubExpressions().get(0)));
|
||||
for (int index = 0; index < subAlarms.size(); index++) {
|
||||
updatedSubAlarms.add(new SubAlarm(subAlarms.get(index).getId(), initialAlarm.getId(), expression.getSubExpressions().get(index+1)));
|
||||
}
|
||||
|
||||
final AlarmUpdatedEvent event = EventProcessingBoltTest.createAlarmUpdatedEvent(initialAlarm, expression, updatedSubAlarms);
|
||||
event.alarmState = currentState;
|
||||
subAlarms = updatedSubAlarms;
|
||||
eventSpout.feed(new Values(event));
|
||||
|
||||
System.out.printf("Send AlarmUpdatedEvent for expression %s%n", expression.getExpression());
|
||||
}
|
||||
if (feedCount > 0) {
|
||||
System.out.println("Feeding metrics...");
|
||||
|
||||
long time = System.currentTimeMillis() / 1000;
|
||||
++goodValueCount;
|
||||
for (final SubAlarm subAlarm : subAlarms) {
|
||||
final MetricDefinitionAndTenantId metricDefinitionAndTenantId =
|
||||
new MetricDefinitionAndTenantId(subAlarm.getExpression().getMetricDefinition(), TEST_ALARM_TENANT_ID);
|
||||
metricSpout.feed(new Values(metricDefinitionAndTenantId,
|
||||
new Metric(metricDefinitionAndTenantId.metricDefinition, time, (double) (goodValueCount == 15 ? 1 : 555))));
|
||||
}
|
||||
|
||||
if (--feedCount == 0)
|
||||
waitCount = 3;
|
||||
|
||||
if (goodValueCount == 15)
|
||||
goodValueCount = 0;
|
||||
} else {
|
||||
System.out.println("Waiting...");
|
||||
if (--waitCount == 0)
|
||||
feedCount = 5;
|
||||
}
|
||||
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
for (int i = 0; alarmsSent != expectedAlarms && i < 60; i++) {
|
||||
try {
|
||||
Thread.sleep(1000);
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
assertEquals(alarmsSent, expectedAlarms);
|
||||
assertEquals(currentState, AlarmState.ALARM);
|
||||
}
|
||||
|
||||
private Map<String, AlarmSubExpression> createSubExpressionMap() {
|
||||
final Map<String, AlarmSubExpression> exprs = new HashMap<>();
|
||||
for (final SubAlarm subAlarm : subAlarms) {
|
||||
exprs.put(subAlarm.getId(), subAlarm.getExpression());
|
||||
}
|
||||
return exprs;
|
||||
}
|
||||
}
|
||||
|
|
@ -4,14 +4,15 @@ import static org.mockito.Mockito.mock;
|
|||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.testng.Assert.assertEquals;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.testng.annotations.BeforeClass;
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
import org.testng.annotations.Test;
|
||||
|
||||
|
|
@ -22,6 +23,7 @@ import backtype.storm.testing.MkTupleParam;
|
|||
import backtype.storm.tuple.Tuple;
|
||||
|
||||
import com.hpcloud.mon.ThresholdingConfiguration;
|
||||
import com.hpcloud.mon.common.event.AlarmUpdatedEvent;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmExpression;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmState;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmSubExpression;
|
||||
|
|
@ -35,6 +37,7 @@ public class AlarmThresholdingBoltTest {
|
|||
|
||||
private static final String ALERT_ROUTING_KEY = "Alert Routing Key";
|
||||
private static final String ALERTS_EXCHANGE = "Alerts";
|
||||
private static final String tenantId = "AAAAABBBBBBCCCCC";
|
||||
|
||||
private AlarmExpression alarmExpression;
|
||||
private Alarm alarm;
|
||||
|
|
@ -44,12 +47,21 @@ public class AlarmThresholdingBoltTest {
|
|||
private AlarmDAO alarmDAO;
|
||||
private AlarmThresholdingBolt bolt;
|
||||
private OutputCollector collector;
|
||||
private final String[] subExpressions = {
|
||||
"avg(cpu{instance_id=123,device=42}, 1) > 5",
|
||||
"max(load{instance_id=123,device=42}, 1) > 8",
|
||||
"sum(diskio{instance_id=123,device=42}, 1) > 5000" };
|
||||
|
||||
@BeforeClass
|
||||
protected void beforeClass() {
|
||||
@BeforeMethod
|
||||
protected void beforeMethod() {
|
||||
final String alarmId = "111111112222222222233333333334";
|
||||
final String tenantId = "AAAAABBBBBBCCCCC";
|
||||
final String expression = "avg(hpcs.compute.cpu{instance_id=123,device=42}, 1) > 5";
|
||||
final StringBuilder builder = new StringBuilder();
|
||||
for (final String subExpression : subExpressions) {
|
||||
if (builder.length() > 0)
|
||||
builder.append(" or ");
|
||||
builder.append(subExpression);
|
||||
}
|
||||
final String expression = builder.toString();
|
||||
alarm = new Alarm();
|
||||
alarm.setName("Test CPU Alarm");
|
||||
alarm.setDescription("Description of Alarm");
|
||||
|
|
@ -60,16 +72,12 @@ public class AlarmThresholdingBoltTest {
|
|||
alarmExpression = new AlarmExpression(expression);
|
||||
final List<AlarmSubExpression> subExpressions = alarmExpression.getSubExpressions();
|
||||
subAlarms = new ArrayList<SubAlarm>(subExpressions.size());
|
||||
int subAlarmId = 4242;
|
||||
for (int i = 0; i < subExpressions.size(); i++) {
|
||||
final SubAlarm subAlarm = new SubAlarm(String.valueOf(subAlarmId++), alarmId, subExpressions.get(i));
|
||||
final SubAlarm subAlarm = new SubAlarm(UUID.randomUUID().toString(), alarmId, subExpressions.get(i));
|
||||
subAlarms.add(subAlarm);
|
||||
}
|
||||
alarm.setSubAlarms(subAlarms);
|
||||
}
|
||||
|
||||
@BeforeMethod
|
||||
protected void beforeMethod() {
|
||||
alarmEventForwarder = mock(AlarmEventForwarder.class);
|
||||
alarmDAO = mock(AlarmDAO.class);
|
||||
bolt = new MockAlarmThreshholdBolt(alarmDAO, alarmEventForwarder);
|
||||
|
|
@ -88,34 +96,124 @@ public class AlarmThresholdingBoltTest {
|
|||
*/
|
||||
public void simpleAlarmCreation() {
|
||||
final SubAlarm subAlarm = subAlarms.get(0);
|
||||
subAlarm.setState(AlarmState.ALARM);
|
||||
when(alarmDAO.findById(alarm.getId())).thenReturn(alarm);
|
||||
MkTupleParam tupleParam = new MkTupleParam();
|
||||
tupleParam.setFields("alarmId", "subAlarm");
|
||||
tupleParam.setStream(Streams.DEFAULT_STREAM_ID);
|
||||
final Tuple tuple = Testing.testTuple(Arrays.asList(alarm.getId(), subAlarm), tupleParam);
|
||||
bolt.execute(tuple);
|
||||
bolt.execute(tuple);
|
||||
verify(collector, times(2)).ack(tuple);
|
||||
final String alarmJson = "{\"alarm-transitioned\":{\"tenantId\":\"AAAAABBBBBBCCCCC\"," +
|
||||
final String alarmId = alarm.getId();
|
||||
when(alarmDAO.findById(alarmId)).thenReturn(alarm);
|
||||
emitSubAlarmStateChange(alarmId, subAlarm, AlarmState.ALARM);
|
||||
for (int i = 1; i < subAlarms.size(); i++) {
|
||||
emitSubAlarmStateChange(alarmId, subAlarms.get(i), AlarmState.OK);
|
||||
}
|
||||
final String alarmJson = "{\"alarm-transitioned\":{\"tenantId\":\"" + tenantId + "\"," +
|
||||
"\"alarmId\":\"111111112222222222233333333334\",\"alarmName\":\"Test CPU Alarm\"," +
|
||||
"\"alarmDescription\":\"Description of Alarm\",\"oldState\":\"OK\",\"newState\":\"ALARM\"," +
|
||||
"\"stateChangeReason\":\"Thresholds were exceeded for the sub-alarms: [avg(hpcs.compute.cpu{device=42, instance_id=123}, 1) > 5.0]\"," +
|
||||
"\"stateChangeReason\":\"Thresholds were exceeded for the sub-alarms: [" + subAlarm.getExpression().getExpression() + "]\"," +
|
||||
"\"timestamp\":1395587091}}";
|
||||
|
||||
verify(alarmEventForwarder, times(1)).send(ALERTS_EXCHANGE, ALERT_ROUTING_KEY, alarmJson);
|
||||
verify(alarmDAO, times(1)).updateState(alarm.getId(), AlarmState.ALARM);
|
||||
verify(alarmDAO, times(1)).updateState(alarmId, AlarmState.ALARM);
|
||||
|
||||
// Now clear the alarm and ensure another notification gets sent out
|
||||
subAlarm.setState(AlarmState.OK);
|
||||
final Tuple clearTuple = Testing.testTuple(Arrays.asList(alarm.getId(), subAlarm), tupleParam);
|
||||
final Tuple clearTuple = createSubAlarmStateChangeTuple(alarmId, subAlarm);
|
||||
bolt.execute(clearTuple);
|
||||
verify(collector, times(1)).ack(clearTuple);
|
||||
final String okJson = "{\"alarm-transitioned\":{\"tenantId\":\"AAAAABBBBBBCCCCC\",\"alarmId\":\"111111112222222222233333333334\",\"alarmName\":\"Test CPU Alarm\",\"alarmDescription\":\"Description of Alarm\",\"oldState\":\"ALARM\",\"newState\":\"OK\",\"stateChangeReason\":\"The alarm threshold(s) have not been exceeded\",\"timestamp\":1395587091}}";
|
||||
verify(alarmEventForwarder, times(1)).send(ALERTS_EXCHANGE, ALERT_ROUTING_KEY, okJson);
|
||||
verify(alarmDAO, times(1)).updateState(alarm.getId(), AlarmState.OK);
|
||||
verify(alarmDAO, times(1)).updateState(alarmId, AlarmState.OK);
|
||||
}
|
||||
|
||||
public void simpleAlarmUpdate() {
|
||||
String alarmId = setUpInitialAlarm();
|
||||
|
||||
// Now send an AlarmUpdatedEvent
|
||||
final Map<String, AlarmSubExpression> empty = new HashMap<>();
|
||||
final String newName = "New Name";
|
||||
final String newDescription = "New Description";
|
||||
final AlarmState newState = AlarmState.OK;
|
||||
boolean newEnabled = false;
|
||||
final AlarmUpdatedEvent event = new AlarmUpdatedEvent(tenantId, alarmId, newName, newDescription, alarm.getAlarmExpression().getExpression(),
|
||||
newState, newEnabled, empty, empty, empty);
|
||||
final Tuple updateTuple = createAlarmUpdateTuple(event);
|
||||
bolt.execute(updateTuple);
|
||||
verify(collector, times(1)).ack(updateTuple);
|
||||
assertEquals(alarm.getName(), newName);
|
||||
assertEquals(alarm.getState(), newState);
|
||||
assertEquals(alarm.isActionsEnabled(), newEnabled);
|
||||
}
|
||||
|
||||
public void complexAlarmUpdate() {
|
||||
String alarmId = setUpInitialAlarm();
|
||||
|
||||
// Now send an AlarmUpdatedEvent
|
||||
final Map<String, AlarmSubExpression> newSubExpressions = new HashMap<>();
|
||||
final Map<String, AlarmSubExpression> oldSubExpressions = new HashMap<>();
|
||||
final Map<String, AlarmSubExpression> changedSubExpressions = new HashMap<>();
|
||||
final String newExpression = subExpressions[1] + " or " +
|
||||
subExpressions[2].replace("max", "avg") + " or " +
|
||||
"sum(diskio{instance_id=123,device=4242}, 1) > 5000";
|
||||
|
||||
final AlarmExpression newAlarmExpression = new AlarmExpression(newExpression);
|
||||
final SubAlarm newSubAlarm = new SubAlarm(UUID.randomUUID().toString(), alarmId, newAlarmExpression.getSubExpressions().get(2));
|
||||
newSubExpressions.put(newSubAlarm.getId(), newSubAlarm.getExpression());
|
||||
final SubAlarm deletedSubAlarm = subAlarms.get(0);
|
||||
oldSubExpressions.put(deletedSubAlarm.getId(), deletedSubAlarm.getExpression());
|
||||
final SubAlarm changedSubAlarm = new SubAlarm(subAlarms.get(2).getId(), alarmId, newAlarmExpression.getSubExpressions().get(1));
|
||||
changedSubExpressions.put(changedSubAlarm.getId(), changedSubAlarm.getExpression());
|
||||
final SubAlarm unChangedSubAlarm = new SubAlarm(subAlarms.get(1).getId(), alarmId, subAlarms.get(1).getExpression());
|
||||
|
||||
emitSubAlarmStateChange(alarmId, changedSubAlarm, AlarmState.OK);
|
||||
emitSubAlarmStateChange(alarmId, unChangedSubAlarm, AlarmState.OK);
|
||||
unChangedSubAlarm.setState(AlarmState.OK);
|
||||
|
||||
final AlarmUpdatedEvent event = new AlarmUpdatedEvent(tenantId, alarmId, alarm.getName(), alarm.getDescription(), newExpression,
|
||||
alarm.getState(), alarm.isActionsEnabled(), oldSubExpressions, changedSubExpressions, newSubExpressions);
|
||||
final Tuple updateTuple = createAlarmUpdateTuple(event);
|
||||
bolt.execute(updateTuple);
|
||||
verify(collector, times(1)).ack(updateTuple);
|
||||
|
||||
final Alarm changedAlarm = bolt.alarms.get(alarmId);
|
||||
assertEquals(changedAlarm.getAlarmExpression(), newAlarmExpression);
|
||||
assertEquals(changedAlarm.getSubAlarms().size(), 3);
|
||||
assertEquals(changedAlarm.getSubAlarm(unChangedSubAlarm.getId()), unChangedSubAlarm);
|
||||
assertEquals(changedAlarm.getSubAlarm(newSubAlarm.getId()), newSubAlarm);
|
||||
changedSubAlarm.setState(AlarmState.OK);
|
||||
assertEquals(changedAlarm.getSubAlarm(changedSubAlarm.getId()), changedSubAlarm);
|
||||
assertEquals(changedSubAlarm.isNoState(), false);
|
||||
}
|
||||
|
||||
private String setUpInitialAlarm() {
|
||||
final String alarmId = alarm.getId();
|
||||
when(alarmDAO.findById(alarmId)).thenReturn(alarm);
|
||||
// Load up the original Alarm
|
||||
emitSubAlarmStateChange(alarmId, subAlarms.get(0), AlarmState.ALARM);
|
||||
return alarmId;
|
||||
}
|
||||
|
||||
private void emitSubAlarmStateChange(String alarmId,
|
||||
final SubAlarm subAlarm, AlarmState state) {
|
||||
// Create a copy so changing the state doesn't directly update the ones in the bolt
|
||||
final SubAlarm toEmit = new SubAlarm(subAlarm.getId(), subAlarm.getAlarmId(), subAlarm.getExpression());
|
||||
toEmit.setState(state);
|
||||
final Tuple tuple = createSubAlarmStateChangeTuple(alarmId, toEmit);
|
||||
bolt.execute(tuple);
|
||||
verify(collector, times(1)).ack(tuple);
|
||||
}
|
||||
|
||||
private Tuple createAlarmUpdateTuple(AlarmUpdatedEvent event) {
|
||||
final MkTupleParam tupleParam = new MkTupleParam();
|
||||
tupleParam.setFields(EventProcessingBolt.ALARM_EVENT_STREAM_FIELDS);
|
||||
tupleParam.setStream(EventProcessingBolt.ALARM_EVENT_STREAM_ID);
|
||||
final Tuple tuple = Testing.testTuple(Arrays.asList(EventProcessingBolt.UPDATED, event.alarmId, event), tupleParam);
|
||||
return tuple;
|
||||
}
|
||||
|
||||
private Tuple createSubAlarmStateChangeTuple(String alarmId, final SubAlarm subAlarm) {
|
||||
final MkTupleParam tupleParam = new MkTupleParam();
|
||||
tupleParam.setFields("alarmId", "subAlarm");
|
||||
tupleParam.setStream(Streams.DEFAULT_STREAM_ID);
|
||||
final Tuple tuple = Testing.testTuple(Arrays.asList(alarmId, subAlarm), tupleParam);
|
||||
return tuple;
|
||||
}
|
||||
|
||||
private class MockAlarmThreshholdBolt extends AlarmThresholdingBolt {
|
||||
|
||||
private static final long serialVersionUID = 1L;
|
||||
|
|
|
|||
|
|
@ -4,15 +4,11 @@ import static org.mockito.Mockito.mock;
|
|||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
|
||||
import java.util.AbstractMap.SimpleEntry;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Map.Entry;
|
||||
import java.util.Set;
|
||||
import java.util.UUID;
|
||||
|
||||
import org.testng.annotations.BeforeMethod;
|
||||
|
|
@ -25,7 +21,6 @@ import backtype.storm.testing.MkTupleParam;
|
|||
import backtype.storm.tuple.Tuple;
|
||||
import backtype.storm.tuple.Values;
|
||||
|
||||
import com.google.common.collect.Sets;
|
||||
import com.hpcloud.mon.common.event.AlarmCreatedEvent;
|
||||
import com.hpcloud.mon.common.event.AlarmDeletedEvent;
|
||||
import com.hpcloud.mon.common.event.AlarmUpdatedEvent;
|
||||
|
|
@ -121,7 +116,7 @@ public class EventProcessingBoltTest {
|
|||
verifyDeletedSubAlarm(subAlarm);
|
||||
}
|
||||
verify(collector, times(1)).emit(EventProcessingBolt.ALARM_EVENT_STREAM_ID,
|
||||
new Values(EventProcessingBolt.DELETED, event.alarmId));
|
||||
new Values(EventProcessingBolt.DELETED, event.alarmId, event));
|
||||
verify(collector, times(1)).ack(tuple);
|
||||
}
|
||||
|
||||
|
|
@ -133,87 +128,76 @@ public class EventProcessingBoltTest {
|
|||
}
|
||||
|
||||
public static AlarmUpdatedEvent createAlarmUpdatedEvent(final Alarm alarm,
|
||||
final AlarmExpression updatedAlarmExpression) {
|
||||
final AlarmExpression updatedAlarmExpression, List<SubAlarm> updatedSubAlarms) {
|
||||
final Alarm updatedAlarm = new Alarm();
|
||||
updatedAlarm.setId(alarm.getId());
|
||||
updatedAlarm.setTenantId(alarm.getTenantId());
|
||||
updatedAlarm.setName(alarm.getName());
|
||||
updatedAlarm.setExpression(updatedAlarmExpression.getExpression());
|
||||
Entry<Map<String, AlarmSubExpression>, Map<String, AlarmSubExpression>> entry =
|
||||
oldAndNewSubExpressionsFor(createAlarmSubExpressionMap(alarm), updatedAlarmExpression);
|
||||
updatedAlarm.setDescription(alarm.getDescription());
|
||||
updatedAlarm.setState(alarm.getState());
|
||||
|
||||
final Map<String, AlarmSubExpression> newAlarmSubExpressions = entry.getValue();
|
||||
Map<String, AlarmSubExpression> changedSubExpressions = new HashMap<>();
|
||||
final List<SubAlarm> toDelete = new ArrayList<>(alarm.getSubAlarms());
|
||||
final Map<String, AlarmSubExpression> newAlarmSubExpressions = new HashMap<>();
|
||||
final Map<String, AlarmSubExpression> updatedSubExpressions = new HashMap<>();
|
||||
for (final SubAlarm newSubAlarm : updatedSubAlarms) {
|
||||
final SubAlarm oldSubAlarm = alarm.getSubAlarm(newSubAlarm.getId());
|
||||
if (oldSubAlarm == null) {
|
||||
newAlarmSubExpressions.put(newSubAlarm.getId(), newSubAlarm.getExpression());
|
||||
}
|
||||
else {
|
||||
toDelete.remove(oldSubAlarm);
|
||||
if (!newSubAlarm.getExpression().equals(oldSubAlarm.getExpression())) {
|
||||
updatedSubExpressions.put(newSubAlarm.getId(), newSubAlarm.getExpression());
|
||||
}
|
||||
}
|
||||
}
|
||||
final Map<String, AlarmSubExpression> deletedSubExpressions = new HashMap<>(toDelete.size());
|
||||
for (final SubAlarm oldSubAlarm : toDelete) {
|
||||
deletedSubExpressions.put(oldSubAlarm.getId(), oldSubAlarm.getExpression());
|
||||
}
|
||||
final AlarmUpdatedEvent event = new AlarmUpdatedEvent(updatedAlarm.getTenantId(), updatedAlarm.getId(),
|
||||
updatedAlarm.getName(), updatedAlarm.getAlarmExpression().getExpression(), alarm.getState(), true, entry.getKey(),
|
||||
changedSubExpressions, newAlarmSubExpressions);
|
||||
updatedAlarm.getName(), updatedAlarm.getDescription(), updatedAlarm.getAlarmExpression().getExpression(), alarm.getState(), true, deletedSubExpressions,
|
||||
updatedSubExpressions, newAlarmSubExpressions);
|
||||
return event;
|
||||
}
|
||||
|
||||
public void testAlarmUpdatedEvent() {
|
||||
final String updatedExpression = "avg(hpcs.compute.cpu{instance_id=123,device=42}, 1) > 5 " +
|
||||
"and max(hpcs.compute.newMem{instance_id=123,device=42}) > 80 " +
|
||||
"and max(hpcs.compute.Mem{instance_id=123,device=42}) > 90 " +
|
||||
"and max(hpcs.compute.newLoad{instance_id=123,device=42}) > 5";
|
||||
|
||||
final AlarmExpression updatedAlarmExpression = new AlarmExpression(updatedExpression);
|
||||
final AlarmUpdatedEvent event = createAlarmUpdatedEvent(alarm, updatedAlarmExpression);
|
||||
final Tuple tuple = createTuple(event);
|
||||
bolt.execute(tuple);
|
||||
verify(collector, times(1)).emit(EventProcessingBolt.ALARM_EVENT_STREAM_ID,
|
||||
new Values(EventProcessingBolt.UPDATED, event.alarmId));
|
||||
|
||||
final List<SubAlarm> updatedSubAlarms = new ArrayList<>();
|
||||
final Map<String, AlarmSubExpression> oldAlarmSubExpressionMap = createAlarmSubExpressionMap(alarm);
|
||||
for (final AlarmSubExpression alarmExpression : updatedAlarmExpression.getSubExpressions()) {
|
||||
String id = find(oldAlarmSubExpressionMap, alarmExpression);
|
||||
if (id == null) {
|
||||
id = find(event.newAlarmSubExpressions, alarmExpression);
|
||||
}
|
||||
final SubAlarm subAlarm = new SubAlarm(id, alarm.getId(), alarmExpression);
|
||||
updatedSubAlarms.add(subAlarm);
|
||||
}
|
||||
updatedSubAlarms.add(subAlarms.get(0));
|
||||
updatedSubAlarms.add(new SubAlarm(subAlarms.get(1).getId(), alarm.getId(), updatedAlarmExpression.getSubExpressions().get(1)));
|
||||
updatedSubAlarms.add(new SubAlarm(UUID.randomUUID().toString(), alarm.getId(), updatedAlarmExpression.getSubExpressions().get(2)));
|
||||
|
||||
verifyDeletedSubAlarm(subAlarms.get(1));
|
||||
verifyDeletedSubAlarm(subAlarms.get(2));
|
||||
verifyAddedSubAlarm(updatedSubAlarms.get(1));
|
||||
verifyAddedSubAlarm(updatedSubAlarms.get(2));
|
||||
final AlarmUpdatedEvent event = createAlarmUpdatedEvent(alarm, updatedAlarmExpression, updatedSubAlarms);
|
||||
|
||||
final Tuple tuple = createTuple(event);
|
||||
bolt.execute(tuple);
|
||||
verify(collector, times(1)).ack(tuple);
|
||||
}
|
||||
|
||||
private String find(
|
||||
final Map<String, AlarmSubExpression> newAlarmSubExpressions,
|
||||
final AlarmSubExpression alarmExpression) {
|
||||
String id = null;
|
||||
for (Entry<String, AlarmSubExpression> entry2 : newAlarmSubExpressions.entrySet()) {
|
||||
if (entry2.getValue().equals(alarmExpression)) {
|
||||
id = entry2.getKey();
|
||||
break;
|
||||
}
|
||||
}
|
||||
return id;
|
||||
}
|
||||
|
||||
private static Entry<Map<String, AlarmSubExpression>, Map<String, AlarmSubExpression>> oldAndNewSubExpressionsFor(
|
||||
Map<String, AlarmSubExpression> oldSubAlarms,
|
||||
final AlarmExpression alarmExpression) {
|
||||
Set<AlarmSubExpression> oldSet = new HashSet<>(oldSubAlarms.values());
|
||||
Set<AlarmSubExpression> newSet = new HashSet<>(alarmExpression.getSubExpressions());
|
||||
|
||||
// Filter old sub expressions
|
||||
Set<AlarmSubExpression> oldExpressions = Sets.difference(oldSet, newSet);
|
||||
oldSubAlarms.values().retainAll(oldExpressions);
|
||||
|
||||
// Identify new sub expressions
|
||||
Map<String, AlarmSubExpression> newSubAlarms = new HashMap<>();
|
||||
Set<AlarmSubExpression> newExpressions = Sets.difference(newSet, oldSet);
|
||||
for (AlarmSubExpression expression : newExpressions)
|
||||
newSubAlarms.put(UUID.randomUUID().toString(), expression);
|
||||
|
||||
return new SimpleEntry<>(oldSubAlarms, newSubAlarms);
|
||||
verifyDeletedSubAlarm(subAlarms.get(2));
|
||||
verifyUpdatedSubAlarm(updatedSubAlarms.get(1));
|
||||
verifyAddedSubAlarm(updatedSubAlarms.get(2));
|
||||
verify(collector, times(1)).emit(EventProcessingBolt.ALARM_EVENT_STREAM_ID,
|
||||
new Values(EventProcessingBolt.UPDATED, event.alarmId, event));
|
||||
}
|
||||
|
||||
private void verifyAddedSubAlarm(final SubAlarm subAlarm) {
|
||||
sendSubAlarm(subAlarm, EventProcessingBolt.CREATED);
|
||||
}
|
||||
|
||||
private void verifyUpdatedSubAlarm(final SubAlarm subAlarm) {
|
||||
sendSubAlarm(subAlarm, EventProcessingBolt.UPDATED);
|
||||
}
|
||||
|
||||
private void sendSubAlarm(final SubAlarm subAlarm, String eventType) {
|
||||
verify(collector, times(1)).emit(EventProcessingBolt.METRIC_SUB_ALARM_EVENT_STREAM_ID,
|
||||
new Values(EventProcessingBolt.CREATED,
|
||||
new Values(eventType,
|
||||
new MetricDefinitionAndTenantId(
|
||||
subAlarm.getExpression().getMetricDefinition(), TENANT_ID), subAlarm));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ import static org.testng.Assert.assertEquals;
|
|||
import static org.testng.Assert.assertNotEquals;
|
||||
import static org.testng.Assert.assertNotNull;
|
||||
import static org.testng.Assert.assertNull;
|
||||
import static org.testng.Assert.assertTrue;
|
||||
import static org.testng.Assert.assertFalse;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
|
|
@ -29,6 +31,7 @@ import backtype.storm.testing.MkTupleParam;
|
|||
import backtype.storm.tuple.Tuple;
|
||||
import backtype.storm.tuple.Values;
|
||||
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmOperator;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmState;
|
||||
import com.hpcloud.mon.common.model.alarm.AlarmSubExpression;
|
||||
import com.hpcloud.mon.common.model.metric.Metric;
|
||||
|
|
@ -199,6 +202,72 @@ public class MetricAggregationBoltTest {
|
|||
assertNotNull(bolt.subAlarmStatsRepos.get(metricDefinitionAndTenantId).get("123"));
|
||||
}
|
||||
|
||||
public void validateMetricDefUpdatedThreshold() {
|
||||
final SubAlarmStats stats = updateEnsureMeasurementsKept(subExpr2, "avg(hpcs.compute.mem{id=5}, 60) >= 80");
|
||||
assertEquals(stats.getSubAlarm().getExpression().getThreshold(), 80.0);
|
||||
}
|
||||
|
||||
public void validateMetricDefUpdatedOperator() {
|
||||
final SubAlarmStats stats = updateEnsureMeasurementsKept(subExpr2, "avg(hpcs.compute.mem{id=5}, 60) < 80");
|
||||
assertEquals(stats.getSubAlarm().getExpression().getOperator(), AlarmOperator.LT);
|
||||
}
|
||||
|
||||
private SubAlarmStats updateEnsureMeasurementsKept(AlarmSubExpression subExpr,
|
||||
String newSubExpression) {
|
||||
final SubAlarmStats stats = updateSubAlarmsStats(subExpr, newSubExpression);
|
||||
final double[] values = stats.getStats().getWindowValues();
|
||||
assertFalse(Double.isNaN(values[0])); // Ensure old measurements weren't flushed
|
||||
return stats;
|
||||
}
|
||||
|
||||
public void validateMetricDefReplacedFunction() {
|
||||
final SubAlarmStats stats = updateEnsureMeasurementsFlushed(subExpr2, "max(hpcs.compute.mem{id=5}, 60) < 80");
|
||||
assertEquals(stats.getSubAlarm().getExpression().getOperator(), AlarmOperator.LT);
|
||||
}
|
||||
|
||||
public void validateMetricDefReplacedPeriods() {
|
||||
final SubAlarmStats stats = updateEnsureMeasurementsFlushed(subExpr2, "avg(hpcs.compute.mem{id=5}, 60) >= 80 times 7");
|
||||
assertEquals(stats.getSubAlarm().getExpression().getPeriods(), 7);
|
||||
}
|
||||
|
||||
public void validateMetricDefReplacedPeriod() {
|
||||
final SubAlarmStats stats = updateEnsureMeasurementsFlushed(subExpr2, "avg(hpcs.compute.mem{id=5}, 120) >= 80");
|
||||
assertEquals(stats.getSubAlarm().getExpression().getPeriod(), 120);
|
||||
}
|
||||
|
||||
private SubAlarmStats updateEnsureMeasurementsFlushed(AlarmSubExpression subExpr,
|
||||
String newSubExpression) {
|
||||
final SubAlarmStats stats = updateSubAlarmsStats(subExpr, newSubExpression);
|
||||
final double[] values = stats.getStats().getWindowValues();
|
||||
assertTrue(Double.isNaN(values[0])); // Ensure old measurements were flushed
|
||||
return stats;
|
||||
}
|
||||
|
||||
private SubAlarmStats updateSubAlarmsStats(AlarmSubExpression subExpr,
|
||||
String newSubExpression) {
|
||||
final MkTupleParam tupleParam = new MkTupleParam();
|
||||
tupleParam.setFields(EventProcessingBolt.METRIC_SUB_ALARM_EVENT_STREAM_FIELDS);
|
||||
tupleParam.setStream(EventProcessingBolt.METRIC_SUB_ALARM_EVENT_STREAM_ID);
|
||||
|
||||
final MetricDefinitionAndTenantId metricDefinitionAndTenantId = new MetricDefinitionAndTenantId(subExpr.getMetricDefinition(), TENANT_ID);
|
||||
assertNull(bolt.subAlarmStatsRepos.get(metricDefinitionAndTenantId));
|
||||
|
||||
bolt.execute(Testing.testTuple(Arrays.asList(EventProcessingBolt.CREATED,
|
||||
metricDefinitionAndTenantId, new SubAlarm("123", "1", subExpr)), tupleParam));
|
||||
final SubAlarmStats oldStats = bolt.subAlarmStatsRepos.get(metricDefinitionAndTenantId).get("123");
|
||||
assertEquals(oldStats.getSubAlarm().getExpression().getThreshold(), 90.0);
|
||||
assertTrue(oldStats.getStats().addValue(80.0, System.currentTimeMillis()/1000));
|
||||
assertFalse(Double.isNaN(oldStats.getStats().getWindowValues()[0]));
|
||||
assertNotNull(bolt.subAlarmStatsRepos.get(metricDefinitionAndTenantId).get("123"));
|
||||
|
||||
final AlarmSubExpression newExpr = AlarmSubExpression.of(newSubExpression);
|
||||
|
||||
bolt.execute(Testing.testTuple(Arrays.asList(EventProcessingBolt.UPDATED,
|
||||
metricDefinitionAndTenantId, new SubAlarm("123", "1", newExpr)), tupleParam));
|
||||
|
||||
return bolt.subAlarmStatsRepos.get(metricDefinitionAndTenantId).get("123");
|
||||
}
|
||||
|
||||
public void validateMetricDefDeleted() {
|
||||
MkTupleParam tupleParam = new MkTupleParam();
|
||||
tupleParam.setFields(EventProcessingBolt.METRIC_ALARM_EVENT_STREAM_FIELDS);
|
||||
|
|
|
|||
|
|
@ -18,10 +18,11 @@ import com.hpcloud.util.Serialization;
|
|||
@Test
|
||||
public class EventDeserializerTest {
|
||||
private static final String ALARM_EXPRESSION = "avg(hpcs.compute{instance_id=5,metric_name=cpu,device=1}, 1) > 5 times 3 OR avg(hpcs.compute{flavor_id=3,metric_name=mem}, 2) < 4 times 3";
|
||||
private static final String ALARM_NAME = "An Alarm";
|
||||
private static final String ALARM_ID = "123";
|
||||
private static final String TENANT_ID = "abc";
|
||||
private EventDeserializer deserializer = new EventDeserializer();
|
||||
private static final String ALARM_NAME = "An Alarm";
|
||||
private static final String ALARM_DESCRIPTION = "An Alarm Description";
|
||||
private static final String ALARM_ID = "123";
|
||||
private static final String TENANT_ID = "abc";
|
||||
private EventDeserializer deserializer = new EventDeserializer();
|
||||
|
||||
public void shouldDeserializeAlarmDeletedEvent() {
|
||||
roundTrip(new AlarmDeletedEvent(TENANT_ID, ALARM_ID, null));
|
||||
|
|
@ -32,7 +33,7 @@ private EventDeserializer deserializer = new EventDeserializer();
|
|||
}
|
||||
|
||||
public void shouldDeserializeAlarmUpdatedEvent() {
|
||||
roundTrip(new AlarmUpdatedEvent(TENANT_ID, ALARM_ID, ALARM_NAME, ALARM_EXPRESSION, null, false, null, null, null));
|
||||
roundTrip(new AlarmUpdatedEvent(TENANT_ID, ALARM_ID, ALARM_NAME, ALARM_DESCRIPTION, ALARM_EXPRESSION, null, false, null, null, null));
|
||||
}
|
||||
|
||||
private void roundTrip(Object event) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue