From 080b11dc54db18225fd4749e2e6da5436bf81d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Tr=C4=99bski?= Date: Tue, 15 Mar 2016 09:12:39 +0100 Subject: [PATCH] (Non)deterministic alarm processing 'deterministic' being part of alarm expressions allows monasca-thresh to determine if given alarms can go back to UNDETERMINED state or not. 'deterministic' means that alarm won't ever transititon to UNDETERMINED state, even if there are no measurements received for long enough. By default, all alarms are assumed to be 'non-deterministic' which means that they can transition to 'UNDETERMINED' state Implements: blueprint alarmonlogs Depends-On: Ia42f9a1be37c31416bdac341b092fe527f860c16 Change-Id: Ibe0839123a15494ad45b809e68600c0acef3d330 --- .../monasca/thresh/domain/model/Alarm.java | 80 ++++- .../monasca/thresh/domain/model/SubAlarm.java | 68 +++- .../thresh/domain/model/SubAlarmStats.java | 50 ++- .../persistence/AlarmDefinitionDAOImpl.java | 36 ++- .../hibernate/AlarmDefinitionSqlImpl.java | 19 +- .../persistence/hibernate/AlarmSqlImpl.java | 11 +- .../thresholding/AlarmCreationBolt.java | 3 +- .../thresholding/MetricAggregationBolt.java | 2 +- .../thresh/ThresholdingEngineTest.java | 296 +++++++++++++++--- .../thresh/domain/model/AlarmTest.java | 97 +++++- .../domain/model/SubAlarmStatsTest.java | 48 ++- .../thresh/domain/model/SubAlarmTest.java | 35 ++- .../persistence/AlarmDAOImplTest.java | 37 +++ .../AlarmDefinitionDAOImplTest.java | 23 +- .../hibernate/AlarmDefinitionSqlImplTest.java | 16 + .../hibernate/AlarmSqlImplTest.java | 84 ++++- .../persistence/hibernate/HibernateUtil.java | 3 +- .../thresholding/AlarmCreationBoltTest.java | 121 +++++-- .../MetricAggregationBoltTest.java | 102 +++++- .../infrastructure/persistence/alarm.sql | 1 + 20 files changed, 982 insertions(+), 150 deletions(-) diff --git a/thresh/src/main/java/monasca/thresh/domain/model/Alarm.java b/thresh/src/main/java/monasca/thresh/domain/model/Alarm.java index 3406861..fab151d 100644 --- a/thresh/src/main/java/monasca/thresh/domain/model/Alarm.java +++ b/thresh/src/main/java/monasca/thresh/domain/model/Alarm.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2014,2016 Hewlett Packard Enterprise Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,6 +33,12 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import javax.annotation.Nullable; + +import com.google.common.base.Predicate; +import com.google.common.collect.FluentIterable; +import com.google.common.collect.Lists; + /** * An alarm comprised of sub-alarms. * @@ -39,6 +46,12 @@ import java.util.UUID; * */ public class Alarm extends AbstractEntity { + private static final Predicate DETERMINISTIC_PREDICATE = new Predicate() { + @Override + public boolean apply(@Nullable final SubAlarm input) { + return input != null && input.isDeterministic(); + } + }; private Map subAlarms; private Set alarmedMetrics = new HashSet<>(); private AlarmState state; @@ -50,7 +63,7 @@ public class Alarm extends AbstractEntity { public Alarm() { } - public Alarm(AlarmDefinition alarmDefinition, AlarmState state) { + public Alarm(AlarmDefinition alarmDefinition) { this.id = UUID.randomUUID().toString(); List subExpressions = alarmDefinition.getSubExpressions(); final List subAlarms = new ArrayList<>(subExpressions.size()); @@ -58,10 +71,15 @@ public class Alarm extends AbstractEntity { subAlarms.add(new SubAlarm(UUID.randomUUID().toString(), id, subExpr)); } setSubAlarms(subAlarms); - this.state = state; + this.state = SubAlarm.getDefaultState(this.isDeterministic()); this.alarmDefinitionId = alarmDefinition.getId(); } + public Alarm(AlarmDefinition alarmDefinition, AlarmState state) { + this(alarmDefinition); + this.state = state; // override state detected from expression, for test purposes + } + public String buildStateChangeReason(AlarmState alarmState) { StringBuilder stringBuilder = new StringBuilder(); for(AlarmTransitionSubAlarm alarmTransitionSubAlarm : transitionSubAlarms){ @@ -132,11 +150,15 @@ public class Alarm extends AbstractEntity { } /** - * Evaluates the {@code alarm}, updating the alarm's state if necessary and returning true if the - * alarm's state changed, else false. + * Evaluates the {@code alarm}, updating the alarm's state if necessary. + * + * @param expression expression to evaluate + * + * @return {@link Boolean#TRUE} if alarm's state has changed, {@link Boolean#FALSE} otherwise */ public boolean evaluate(AlarmExpression expression) { transitionSubAlarms.clear(); + AlarmState initialState = state; boolean uninitialized = false; @@ -153,16 +175,18 @@ public class Alarm extends AbstractEntity { if (AlarmState.UNDETERMINED.equals(initialState)) { return false; } + state = AlarmState.UNDETERMINED; stateChangeReason = buildStateChangeReason(state); return true; } - Map subExpressionValues = - new HashMap(); + Map subExpressionValues = new HashMap<>(subAlarms.size()); for (SubAlarm subAlarm : subAlarms.values()) { - subExpressionValues.put(subAlarm.getExpression(), - AlarmState.ALARM.equals(subAlarm.getState())); + subExpressionValues.put( + subAlarm.getExpression(), + AlarmState.ALARM.equals(subAlarm.getState()) + ); } // Handle ALARM state @@ -230,7 +254,7 @@ public class Alarm extends AbstractEntity { } public void setSubAlarms(List subAlarms) { - this.subAlarms = new HashMap(); + this.subAlarms = new HashMap<>(); for (SubAlarm subAlarm : subAlarms) { this.subAlarms.put(subAlarm.getId(), subAlarm); } @@ -294,4 +318,42 @@ public class Alarm extends AbstractEntity { public void setTransitionSubAlarms(List transitionSubAlarms) { this.transitionSubAlarms = transitionSubAlarms; } + + /** + * Returns list of sub alarms which are deterministic. + * + * @return list of deterministic {@link SubAlarm} + * + * @see SubAlarm#isDeterministic() + * @see #isDeterministic() + */ + public List getDeterministicSubAlarms() { + if (this.subAlarms == null || this.subAlarms.isEmpty()) { + return Lists.newArrayList(); + } + return FluentIterable.from(this.getSubAlarms()) + .filter(DETERMINISTIC_PREDICATE) + .toList(); + } + + /** + * Verifies if given alarm is deterministic. + * + * All sub alarms need to be deterministic for entire + * alarm to be such. Otherwise alarm is non-deterministic + * (i.e. if at least one sub alarm is non-deterministic). + * + * @return true/false + * + * @see SubAlarm#isDeterministic() + */ + public boolean isDeterministic() { + for (final SubAlarm subAlarm : this.subAlarms.values()) { + if (!subAlarm.isDeterministic()) { + return false; + } + } + return true; + } + } diff --git a/thresh/src/main/java/monasca/thresh/domain/model/SubAlarm.java b/thresh/src/main/java/monasca/thresh/domain/model/SubAlarm.java index 0ce5d7b..9572b13 100644 --- a/thresh/src/main/java/monasca/thresh/domain/model/SubAlarm.java +++ b/thresh/src/main/java/monasca/thresh/domain/model/SubAlarm.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -37,13 +38,9 @@ public class SubAlarm extends AbstractEntity implements Serializable { private AlarmState state; private boolean noState; private List currentValues; - /** - * Whether metrics for this sub-alarm are received sporadically. - */ - private boolean sporadicMetric; public SubAlarm(String id, String alarmId, SubExpression expression) { - this(id, alarmId, expression, AlarmState.UNDETERMINED); + this(id, alarmId, expression, SubAlarm.initialStateFromExpression(expression)); } // Need this for kryo serialization/deserialization. Fixes a bug in default java @@ -144,12 +141,17 @@ public class SubAlarm extends AbstractEntity implements Serializable { return result; } - public boolean isSporadicMetric() { - return sporadicMetric; - } - - public void setSporadicMetric(boolean sporadicMetric) { - this.sporadicMetric = sporadicMetric; + /** + * Determines if {@link SubAlarm} is deterministic. + * + * Is {@link SubAlarm} deterministic or not depends + * on underlying expression. + * + * @return true/false + * @see AlarmSubExpression#isDeterministic() + */ + public boolean isDeterministic() { + return this.expression.isDeterministic(); } public void setState(AlarmState state) { @@ -166,7 +168,8 @@ public class SubAlarm extends AbstractEntity implements Serializable { @Override public String toString() { - return String.format("SubAlarm [id=%s, alarmId=%s, alarmSubExpressionId=%s, expression=%s, state=%s, noState=%s, currentValues:[", id, + return String.format("SubAlarm [id=%s, alarmId=%s, alarmSubExpressionId=%s, expression=%s, " + + "state=%s, noState=%s, currentValues:[", id, alarmId, alarmSubExpressionId, expression, state, noState) + currentValues + "]]"; } @@ -223,4 +226,45 @@ public class SubAlarm extends AbstractEntity implements Serializable { return false; } } + + /** + * Computes initial state for an {@link SubAlarm} based on + * underlying {@link SubExpression}. + * + * @param expr sub expression + * + * @return initial state for an sub alarm + * + * @see SubExpression#getAlarmSubExpression() + * @see #getDefaultState(boolean) + */ + private static AlarmState initialStateFromExpression(final SubExpression expr) { + final AlarmSubExpression subExpression = expr.getAlarmSubExpression(); + return getDefaultState(subExpression.isDeterministic()); + } + + /** + * Returns default {@link AlarmState} for {@link AlarmSubExpression#DEFAULT_DETERMINISTIC} + * value ({@value AlarmSubExpression#DEFAULT_DETERMINISTIC}). + * + * @return default state + */ + public static AlarmState getDefaultState() { + return getDefaultState(AlarmSubExpression.DEFAULT_DETERMINISTIC); + } + + /** + * Returns default {@link AlarmState} sub alarm should fallback to + * + * If deterministic is equal to {@link Boolean#TRUE}, {@link AlarmState#OK} is returned, + * otherwise default state is {@link AlarmState#UNDETERMINED}. + * + * @param deterministic is sub alarm deterministic + * + * @return default state according to deterministic flag + */ + public static AlarmState getDefaultState(final boolean deterministic) { + return deterministic ? AlarmState.OK : AlarmState.UNDETERMINED; + } + } diff --git a/thresh/src/main/java/monasca/thresh/domain/model/SubAlarmStats.java b/thresh/src/main/java/monasca/thresh/domain/model/SubAlarmStats.java index 8ecc0d0..945762b 100644 --- a/thresh/src/main/java/monasca/thresh/domain/model/SubAlarmStats.java +++ b/thresh/src/main/java/monasca/thresh/domain/model/SubAlarmStats.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -120,23 +121,33 @@ public class SubAlarmStats { * @param alarmDelay How long to give metrics a chance to arrive */ boolean evaluate(final long now, long alarmDelay) { - + final boolean shouldEvaluate = this.stats.shouldEvaluate(now, alarmDelay); final AlarmState newState; + if (immediateAlarmEvaluate()) { newState = AlarmState.ALARM; - } - else { - if (!stats.shouldEvaluate(now, alarmDelay)) { + } else { + if (!shouldEvaluate) { return false; } - newState = determineAlarmStateUsingView(); + newState = this.determineAlarmStateUsingView(); } - if (shouldSendStateChange(newState) && - (stats.shouldEvaluate(now, alarmDelay) || - (newState == AlarmState.ALARM && this.subAlarm.canEvaluateImmediately()))) { + + final boolean shouldSendStateChange = this.shouldSendStateChange(newState); + final boolean immediateAlarmTransition = + newState == AlarmState.ALARM && this.subAlarm.canEvaluateImmediately(); + + if (shouldSendStateChange && (shouldEvaluate || immediateAlarmTransition)) { + logger.debug("SubAlarm[deterministic={}] {} transitions from {} to {}", + this.getSubAlarm().isDeterministic(), + this.getSubAlarm().getId(), + this.getSubAlarm().getState(), + newState + ); setSubAlarmState(newState); return true; } + return false; } @@ -166,10 +177,25 @@ public class SubAlarmStats { } // Window is empty at this point - emptyWindowObservations++; - if ((emptyWindowObservations >= emptyWindowObservationThreshold) - && shouldSendStateChange(AlarmState.UNDETERMINED) && !subAlarm.isSporadicMetric()) { - return AlarmState.UNDETERMINED; + this.emptyWindowObservations++; + final boolean emptyWindowThresholdExceeded = this.emptyWindowObservations >= + this.emptyWindowObservationThreshold; + + if (emptyWindowThresholdExceeded && this.shouldSendStateChange(AlarmState.UNDETERMINED)) { + final boolean isDeterministic = this.subAlarm.isDeterministic(); + final AlarmState state = SubAlarm.getDefaultState(isDeterministic); + final AlarmState subAlarmState = this.subAlarm.getState(); + + logger.debug( + "SubAlarm[deterministic={}] {} exceeded empty window threshold {}, transition to {} from {}", + isDeterministic, + this.subAlarm.getId(), + this.emptyWindowObservationThreshold, + state, + subAlarmState + ); + return state; + } // Hasn't transitioned to UNDETERMINED yet, so use the current state diff --git a/thresh/src/main/java/monasca/thresh/infrastructure/persistence/AlarmDefinitionDAOImpl.java b/thresh/src/main/java/monasca/thresh/infrastructure/persistence/AlarmDefinitionDAOImpl.java index 2280454..5d8797e 100644 --- a/thresh/src/main/java/monasca/thresh/infrastructure/persistence/AlarmDefinitionDAOImpl.java +++ b/thresh/src/main/java/monasca/thresh/infrastructure/persistence/AlarmDefinitionDAOImpl.java @@ -17,6 +17,8 @@ package monasca.thresh.infrastructure.persistence; +import com.google.common.base.Function; + import monasca.common.model.alarm.AggregateFunction; import monasca.common.model.alarm.AlarmExpression; import monasca.common.model.alarm.AlarmOperator; @@ -41,9 +43,27 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import javax.annotation.Nullable; import javax.inject.Inject; public class AlarmDefinitionDAOImpl implements AlarmDefinitionDAO { + private static Function BOOLEAN_MAPPER_FUNCTION = new Function() { + @Nullable + @Override + public Boolean apply(@Nullable final Object input) { + /* + For connection established with actual database boolean + values are returned from database as actual Boolean, + however unit test are written in H2 and there booleans + are returned as Byte + */ + assert input != null; + if (input instanceof Boolean) { + return (Boolean) input; + } + return "1".equals(input.toString()); + } + }; private static final String SUB_ALARM_SQL = "select sad.*, sadd.* from sub_alarm_definition sad " + "left outer join sub_alarm_definition_dimension sadd on sadd.sub_alarm_definition_id=sad.id " + @@ -89,12 +109,24 @@ public class AlarmDefinitionDAOImpl implements AlarmDefinitionDAO { // Need to convert the results appropriately based on type. Integer period = Conversions.variantToInteger(row.get("period")); Integer periods = Conversions.variantToInteger(row.get("periods")); + Boolean deterministic = BOOLEAN_MAPPER_FUNCTION.apply(row.get("is_deterministic")); Map dimensions = new HashMap<>(); while (addedDimension(dimensions, id, rows, index)) { index++; } - subExpressions.add(new SubExpression(id, new AlarmSubExpression(function, - new MetricDefinition(metricName, dimensions), operator, threshold, period, periods))); + subExpressions.add( + new SubExpression(id, + new AlarmSubExpression( + function, + new MetricDefinition(metricName, dimensions), + operator, + threshold, + period, + periods, + deterministic + ) + ) + ); } return subExpressions; diff --git a/thresh/src/main/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmDefinitionSqlImpl.java b/thresh/src/main/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmDefinitionSqlImpl.java index f28e392..be905c5 100644 --- a/thresh/src/main/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmDefinitionSqlImpl.java +++ b/thresh/src/main/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmDefinitionSqlImpl.java @@ -88,10 +88,10 @@ public class AlarmDefinitionSqlImpl alarmDefDb.getTenantId(), alarmDefDb.getName(), alarmDefDb.getDescription(), - new AlarmExpression(alarmDefDb.getExpression()), + AlarmExpression.of(alarmDefDb.getExpression()), alarmDefDb.getSeverity().name(), actionEnable, - this.findSubExpressions(session, alarmDefDb.getId()), // TODO add reverse model assoc and use it here + this.findSubExpressions(session, alarmDefDb.getId()), matchBy.isEmpty() ? Collections.emptyList() : Lists.newArrayList(matchBy) )); @@ -116,25 +116,26 @@ public class AlarmDefinitionSqlImpl try { session = sessionFactory.openSession(); - AlarmDefinitionDb alarmDefDb = (AlarmDefinitionDb) session.get(AlarmDefinitionDb.class, id); + AlarmDefinitionDb alarmDefDb = session.get(AlarmDefinitionDb.class, id); if (alarmDefDb != null) { final Collection matchBy = alarmDefDb.getMatchByAsCollection(); - boolean actionEnable = alarmDefDb.isActionsEnabled(); + final boolean actionEnabled = alarmDefDb.isActionsEnabled(); + final AlarmExpression expression = AlarmExpression.of(alarmDefDb.getExpression()); alarmDefinition = new AlarmDefinition( alarmDefDb.getId(), alarmDefDb.getTenantId(), alarmDefDb.getName(), alarmDefDb.getDescription(), - new AlarmExpression(alarmDefDb.getExpression()), + expression, alarmDefDb.getSeverity().name(), - actionEnable, null, + actionEnabled, + this.findSubExpressions(session, id), matchBy.isEmpty() ? Collections.emptyList() : Lists.newArrayList(matchBy) ); - alarmDefinition.setSubExpressions(findSubExpressions(session, alarmDefinition.getId())); } return alarmDefinition; @@ -207,6 +208,7 @@ public class AlarmDefinitionSqlImpl final Double threshold = def.getThreshold(); final Integer period = def.getPeriod(); final Integer periods = def.getPeriods(); + final Boolean deterministic = def.isDeterministic(); Map dimensions = dimensionMap.get(id); @@ -222,7 +224,8 @@ public class AlarmDefinitionSqlImpl operator, threshold, period, - periods + periods, + deterministic ) ) ); diff --git a/thresh/src/main/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmSqlImpl.java b/thresh/src/main/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmSqlImpl.java index 4b1a549..3a0f632 100644 --- a/thresh/src/main/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmSqlImpl.java +++ b/thresh/src/main/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmSqlImpl.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 FUJITSU LIMITED + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -194,7 +194,7 @@ public class AlarmSqlImpl final DateTime now = DateTime.now(); final AlarmDb alarm = new AlarmDb( newAlarm.getId(), - (AlarmDefinitionDb) session.get(AlarmDefinitionDb.class, newAlarm.getAlarmDefinitionId()), + session.get(AlarmDefinitionDb.class, newAlarm.getAlarmDefinitionId()), newAlarm.getState(), null, null, @@ -208,7 +208,7 @@ public class AlarmSqlImpl for (final SubAlarm subAlarm : newAlarm.getSubAlarms()) { session.save(new SubAlarmDb() .setAlarm(alarm) - .setSubExpression((SubAlarmDefinitionDb) session.get(SubAlarmDefinitionDb.class, subAlarm.getAlarmSubExpressionId())) + .setSubExpression(session.get(SubAlarmDefinitionDb.class, subAlarm.getAlarmSubExpressionId())) .setExpression(subAlarm.getExpression().getExpression()) .setUpdatedAt(now) .setCreatedAt(now) @@ -376,6 +376,11 @@ public class AlarmSqlImpl final List alarmList, final LookupHelper lookupHelper) { final List alarms = Lists.newArrayListWithCapacity(alarmList.size()); + + if (alarmList.isEmpty()) { + return alarms; + } + List subAlarms = null; String prevAlarmId = null; diff --git a/thresh/src/main/java/monasca/thresh/infrastructure/thresholding/AlarmCreationBolt.java b/thresh/src/main/java/monasca/thresh/infrastructure/thresholding/AlarmCreationBolt.java index 9fb6465..401c953 100644 --- a/thresh/src/main/java/monasca/thresh/infrastructure/thresholding/AlarmCreationBolt.java +++ b/thresh/src/main/java/monasca/thresh/infrastructure/thresholding/AlarmCreationBolt.java @@ -25,7 +25,6 @@ import backtype.storm.tuple.Fields; import backtype.storm.tuple.Tuple; import backtype.storm.tuple.Values; -import monasca.common.model.alarm.AlarmState; import monasca.common.model.alarm.AlarmSubExpression; import monasca.common.model.event.AlarmDefinitionDeletedEvent; import monasca.common.model.event.AlarmDefinitionUpdatedEvent; @@ -359,7 +358,7 @@ public class AlarmCreationBolt extends BaseRichBolt { alarmDefinition, metricDefinitionAndTenantId); final List result = new LinkedList<>(); if (waitingAlarms.isEmpty()) { - final Alarm newAlarm = new Alarm(alarmDefinition, AlarmState.UNDETERMINED); + final Alarm newAlarm = new Alarm(alarmDefinition); newAlarm.addAlarmedMetric(metricDefinitionAndTenantId); reuseExistingMetric(newAlarm, alarmDefinition, existingAlarms); if (alarmIsComplete(newAlarm)) { diff --git a/thresh/src/main/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBolt.java b/thresh/src/main/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBolt.java index 7f23836..0f03951 100644 --- a/thresh/src/main/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBolt.java +++ b/thresh/src/main/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBolt.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -235,7 +236,6 @@ public class MetricAggregationBolt extends BaseRichBolt { new SubAlarm(original.getId(), original.getAlarmId(), new SubExpression( original.getAlarmSubExpressionId(), original.getExpression()), original.getState()); newSubAlarm.setNoState(original.isNoState()); - newSubAlarm.setSporadicMetric(original.isSporadicMetric()); newSubAlarm.setCurrentValues(original.getCurrentValues()); return newSubAlarm; } diff --git a/thresh/src/test/java/monasca/thresh/ThresholdingEngineTest.java b/thresh/src/test/java/monasca/thresh/ThresholdingEngineTest.java index 491c834..2ea7406 100644 --- a/thresh/src/test/java/monasca/thresh/ThresholdingEngineTest.java +++ b/thresh/src/test/java/monasca/thresh/ThresholdingEngineTest.java @@ -30,9 +30,7 @@ import backtype.storm.Config; import backtype.storm.testing.FeederSpout; import backtype.storm.tuple.Fields; import backtype.storm.tuple.Values; - import com.google.inject.AbstractModule; - import monasca.common.configuration.KafkaProducerConfiguration; import monasca.common.model.alarm.AlarmExpression; import monasca.common.model.alarm.AlarmState; @@ -54,7 +52,6 @@ import monasca.thresh.infrastructure.thresholding.AlarmEventForwarder; import monasca.thresh.infrastructure.thresholding.MetricFilteringBolt; import monasca.thresh.infrastructure.thresholding.MetricSpout; import monasca.thresh.infrastructure.thresholding.ProducerModule; - import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.testng.annotations.AfterMethod; @@ -80,13 +77,23 @@ public class ThresholdingEngineTest extends TopologyTestCase { private static final String TEST_ALARM_TENANT_ID = "bob"; private static final String TEST_ALARM_NAME = "test-alarm"; private static final String TEST_ALARM_DESCRIPTION = "Description of test-alarm"; + private static final String DET_TEST_ALARM_TENANT_ID = TEST_ALARM_TENANT_ID; + private static final String DET_TEST_ALARM_NAME = "non-det-test-alarm"; + private static final String DET_TEST_ALARM_DESCRIPTION = "Description of non-det-test-alarm"; + private static final String MIXED_TEST_ALARM_TENANT_ID = TEST_ALARM_TENANT_ID; + private static final String MIXED_TEST_ALARM_NAME = "mixed-test-alarm"; + private static final String MIXED_TEST_ALARM_DESCRIPTION = "Description of mixed-test-alarm"; private AlarmDefinition alarmDefinition; + private AlarmDefinition deterministicAlarmDefinition; + private AlarmDefinition mixedAlarmDefinition; private FeederSpout metricSpout; private FeederSpout eventSpout; private AlarmDAO alarmDAO; private AlarmDefinitionDAO alarmDefinitionDAO; private MetricDefinition cpuMetricDef; private MetricDefinition memMetricDef; + private MetricDefinition logErrorMetricDef; + private MetricDefinition logWarningMetricDef; private Map extraMemMetricDefDimensions; private AlarmEventForwarder alarmEventForwarder; @@ -99,9 +106,17 @@ public class ThresholdingEngineTest extends TopologyTestCase { // Fixtures final AlarmExpression expression = new AlarmExpression("max(cpu{id=5}) >= 3 or max(mem{id=5}) >= 5"); + final AlarmExpression expression2 = AlarmExpression.of( + "count(log.error{id=5},deterministic) >= 1 OR count(log.warning{id=5},deterministic) >= 1" + ); + final AlarmExpression expression3 = AlarmExpression.of( + "max(cpu{id=5}) >= 3 AND count(log.warning{id=5},deterministic) >= 1" + ); cpuMetricDef = expression.getSubExpressions().get(0).getMetricDefinition(); memMetricDef = expression.getSubExpressions().get(1).getMetricDefinition(); + logErrorMetricDef = expression2.getSubExpressions().get(0).getMetricDefinition(); + logWarningMetricDef = expression2.getSubExpressions().get(1).getMetricDefinition(); extraMemMetricDefDimensions = new HashMap<>(memMetricDef.dimensions); extraMemMetricDefDimensions.put("Group", "group A"); @@ -109,6 +124,24 @@ public class ThresholdingEngineTest extends TopologyTestCase { alarmDefinition = new AlarmDefinition(TEST_ALARM_TENANT_ID, TEST_ALARM_NAME, TEST_ALARM_DESCRIPTION, expression, "LOW", true, new ArrayList()); + this.deterministicAlarmDefinition = new AlarmDefinition( + DET_TEST_ALARM_TENANT_ID, + DET_TEST_ALARM_NAME, + DET_TEST_ALARM_DESCRIPTION, + expression2, + "LOW", + true, + new ArrayList() + ); + this.mixedAlarmDefinition = new AlarmDefinition( + MIXED_TEST_ALARM_TENANT_ID, + MIXED_TEST_ALARM_NAME, + MIXED_TEST_ALARM_DESCRIPTION, + expression3, + "LOW", + true, + new ArrayList() + ); // Mocks alarmDAO = mock(AlarmDAO.class); @@ -150,36 +183,160 @@ public class ThresholdingEngineTest extends TopologyTestCase { cluster = null; } - public void testWithInitialAlarmDefinition() throws Exception { - when(alarmDefinitionDAO.findById(alarmDefinition.getId())).thenReturn(alarmDefinition); - when(alarmDefinitionDAO.listAll()).thenReturn(Arrays.asList(alarmDefinition)); - shouldThreshold(null, false); + public void testWithInitialAlarmDefinition_NonDeterministic() throws Exception { + this.testWithInitialAlarmDefinition(this.alarmDefinition, new ThresholdSpec( + this.alarmDefinition.getId(), + null, + TEST_ALARM_NAME, + TEST_ALARM_DESCRIPTION, + TEST_ALARM_TENANT_ID + )); } - public void testWithInitialAlarm() throws Exception { - when(alarmDefinitionDAO.findById(alarmDefinition.getId())).thenReturn(alarmDefinition); - when(alarmDefinitionDAO.listAll()).thenReturn(Arrays.asList(alarmDefinition)); - final Alarm alarm = new Alarm(alarmDefinition, AlarmState.UNDETERMINED); + public void testWithInitialAlarmDefinition_Mixed() throws Exception { + this.testWithInitialAlarmDefinition(this.mixedAlarmDefinition, new ThresholdSpec( + this.mixedAlarmDefinition.getId(), + null, + MIXED_TEST_ALARM_NAME, + MIXED_TEST_ALARM_DESCRIPTION, + MIXED_TEST_ALARM_TENANT_ID + )); + } + + public void testWithInitialAlarmDefinition_Deterministic() throws Exception { + this.testWithInitialAlarmDefinition(this.deterministicAlarmDefinition, new ThresholdSpec( + this.deterministicAlarmDefinition.getId(), + null, + DET_TEST_ALARM_NAME, + DET_TEST_ALARM_DESCRIPTION, + DET_TEST_ALARM_TENANT_ID + )); + } + + public void testWithInitialAlarm_NonDeterministic() throws Exception { + final Alarm alarm = new Alarm(this.alarmDefinition); alarm.addAlarmedMetric(new MetricDefinitionAndTenantId(cpuMetricDef, TEST_ALARM_TENANT_ID)); alarm.addAlarmedMetric(new MetricDefinitionAndTenantId(memMetricDef, TEST_ALARM_TENANT_ID)); - when(alarmDAO.listAll()).thenReturn(Arrays.asList(alarm)); - when(alarmDAO.findById(alarm.getId())).thenReturn(alarm); - when(alarmDAO.findForAlarmDefinitionId(alarmDefinition.getId())).thenReturn(Arrays.asList(alarm)); - shouldThreshold(alarm.getId(), true); + + this.testWithInitialAlarm( + this.alarmDefinition, + alarm, + new ThresholdSpec( + this.alarmDefinition.getId(), + alarm.getId(), + TEST_ALARM_NAME, + TEST_ALARM_DESCRIPTION, + TEST_ALARM_TENANT_ID, + true + )); } - public void testWithAlarmDefinitionCreatedEvent() throws Exception { + public void testWithInitialAlarm_Mixed() throws Exception { + final Alarm alarm = new Alarm(this.mixedAlarmDefinition); + alarm.addAlarmedMetric(new MetricDefinitionAndTenantId(cpuMetricDef, MIXED_TEST_ALARM_TENANT_ID)); + alarm.addAlarmedMetric(new MetricDefinitionAndTenantId(logWarningMetricDef, MIXED_TEST_ALARM_TENANT_ID)); + + this.testWithInitialAlarm( + this.mixedAlarmDefinition, + alarm, + new ThresholdSpec( + this.mixedAlarmDefinition.getId(), + alarm.getId(), + MIXED_TEST_ALARM_NAME, + MIXED_TEST_ALARM_DESCRIPTION, + MIXED_TEST_ALARM_TENANT_ID + )); + } + + public void testWithInitialAlarm_Deterministic() throws Exception { + final Alarm alarm = new Alarm(this.deterministicAlarmDefinition); + alarm.addAlarmedMetric(new MetricDefinitionAndTenantId(logErrorMetricDef, DET_TEST_ALARM_TENANT_ID)); + alarm.addAlarmedMetric(new MetricDefinitionAndTenantId(logWarningMetricDef, DET_TEST_ALARM_TENANT_ID)); + + this.testWithInitialAlarm( + this.deterministicAlarmDefinition, + alarm, + new ThresholdSpec( + this.deterministicAlarmDefinition.getId(), + alarm.getId(), + DET_TEST_ALARM_NAME, + DET_TEST_ALARM_DESCRIPTION, + DET_TEST_ALARM_TENANT_ID + )); + } + + public void testWithAlarmDefinitionCreatedEvent_NonDeterministic() throws Exception { + this.testWithAlarmDefinitionCreatedEvent( + this.alarmDefinition, + new ThresholdSpec( + this.alarmDefinition.getId(), + null, + TEST_ALARM_NAME, + TEST_ALARM_DESCRIPTION, + TEST_ALARM_TENANT_ID + ) + ); + } + + public void testWithAlarmDefinitionCreatedEvent_Mixed() throws Exception { + this.testWithAlarmDefinitionCreatedEvent( + this.mixedAlarmDefinition, + new ThresholdSpec( + this.mixedAlarmDefinition.getId(), + null, + MIXED_TEST_ALARM_NAME, + MIXED_TEST_ALARM_DESCRIPTION, + MIXED_TEST_ALARM_TENANT_ID + ) + ); + } + + public void testWithAlarmDefinitionCreatedEvent_Deterministic() throws Exception { + this.testWithAlarmDefinitionCreatedEvent( + this.deterministicAlarmDefinition, + new ThresholdSpec( + this.deterministicAlarmDefinition.getId(), + null, + DET_TEST_ALARM_NAME, + DET_TEST_ALARM_DESCRIPTION, + DET_TEST_ALARM_TENANT_ID + ) + ); + } + + private void testWithAlarmDefinitionCreatedEvent(final AlarmDefinition alarmDefinition, + final ThresholdSpec thresholdSpec) throws Exception { when(alarmDefinitionDAO.listAll()).thenReturn(new ArrayList()); when(alarmDefinitionDAO.findById(alarmDefinition.getId())).thenReturn(alarmDefinition); final AlarmDefinitionCreatedEvent event = - new AlarmDefinitionCreatedEvent(alarmDefinition.getTenantId(), alarmDefinition.getId(), - alarmDefinition.getName(), alarmDefinition.getDescription(), alarmDefinition - .getAlarmExpression().getExpression(), - createSubExpressionMap(alarmDefinition.getAlarmExpression()), Arrays.asList("id")); + new AlarmDefinitionCreatedEvent(alarmDefinition.getTenantId(), alarmDefinition.getId(), + alarmDefinition.getName(), alarmDefinition.getDescription(), alarmDefinition + .getAlarmExpression().getExpression(), + createSubExpressionMap(alarmDefinition.getAlarmExpression()), Arrays.asList("id")); eventSpout.feed(new Values(event)); - shouldThreshold(null, false); + shouldThreshold(thresholdSpec); } + private void testWithInitialAlarmDefinition(final AlarmDefinition alarmDefinition, + final ThresholdSpec thresholdSpec) throws Exception { + when(alarmDefinitionDAO.findById(alarmDefinition.getId())).thenReturn(alarmDefinition); + when(alarmDefinitionDAO.listAll()).thenReturn(Arrays.asList(alarmDefinition)); + shouldThreshold(thresholdSpec); + } + + private void testWithInitialAlarm(final AlarmDefinition alarmDefinition, + final Alarm alarm, + final ThresholdSpec thresholdSpec) throws Exception { + when(alarmDefinitionDAO.findById(alarmDefinition.getId())).thenReturn(alarmDefinition); + when(alarmDefinitionDAO.listAll()).thenReturn(Arrays.asList(alarmDefinition)); + + when(alarmDAO.listAll()).thenReturn(Arrays.asList(alarm)); + when(alarmDAO.findById(alarm.getId())).thenReturn(alarm); + when(alarmDAO.findForAlarmDefinitionId(alarmDefinition.getId())).thenReturn(Arrays.asList(alarm)); + shouldThreshold(thresholdSpec); + } + + private Map createSubExpressionMap(AlarmExpression alarmExpression) { final Map subExprMap = new HashMap<>(); for (final AlarmSubExpression subExpr : alarmExpression.getSubExpressions()) { @@ -192,11 +349,10 @@ public class ThresholdingEngineTest extends TopologyTestCase { return UUID.randomUUID().toString(); } - private void shouldThreshold(final String expectedAlarmId, - final boolean hasExtraMetric) throws Exception { + private void shouldThreshold(final ThresholdSpec thresholdSpec) throws Exception { System.out.println("Starting topology"); startTopology(); - previousState = AlarmState.UNDETERMINED; + previousState = thresholdSpec.isDeterministic ? AlarmState.OK : AlarmState.UNDETERMINED; expectedState = AlarmState.ALARM; alarmsSent = 0; MetricFilteringBolt.clearMetricDefinitions(); @@ -206,24 +362,27 @@ public class ThresholdingEngineTest extends TopologyTestCase { AlarmStateTransitionedEvent event = Serialization.fromJson((String) args[0]); alarmsSent++; System.out.printf("Alarm transitioned from %s to %s%n", event.oldState, event.newState); - assertEquals(event.alarmDefinitionId, alarmDefinition.getId()); - assertEquals(event.alarmName, TEST_ALARM_NAME); - assertEquals(event.tenantId, TEST_ALARM_TENANT_ID); - if (expectedAlarmId != null) { - assertEquals(event.alarmId, expectedAlarmId); + assertEquals(event.alarmDefinitionId, thresholdSpec.alarmDefinitionId); + assertEquals(event.alarmName, thresholdSpec.alarmName); + assertEquals(event.tenantId, thresholdSpec.alarmTenantId); + if (thresholdSpec.alarmId != null) { + assertEquals(event.alarmId, thresholdSpec.alarmId); } assertEquals(event.oldState, previousState); assertEquals(event.newState, expectedState); - assertEquals(event.metrics.size(), hasExtraMetric ? 3 : 2); + assertEquals(event.metrics.size(), thresholdSpec.hasExtraMetric ? 3 : 2); for (MetricDefinition md : event.metrics) { if (md.name.equals(cpuMetricDef.name)) { assertEquals(cpuMetricDef, md); - } - else if (md.name.equals(memMetricDef.name)) { + } else if (md.name.equals(logErrorMetricDef.name)) { + assertEquals(logErrorMetricDef, md); + } else if (md.name.equals(logWarningMetricDef.name)) { + assertEquals(logWarningMetricDef, md); + } else if (md.name.equals(memMetricDef.name)) { if (md.dimensions.size() == extraMemMetricDefDimensions.size()) { assertEquals(extraMemMetricDefDimensions, md.dimensions); } - else if (hasExtraMetric) { + else if (thresholdSpec.hasExtraMetric) { assertEquals(memMetricDef, md); } else { @@ -257,14 +416,7 @@ public class ThresholdingEngineTest extends TopologyTestCase { System.out.println("Feeding metrics..."); long time = System.currentTimeMillis(); - final MetricDefinitionAndTenantId cpuMtid = new MetricDefinitionAndTenantId(cpuMetricDef, - TEST_ALARM_TENANT_ID); - metricSpout.feed(new Values(new TenantIdAndMetricName(cpuMtid), time, new Metric(cpuMetricDef.name, cpuMetricDef.dimensions, - time, (double) (++goodValueCount == 15 ? 1 : 555), null))); - final MetricDefinitionAndTenantId memMtid = new MetricDefinitionAndTenantId(memMetricDef, - TEST_ALARM_TENANT_ID); - metricSpout.feed(new Values(new TenantIdAndMetricName(memMtid), time, new Metric(memMetricDef.name, extraMemMetricDefDimensions, - time, (double) (goodValueCount == 15 ? 1 : 555), null))); + goodValueCount = this.feedMetrics(thresholdSpec, goodValueCount, time); if (--feedCount == 0) { waitCount = 3; @@ -302,4 +454,66 @@ public class ThresholdingEngineTest extends TopologyTestCase { assertTrue(alarmsSent > 0, "Not enough alarms"); System.out.println("All expected Alarms received"); } + + private int feedMetrics(final ThresholdSpec thresholdSpec, int goodValueCount, final long time) { + + final MetricDefinitionAndTenantId cpuMtid = new MetricDefinitionAndTenantId(cpuMetricDef, + thresholdSpec.alarmTenantId); + metricSpout.feed(new Values(new TenantIdAndMetricName(cpuMtid), time, new Metric(cpuMetricDef.name, cpuMetricDef.dimensions, + time, (double) (++goodValueCount == 15 ? 1 : 555), null))); + + final MetricDefinitionAndTenantId memMtid = new MetricDefinitionAndTenantId(memMetricDef, + thresholdSpec.alarmTenantId); + metricSpout.feed(new Values(new TenantIdAndMetricName(memMtid), time, new Metric(memMetricDef.name, extraMemMetricDefDimensions, + time, (double) (goodValueCount == 15 ? 1 : 555), null))); + + final MetricDefinitionAndTenantId logErrorMtid = new MetricDefinitionAndTenantId(logErrorMetricDef, + thresholdSpec.alarmTenantId); + metricSpout.feed(new Values(new TenantIdAndMetricName(logErrorMtid), time, new Metric(logErrorMetricDef.name, logErrorMetricDef.dimensions, + time, (double) (goodValueCount == 15 ? 1 : 555), null))); + + final MetricDefinitionAndTenantId logWarningMtid = new MetricDefinitionAndTenantId(logWarningMetricDef, + thresholdSpec.alarmTenantId); + metricSpout.feed(new Values(new TenantIdAndMetricName(logWarningMtid), time, new Metric(logWarningMetricDef.name, logWarningMetricDef.dimensions, + time, (double) (goodValueCount == 15 ? 1 : 555), null))); + + return goodValueCount; + } + + private class ThresholdSpec { + String alarmDefinitionId; + String alarmId; + String alarmName; + String alarmDescription; + String alarmTenantId; + boolean hasExtraMetric; + boolean isDeterministic; + + ThresholdSpec(final String alarmDefinitionId, + final String alarmId, + final String alarmName, + final String alarmDescription, + final String alarmTenantId) { + this(alarmDefinitionId, alarmId, alarmName, alarmDescription, alarmTenantId, false); + } + + ThresholdSpec(final String alarmDefinitionId, + final String alarmId, + final String alarmName, + final String alarmDescription, + final String alarmTenantId, + final boolean hasExtraMetric) { + this.alarmDefinitionId = alarmDefinitionId; + this.alarmId = alarmId; + this.alarmName = alarmName; + this.alarmDescription = alarmDescription; + this.alarmTenantId = alarmTenantId; + this.hasExtraMetric = hasExtraMetric; + + this.isDeterministic = alarmDefinitionId.equals(deterministicAlarmDefinition.getId()); + + } + + } + } diff --git a/thresh/src/test/java/monasca/thresh/domain/model/AlarmTest.java b/thresh/src/test/java/monasca/thresh/domain/model/AlarmTest.java index 49b2801..4e8e326 100644 --- a/thresh/src/test/java/monasca/thresh/domain/model/AlarmTest.java +++ b/thresh/src/test/java/monasca/thresh/domain/model/AlarmTest.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -41,6 +42,7 @@ import java.util.UUID; @Test public class AlarmTest { + public void shouldBeUndeterminedIfAnySubAlarmIsUndetermined() { AlarmExpression expr = new AlarmExpression( @@ -58,8 +60,7 @@ public class AlarmTest { private Alarm createAlarm(AlarmExpression expr) { final AlarmDefinition alarmDefinition = new AlarmDefinition("42", "Test Def", "", expr, "LOW", true, new ArrayList(0)); - Alarm alarm = new Alarm(alarmDefinition, AlarmState.UNDETERMINED); - return alarm; + return new Alarm(alarmDefinition); } public void shouldEvaluateExpressionWithBooleanAnd() { @@ -174,7 +175,7 @@ public class AlarmTest { new AlarmSubExpression(AggregateFunction.MAX, metricDefinition, AlarmOperator.GT, 1, 60, 1); final SubAlarm subAlarm = new SubAlarm("123", "456", new SubExpression(UUID.randomUUID().toString(), ase)); final Map subExpressionValues = - new HashMap(); + new HashMap<>(); subExpressionValues.put(subAlarm.getExpression(), true); assertEquals(expression.getSubExpressions().get(0).getMetricDefinition().hashCode(), metricDefinition.hashCode()); @@ -182,4 +183,94 @@ public class AlarmTest { // Handle ALARM state assertTrue(expression.evaluate(subExpressionValues)); } + + public void testShouldInitiallyProceedToOKIfAllSubAlarmsAreDeterministic() { + final String expression1 = "count(log.error{path=/var/log/test.log}, deterministic, 1) > 10"; + final String expression2 = "count(log.warning{path=/var/log/test.log}, deterministic, 1) > 5"; + final String expression = String.format("%s or %s", expression1, expression2); + + final AlarmExpression expr = new AlarmExpression(expression); + final Alarm alarm = this.createAlarm(expr); + + assertFalse(alarm.evaluate(expr)); + assertTrue(alarm.isDeterministic()); + assertEquals(alarm.getState(), AlarmState.OK); + } + + public void testShouldNotInitiallyProceedToOKIfNotAllSubAlarmsAreDeterministic() { + final String expression1 = "count(log.error{path=/var/log/test.log}, deterministic, 1) > 10"; + final String expression2 = "count(log.warning{path=/var/log/test.log}, deterministic, 1) > 5"; + final String expression3 = "count(log.debug{path=/var/log/test.log}, 1) > 1"; + final String expression = String.format("(%s or %s) and %s", + expression1, + expression2, + expression3 + ); + + final AlarmExpression expr = new AlarmExpression(expression); + final Alarm alarm = this.createAlarm(expr); + + assertFalse(alarm.evaluate(expr)); + assertFalse(alarm.isDeterministic()); + assertEquals(alarm.getState(), AlarmState.UNDETERMINED); + } + + public void testShouldStayInAlarmDeterministic() { + final String expression = "count(log.error{path=/var/log/test.log}, deterministic, 1) > 5"; + final AlarmExpression expr = new AlarmExpression(expression); + final Alarm alarm = this.createAlarm(expr); + final Iterator iter = alarm.getSubAlarms().iterator(); + + alarm.setState(AlarmState.ALARM); + + SubAlarm subAlarm1 = iter.next(); + subAlarm1.setState(AlarmState.ALARM); + + assertFalse(alarm.evaluate(expr)); + assertTrue(alarm.isDeterministic()); + assertEquals(alarm.getState(), AlarmState.ALARM); + } + + public void testShouldStayInOkDeterministic() { + final String expression = "count(log.error{path=/var/log/test.log}, deterministic, 1) > 5"; + final AlarmExpression expr = new AlarmExpression(expression); + final Alarm alarm = this.createAlarm(expr); + + alarm.setState(AlarmState.OK); + + assertFalse(alarm.evaluate(expr)); + assertTrue(alarm.isDeterministic()); + assertEquals(alarm.getState(), AlarmState.OK); + } + + public void testShouldEnterOkFromAlarmDeterministic() { + final String expression = "count(log.error{path=/var/log/test.log}, deterministic, 1) > 5"; + final AlarmExpression expr = new AlarmExpression(expression); + final Alarm alarm = this.createAlarm(expr); + final Iterator iter = alarm.getSubAlarms().iterator(); + + alarm.setState(AlarmState.ALARM); + + SubAlarm subAlarm1 = iter.next(); + subAlarm1.setState(AlarmState.OK); + + assertTrue(alarm.evaluate(expr)); + assertTrue(alarm.isDeterministic()); + assertEquals(alarm.getState(), AlarmState.OK); + } + + public void testShouldInitiallySetOKForDeterministic() { + assertEquals( + this.createAlarm(AlarmExpression.of("count(log.error,deterministic) > 2")).getState(), + AlarmState.OK + ); + } + + public void testShouldInitiallySetUndeterminedForNonDeterministic() { + assertEquals( + this.createAlarm(AlarmExpression.of("count(log.error) > 2")).getState(), + AlarmState.UNDETERMINED + ); + } + } diff --git a/thresh/src/test/java/monasca/thresh/domain/model/SubAlarmStatsTest.java b/thresh/src/test/java/monasca/thresh/domain/model/SubAlarmStatsTest.java index bec1fcb..92659b3 100644 --- a/thresh/src/test/java/monasca/thresh/domain/model/SubAlarmStatsTest.java +++ b/thresh/src/test/java/monasca/thresh/domain/model/SubAlarmStatsTest.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,17 +19,17 @@ package monasca.thresh.domain.model; import static org.testng.Assert.assertEquals; -import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertNotEquals; import static org.testng.Assert.assertTrue; -import monasca.common.model.alarm.AlarmState; -import monasca.common.model.alarm.AlarmSubExpression; - -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; import java.util.UUID; +import monasca.common.model.alarm.AlarmState; +import monasca.common.model.alarm.AlarmSubExpression; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + @Test public class SubAlarmStatsTest { private SubExpression expression; @@ -141,6 +142,7 @@ public class SubAlarmStatsTest { new SubExpression(UUID.randomUUID().toString(), AlarmSubExpression.of("max(hpcs.compute.cpu{id=5}, 60) > 3 times 3")); subAlarm = new SubAlarm("123", "1", expression); + assertEquals(subAlarm.getState(), AlarmState.UNDETERMINED); subAlarm.setNoState(true); subAlarmStats = new SubAlarmStats(subAlarm, expression.getAlarmSubExpression().getPeriod()); @@ -223,6 +225,7 @@ public class SubAlarmStatsTest { new SubExpression(UUID.randomUUID().toString(), AlarmSubExpression.of("avg(hpcs.compute.cpu{id=5}) > 3 times 3")); subAlarm = new SubAlarm("123", "1", expression); + assertEquals(subAlarm.getState(), AlarmState.UNDETERMINED); SubAlarmStats saStats = new SubAlarmStats(subAlarm, (System.currentTimeMillis() / 1000) + 60); assertEquals(saStats.emptyWindowObservationThreshold, 6); } @@ -257,6 +260,7 @@ public class SubAlarmStatsTest { AlarmSubExpression.of("sum(hpcs.compute.mem{id=5}, 120) >= 96")); final SubAlarm subAlarm = new SubAlarm("42", "4242", subExpr); + assertEquals(subAlarm.getState(), AlarmState.UNDETERMINED); long t1 = 0; final SubAlarmStats stats = new SubAlarmStats(subAlarm, t1 + subExpr.getAlarmSubExpression().getPeriod()); @@ -274,4 +278,36 @@ public class SubAlarmStatsTest { } } } + + public void shouldNotAllowSubAlarmTransitionFromOkToUndetermined_Deterministic() { + final String expression = "sum(log.error{path=/var/log/test.log},deterministic,240) >= 100"; + final SubExpression subExpr = new SubExpression( + UUID.randomUUID().toString(), + AlarmSubExpression.of(expression) + ); + + final SubAlarm subAlarm = new SubAlarm("42", "4242", subExpr); + final SubAlarmStats stats = new SubAlarmStats(subAlarm, subExpr.getAlarmSubExpression().getPeriod()); + + // initially in OK because deterministic + assertTrue(stats.getSubAlarm().isDeterministic()); + assertEquals(stats.getSubAlarm().getState(), AlarmState.OK); + + int t1 = 0; + for (int i = 0; i < 1080; i++) { + t1++; + stats.getStats().addValue(1.0, t1); + if ((t1 % 60) == 2) { + stats.evaluateAndSlideWindow(t1, 1); + if (i <= subExpr.getAlarmSubExpression().getPeriod()) { + // Haven't waited long enough to evaluate, + // but this is deterministic sub alarm, so it should be in ok + assertEquals(stats.getSubAlarm().getState(), AlarmState.OK); + } else { + assertEquals(stats.getSubAlarm().getState(), AlarmState.ALARM); + } + } + } + } + } diff --git a/thresh/src/test/java/monasca/thresh/domain/model/SubAlarmTest.java b/thresh/src/test/java/monasca/thresh/domain/model/SubAlarmTest.java index c8e1080..067e417 100644 --- a/thresh/src/test/java/monasca/thresh/domain/model/SubAlarmTest.java +++ b/thresh/src/test/java/monasca/thresh/domain/model/SubAlarmTest.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2015 Hewlett-Packard Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,12 +19,15 @@ package monasca.thresh.domain.model; import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertFalse; +import static org.testng.Assert.assertTrue; -import monasca.common.model.alarm.AlarmSubExpression; +import java.util.UUID; import org.testng.annotations.Test; -import java.util.UUID; +import monasca.common.model.alarm.AlarmState; +import monasca.common.model.alarm.AlarmSubExpression; @Test public class SubAlarmTest { @@ -55,11 +59,34 @@ public class SubAlarmTest { checkExpression("min(hpcs.compute.cpu{id=5}, 60) <= 3 times 3", true); } + public void shouldBeDeterministicIfKeywordFound() { + final SubAlarm subAlarm = this.getSubAlarm("count(log.error{},deterministic,1)>1"); + assertTrue(subAlarm.isDeterministic()); + assertEquals(subAlarm.getState(), AlarmState.OK); + } + + public void shouldBeNonDeterministicByDefault() { + final SubAlarm subAlarm = this.getSubAlarm("min(hpcs.compute.cpu{id=5}, 60) > 3 times 3"); + assertFalse(subAlarm.isDeterministic()); + assertEquals(subAlarm.getState(), AlarmState.UNDETERMINED); + } + + public void verifyDefaultAlarmState() { + assertEquals(AlarmState.UNDETERMINED, SubAlarm.getDefaultState(false)); + assertEquals(AlarmState.OK, SubAlarm.getDefaultState(true)); + } + private void checkExpression(String expressionString, boolean expected) { + final SubAlarm subAlarm = this.getSubAlarm(expressionString); + assertEquals(subAlarm.canEvaluateImmediately(), expected); + assertEquals(subAlarm.getState(), AlarmState.UNDETERMINED); + assertFalse(subAlarm.isDeterministic()); + } + + private SubAlarm getSubAlarm(final String expressionString) { final SubExpression expression = new SubExpression(UUID.randomUUID().toString(), AlarmSubExpression.of(expressionString)); - final SubAlarm subAlarm = new SubAlarm(UUID.randomUUID().toString(), "1", expression); - assertEquals(subAlarm.canEvaluateImmediately(), expected); + return new SubAlarm(UUID.randomUUID().toString(), "1", expression); } } diff --git a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/AlarmDAOImplTest.java b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/AlarmDAOImplTest.java index d236949..09bcc38 100644 --- a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/AlarmDAOImplTest.java +++ b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/AlarmDAOImplTest.java @@ -57,8 +57,12 @@ public class AlarmDAOImplTest { private static String ALARM_NAME = "90% CPU"; private static String ALARM_DESCR = "Description for " + ALARM_NAME; private static Boolean ALARM_ENABLED = Boolean.TRUE; + private static String DETERMINISTIC_ALARM_NAME = "count(log.error)"; + private static String DETERMINISTIC_ALARM_DESCRIPTION = "Description for " + ALARM_NAME; + private static Boolean DETERMINISTIC_ALARM_ENABLED = Boolean.TRUE; private MetricDefinitionAndTenantId newMetric; + private AlarmDefinition deterministicAlarmDef; private AlarmDefinition alarmDef; private DBI db; @@ -97,6 +101,18 @@ public class AlarmDAOImplTest { expr), "LOW", ALARM_ENABLED, new ArrayList()); AlarmDefinitionDAOImplTest.insertAlarmDefinition(handle, alarmDef); + final String deterministicExpr = "count(log.error{path=/var/log/test},deterministic,20) > 5"; + this.deterministicAlarmDef = new AlarmDefinition( + TENANT_ID, + DETERMINISTIC_ALARM_NAME, + DETERMINISTIC_ALARM_DESCRIPTION, + new AlarmExpression(deterministicExpr), + "HIGH", + DETERMINISTIC_ALARM_ENABLED, + new ArrayList() + ); + AlarmDefinitionDAOImplTest.insertAlarmDefinition(handle, this.deterministicAlarmDef); + final Map dimensions = new HashMap(); dimensions.put("first", "first_value"); dimensions.put("second", "second_value"); @@ -234,4 +250,25 @@ public class AlarmDAOImplTest { List> rows = handle.select("select * from metric_dimension"); assertEquals(2, rows.size()); } + + public void shouldPersistDeterministic() { + final Alarm alarm1 = new Alarm(this.deterministicAlarmDef); + final MetricDefinition definition = this.deterministicAlarmDef + .getSubExpressions() + .get(0) + .getAlarmSubExpression() + .getMetricDefinition(); + final MetricDefinitionAndTenantId mtid = new MetricDefinitionAndTenantId( + definition, + TENANT_ID + ); + + alarm1.addAlarmedMetric(mtid); + dao.createAlarm(alarm1); + + final Alarm byId = dao.findById(alarm1.getId()); + + assertEquals(byId, alarm1); + assertEquals(1, byId.getDeterministicSubAlarms().size()); + } } diff --git a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/AlarmDefinitionDAOImplTest.java b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/AlarmDefinitionDAOImplTest.java index e966df1..a41186b 100644 --- a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/AlarmDefinitionDAOImplTest.java +++ b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/AlarmDefinitionDAOImplTest.java @@ -97,6 +97,12 @@ public class AlarmDefinitionDAOImplTest { new AlarmDefinition(TENANT_ID, ALARM_NAME, ALARM_DESCR, expression3, "LOW", false, Arrays.asList("hostname", "dev")); insertAndCheck(alarmDefinition3); + + final AlarmExpression expression4 = new AlarmExpression("max(cpu,deterministic) > 90"); + final AlarmDefinition alarmDefinition4 = + new AlarmDefinition(TENANT_ID, ALARM_NAME, ALARM_DESCR, expression4, "LOW", + false, Arrays.asList("hostname", "dev")); + insertAndCheck(alarmDefinition4); } public void testListAll() { @@ -116,6 +122,16 @@ public class AlarmDefinitionDAOImplTest { insertAlarmDefinition(handle, alarmDefinition2); verifyListAllMatches(alarmDefinition, alarmDefinition2); + + final AlarmExpression expression3 = new AlarmExpression( + "max(cpu{service=swift}, deterministic) > 90" + ); + final AlarmDefinition alarmDefinition3 = + new AlarmDefinition(TENANT_ID, ALARM_NAME, ALARM_DESCR, expression3, "LOW", + false, Arrays.asList("fred", "barney", "wilma", "betty", "scooby", "doo")); + insertAlarmDefinition(handle, alarmDefinition3); + + verifyListAllMatches(alarmDefinition, alarmDefinition2, alarmDefinition3); } private void insertAndCheck(final AlarmDefinition alarmDefinition) { @@ -155,10 +171,13 @@ public class AlarmDefinitionDAOImplTest { handle .insert( "insert into sub_alarm_definition (id, alarm_definition_id, function, metric_name, operator, " - + "threshold, period, periods, created_at, updated_at) values (?, ?, ?, ?, ?, ?, ?, ?, NOW(), NOW())", + + "threshold, period, periods, is_deterministic, created_at, updated_at) " + + "values (?, ?, ?, ?, ?, ?, ?, ?, ?, NOW(), NOW())", subExpression.getId(), alarmDefinition.getId(), alarmSubExpr.getFunction().name(), alarmSubExpr.getMetricDefinition().name, alarmSubExpr.getOperator().name(), - alarmSubExpr.getThreshold(), alarmSubExpr.getPeriod(), alarmSubExpr.getPeriods()); + alarmSubExpr.getThreshold(), alarmSubExpr.getPeriod(), alarmSubExpr.getPeriods(), + alarmSubExpr.isDeterministic() + ); for (final Map.Entry entry : alarmSubExpr.getMetricDefinition().dimensions.entrySet()) { handle .insert( diff --git a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmDefinitionSqlImplTest.java b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmDefinitionSqlImplTest.java index dfdb0b5..9b4a2fb 100644 --- a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmDefinitionSqlImplTest.java +++ b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmDefinitionSqlImplTest.java @@ -80,6 +80,12 @@ public class AlarmDefinitionSqlImplTest { new AlarmDefinition(TENANT_ID, ALARM_NAME, ALARM_DESCR, expression3, "LOW", false, Arrays.asList("hostname", "dev")); insertAndCheck(alarmDefinition3); + + final AlarmExpression expression4 = new AlarmExpression("max(cpu,deterministic) > 90"); + final AlarmDefinition alarmDefinition4 = + new AlarmDefinition(TENANT_ID, ALARM_NAME, ALARM_DESCR, expression4, "LOW", + false, Arrays.asList("hostname", "dev")); + insertAndCheck(alarmDefinition4); } public void testListAll() { @@ -99,6 +105,16 @@ public class AlarmDefinitionSqlImplTest { HibernateUtil.insertAlarmDefinition(sessionFactory.openSession(), alarmDefinition2); verifyListAllMatches(alarmDefinition, alarmDefinition2); + + final AlarmExpression expression3 = new AlarmExpression( + "max(cpu{service=swift}, deterministic) > 90" + ); + final AlarmDefinition alarmDefinition3 = + new AlarmDefinition(TENANT_ID, ALARM_NAME, ALARM_DESCR, expression3, "LOW", + false, Arrays.asList("fred", "barney", "wilma", "betty", "scooby", "doo")); + HibernateUtil.insertAlarmDefinition(sessionFactory.openSession(), alarmDefinition3); + + verifyListAllMatches(alarmDefinition, alarmDefinition2, alarmDefinition3); } private void insertAndCheck(final AlarmDefinition alarmDefinition) { diff --git a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmSqlImplTest.java b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmSqlImplTest.java index 1cce114..fd37a73 100644 --- a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmSqlImplTest.java +++ b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/AlarmSqlImplTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 FUJITSU LIMITED + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except * in compliance with the License. You may obtain a copy of the License at @@ -18,13 +18,18 @@ import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertTrue; import static org.testng.AssertJUnit.assertNull; -import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import com.beust.jcommander.internal.Lists; import com.beust.jcommander.internal.Maps; +import org.hibernate.SessionFactory; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + import monasca.common.model.alarm.AggregateFunction; import monasca.common.model.alarm.AlarmExpression; import monasca.common.model.alarm.AlarmState; @@ -36,11 +41,6 @@ import monasca.thresh.domain.model.SubAlarm; import monasca.thresh.domain.model.SubExpression; import monasca.thresh.domain.service.AlarmDAO; -import org.hibernate.SessionFactory; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - /** * Test scenarios for AlarmSqlRepoImpl. * @@ -53,8 +53,12 @@ public class AlarmSqlImplTest { private static String ALARM_NAME = "90% CPU"; private static String ALARM_DESCR = "Description for " + ALARM_NAME; private static Boolean ALARM_ENABLED = Boolean.TRUE; + private static String DETERMINISTIC_ALARM_NAME = "count(log.error)"; + private static String DETERMINISTIC_ALARM_DESCRIPTION = "Description for " + ALARM_NAME; + private static Boolean DETERMINISTIC_ALARM_ENABLED = Boolean.TRUE; private MetricDefinitionAndTenantId newMetric; private AlarmDefinition alarmDef; + private AlarmDefinition deterministicAlarmDef; private SessionFactory sessionFactory; private AlarmDAO dao; @@ -73,9 +77,29 @@ public class AlarmSqlImplTest { } protected void prepareData() { - final String expr = "avg(load{first=first_value}) > 10 and max(cpu) < 90"; - alarmDef = new AlarmDefinition(TENANT_ID, ALARM_NAME, ALARM_DESCR, new AlarmExpression(expr), "LOW", ALARM_ENABLED, new ArrayList()); - HibernateUtil.insertAlarmDefinition(this.sessionFactory.openSession(), alarmDef); + this.alarmDef = new AlarmDefinition( + TENANT_ID, + ALARM_NAME, + ALARM_DESCR, + new AlarmExpression("avg(load{first=first_value}) > 10 and max(cpu) < 90"), + "LOW", + ALARM_ENABLED, + Lists.newArrayList() + ); + this.deterministicAlarmDef = new AlarmDefinition( + TENANT_ID, + DETERMINISTIC_ALARM_NAME, + DETERMINISTIC_ALARM_DESCRIPTION, + new AlarmExpression("count(log.error{path=/var/log/test},deterministic,20) > 5"), + "HIGH", + DETERMINISTIC_ALARM_ENABLED, + Lists.newArrayList() + ); + + HibernateUtil.insertAlarmDefinition(this.sessionFactory.openSession(), + this.alarmDef); + HibernateUtil.insertAlarmDefinition(this.sessionFactory.openSession(), + this.deterministicAlarmDef); final Map dimensions = new HashMap(); dimensions.put("first", "first_value"); @@ -186,4 +210,44 @@ public class AlarmSqlImplTest { assertNull(dao.findById(newAlarm.getId())); } + + @Test(groups = "orm") + public void shouldFindNonDeterministicAlarmDefinition() { + final MetricDefinition definition = this.deterministicAlarmDef + .getSubExpressions() + .get(0) + .getAlarmSubExpression() + .getMetricDefinition(); + + final MetricDefinitionAndTenantId mtid = new MetricDefinitionAndTenantId( + definition, + TENANT_ID + ); + + // no alarms in the beginning + this.verifyAlarmList(dao.findForAlarmDefinitionId(this.deterministicAlarmDef.getId())); + + // create two alarms + final Alarm firstAlarm = new Alarm(this.deterministicAlarmDef, AlarmState.OK); + final Alarm secondAlarm = new Alarm(this.deterministicAlarmDef, AlarmState.UNDETERMINED); + + secondAlarm.addAlarmedMetric(mtid); + firstAlarm.addAlarmedMetric(mtid); + + dao.createAlarm(secondAlarm); + dao.createAlarm(firstAlarm); + + // and check + final List found = dao.findForAlarmDefinitionId(this.deterministicAlarmDef.getId()); + this.verifyAlarmList( + found, + firstAlarm, + secondAlarm + ); + + // ensure deterministic has been saved + for (final Alarm alarm : found) { + assertEquals(1, alarm.getDeterministicSubAlarms().size()); + } + } } diff --git a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/HibernateUtil.java b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/HibernateUtil.java index dd889cc..5b41981 100644 --- a/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/HibernateUtil.java +++ b/thresh/src/test/java/monasca/thresh/infrastructure/persistence/hibernate/HibernateUtil.java @@ -135,7 +135,8 @@ class HibernateUtil { alarmSubExpr.getPeriod(), alarmSubExpr.getPeriods(), now, - now + now, + alarmSubExpr.isDeterministic() ); session.save(subAlarmDef); diff --git a/thresh/src/test/java/monasca/thresh/infrastructure/thresholding/AlarmCreationBoltTest.java b/thresh/src/test/java/monasca/thresh/infrastructure/thresholding/AlarmCreationBoltTest.java index c1289a0..573c6a2 100644 --- a/thresh/src/test/java/monasca/thresh/infrastructure/thresholding/AlarmCreationBoltTest.java +++ b/thresh/src/test/java/monasca/thresh/infrastructure/thresholding/AlarmCreationBoltTest.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -136,6 +137,7 @@ public class AlarmCreationBoltTest { public void testmetricFitsInAlarmDefinition() { final AlarmDefinition alarmDefinition = createAlarmDefinition("max(cpu{service=2}) > 90 and max(load_avg) > 10", "hostname"); + final MetricDefinitionAndTenantId goodCpu = new MetricDefinitionAndTenantId(build("cpu", "hostname", "eleanore", "service", "2", "other", "vivi"), TENANT_ID); @@ -170,6 +172,26 @@ public class AlarmCreationBoltTest { final MetricDefinitionAndTenantId badCpuWrongTenant = new MetricDefinitionAndTenantId(build("cpu"), TENANT_ID + "2"); assertFalse(bolt.validMetricDefinition(alarmDefinition, badCpuWrongTenant)); + + // check deterministic + final AlarmDefinition deterministicAlarmDefinition = + createAlarmDefinition("count(log.error{},deterministic) > 2", "hostname"); + + // deterministic, same tenant, with dimensions + MetricDefinitionAndTenantId validLogError = + new MetricDefinitionAndTenantId(build("log.error", "hostname", "eleanore", "path", + "/var/log/test.log"), TENANT_ID); + assertTrue(bolt.validMetricDefinition(deterministicAlarmDefinition, validLogError)); + + // deterministic, same tenant, no dimensions + MetricDefinitionAndTenantId invalidLogError = + new MetricDefinitionAndTenantId(build("log.error"), TENANT_ID); + assertTrue(bolt.validMetricDefinition(deterministicAlarmDefinition, invalidLogError)); + + // deterministic, different tenant + invalidLogError = + new MetricDefinitionAndTenantId(build("log.error"), TENANT_ID + "234"); + assertFalse(bolt.validMetricDefinition(deterministicAlarmDefinition, invalidLogError)); } public void testMetricFitsInAlarm() { @@ -177,7 +199,8 @@ public class AlarmCreationBoltTest { createAlarmDefinition("max(cpu{service=2}) > 90 and max(load_avg{service=2}) > 10", "hostname"); - final Alarm alarm = new Alarm(alarmDefinition, AlarmState.ALARM); + final Alarm alarm = new Alarm(alarmDefinition); + alarm.setState(AlarmState.ALARM); final Iterator iterator = alarm.getSubAlarms().iterator(); final SubAlarm cpu = iterator.next(); @@ -329,20 +352,28 @@ public class AlarmCreationBoltTest { bolt.execute(tuple); } + public void testCreateSimpleDeterministicAlarm() { + this.runCreateComplexAlarm(true); + } + + public void testCreateSimpleDeterministicAlarmWithMatchBy() { + this.runCreateComplexAlarm(true, "hostname"); + } + public void testCreateSimpleAlarmWithMatchBy() { - runCreateSimpleAlarm("hostname"); + this.runCreateSimpleAlarm(false, "hostname"); } public void testCreateSimpleAlarm() { - runCreateSimpleAlarm(); + this.runCreateSimpleAlarm(false); } public void testCreateComplexAlarmWithMatchBy() { - runCreateComplexAlarm("hostname"); + this.runCreateComplexAlarm(false, "hostname"); } public void testCreateComplexAlarm() { - runCreateComplexAlarm(); + this.runCreateComplexAlarm(false); } public void testFinishesMultipleAlarms() { @@ -378,6 +409,7 @@ public class AlarmCreationBoltTest { } public void testReuseMetricFromExistingAlarm() { + final boolean deterministic = false; final String expression = "max(cpu{service=vivi}) > 90"; final String[] matchBy = new String[] { "hostname", "amplifier" }; final AlarmDefinition alarmDefinition = createAlarmDefinition(expression, matchBy); @@ -389,7 +421,7 @@ public class AlarmCreationBoltTest { alarmDefinition.getId()); assertEquals(this.createdAlarms.size(), 1); - verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric, TENANT_ID)); final MetricDefinition metric2 = @@ -400,7 +432,7 @@ public class AlarmCreationBoltTest { assertEquals(this.createdAlarms.size(), 1, "A second alarm was created instead of the metric fitting into the first"); - verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric, TENANT_ID), new MetricDefinitionAndTenantId(metric2, TENANT_ID)); @@ -412,12 +444,13 @@ public class AlarmCreationBoltTest { assertEquals(this.createdAlarms.size(), 2); - verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric3, TENANT_ID), new MetricDefinitionAndTenantId(metric2, TENANT_ID)); } public void testUseMetricInExistingAlarm() { + final boolean deterministic = false; final String expression = "max(cpu{service=vivi}) > 90"; final String[] matchBy = new String[] { "hostname", "amplifier" }; final AlarmDefinition alarmDefinition = createAlarmDefinition(expression, matchBy); @@ -429,7 +462,7 @@ public class AlarmCreationBoltTest { alarmDefinition.getId()); assertEquals(this.createdAlarms.size(), 1); - verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric, TENANT_ID)); final MetricDefinition metric3 = @@ -440,7 +473,7 @@ public class AlarmCreationBoltTest { assertEquals(this.createdAlarms.size(), 2); - verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric3, TENANT_ID)); final MetricDefinition metric2 = @@ -451,17 +484,17 @@ public class AlarmCreationBoltTest { assertEquals(this.createdAlarms.size(), 2, "A third alarm was created instead of the metric fitting into the first two"); - verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric, TENANT_ID), new MetricDefinitionAndTenantId(metric2, TENANT_ID)); - verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric3, TENANT_ID), new MetricDefinitionAndTenantId(metric2, TENANT_ID)); } public void testDeletedAlarm() { - final AlarmDefinition alarmDefinition = runCreateSimpleAlarm(); + final AlarmDefinition alarmDefinition = runCreateSimpleAlarm(false); assertEquals(this.createdAlarms.size(), 1); final Alarm alarmToDelete = this.createdAlarms.get(0); this.createdAlarms.clear(); @@ -486,7 +519,7 @@ public class AlarmCreationBoltTest { bolt.execute(tuple); // Make sure the alarm gets created again - createAlarms(alarmDefinition); + createAlarms(alarmDefinition, false); } private void testMultipleExpressions(final List metricDefinitionsToSend, @@ -501,15 +534,18 @@ public class AlarmCreationBoltTest { assertEquals(this.createdAlarms.size(), numAlarms); } - private AlarmDefinition runCreateSimpleAlarm(final String... matchBy) { - - final String expression = "max(cpu{service=2}) > 90"; + private AlarmDefinition runCreateSimpleAlarm(final Boolean deterministic, final String... matchBy) { + final String expression = String.format( + "max(cpu{service=2}%s) > 90", (deterministic ? ",deterministic" : "") + ); final AlarmDefinition alarmDefinition = createAlarmDefinition(expression, matchBy); - createAlarms(alarmDefinition, matchBy); + this.createAlarms(alarmDefinition, deterministic, matchBy); return alarmDefinition; } - private void createAlarms(final AlarmDefinition alarmDefinition, final String... matchBy) { + private void createAlarms(final AlarmDefinition alarmDefinition, + final Boolean deterministic, + final String... matchBy) { final MetricDefinition metric = build("cpu", "hostname", "eleanore", "service", "2", "other", "vivi"); @@ -517,8 +553,13 @@ public class AlarmCreationBoltTest { alarmDefinition.getId()); assertEquals(this.createdAlarms.size(), 1); - verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, collector, - new MetricDefinitionAndTenantId(metric, TENANT_ID)); + this.verifyCreatedAlarm( + this.createdAlarms.get(0), + alarmDefinition, + deterministic, + collector, + new MetricDefinitionAndTenantId(metric, TENANT_ID) + ); final MetricDefinition metric2 = build("cpu", "hostname", "vivi", "service", "2", "other", "eleanore"); @@ -531,7 +572,7 @@ public class AlarmCreationBoltTest { assertEquals(this.createdAlarms.size(), 2, "The metric was fitted into the first alarm instead of creating a new alarm"); - verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric2, TENANT_ID)); // Now send a metric that must fit into the just created alarm to test that @@ -544,14 +585,23 @@ public class AlarmCreationBoltTest { assertEquals(this.createdAlarms.size(), 2, "The metric created a new alarm instead of fitting into the second"); - verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, collector, + verifyCreatedAlarm(this.createdAlarms.get(1), alarmDefinition, deterministic, collector, new MetricDefinitionAndTenantId(metric2, TENANT_ID), new MetricDefinitionAndTenantId(metric3, TENANT_ID)); } } - private void runCreateComplexAlarm(final String... matchBy) { - final AlarmDefinition alarmDefinition = - createAlarmDefinition("max(cpu{service=2}) > 90 or max(load.avg{service=2}) > 5", matchBy); + private void runCreateComplexAlarm(final Boolean deterministic, final String... matchBy) { + final String rawExpression = "max(cpu{service=2}%s) > 90 or max(load.avg{service=2}%s) > 5"; + final String expression; + final AlarmDefinition alarmDefinition; + + if (deterministic) { + expression = String.format(rawExpression, ",deterministic", ",deterministic"); + } else { + expression = String.format(rawExpression, "", ""); + } + + alarmDefinition = this.createAlarmDefinition(expression, matchBy); final MetricDefinition cpuMetric = build("cpu", "hostname", "eleanore", "service", "2", "other", "vivi"); @@ -571,7 +621,8 @@ public class AlarmCreationBoltTest { bolt.handleNewMetricDefinition(loadAvgMtid, alarmDefinition.getId()); assertEquals(this.createdAlarms.size(), 1); - verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, collector, cpuMtid, loadAvgMtid); + verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, deterministic, collector, + cpuMtid, loadAvgMtid); // Send it again to ensure it handles case where the metric is sent after // the alarm has been created. @@ -580,7 +631,8 @@ public class AlarmCreationBoltTest { assertEquals(this.createdAlarms.size(), 1); // Make sure it did not get added to the existing alarm - verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, collector, cpuMtid, loadAvgMtid); + verifyCreatedAlarm(this.createdAlarms.get(0), alarmDefinition, deterministic, collector, + cpuMtid, loadAvgMtid); } private AlarmDefinition createAlarmDefinition(final String expression, final String... matchBy) { @@ -593,10 +645,17 @@ public class AlarmCreationBoltTest { return alarmDefinition; } - private void verifyCreatedAlarm(final Alarm newAlarm, final AlarmDefinition alarmDefinition, - final OutputCollector collector, MetricDefinitionAndTenantId... mtids) { + private void verifyCreatedAlarm(final Alarm newAlarm, + final AlarmDefinition alarmDefinition, + final Boolean deterministic, + final OutputCollector collector, + MetricDefinitionAndTenantId... mtids) { + + final AlarmState expectedState = deterministic ? AlarmState.OK : AlarmState.UNDETERMINED; + assertEquals(newAlarm.getState(), expectedState); + final String alarmId = newAlarm.getId(); - final Alarm expectedAlarm = new Alarm(alarmDefinition, AlarmState.UNDETERMINED); + final Alarm expectedAlarm = new Alarm(alarmDefinition); expectedAlarm.setId(alarmId); final List expectedSubAlarms = new LinkedList<>(); for (final SubAlarm expectedSubAlarm : expectedAlarm.getSubAlarms()) { diff --git a/thresh/src/test/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBoltTest.java b/thresh/src/test/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBoltTest.java index 0805c21..b3cfb9d 100644 --- a/thresh/src/test/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBoltTest.java +++ b/thresh/src/test/java/monasca/thresh/infrastructure/thresholding/MetricAggregationBoltTest.java @@ -1,5 +1,6 @@ /* * Copyright (c) 2014 Hewlett-Packard Development Company, L.P. + * Copyright 2016 FUJITSU LIMITED * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -29,6 +30,7 @@ import static org.testng.Assert.assertNull; import static org.testng.Assert.assertTrue; import monasca.common.model.alarm.AlarmOperator; import monasca.common.model.alarm.AlarmState; + import monasca.common.model.alarm.AlarmSubExpression; import monasca.common.model.metric.Metric; import monasca.common.model.metric.MetricDefinition; @@ -69,12 +71,15 @@ public class MetricAggregationBoltTest { private SubAlarm subAlarm1; private SubAlarm subAlarm2; private SubAlarm subAlarm3; + private SubAlarm subAlarm4; private SubExpression subExpr1; private SubExpression subExpr2; private SubExpression subExpr3; + private SubExpression subExpr4; private MetricDefinition metricDef1; private MetricDefinition metricDef2; private MetricDefinition metricDef3; + private MetricDefinition metricDef4; @BeforeClass protected void beforeClass() { @@ -83,21 +88,27 @@ public class MetricAggregationBoltTest { subExpr1 = new SubExpression("444", AlarmSubExpression.of("avg(hpcs.compute.cpu{id=5}, 60) >= 90 times 3")); subExpr2 = new SubExpression("555", AlarmSubExpression.of("avg(hpcs.compute.mem{id=5}, 60) >= 90")); subExpr3 = new SubExpression("666", AlarmSubExpression.of("max(hpcs.compute.mem{id=5}, 60) >= 96")); + subExpr4 = new SubExpression("777", AlarmSubExpression.of( + "count(log.error{id=5},deterministic,60) >= 5") + ); metricDef1 = subExpr1.getAlarmSubExpression().getMetricDefinition(); metricDef2 = subExpr2.getAlarmSubExpression().getMetricDefinition(); metricDef3 = subExpr3.getAlarmSubExpression().getMetricDefinition(); + metricDef4 = subExpr4.getAlarmSubExpression().getMetricDefinition(); } @BeforeMethod protected void beforeMethod() { // Fixtures - subAlarm1 = new SubAlarm(ALARM_ID_1, "1", subExpr1, AlarmState.UNDETERMINED); - subAlarm2 = new SubAlarm("456", "1", subExpr2, AlarmState.UNDETERMINED); - subAlarm3 = new SubAlarm("789", "2", subExpr3, AlarmState.UNDETERMINED); + subAlarm1 = new SubAlarm(ALARM_ID_1, "1", subExpr1); + subAlarm2 = new SubAlarm("456", "1", subExpr2); + subAlarm3 = new SubAlarm("789", "2", subExpr3); + subAlarm4 = new SubAlarm("666", "3", subExpr4); subAlarms = new ArrayList<>(); subAlarms.add(subAlarm1); subAlarms.add(subAlarm2); subAlarms.add(subAlarm3); + subAlarms.add(subAlarm4); final ThresholdingConfiguration config = new ThresholdingConfiguration(); config.alarmDelay = 10; @@ -111,6 +122,7 @@ public class MetricAggregationBoltTest { sendSubAlarmCreated(metricDef1, subAlarm1); sendSubAlarmCreated(metricDef2, subAlarm2); + sendSubAlarmCreated(metricDef4, subAlarm4); long t1 = System.currentTimeMillis(); @@ -122,6 +134,10 @@ public class MetricAggregationBoltTest { metricDef2.name, metricDef2.dimensions, t1, 50, null)); bolt.aggregateValues(new MetricDefinitionAndTenantId(metricDef2, TENANT_ID), new Metric( metricDef2.name, metricDef2.dimensions, t1, 40, null)); + bolt.aggregateValues(new MetricDefinitionAndTenantId(metricDef4, TENANT_ID), new Metric( + metricDef4.name, metricDef4.dimensions, t1, 1, null)); + bolt.aggregateValues(new MetricDefinitionAndTenantId(metricDef4, TENANT_ID), new Metric( + metricDef4.name, metricDef4.dimensions, t1, 1, null)); SubAlarmStatsRepository orCreateSubAlarmStatsRepo = bolt.getOrCreateSubAlarmStatsRepo(new MetricDefinitionAndTenantId(metricDef1, TENANT_ID)); SubAlarmStats alarmData = @@ -133,6 +149,11 @@ public class MetricAggregationBoltTest { bolt.getOrCreateSubAlarmStatsRepo(new MetricDefinitionAndTenantId(metricDef2, TENANT_ID)) .get(subAlarm2.getId()); assertEquals(alarmData.getStats().getValue(t1/1000), 45.0); + + alarmData = + bolt.getOrCreateSubAlarmStatsRepo(new MetricDefinitionAndTenantId(metricDef4, TENANT_ID)) + .get(subAlarm4.getId()); + assertEquals(alarmData.getStats().getValue(t1/1000), 2.0); } public void shouldEvaluateAlarms() { @@ -144,6 +165,7 @@ public class MetricAggregationBoltTest { sendSubAlarmCreated(metricDef1, subAlarm1); sendSubAlarmCreated(metricDef2, subAlarm2); sendSubAlarmCreated(metricDef3, subAlarm3); + sendSubAlarmCreated(metricDef4, subAlarm4); // Send metrics for subAlarm1 bolt.execute(createMetricTuple(metricDef1, new Metric(metricDef1, t1, 100000, null))); @@ -157,6 +179,7 @@ public class MetricAggregationBoltTest { assertEquals(subAlarm1.getState(), AlarmState.OK); assertEquals(subAlarm2.getState(), AlarmState.UNDETERMINED); assertEquals(subAlarm3.getState(), AlarmState.UNDETERMINED); + assertEquals(subAlarm4.getState(), AlarmState.OK); // deterministic verify(collector, times(1)).emit(new Values(subAlarm1.getAlarmId(), subAlarm1)); // Have to reset the mock so it can tell the difference when subAlarm2 and subAlarm3 are emitted @@ -175,9 +198,11 @@ public class MetricAggregationBoltTest { assertEquals(subAlarm1.getState(), AlarmState.ALARM); assertEquals(subAlarm2.getState(), AlarmState.ALARM); assertEquals(subAlarm3.getState(), AlarmState.OK); + assertEquals(subAlarm4.getState(), AlarmState.OK); verify(collector, times(1)).emit(new Values(subAlarm1.getAlarmId(), subAlarm1)); verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2)); verify(collector, times(1)).emit(new Values(subAlarm3.getAlarmId(), subAlarm3)); + verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); } public void shouldImmediatelyEvaluateSubAlarm() { @@ -188,16 +213,23 @@ public class MetricAggregationBoltTest { bolt.setCurrentTime(t1); sendSubAlarmCreated(metricDef2, subAlarm2); sendSubAlarmCreated(metricDef3, subAlarm3); + sendSubAlarmCreated(metricDef4, subAlarm4); // Send metric for subAlarm2 and subAlarm3 bolt.execute(createMetricTuple(metricDef3, new Metric(metricDef3, t1 + 1000, 100000, null))); + for (int i = 0; i < 5; i++) { + bolt.execute(createMetricTuple(metricDef4, new Metric(metricDef4, t1 + 1000 + i, 1, null))); + } // subAlarm2 is AVG so it can't be evaluated immediately like the MAX for subalarm3 + // or count for subAlarm4 assertEquals(subAlarm2.getState(), AlarmState.UNDETERMINED); assertEquals(subAlarm3.getState(), AlarmState.ALARM); + assertEquals(subAlarm4.getState(), AlarmState.ALARM); verify(collector, never()).emit(new Values(subAlarm2.getAlarmId(), subAlarm2)); verify(collector, times(1)).emit(new Values(subAlarm3.getAlarmId(), subAlarm3)); + verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); // Have to reset the mock so it can tell the difference when subAlarm2 and subAlarm3 are emitted // again. @@ -209,8 +241,11 @@ public class MetricAggregationBoltTest { assertEquals(subAlarm2.getState(), AlarmState.ALARM); assertEquals(subAlarm3.getState(), AlarmState.ALARM); + assertEquals(subAlarm4.getState(), AlarmState.ALARM); + verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2)); verify(collector, never()).emit(new Values(subAlarm3.getAlarmId(), subAlarm3)); + verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); // Have to reset the mock so it can tell the difference when subAlarm2 and subAlarm3 are emitted // again. @@ -230,8 +265,10 @@ public class MetricAggregationBoltTest { assertEquals(subAlarm2.getState(), AlarmState.OK); assertEquals(subAlarm3.getState(), AlarmState.OK); + assertEquals(subAlarm4.getState(), AlarmState.ALARM); // still in alarm verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2)); verify(collector, times(1)).emit(new Values(subAlarm3.getAlarmId(), subAlarm3)); + verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); // Have to reset the mock so it can tell the difference when subAlarm2 and subAlarm3 are emitted // again. @@ -260,10 +297,13 @@ public class MetricAggregationBoltTest { // Ensure that subAlarm3 is still ALARM. subAlarm2 is still OK but because the metric // that triggered ALARM is in the future bucket + // subAlarm4 gets back to OK due to missing metrics assertEquals(subAlarm2.getState(), AlarmState.OK); assertEquals(subAlarm3.getState(), AlarmState.ALARM); + assertEquals(subAlarm4.getState(), AlarmState.OK); verify(collector, never()).emit(new Values(subAlarm2.getAlarmId(), subAlarm2)); verify(collector, never()).emit(new Values(subAlarm3.getAlarmId(), subAlarm3)); + verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); } private void sendTickTuple() { @@ -355,6 +395,32 @@ public class MetricAggregationBoltTest { verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2)); } + public void shouldNeverLeaveOkIfThresholdNotExceededForNonDeterministic() { + long t1 = 50000; + bolt.setCurrentTime(t1); + sendSubAlarmCreated(metricDef4, subAlarm4); + bolt.execute(createMetricTuple(metricDef4, new Metric(metricDef4, t1, 1.0, null))); + t1 += 1000; + bolt.execute(createMetricTuple(metricDef4, new Metric(metricDef4, t1, 1.0, null))); + + bolt.setCurrentTime(t1 += 60000); + sendTickTuple(); + assertEquals(subAlarm4.getState(), AlarmState.OK); + + bolt.setCurrentTime(t1 += 60000); + sendTickTuple(); + assertEquals(subAlarm4.getState(), AlarmState.OK); + verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); + + // Have to reset the mock so it can tell the difference when subAlarm4 is emitted again. + reset(collector); + + bolt.setCurrentTime(t1 += 60000); + sendTickTuple(); + assertEquals(subAlarm4.getState(), AlarmState.OK); + verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); + } + public void shouldSendUndeterminedOnStartup() { long t1 = 14000; bolt.setCurrentTime(t1); @@ -385,6 +451,36 @@ public class MetricAggregationBoltTest { verify(collector, times(1)).emit(new Values(subAlarm2.getAlarmId(), subAlarm2)); } + public void shouldSendOKOnStartupForNonDeterministic() { + long t1 = 14000; + bolt.setCurrentTime(t1); + sendSubAlarmCreated(metricDef4, subAlarm4); + + final MkTupleParam tupleParam = new MkTupleParam(); + tupleParam.setStream(MetricAggregationBolt.METRIC_AGGREGATION_CONTROL_STREAM); + final Tuple lagTuple = + Testing.testTuple(Arrays.asList(MetricAggregationBolt.METRICS_BEHIND), tupleParam); + bolt.execute(lagTuple); + verify(collector, times(1)).ack(lagTuple); + + t1 += 60000; + bolt.setCurrentTime(t1); + sendTickTuple(); + verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); + + t1 += 60000; + bolt.setCurrentTime(t1); + sendTickTuple(); + verify(collector, never()).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); + + t1 += 60000; + bolt.setCurrentTime(t1); + sendTickTuple(); + assertEquals(subAlarm4.getState(), AlarmState.OK); + + verify(collector, times(1)).emit(new Values(subAlarm4.getAlarmId(), subAlarm4)); + } + private Tuple createTickTuple() { final MkTupleParam tupleParam = new MkTupleParam(); tupleParam.setComponent(Constants.SYSTEM_COMPONENT_ID); diff --git a/thresh/src/test/resources/monasca/thresh/infrastructure/persistence/alarm.sql b/thresh/src/test/resources/monasca/thresh/infrastructure/persistence/alarm.sql index 43211b9..908c74d 100644 --- a/thresh/src/test/resources/monasca/thresh/infrastructure/persistence/alarm.sql +++ b/thresh/src/test/resources/monasca/thresh/infrastructure/persistence/alarm.sql @@ -23,6 +23,7 @@ CREATE TABLE `sub_alarm_definition` ( `threshold` double NOT NULL, `period` int(11) NOT NULL, `periods` int(11) NOT NULL, + `is_deterministic` tinyint(1) NOT NULL DEFAULT '0', `created_at` datetime NOT NULL, `updated_at` datetime NOT NULL, PRIMARY KEY (`id`)