From 85721b65b4bc5a9703aee041fd3ed7e4811ef272 Mon Sep 17 00:00:00 2001 From: Matthias Runge Date: Thu, 12 Aug 2021 09:50:37 +0000 Subject: [PATCH] Revert "Remove autoscaling until gate issues sorted out" This reverts commit 34e514a4824a5e47a895222fdb42f0f7d7b0123d. In general, we'd want to check autoscaling from end to end. Until now this plugin could only scale up the stack by adding an instance. With this change, scaling down of instance is also tested using a scaledown policy and an alarm which is triggered if instance CPU usage falls below the threshold. Depends-On: https://review.opendev.org/865386 Depends-On: https://review.opendev.org/868105 Depends-On: https://review.opendev.org/c/openstack/telemetry-tempest-plugin/+/904746 Co-authored-by: Yadnesh Kulkarni Change-Id: I7f19dcaf327b636ef8328c6634d295414b7c8e65 --- .../autoscaling.yaml | 163 ++++++++++++++++++ .../create_stack.json | 48 +++++- 2 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 telemetry_tempest_plugin/scenario/telemetry_integration_gabbits/autoscaling.yaml diff --git a/telemetry_tempest_plugin/scenario/telemetry_integration_gabbits/autoscaling.yaml b/telemetry_tempest_plugin/scenario/telemetry_integration_gabbits/autoscaling.yaml new file mode 100644 index 0000000..6b87b2b --- /dev/null +++ b/telemetry_tempest_plugin/scenario/telemetry_integration_gabbits/autoscaling.yaml @@ -0,0 +1,163 @@ +defaults: + request_headers: + x-auth-token: $ENVIRON['USER_TOKEN'] + +tests: + - name: list alarms none + desc: Lists alarms, none yet exist + verbose: all + url: $ENVIRON['AODH_SERVICE_URL']/v2/alarms + method: GET + response_strings: + - "[]" + + - name: list servers none + desc: List servers, none yet exists + verbose: all + url: $ENVIRON['NOVA_SERVICE_URL']/servers + method: GET + response_strings: + - "[]" + + - name: create stack + desc: Create an autoscaling stack + verbose: all + url: $ENVIRON['HEAT_SERVICE_URL']/stacks + method: POST + request_headers: + content-type: application/json + data: <@create_stack.json + status: 201 + + - name: control stack status + desc: Checks the stack have been created successfully + url: $ENVIRON['HEAT_SERVICE_URL']/stacks/$ENVIRON['STACK_NAME'] + redirects: true + verbose: all + method: GET + status: 200 + poll: + count: 300 + delay: 1 + response_json_paths: + $.stack.stack_status: "CREATE_COMPLETE" + + - name: list servers grow + verbose: all + desc: Wait the autoscaling stack grow to two servers + url: $ENVIRON['NOVA_SERVICE_URL']/servers/detail + method: GET + poll: + count: 600 + delay: 1 + response_json_paths: + $.servers[0].metadata.'metering.server_group': $RESPONSE['$.stack.id'] + $.servers[1].metadata.'metering.server_group': $RESPONSE['$.stack.id'] + $.servers[0].status: ACTIVE + $.servers[1].status: ACTIVE + $.servers.`len`: 2 + + - name: check gnocchi resources + desc: Check the gnocchi resources for this two servers exists + url: $ENVIRON['GNOCCHI_SERVICE_URL']/v1/search/resource/instance + verbose: all + method: POST + request_headers: + content-type: application/json + data: + =: + server_group: $RESPONSE['$.servers[0].metadata."metering.server_group"'] + poll: + count: 600 + delay: 1 + response_json_paths: + $.`len`: 2 + + - name: check alarm cpu_alarm_high ALARM + verbose: all + desc: Check the aodh alarm and its state + url: $ENVIRON['AODH_SERVICE_URL']/v2/alarms?sort=name%3Aasc + method: GET + poll: + count: 600 + delay: 5 + response_strings: + - "$ENVIRON['STACK_NAME']-cpu_alarm_high" + response_json_paths: + $[0].state: alarm + + - name: check alarm cpu_alarm_high is OK + verbose: all + desc: Check the aodh alarm and its state + url: $ENVIRON['AODH_SERVICE_URL']/v2/alarms?sort=name%3Aasc + method: GET + poll: + count: 600 + delay: 5 + response_strings: + - "$ENVIRON['STACK_NAME']-cpu_alarm_high-" + response_json_paths: + $[0].state: ok + + + - name: check alarm cpu_alarm_low is ALARM + verbose: all + desc: Check the aodh alarm and its state + url: $ENVIRON['AODH_SERVICE_URL']/v2/alarms?sort=name%3Aasc + method: GET + poll: + count: 600 + delay: 5 + response_strings: + - "$ENVIRON['STACK_NAME']-cpu_alarm_low-" + response_json_paths: + $[1].state: alarm + + - name: list servers shrink + verbose: all + desc: Wait for the autoscaling stack to delete one server + url: $ENVIRON['NOVA_SERVICE_URL']/servers/detail + method: GET + poll: + count: 600 + delay: 1 + response_json_paths: + $.servers[0].metadata.'metering.server_group': $HISTORY['control stack status'].$RESPONSE['$.stack.id'] + $.servers[0].status: ACTIVE + $.servers.`len`: 1 + + - name: get stack location + desc: Get the stack location + url: $ENVIRON['HEAT_SERVICE_URL']/stacks/$ENVIRON['STACK_NAME'] + method: GET + status: 302 + + - name: delete stack + desc: Delete the stack + url: $LOCATION + method: DELETE + status: 204 + + - name: get deleted stack + desc: Check the stack have been deleted + url: $ENVIRON['HEAT_SERVICE_URL']/stacks/$ENVIRON['STACK_NAME'] + redirects: true + method: GET + poll: + count: 300 + delay: 5 + status: 404 + + - name: list alarms deleted + desc: List alarms, no more exist + url: $ENVIRON['AODH_SERVICE_URL']/v2/alarms + method: GET + response_strings: + - "[]" + + - name: list servers deleted + desc: List servers, no more exists + url: $ENVIRON['NOVA_SERVICE_URL']/servers + method: GET + response_strings: + - "[]" diff --git a/telemetry_tempest_plugin/scenario/telemetry_integration_gabbits/create_stack.json b/telemetry_tempest_plugin/scenario/telemetry_integration_gabbits/create_stack.json index 429ba49..6bb6d30 100644 --- a/telemetry_tempest_plugin/scenario/telemetry_integration_gabbits/create_stack.json +++ b/telemetry_tempest_plugin/scenario/telemetry_integration_gabbits/create_stack.json @@ -23,7 +23,7 @@ "#!/bin/sh\n", "echo 'Loading CPU'\n", "set -v\n", - "cat /dev/urandom > /dev/null\n" + "cat /dev/urandom > /dev/null & sleep 120 ; kill $! \n" ]]} } } @@ -34,18 +34,18 @@ "properties": { "adjustment_type": "change_in_capacity", "auto_scaling_group_id": { "get_resource": "asg" }, - "cooldown": 2, + "cooldown": 60, "scaling_adjustment": 1 } }, "cpu_alarm_high": { - "type": "OS::Ceilometer::GnocchiAggregationByResourcesAlarm", + "type": "OS::Aodh::GnocchiAggregationByResourcesAlarm", "properties": { "description": "Scale-up if the mean CPU > 10% on 1 minute", "metric": "$ENVIRON["CEILOMETER_METRIC_NAME"]", "aggregation_method": "$ENVIRON["GNOCCHI_AGGREGATION_METHOD"]", "granularity": $ENVIRON["AODH_GRANULARITY"], - "evaluation_periods": 1, + "evaluation_periods": 2, "threshold": $ENVIRON["AODH_THRESHOLD"], "comparison_operator": "gt", "alarm_actions": [ @@ -68,6 +68,46 @@ } } } + }, + "web_server_scaledown_policy": { + "type": "OS::Heat::ScalingPolicy", + "properties": { + "adjustment_type": "change_in_capacity", + "auto_scaling_group_id": { "get_resource": "asg" }, + "cooldown": 60, + "scaling_adjustment": -1 + } + }, + "cpu_alarm_low": { + "type": "OS::Aodh::GnocchiAggregationByResourcesAlarm", + "properties": { + "description": "Scale-down if the mean CPU < 10% on 1 minute", + "metric": "$ENVIRON["CEILOMETER_METRIC_NAME"]", + "aggregation_method": "$ENVIRON["GNOCCHI_AGGREGATION_METHOD"]", + "granularity": $ENVIRON["AODH_GRANULARITY"], + "evaluation_periods": 2, + "threshold": $ENVIRON["AODH_THRESHOLD"], + "resource_type": "instance", + "comparison_operator": "lt", + "alarm_actions": [ + { + "str_replace": { + "template": "trust+url", + "params": { + "url": { "get_attr": [ "web_server_scaledown_policy", "signal_url" ] } + } + } + } + ], + "query": { + "str_replace": { + "template": "{\"and\": [{\"=\": {\"server_group\": \"stack_id\"}}, {\"=\": {\"ended_at\": null}}]}", + "params": { + "stack_id": { "get_param": "OS::stack_id" } + } + } + } + } } } }