From d8e73f3bdef86041ab8486e8320ca497e80941bf Mon Sep 17 00:00:00 2001 From: Flint Calvin Date: Tue, 21 Jun 2016 19:44:02 +0000 Subject: [PATCH] Added several Swift aggregations (including a new usage component for calculating rate changes). Also fixed some pep8 issues. Change-Id: I46685d39ace663595aa524f04d8d35a71c9432c3 --- devstack/files/monasca-transform/driver.py | 4 +- .../monasca-transform/monasca-transform.conf | 2 +- .../files/monasca-transform/service_runner.py | 5 +- etc/monasca-transform.conf | 2 +- .../component/setter/rollup_quantity.py | 31 ++- .../component/usage/calculate_rate.py | 148 ++++++++++++++ .../component/usage/fetch_quantity.py | 43 ++-- .../component/usage/fetch_quantity_util.py | 8 +- .../pre_transform_specs.json | 5 +- .../transform_specs/transform_specs.json | 4 + setup.cfg | 1 + .../test_data_driven_specs.py | 45 ++++- .../driver/first_attempt_at_spark_test.py | 188 +++++++++++++++++- .../test_resources/kafka_data/kafka_data.txt | 70 ++++--- .../test_resources/mock_component_manager.py | 9 + tests/unit/usage/test_vm_cpu_allocated_agg.py | 12 -- 16 files changed, 493 insertions(+), 84 deletions(-) create mode 100644 monasca_transform/component/usage/calculate_rate.py diff --git a/devstack/files/monasca-transform/driver.py b/devstack/files/monasca-transform/driver.py index 9bf59e2..3f6eeff 100644 --- a/devstack/files/monasca-transform/driver.py +++ b/devstack/files/monasca-transform/driver.py @@ -12,10 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +from monasca_transform.driver.mon_metrics_kafka import invoke + activate_this_file = "/opt/monasca/transform/venv/bin/activate_this.py" execfile(activate_this_file, dict(__file__=activate_this_file)) -from monasca_transform.driver.mon_metrics_kafka import invoke - invoke() diff --git a/devstack/files/monasca-transform/monasca-transform.conf b/devstack/files/monasca-transform/monasca-transform.conf index 4cfd277..f2d05d3 100644 --- a/devstack/files/monasca-transform/monasca-transform.conf +++ b/devstack/files/monasca-transform/monasca-transform.conf @@ -54,7 +54,7 @@ spark_home = /opt/spark/current spark_python_files = /opt/monasca/transform/lib/monasca-transform.zip # How often the stream should be read (in seconds) -stream_interval = 120 +stream_interval = 300 # The working directory for monasca-transform work_dir = /var/run/monasca/transform diff --git a/devstack/files/monasca-transform/service_runner.py b/devstack/files/monasca-transform/service_runner.py index c19de04..758422d 100644 --- a/devstack/files/monasca-transform/service_runner.py +++ b/devstack/files/monasca-transform/service_runner.py @@ -14,11 +14,12 @@ import sys +from monasca_transform.service.transform_service import main_service + + activate_this_file = "/opt/monasca/transform/venv/bin/activate_this.py" execfile(activate_this_file, dict(__file__=activate_this_file)) -from monasca_transform.service.transform_service import main_service - def main(): main_service() diff --git a/etc/monasca-transform.conf b/etc/monasca-transform.conf index 57e9692..a6894e4 100644 --- a/etc/monasca-transform.conf +++ b/etc/monasca-transform.conf @@ -60,7 +60,7 @@ spark_home = /opt/spark/current spark_python_files = /opt/stack/monasca-transform/dist/monasca_transform-0.0.1.egg # How often the stream should be read (in seconds) -stream_interval = 120 +stream_interval = 300 # The working directory for monasca-transform work_dir = /opt/stack/monasca-transform diff --git a/monasca_transform/component/setter/rollup_quantity.py b/monasca_transform/component/setter/rollup_quantity.py index daf9cf6..e3f610d 100644 --- a/monasca_transform/component/setter/rollup_quantity.py +++ b/monasca_transform/component/setter/rollup_quantity.py @@ -163,9 +163,28 @@ class RollupQuantity(SetterComponent): transform_spec_df = transform_context.transform_spec_df_info + # get rollup operation (sum, max, avg, min) + agg_params = transform_spec_df.select( + "aggregation_params_map.setter_rollup_operation").\ + collect()[0].asDict() + setter_rollup_operation = agg_params["setter_rollup_operation"] + + instance_usage_trans_df = RollupQuantity.setter_by_operation( + transform_context, + instance_usage_df, + setter_rollup_operation) + + return instance_usage_trans_df + + @staticmethod + def setter_by_operation(transform_context, instance_usage_df, + setter_rollup_operation): + + transform_spec_df = transform_context.transform_spec_df_info + # get fields we want to group by for a rollup agg_params = transform_spec_df.select( - "aggregation_params_map.setter_rollup_group_by_list").\ + "aggregation_params_map.setter_rollup_group_by_list"). \ collect()[0].asDict() setter_rollup_group_by_list = agg_params["setter_rollup_group_by_list"] @@ -178,14 +197,8 @@ class RollupQuantity(SetterComponent): aggregation_period) # group by columns list - group_by_columns_list = group_by_period_list + \ - setter_rollup_group_by_list - - # get rollup operation (sum, max, avg, min) - agg_params = transform_spec_df.select( - "aggregation_params_map.setter_rollup_operation").\ - collect()[0].asDict() - setter_rollup_operation = agg_params["setter_rollup_operation"] + group_by_columns_list = \ + group_by_period_list + setter_rollup_group_by_list # perform rollup operation instance_usage_json_rdd = RollupQuantity._rollup_quantity( diff --git a/monasca_transform/component/usage/calculate_rate.py b/monasca_transform/component/usage/calculate_rate.py new file mode 100644 index 0000000..5f13fb5 --- /dev/null +++ b/monasca_transform/component/usage/calculate_rate.py @@ -0,0 +1,148 @@ +# Copyright 2016 Hewlett Packard Enterprise Development Company LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from pyspark.sql import SQLContext + +from monasca_transform.component import Component +from monasca_transform.component.setter.rollup_quantity import RollupQuantity +from monasca_transform.component.usage.fetch_quantity import FetchQuantity +from monasca_transform.component.usage import UsageComponent +from monasca_transform.transform.transform_utils import InstanceUsageUtils + +import json + + +class CalculateRateException(Exception): + """Exception thrown when calculating rate + Attributes: + value: string representing the error + """ + + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +class CalculateRate(UsageComponent): + + @staticmethod + def usage(transform_context, record_store_df): + """component which groups together record store records by + provided group by columns list,sorts within the group by event + timestamp field, calculates the rate of change between the + oldest and latest values, and returns the resultant value as an + instance usage dataframe + """ + instance_usage_data_json_list = [] + + transform_spec_df = transform_context.transform_spec_df_info + + # get aggregated metric name + agg_params = transform_spec_df.select( + "aggregation_params_map.aggregated_metric_name"). \ + collect()[0].asDict() + aggregated_metric_name = agg_params["aggregated_metric_name"] + + # get aggregation period + agg_params = transform_spec_df.select( + "aggregation_params_map.aggregation_period").collect()[0].asDict() + aggregation_period = agg_params["aggregation_period"] + + # Fetch the oldest quantities + latest_instance_usage_df = \ + FetchQuantity().usage_by_operation(transform_context, + record_store_df, + "avg") + + # Roll up the latest quantities + latest_rolled_up_instance_usage_df = \ + RollupQuantity().setter_by_operation(transform_context, + latest_instance_usage_df, + "sum") + + # Fetch the oldest quantities + oldest_instance_usage_df = \ + FetchQuantity().usage_by_operation(transform_context, + record_store_df, + "oldest") + + # Roll up the oldest quantities + oldest_rolled_up_instance_usage_df = \ + RollupQuantity().setter_by_operation(transform_context, + oldest_instance_usage_df, + "sum") + + # Calculate the rate change by percentage + oldest_dict = oldest_rolled_up_instance_usage_df.collect()[0].asDict() + oldest_quantity = oldest_dict['quantity'] + + latest_dict = latest_rolled_up_instance_usage_df.collect()[0].asDict() + latest_quantity = latest_dict['quantity'] + + rate_percentage = \ + ((latest_quantity - oldest_quantity) / oldest_quantity) * 100 + + # create a new instance usage dict + instance_usage_dict = {"tenant_id": "all", + "user_id": "all", + "resource_uuid": "all", + "geolocation": "all", + "region": "all", + "zone": "all", + "host": "all", + "project_id": "all", + "aggregated_metric_name": + aggregated_metric_name, + "quantity": rate_percentage, + "firstrecord_timestamp_unix": + oldest_dict["firstrecord_timestamp_unix"], + "firstrecord_timestamp_string": + oldest_dict["firstrecord_timestamp_string"], + "lastrecord_timestamp_unix": + latest_dict["lastrecord_timestamp_unix"], + "lastrecord_timestamp_string": + latest_dict["lastrecord_timestamp_string"], + "record_count": oldest_dict["record_count"] + + latest_dict["record_count"], + "service_group": + Component.DEFAULT_UNAVAILABLE_VALUE, + "service_id": + Component.DEFAULT_UNAVAILABLE_VALUE, + "usage_date": latest_dict["usage_date"], + "usage_hour": latest_dict["usage_hour"], + "usage_minute": latest_dict["usage_minute"], + "aggregation_period": aggregation_period, + "processing_meta": + {"event_type": + latest_dict.get("event_type", + Component. + DEFAULT_UNAVAILABLE_VALUE)} + } + + instance_usage_data_json = json.dumps(instance_usage_dict) + instance_usage_data_json_list.append(instance_usage_data_json) + spark_context = record_store_df.rdd.context + + instance_usage_rdd = \ + spark_context.parallelize(instance_usage_data_json_list) + + sql_context = SQLContext\ + .getOrCreate(record_store_df.rdd.context) + instance_usage_df = InstanceUsageUtils.create_df_from_json_rdd( + sql_context, + instance_usage_rdd) + + return instance_usage_df diff --git a/monasca_transform/component/usage/fetch_quantity.py b/monasca_transform/component/usage/fetch_quantity.py index 2bec411..471ab77 100644 --- a/monasca_transform/component/usage/fetch_quantity.py +++ b/monasca_transform/component/usage/fetch_quantity.py @@ -299,8 +299,23 @@ class FetchQuantity(UsageComponent): collect()[0].asDict() usage_fetch_operation = agg_params["usage_fetch_operation"] + instance_usage_df = FetchQuantity.usage_by_operation( + transform_context, record_store_df, usage_fetch_operation) + + return instance_usage_df + + @staticmethod + def usage_by_operation(transform_context, record_store_df, + usage_fetch_operation): + """component which groups together record store records by + provided group by columns list , sorts within the group by event + timestamp field, applies group stats udf and returns the latest + quantity as a instance usage dataframe + """ + transform_spec_df = transform_context.transform_spec_df_info + # check if operation is valid - if not FetchQuantity.\ + if not FetchQuantity. \ _is_valid_fetch_operation(usage_fetch_operation): raise FetchQuantityException( "Operation %s is not supported" % usage_fetch_operation) @@ -314,7 +329,7 @@ class FetchQuantity(UsageComponent): # get what we want to group by agg_params = transform_spec_df.select( - "aggregation_params_map.aggregation_group_by_list").\ + "aggregation_params_map.aggregation_group_by_list"). \ collect()[0].asDict() aggregation_group_by_list = agg_params["aggregation_group_by_list"] @@ -323,8 +338,8 @@ class FetchQuantity(UsageComponent): aggregation_group_by_list instance_usage_json_rdd = None - if (usage_fetch_operation == "latest" - or usage_fetch_operation == "oldest"): + if (usage_fetch_operation == "latest" or + usage_fetch_operation == "oldest"): grouped_rows_rdd = None @@ -342,7 +357,7 @@ class FetchQuantity(UsageComponent): # high number is adversely affecting performance num_of_groups = 100 grouped_rows_rdd = \ - GroupSortbyTimestampPartition.\ + GroupSortbyTimestampPartition. \ fetch_group_latest_oldest_quantity( record_store_df, transform_spec_df, group_by_columns_list, @@ -350,15 +365,15 @@ class FetchQuantity(UsageComponent): else: # group using key-value pair RDD's groupByKey() grouped_rows_rdd = \ - GroupSortbyTimestamp.\ + GroupSortbyTimestamp. \ fetch_group_latest_oldest_quantity( record_store_df, transform_spec_df, group_by_columns_list) grouped_data_rdd_with_operation = grouped_rows_rdd.map( lambda x: - GroupedDataWithOperation(x, - str(usage_fetch_operation))) + GroupedDataWithOperation(x, + str(usage_fetch_operation))) instance_usage_json_rdd = \ grouped_data_rdd_with_operation.map( @@ -367,10 +382,10 @@ class FetchQuantity(UsageComponent): record_store_df_int = \ record_store_df.select( - record_store_df.event_timestamp_unix - .alias("event_timestamp_unix_for_min"), - record_store_df.event_timestamp_unix - .alias("event_timestamp_unix_for_max"), + record_store_df.event_timestamp_unix.alias( + "event_timestamp_unix_for_min"), + record_store_df.event_timestamp_unix.alias( + "event_timestamp_unix_for_max"), "*") # for standard sum, max, min, avg operations on grouped data @@ -385,8 +400,8 @@ class FetchQuantity(UsageComponent): grouped_data_rdd_with_operation = grouped_record_store_df.map( lambda x: - GroupedDataWithOperation(x, - str(usage_fetch_operation))) + GroupedDataWithOperation(x, + str(usage_fetch_operation))) instance_usage_json_rdd = grouped_data_rdd_with_operation.map( FetchQuantity._get_quantity) diff --git a/monasca_transform/component/usage/fetch_quantity_util.py b/monasca_transform/component/usage/fetch_quantity_util.py index f9d40dc..55c0569 100644 --- a/monasca_transform/component/usage/fetch_quantity_util.py +++ b/monasca_transform/component/usage/fetch_quantity_util.py @@ -251,10 +251,10 @@ class FetchQuantityUtil(UsageComponent): # quant_idle_perc_calc_df = quant_idle_perc_df.select( col("quantity_df_alias.*"), - when(col("idle_perc_df_alias.quantity") - != 0.0, - ceil(((100.0 - col("idle_perc_df_alias.quantity"))) - * col("quantity_df_alias.quantity") / 100.0)) + when(col("idle_perc_df_alias.quantity") != 0.0, + ceil(((100.0 - col( + "idle_perc_df_alias.quantity"))) * col( + "quantity_df_alias.quantity") / 100.0)) .otherwise(col("quantity_df_alias.quantity")) .alias("utilized_quantity"), diff --git a/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json b/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json index c178b7c..d763c51 100644 --- a/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json +++ b/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json @@ -5,9 +5,10 @@ {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_space_mb","metric_id_list":["disk_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_used_space_mb","metric_id_list":["disk_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.disk.total_allocated_gb","metric_id_list":["nova_disk_total_allocated_gb_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} -{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.disk.allocation","metric_id_list":["vm_disk_allocation_all","vm_disk_allocation_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"} +{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.disk.allocation","metric_id_list":["vm_disk_allocation_all","vm_disk_allocation_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.total_logical_cores","metric_id_list":["cpu_total_all","cpu_total_host","cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.idle_perc","metric_id_list":["cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id","resource_id"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"} -{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"} +{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all","swift_usage_rate","swift_usage_host"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"} +{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.avail","metric_id_list":["swift_avail_all","swift_avail_host"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"} diff --git a/monasca_transform/data_driven_specs/transform_specs/transform_specs.json b/monasca_transform/data_driven_specs/transform_specs/transform_specs.json index b553f30..45ba3eb 100644 --- a/monasca_transform/data_driven_specs/transform_specs/transform_specs.json +++ b/monasca_transform/data_driven_specs/transform_specs/transform_specs.json @@ -17,3 +17,7 @@ {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_all"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_host"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"calculate_rate","setters":["set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.rate_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"setter_rollup_group_by_list": [],"dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_rate","metric_id":"swift_usage_rate"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_avail_all","metric_id":"swift_avail_all"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_avail_all","metric_id":"swift_avail_host"} diff --git a/setup.cfg b/setup.cfg index 586dbe1..28d28cb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -20,6 +20,7 @@ packages = [entry_points] monasca_transform.usage = + calculate_rate = monasca_transform.component.usage.calculate_rate:CalculateRate fetch_quantity = monasca_transform.component.usage.fetch_quantity:FetchQuantity fetch_quantity_util = monasca_transform.component.usage.fetch_quantity_util:FetchQuantityUtil diff --git a/tests/unit/data_driven_specs/test_data_driven_specs.py b/tests/unit/data_driven_specs/test_data_driven_specs.py index 5e8d9b7..9d99ec0 100644 --- a/tests/unit/data_driven_specs/test_data_driven_specs.py +++ b/tests/unit/data_driven_specs/test_data_driven_specs.py @@ -79,6 +79,14 @@ class TestDataDrivenSpecsRepo(SparkContextTest): metric_id='swift_usage_all', expected_agg_metric_name='swiftlm.diskusage.val.size_agg', transform_specs_dataframe=transform_specs_data_frame) + self.check_metric( + metric_id='swift_avail_all', + expected_agg_metric_name='swiftlm.diskusage.val.avail_agg', + transform_specs_dataframe=transform_specs_data_frame) + self.check_metric( + metric_id='swift_usage_rate', + expected_agg_metric_name='swiftlm.diskusage.rate_agg', + transform_specs_dataframe=transform_specs_data_frame) def check_metric(self, metric_id=None, expected_agg_metric_name=None, transform_specs_dataframe=None): @@ -107,7 +115,8 @@ class TestDataDrivenSpecsRepo(SparkContextTest): u'vm.mem.total_mb', u'vm.mem.used_mb', u'nova.vm.disk.total_allocated_gb', u'vm.disk.allocation', u'vm.cpu.utilization_perc', - u'swiftlm.diskusage.host.val.size']), + u'swiftlm.diskusage.host.val.size', + u'swiftlm.diskusage.host.val.avail']), Counter([row.event_type for row in pre_transform_specs_data_frame.collect()])) @@ -295,7 +304,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest): self.check_list_field_for_row( row=vm_disk_allocation_all_row, field_name='required_raw_fields_list', - expected_list=['creation_time', 'tenantId', 'resource_id'], + expected_list=['creation_time', 'tenant_id', 'resource_id'], ) self.check_dict_field_for_row( row=vm_disk_allocation_all_row, @@ -346,7 +355,8 @@ class TestDataDrivenSpecsRepo(SparkContextTest): self.check_list_field_for_row( row=swiftlm_diskusage_all_row, field_name='metric_id_list', - expected_list=['swift_usage_all'] + expected_list=['swift_usage_all', 'swift_usage_host', + 'swift_usage_rate'] ) self.check_list_field_for_row( row=swiftlm_diskusage_all_row, @@ -367,6 +377,35 @@ class TestDataDrivenSpecsRepo(SparkContextTest): expected_value='host_metrics' ) + # swiftlm.diskusage.host.val.avail + event_type = 'swiftlm.diskusage.host.val.avail' + swiftlm_diskavail_all_row = self.get_row_for_event_type( + event_type=event_type, + pre_transform_specs_data_frame=pre_transform_specs_data_frame) + self.check_list_field_for_row( + row=swiftlm_diskavail_all_row, + field_name='metric_id_list', + expected_list=['swift_avail_all', 'swift_avail_host'] + ) + self.check_list_field_for_row( + row=swiftlm_diskavail_all_row, + field_name='required_raw_fields_list', + expected_list=['creation_time', 'hostname', 'mount', + 'device'], + ) + self.check_dict_field_for_row( + row=swiftlm_diskavail_all_row, + field_name='event_processing_params', + expected_dict={ + "set_default_zone_to": "1", + "set_default_geolocation_to": "1", + "set_default_region_to": "W"}) + self.check_value_field_for_row( + row=swiftlm_diskavail_all_row, + field_name='service_id', + expected_value='host_metrics' + ) + def get_row_for_event_type(self, event_type=None, pre_transform_specs_data_frame=None): diff --git a/tests/unit/driver/first_attempt_at_spark_test.py b/tests/unit/driver/first_attempt_at_spark_test.py index 01f4d54..c39a120 100644 --- a/tests/unit/driver/first_attempt_at_spark_test.py +++ b/tests/unit/driver/first_attempt_at_spark_test.py @@ -114,7 +114,7 @@ class SparkTest(SparkContextTest): result = simple_count_transform(rdd_monasca_with_offsets) # Verify it worked - self.assertEqual(result, 347) + self.assertEqual(result, 363) # Call the primary method in mon_metrics_kafka MonMetricsKafkaProcessor.rdd_to_recordstore( @@ -972,13 +972,14 @@ class SparkTest(SparkContextTest): .get('metric').get('value_meta') .get('lastrecord_timestamp')) - # Verify swiftlm.diskusage.val.size_agg + # Verify swiftlm.diskusage.val.size_agg for all hosts used_swift_agg_metric = [ value for value in metrics if value.get('metric').get('name') == - 'swiftlm.diskusage.val.size_agg'][0] + 'swiftlm.diskusage.val.size_agg' and + value.get('metric').get('dimensions').get('host') == 'all'][0] - self.assertEqual(360000000000342.0, + self.assertEqual(5291.0, used_swift_agg_metric.get('metric').get('value')) self.assertEqual('useast', used_swift_agg_metric.get('meta').get('region')) @@ -986,23 +987,196 @@ class SparkTest(SparkContextTest): used_swift_agg_metric.get('meta').get('tenantId')) self.assertEqual('all', used_swift_agg_metric.get('metric').get('dimensions') - .get('host')) + .get('project_id')) + self.assertEqual('hourly', + used_swift_agg_metric.get('metric').get('dimensions') + .get('aggregation_period')) + self.assertEqual(17.0, + used_swift_agg_metric.get('metric').get('value_meta') + .get('record_count')) + self.assertEqual('2016-06-10 20:27:01', + used_swift_agg_metric.get('metric').get('value_meta') + .get('firstrecord_timestamp')) + self.assertEqual('2016-06-10 20:27:01', + used_swift_agg_metric.get('metric').get('value_meta') + .get('lastrecord_timestamp')) + + # Verify swiftlm.diskusage.val.size_agg for host a + used_swift_agg_metric = [ + value for value in metrics + if value.get('metric').get('name') == + 'swiftlm.diskusage.val.size_agg' and + value.get('metric').get('dimensions').get('host') == 'a'][0] + + self.assertEqual(2848.0, + used_swift_agg_metric.get('metric').get('value')) + self.assertEqual('useast', + used_swift_agg_metric.get('meta').get('region')) + self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id, + used_swift_agg_metric.get('meta').get('tenantId')) self.assertEqual('all', used_swift_agg_metric.get('metric').get('dimensions') .get('project_id')) self.assertEqual('hourly', used_swift_agg_metric.get('metric').get('dimensions') .get('aggregation_period')) - self.assertEqual(18.0, + self.assertEqual(8.0, used_swift_agg_metric.get('metric').get('value_meta') .get('record_count')) self.assertEqual('2016-06-10 20:27:01', used_swift_agg_metric.get('metric').get('value_meta') .get('firstrecord_timestamp')) - self.assertEqual('2016-06-10 20:27:20', + self.assertEqual('2016-06-10 20:27:01', used_swift_agg_metric.get('metric').get('value_meta') .get('lastrecord_timestamp')) + # Verify swiftlm.diskusage.val.size_agg for host b + used_swift_agg_metric = [ + value for value in metrics + if value.get('metric').get('name') == + 'swiftlm.diskusage.val.size_agg' and + value.get('metric').get('dimensions').get('host') == 'b'][0] + + self.assertEqual(2443.0, + used_swift_agg_metric.get('metric').get('value')) + self.assertEqual('useast', + used_swift_agg_metric.get('meta').get('region')) + self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id, + used_swift_agg_metric.get('meta').get('tenantId')) + self.assertEqual('all', + used_swift_agg_metric.get('metric').get('dimensions') + .get('project_id')) + self.assertEqual('hourly', + used_swift_agg_metric.get('metric').get('dimensions') + .get('aggregation_period')) + self.assertEqual(9.0, + used_swift_agg_metric.get('metric').get('value_meta') + .get('record_count')) + self.assertEqual('2016-06-10 20:27:01', + used_swift_agg_metric.get('metric').get('value_meta') + .get('firstrecord_timestamp')) + self.assertEqual('2016-06-10 20:27:01', + used_swift_agg_metric.get('metric').get('value_meta') + .get('lastrecord_timestamp')) + + # Verify swiftlm.diskusage.val.avail_agg for all hosts + avail_swift_agg_metric = [ + value for value in metrics + if value.get('metric').get('name') == + 'swiftlm.diskusage.val.avail_agg' and + value.get('metric').get('dimensions').get('host') == 'all'][0] + + self.assertEqual(5291.0, + avail_swift_agg_metric.get('metric').get('value')) + self.assertEqual('useast', + avail_swift_agg_metric.get('meta').get('region')) + self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id, + avail_swift_agg_metric.get('meta').get('tenantId')) + self.assertEqual('all', + avail_swift_agg_metric.get('metric').get('dimensions') + .get('project_id')) + self.assertEqual('hourly', + avail_swift_agg_metric.get('metric').get('dimensions') + .get('aggregation_period')) + self.assertEqual(17.0, + avail_swift_agg_metric.get('metric').get('value_meta') + .get('record_count')) + self.assertEqual('2016-06-10 20:27:01', + avail_swift_agg_metric.get('metric').get('value_meta') + .get('firstrecord_timestamp')) + self.assertEqual('2016-06-10 20:27:01', + avail_swift_agg_metric.get('metric').get('value_meta') + .get('lastrecord_timestamp')) + + # Verify swiftlm.diskusage.val.avail_agg for host a + avail_swift_agg_metric = [ + value for value in metrics + if value.get('metric').get('name') == + 'swiftlm.diskusage.val.avail_agg' and + value.get('metric').get('dimensions').get('host') == 'a'][0] + + self.assertEqual(2848.0, + avail_swift_agg_metric.get('metric').get('value')) + self.assertEqual('useast', + avail_swift_agg_metric.get('meta').get('region')) + self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id, + avail_swift_agg_metric.get('meta').get('tenantId')) + self.assertEqual('all', + avail_swift_agg_metric.get('metric').get('dimensions') + .get('project_id')) + self.assertEqual('hourly', + avail_swift_agg_metric.get('metric').get('dimensions') + .get('aggregation_period')) + self.assertEqual(8.0, + avail_swift_agg_metric.get('metric').get('value_meta') + .get('record_count')) + self.assertEqual('2016-06-10 20:27:01', + avail_swift_agg_metric.get('metric').get('value_meta') + .get('firstrecord_timestamp')) + self.assertEqual('2016-06-10 20:27:01', + avail_swift_agg_metric.get('metric').get('value_meta') + .get('lastrecord_timestamp')) + + # Verify swiftlm.diskusage.val.avail_agg for host b + avail_swift_agg_metric = [ + value for value in metrics + if value.get('metric').get('name') == + 'swiftlm.diskusage.val.avail_agg' and + value.get('metric').get('dimensions').get('host') == 'b'][0] + + self.assertEqual(2443.0, + avail_swift_agg_metric.get('metric').get('value')) + self.assertEqual('useast', + avail_swift_agg_metric.get('meta').get('region')) + self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id, + avail_swift_agg_metric.get('meta').get('tenantId')) + self.assertEqual('all', + avail_swift_agg_metric.get('metric').get('dimensions') + .get('project_id')) + self.assertEqual('hourly', + avail_swift_agg_metric.get('metric').get('dimensions') + .get('aggregation_period')) + self.assertEqual(9.0, + avail_swift_agg_metric.get('metric').get('value_meta') + .get('record_count')) + self.assertEqual('2016-06-10 20:27:01', + avail_swift_agg_metric.get('metric').get('value_meta') + .get('firstrecord_timestamp')) + self.assertEqual('2016-06-10 20:27:01', + avail_swift_agg_metric.get('metric').get('value_meta') + .get('lastrecord_timestamp')) + + # Verify swiftlm.diskusage.rate_agg metrics + diskusage_rate_agg_metric = [ + value for value in metrics + if value.get('metric').get('name') == + 'swiftlm.diskusage.rate_agg'][0] + + self.assertEqual(15.650273224043715, + diskusage_rate_agg_metric.get('metric').get('value')) + self.assertEqual('useast', + diskusage_rate_agg_metric.get('meta').get('region')) + self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id, + diskusage_rate_agg_metric.get('meta').get('tenantId')) + self.assertEqual('all', + diskusage_rate_agg_metric.get('metric') + .get('dimensions').get('host')) + self.assertEqual('all', + diskusage_rate_agg_metric.get('metric') + .get('dimensions').get('project_id')) + self.assertEqual('hourly', + diskusage_rate_agg_metric.get('metric') + .get('dimensions').get('aggregation_period')) + self.assertEqual(34.0, + diskusage_rate_agg_metric.get('metric') + .get('value_meta').get('record_count')) + self.assertEqual('2016-06-10 20:27:01', + diskusage_rate_agg_metric.get('metric') + .get('value_meta').get('firstrecord_timestamp')) + self.assertEqual('2016-06-10 20:27:01', + diskusage_rate_agg_metric.get('metric') + .get('value_meta').get('lastrecord_timestamp')) + def simple_count_transform(rdd): return rdd.count() diff --git a/tests/unit/test_resources/kafka_data/kafka_data.txt b/tests/unit/test_resources/kafka_data/kafka_data.txt index 11ee4c4..08b8cea 100644 --- a/tests/unit/test_resources/kafka_data/kafka_data.txt +++ b/tests/unit/test_resources/kafka_data/kafka_data.txt @@ -296,15 +296,15 @@ ('','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0006-mgmt","cluster":"compute"},"timestamp":1463498061000,"value":82.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498062}') ('','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0003-mgmt","cluster":"compute"},"timestamp":1463498078000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498080}') ('','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0002-mgmt","cluster":"compute"},"timestamp":1463498084000,"value":132.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498086}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"c","component":"vm","service":"compute","hostname":"dummy1","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":28.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"d","component":"vm","service":"compute","hostname":"dummy2","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":30.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"e","component":"vm","service":"compute","hostname":"dummy3","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":32.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') -('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"c","component":"vm","service":"compute","hostname":"dummy1","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":28.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"d","component":"vm","service":"compute","hostname":"dummy2","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":30.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"e","component":"vm","service":"compute","hostname":"dummy3","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":32.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') +('','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}') ('','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"7b805207-3c97-4ef9-8b28-3fac171f2a96","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283861000,"value":5.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}') ('','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"a07858d1-aa13-4259-9873-f23170bc9e1b","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283895000,"value":35.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}') ('','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"4cd09eac-854d-49eb-993d-914271f8e1fe","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283896000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}') @@ -327,21 +327,37 @@ ('','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"b","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316994000,"value":5120.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316995}') ('','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"c","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317024000,"value":2048.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317025}') ('','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"d","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317054000,"value":6144.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317055}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000006},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000008},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000010},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000012},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000014},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000016},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000018},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000020},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000022},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000024},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000030},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000032},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000034},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000036},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":904},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":804},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":928},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421004,"value":1128},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421001,"value":916},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":901},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":602},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":926},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1126},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":904},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":804},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":928},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421004,"value":1128},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421001,"value":916},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":901},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":602},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":926},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1126},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') diff --git a/tests/unit/test_resources/mock_component_manager.py b/tests/unit/test_resources/mock_component_manager.py index 49979c3..7a801be 100644 --- a/tests/unit/test_resources/mock_component_manager.py +++ b/tests/unit/test_resources/mock_component_manager.py @@ -25,6 +25,8 @@ from monasca_transform.component.setter.set_aggregated_metric_name \ import SetAggregatedMetricName from monasca_transform.component.setter.set_aggregated_period \ import SetAggregatedPeriod +from monasca_transform.component.usage.calculate_rate \ + import CalculateRate from monasca_transform.component.usage.fetch_quantity \ import FetchQuantity from monasca_transform.component.usage.fetch_quantity_util \ @@ -49,6 +51,13 @@ class MockComponentManager(object): 'FetchQuantityUtil', FetchQuantityUtil(), None), + Extension( + 'calculate_rate', + 'monasca_transform.component.usage.' + 'calculate_rate:' + 'CalculateRate', + CalculateRate(), + None), ]) @staticmethod diff --git a/tests/unit/usage/test_vm_cpu_allocated_agg.py b/tests/unit/usage/test_vm_cpu_allocated_agg.py index 9a76e2e..92b1729 100644 --- a/tests/unit/usage/test_vm_cpu_allocated_agg.py +++ b/tests/unit/usage/test_vm_cpu_allocated_agg.py @@ -138,12 +138,6 @@ class TestVmCpuAllocatedAgg(SparkContextTest): rdd_monasca_with_offsets = rdd_monasca.map( lambda x: RddTransformContext(x, transform_context)) - # Do something simple with the RDD - result = simple_count_transform(rdd_monasca_with_offsets) - - # Verify it worked - self.assertEqual(result, 347) - # Call the primary method in mon_metrics_kafka MonMetricsKafkaProcessor.rdd_to_recordstore( rdd_monasca_with_offsets) @@ -317,12 +311,6 @@ class TestVmCpuAllocatedAgg(SparkContextTest): rdd_monasca_with_offsets = rdd_monasca.map( lambda x: RddTransformContext(x, transform_context)) - # Do something simple with the RDD - result = simple_count_transform(rdd_monasca_with_offsets) - - # Verify it worked - self.assertEqual(result, 347) - # Call the primary method in mon_metrics_kafka MonMetricsKafkaProcessor.rdd_to_recordstore( rdd_monasca_with_offsets)