From 3cdb0d16878fbaff266e98de01c4d03f4358679b Mon Sep 17 00:00:00 2001 From: Flint Calvin Date: Tue, 6 Sep 2016 20:52:49 +0000 Subject: [PATCH] Made corrections such that swiftlm.diskusage.rate_agg is now correctly based on swiftlm.diskusage.host.val.avail (instead of incorrectly being based on swiftlm.diskusage.host.val.size). Change-Id: If17853e166c050cefbf390791a8696ce520fca96 --- .../setter/pre_hourly_calculate_rate.py | 2 +- .../component/usage/calculate_rate.py | 2 +- .../pre_transform_specs.json | 4 +-- .../transform_specs/transform_specs.json | 6 ++--- .../test_data_driven_specs.py | 8 +++--- .../driver/first_attempt_at_spark_test.py | 2 +- .../test_pre_hourly_processor_agg.py | 2 +- .../test_resources/kafka_data/kafka_data.txt | 26 +++++++++---------- .../metrics_pre_hourly_data.txt | 6 ++--- 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/monasca_transform/component/setter/pre_hourly_calculate_rate.py b/monasca_transform/component/setter/pre_hourly_calculate_rate.py index 5de618e..6ad1e14 100644 --- a/monasca_transform/component/setter/pre_hourly_calculate_rate.py +++ b/monasca_transform/component/setter/pre_hourly_calculate_rate.py @@ -58,7 +58,7 @@ class PreHourlyCalculateRate(SetterComponent): "processing_meta"]["latest_quantity"]) rate_percentage = 100 * ( - (latest_quantity - oldest_quantity) / oldest_quantity) + (oldest_quantity - latest_quantity) / oldest_quantity) except Exception as e: raise PreHourlyCalculateRateException( "Exception occurred in pre-hourly rate calculation. Error: %s" diff --git a/monasca_transform/component/usage/calculate_rate.py b/monasca_transform/component/usage/calculate_rate.py index eff7e7f..9a03ea8 100644 --- a/monasca_transform/component/usage/calculate_rate.py +++ b/monasca_transform/component/usage/calculate_rate.py @@ -93,7 +93,7 @@ class CalculateRate(UsageComponent): latest_quantity = float(latest_dict['quantity']) rate_percentage = \ - ((latest_quantity - oldest_quantity) / oldest_quantity) * 100 + ((oldest_quantity - latest_quantity) / oldest_quantity) * 100 # create a new instance usage dict instance_usage_dict = {"tenant_id": diff --git a/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json b/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json index ead4304..e1838b3 100644 --- a/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json +++ b/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json @@ -12,6 +12,6 @@ {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id","resource_id"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.cpu.total_allocated","metric_id_list":["nova_vm_cpu_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} -{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all","swift_usage_rate","swift_usage_host"],"required_raw_fields_list":["creation_time", "hostname", "mount"],"service_id":"host_metrics"} -{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.avail","metric_id_list":["swift_avail_all","swift_avail_host"],"required_raw_fields_list":["creation_time", "hostname", "mount"],"service_id":"host_metrics"} +{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_total_all","swift_total_host"],"required_raw_fields_list":["creation_time", "hostname", "mount"],"service_id":"host_metrics"} +{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.avail","metric_id_list":["swift_avail_all","swift_avail_host","swift_usage_rate"],"required_raw_fields_list":["creation_time", "hostname", "mount"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"storage.objects.size","metric_id_list":["storage_objects_size_all"],"required_raw_fields_list":["creation_time", "project_id"],"service_id":"host_metrics"} diff --git a/monasca_transform/data_driven_specs/transform_specs/transform_specs.json b/monasca_transform/data_driven_specs/transform_specs/transform_specs.json index da0be85..5914189 100644 --- a/monasca_transform/data_driven_specs/transform_specs/transform_specs.json +++ b/monasca_transform/data_driven_specs/transform_specs/transform_specs.json @@ -18,9 +18,9 @@ {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"nova.vm.cpu.total_allocated_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"nova_vm_cpu_total_all","metric_id":"nova_vm_cpu_total_all"} -{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_all"} -{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_usage_host","metric_id":"swift_usage_host"} -{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"calculate_rate","setters":["set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.rate_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"filter_by_list": [],"setter_rollup_group_by_list": [],"dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"rate","pre_hourly_group_by_list":["default"]},"metric_group":"swift_usage_rate","metric_id":"swift_usage_rate"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_total_all","metric_id":"swift_total_all"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_total_host","metric_id":"swift_total_host"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_avail_all","metric_id":"swift_avail_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_avail_host","metric_id":"swift_avail_host"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"calculate_rate","setters":["set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.rate_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"filter_by_list": [],"setter_rollup_group_by_list": [],"dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"rate","pre_hourly_group_by_list":["default"]},"metric_group":"swift_avail_rate","metric_id":"swift_usage_rate"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"storage.objects.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"storage_objects_size_all","metric_id":"storage_objects_size_all"} diff --git a/tests/unit/data_driven_specs/test_data_driven_specs.py b/tests/unit/data_driven_specs/test_data_driven_specs.py index 01f4b4d..560a9ac 100644 --- a/tests/unit/data_driven_specs/test_data_driven_specs.py +++ b/tests/unit/data_driven_specs/test_data_driven_specs.py @@ -84,7 +84,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest): expected_agg_metric_name='vm.cpu.utilization_perc_agg', transform_specs_dataframe=transform_specs_data_frame) self.check_metric( - metric_id='swift_usage_all', + metric_id='swift_total_all', expected_agg_metric_name='swiftlm.diskusage.val.size_agg', transform_specs_dataframe=transform_specs_data_frame) self.check_metric( @@ -369,8 +369,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest): self.check_list_field_for_row( row=swiftlm_diskusage_all_row, field_name='metric_id_list', - expected_list=['swift_usage_all', 'swift_usage_host', - 'swift_usage_rate'] + expected_list=['swift_total_all', 'swift_total_host'] ) self.check_list_field_for_row( row=swiftlm_diskusage_all_row, @@ -398,7 +397,8 @@ class TestDataDrivenSpecsRepo(SparkContextTest): self.check_list_field_for_row( row=swiftlm_diskavail_all_row, field_name='metric_id_list', - expected_list=['swift_avail_all', 'swift_avail_host'] + expected_list=['swift_avail_all', 'swift_avail_host', + 'swift_usage_rate'] ) self.check_list_field_for_row( row=swiftlm_diskavail_all_row, diff --git a/tests/unit/driver/first_attempt_at_spark_test.py b/tests/unit/driver/first_attempt_at_spark_test.py index f72f151..a865455 100644 --- a/tests/unit/driver/first_attempt_at_spark_test.py +++ b/tests/unit/driver/first_attempt_at_spark_test.py @@ -1160,7 +1160,7 @@ class SparkTest(SparkContextTest): if value.get('metric').get('name') == 'swiftlm.diskusage.rate_agg'][0] - self.assertEqual(23.610017325563085, + self.assertEqual(24.990442207212947, diskusage_rate_agg_metric.get('metric').get('value')) self.assertEqual('useast', diskusage_rate_agg_metric.get('meta').get('region')) diff --git a/tests/unit/processor/test_pre_hourly_processor_agg.py b/tests/unit/processor/test_pre_hourly_processor_agg.py index b9a8e10..d816c0a 100644 --- a/tests/unit/processor/test_pre_hourly_processor_agg.py +++ b/tests/unit/processor/test_pre_hourly_processor_agg.py @@ -168,7 +168,7 @@ class TestPreHourlyProcessorAgg(SparkContextTest): if value.get('metric').get('name') == 'swiftlm.diskusage.rate_agg'][0] self.assertTrue(swift_disk_rate_agg_metric is not None) - self.assertEqual(59.36612021857923, + self.assertEqual(37.25140584281991, swift_disk_rate_agg_metric .get('metric').get('value')) self.assertEqual('2016-06-10 20:27:02', diff --git a/tests/unit/test_resources/kafka_data/kafka_data.txt b/tests/unit/test_resources/kafka_data/kafka_data.txt index ee61697..f338d25 100644 --- a/tests/unit/test_resources/kafka_data/kafka_data.txt +++ b/tests/unit/test_resources/kafka_data/kafka_data.txt @@ -344,23 +344,23 @@ ('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":904},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":804},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":928},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":1128},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421004,"value":1128},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":928},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":804},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":1004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421004,"value":904},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421001,"value":916},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":901},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":602},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1126},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":926},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1126},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') -('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":602},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":901},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308000000,"value":1.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') ('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308020000,"value":2.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') ('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308030000,"value":3.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') diff --git a/tests/unit/test_resources/metrics_pre_hourly_data/metrics_pre_hourly_data.txt b/tests/unit/test_resources/metrics_pre_hourly_data/metrics_pre_hourly_data.txt index 08aeaed..3f20bdf 100644 --- a/tests/unit/test_resources/metrics_pre_hourly_data/metrics_pre_hourly_data.txt +++ b/tests/unit/test_resources/metrics_pre_hourly_data/metrics_pre_hourly_data.txt @@ -4,6 +4,6 @@ ('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"mem.usable_mb_agg","record_count":20.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"mem_usable_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":10283.1}) ('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"mem.usable_mb_agg","record_count":20.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"mem_usable_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":10283.1}) ('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"mem.usable_mb_agg","record_count":20.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:39:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"mem_usable_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-20 11:49:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466423384.0,"quantity":10283.1}) -('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:27:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "4575.0", "latest_timestamp_string": "2016-06-10 20:27:02", "latest_quantity": "5291.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:27:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":15.6502}) -('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:37:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "5575.0", "latest_timestamp_string": "2016-06-10 20:37:02", "latest_quantity": "6291.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:37:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":16.6502}) -('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:47:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "6575.0", "latest_timestamp_string": "2016-06-10 20:47:02", "latest_quantity": "7291.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:47:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":17.6502}) +('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:27:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "7291.0", "latest_timestamp_string": "2016-06-10 20:27:02", "latest_quantity": "6575.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:27:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":15.6502}) +('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:37:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "6291.0", "latest_timestamp_string": "2016-06-10 20:37:02", "latest_quantity": "5575.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:37:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":16.6502}) +('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:47:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "5291.0", "latest_timestamp_string": "2016-06-10 20:47:02", "latest_quantity": "4575.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:47:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":17.6502})