From accbacb19e2707253b70e6e025b64eac38e2e62e Mon Sep 17 00:00:00 2001 From: Flint Calvin Date: Wed, 3 Aug 2016 23:11:24 +0000 Subject: [PATCH] Reintroduced some aggregations which were apparently lost when we transitioned to the upstream Monasca-Transform OpenStack repo. Specifically, the missing aggregations were those for the nova.vm.cpu.total_allocated and nova.vm.mem.total_allocated_mb source metrics. This set of changes also includes the resolution of a couple pre-existing pep8 errors. Change-Id: I84bf19b674aeadcd0d27799a887d0b89d0381550 --- .../pre_transform_specs.json | 2 + .../transform_specs/transform_specs.json | 4 +- monasca_transform/db/db_utils.py | 13 +++ .../test_data_driven_specs.py | 67 ++++++++++++++- tests/unit/db/test_db_util.py | 13 +++ .../driver/first_attempt_at_spark_test.py | 84 ++++++++++++++++++- .../test_resources/kafka_data/kafka_data.txt | 23 +++++ 7 files changed, 203 insertions(+), 3 deletions(-) diff --git a/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json b/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json index d7eecf2..fd26701 100644 --- a/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json +++ b/monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json @@ -2,6 +2,7 @@ {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"mem.usable_mb","metric_id_list":["mem_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.mem.total_mb","metric_id_list":["vm_mem_total_mb_all","vm_mem_total_mb_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.mem.used_mb","metric_id_list":["vm_mem_used_mb_all","vm_mem_used_mb_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"} +{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.mem.total_allocated_mb","metric_id_list":["nova_vm_mem_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_space_mb","metric_id_list":["disk_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_used_space_mb","metric_id_list":["disk_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.disk.total_allocated_gb","metric_id_list":["nova_disk_total_allocated_gb_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} @@ -10,5 +11,6 @@ {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.idle_perc","metric_id_list":["cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id","resource_id"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"} +{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.cpu.total_allocated","metric_id_list":["nova_vm_cpu_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all","swift_usage_rate","swift_usage_host"],"required_raw_fields_list":["creation_time", "hostname", "mount"],"service_id":"host_metrics"} {"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.avail","metric_id_list":["swift_avail_all","swift_avail_host"],"required_raw_fields_list":["creation_time", "hostname", "mount"],"service_id":"host_metrics"} diff --git a/monasca_transform/data_driven_specs/transform_specs/transform_specs.json b/monasca_transform/data_driven_specs/transform_specs/transform_specs.json index 5028790..1110f3e 100644 --- a/monasca_transform/data_driven_specs/transform_specs/transform_specs.json +++ b/monasca_transform/data_driven_specs/transform_specs/transform_specs.json @@ -4,9 +4,10 @@ {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_mem_total_mb_project","metric_id":"vm_mem_total_mb_project"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.mem.used_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_mem_used_mb_all","metric_id":"vm_mem_used_mb_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.mem.used_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_mem_used_mb_project","metric_id":"vm_mem_used_mb_project"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"nova.vm.mem.total_allocated_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"nova_vm_mem_total_all","metric_id":"nova_vm_mem_total_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"disk.total_space_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [{"field_to_filter": "host","filter_expression": "-comp[0-9]+-","filter_operation": "include"}],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"disk_total_all","metric_id":"disk_total_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"disk.total_used_space_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [{"field_to_filter": "host","filter_expression": "-comp[0-9]+-","filter_operation": "include"}],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"disk_usable_all","metric_id":"disk_usable_all"} -{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"nova.vm.disk.total_allocated_gb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"nova_disk_total_allocated_gb_all","metric_id":"nova_disk_total_allocated_gb_all"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"nova.vm.disk.total_allocated_gb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"nova_disk_total_allocated_gb_all","metric_id":"nova_disk_total_allocated_gb_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.disk.allocation_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_disk_allocation_all","metric_id":"vm_disk_allocation_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.disk.allocation_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_disk_allocation_project","metric_id":"vm_disk_allocation_project"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"cpu.total_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [{"field_to_filter": "host","filter_expression": "-comp[0-9]+-","filter_operation": "include"}],"setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"cpu_total_all","metric_id":"cpu_total_all"} @@ -16,6 +17,7 @@ {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vcpus_all","metric_id":"vcpus_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"} +{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"nova.vm.cpu.total_allocated_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"nova_vm_cpu_total_all","metric_id":"nova_vm_cpu_total_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_all"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_host"} {"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"calculate_rate","setters":["set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.rate_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"filter_by_list": [],"setter_rollup_group_by_list": [],"dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_usage_rate","metric_id":"swift_usage_rate"} diff --git a/monasca_transform/db/db_utils.py b/monasca_transform/db/db_utils.py index 51b1ec1..4d63087 100644 --- a/monasca_transform/db/db_utils.py +++ b/monasca_transform/db/db_utils.py @@ -1,3 +1,16 @@ +# (c) Copyright 2016 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. from oslo_config import cfg diff --git a/tests/unit/data_driven_specs/test_data_driven_specs.py b/tests/unit/data_driven_specs/test_data_driven_specs.py index 06797ec..2a6b953 100644 --- a/tests/unit/data_driven_specs/test_data_driven_specs.py +++ b/tests/unit/data_driven_specs/test_data_driven_specs.py @@ -51,6 +51,14 @@ class TestDataDrivenSpecsRepo(SparkContextTest): metric_id='disk_usable_all', expected_agg_metric_name='disk.total_used_space_mb_agg', transform_specs_dataframe=transform_specs_data_frame) + self.check_metric( + metric_id='nova_vm_cpu_total_all', + expected_agg_metric_name='nova.vm.cpu.total_allocated_agg', + transform_specs_dataframe=transform_specs_data_frame) + self.check_metric( + metric_id='nova_vm_mem_total_all', + expected_agg_metric_name='nova.vm.mem.total_allocated_mb_agg', + transform_specs_dataframe=transform_specs_data_frame) self.check_metric( metric_id='vcpus_all', expected_agg_metric_name='vcpus_agg', @@ -111,7 +119,8 @@ class TestDataDrivenSpecsRepo(SparkContextTest): u'mem.total_mb', u'disk.total_used_space_mb', u'disk.total_space_mb', u'cpu.total_logical_cores', - u'cpu.idle_perc', u'vcpus', + u'cpu.idle_perc', u'nova.vm.cpu.total_allocated', + u'nova.vm.mem.total_allocated_mb', u'vcpus', u'vm.mem.total_mb', u'vm.mem.used_mb', u'nova.vm.disk.total_allocated_gb', u'vm.disk.allocation', u'vm.cpu.utilization_perc', @@ -404,6 +413,62 @@ class TestDataDrivenSpecsRepo(SparkContextTest): expected_value='host_metrics' ) + # nova.vm.cpu.total_allocated + event_type = 'nova.vm.cpu.total_allocated' + nova_vm_cpu_total_all_row = self.get_row_for_event_type( + event_type=event_type, + pre_transform_specs_data_frame=pre_transform_specs_data_frame) + self.check_list_field_for_row( + row=nova_vm_cpu_total_all_row, + field_name='metric_id_list', + expected_list=['nova_vm_cpu_total_all'] + ) + self.check_list_field_for_row( + row=nova_vm_cpu_total_all_row, + field_name='required_raw_fields_list', + expected_list=['creation_time'], + ) + self.check_dict_field_for_row( + row=nova_vm_cpu_total_all_row, + field_name='event_processing_params', + expected_dict={ + "set_default_zone_to": "1", + "set_default_geolocation_to": "1", + "set_default_region_to": "W"}) + self.check_value_field_for_row( + row=nova_vm_cpu_total_all_row, + field_name='service_id', + expected_value='host_metrics' + ) + + # nova.vm.mem.total_allocated_mb + event_type = 'nova.vm.mem.total_allocated_mb' + nova_vm_mem_total_all_row = self.get_row_for_event_type( + event_type=event_type, + pre_transform_specs_data_frame=pre_transform_specs_data_frame) + self.check_list_field_for_row( + row=nova_vm_mem_total_all_row, + field_name='metric_id_list', + expected_list=['nova_vm_mem_total_all'] + ) + self.check_list_field_for_row( + row=nova_vm_mem_total_all_row, + field_name='required_raw_fields_list', + expected_list=['creation_time'], + ) + self.check_dict_field_for_row( + row=nova_vm_mem_total_all_row, + field_name='event_processing_params', + expected_dict={ + "set_default_zone_to": "1", + "set_default_geolocation_to": "1", + "set_default_region_to": "W"}) + self.check_value_field_for_row( + row=nova_vm_mem_total_all_row, + field_name='service_id', + expected_value='host_metrics' + ) + def get_row_for_event_type(self, event_type=None, pre_transform_specs_data_frame=None): diff --git a/tests/unit/db/test_db_util.py b/tests/unit/db/test_db_util.py index 9a1fc05..5e4bb1b 100644 --- a/tests/unit/db/test_db_util.py +++ b/tests/unit/db/test_db_util.py @@ -1,3 +1,16 @@ +# (c) Copyright 2016 Hewlett Packard Enterprise Development LP +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. from oslo_config import cfg from oslo_config.fixture import Config import unittest diff --git a/tests/unit/driver/first_attempt_at_spark_test.py b/tests/unit/driver/first_attempt_at_spark_test.py index 5b854fb..d36a98a 100644 --- a/tests/unit/driver/first_attempt_at_spark_test.py +++ b/tests/unit/driver/first_attempt_at_spark_test.py @@ -114,7 +114,7 @@ class SparkTest(SparkContextTest): result = simple_count_transform(rdd_monasca_with_offsets) # Verify it worked - self.assertEqual(result, 363) + self.assertEqual(result, 386) # Call the primary method in mon_metrics_kafka MonMetricsKafkaProcessor.rdd_to_recordstore( @@ -1187,6 +1187,88 @@ class SparkTest(SparkContextTest): .get('value_meta') .get('lastrecord_timestamp_string')) + # Verify nova.vm.cpu.total_allocated_agg metrics + nova_vm_cpu_total_alloc_agg_metric = [ + value for value in metrics + if value.get('metric').get('name') == + 'nova.vm.cpu.total_allocated_agg'][0] + + self.assertTrue(nova_vm_cpu_total_alloc_agg_metric is not None) + + self.assertEqual(8.0, + nova_vm_cpu_total_alloc_agg_metric + .get('metric').get('value')) + self.assertEqual('useast', + nova_vm_cpu_total_alloc_agg_metric + .get('meta').get('region')) + + self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id, + nova_vm_cpu_total_alloc_agg_metric + .get('meta').get('tenantId')) + self.assertEqual('all', + nova_vm_cpu_total_alloc_agg_metric + .get('metric').get('dimensions').get('host')) + self.assertEqual('all', + nova_vm_cpu_total_alloc_agg_metric.get('metric') + .get('dimensions').get('project_id')) + self.assertEqual('hourly', + nova_vm_cpu_total_alloc_agg_metric + .get('metric').get('dimensions') + .get('aggregation_period')) + + self.assertEqual(14.0, + nova_vm_cpu_total_alloc_agg_metric + .get('metric').get('value_meta').get('record_count')) + self.assertEqual('2016-01-20 16:40:00', + nova_vm_cpu_total_alloc_agg_metric + .get('metric').get('value_meta') + .get('firstrecord_timestamp_string')) + self.assertEqual('2016-01-20 16:40:46', + nova_vm_cpu_total_alloc_agg_metric + .get('metric').get('value_meta') + .get('lastrecord_timestamp_string')) + + # Verify nova.vm.mem.total_allocated_mb_agg metrics + nova_vm_mem_total_alloc_agg_metric = [ + value for value in metrics + if value.get('metric').get('name') == + 'nova.vm.mem.total_allocated_mb_agg'][0] + + self.assertTrue(nova_vm_mem_total_alloc_agg_metric is not None) + + self.assertEqual(9728.0, + nova_vm_mem_total_alloc_agg_metric + .get('metric').get('value')) + self.assertEqual('useast', + nova_vm_mem_total_alloc_agg_metric + .get('meta').get('region')) + + self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id, + nova_vm_mem_total_alloc_agg_metric + .get('meta').get('tenantId')) + self.assertEqual('all', + nova_vm_mem_total_alloc_agg_metric + .get('metric').get('dimensions').get('host')) + self.assertEqual('all', + nova_vm_mem_total_alloc_agg_metric.get('metric') + .get('dimensions').get('project_id')) + self.assertEqual('hourly', + nova_vm_mem_total_alloc_agg_metric + .get('metric').get('dimensions') + .get('aggregation_period')) + + self.assertEqual(9.0, + nova_vm_mem_total_alloc_agg_metric + .get('metric').get('value_meta').get('record_count')) + self.assertEqual('2016-01-20 16:40:00', + nova_vm_mem_total_alloc_agg_metric + .get('metric').get('value_meta') + .get('firstrecord_timestamp_string')) + self.assertEqual('2016-01-20 16:40:46', + nova_vm_mem_total_alloc_agg_metric + .get('metric').get('value_meta') + .get('lastrecord_timestamp_string')) + def simple_count_transform(rdd): return rdd.count() diff --git a/tests/unit/test_resources/kafka_data/kafka_data.txt b/tests/unit/test_resources/kafka_data/kafka_data.txt index e379d05..c144aaa 100644 --- a/tests/unit/test_resources/kafka_data/kafka_data.txt +++ b/tests/unit/test_resources/kafka_data/kafka_data.txt @@ -361,3 +361,26 @@ ('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') ('','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308000000,"value":1.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308020000,"value":2.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308030000,"value":3.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308032000,"value":3.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308038000,"value":2.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308042000,"value":1.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0002-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308005000,"value":1.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0003-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308010000,"value":2.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0004-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308015000,"value":2.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0003-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308025000,"value":3.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0004-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308034000,"value":2.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0002-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308036000,"value":2.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0003-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308040000,"value":1.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.cpu.total_allocated","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0002-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308046000,"value":3.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833584}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308000000,"value":1024.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308005000,"value":1536.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0001-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308040000,"value":2048.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0002-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308000010,"value":1024.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0003-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308000020,"value":1024.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0004-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308000030,"value":2048.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0005-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308000040,"value":3072.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0002-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308006000,"value":3072.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}') +('','{"metric":{"name":"nova.vm.mem.total_allocated_mb","dimensions":{"cluster":"compute","hostname":"padawan-ccp-comp0002-mgmt","component":"vm","service":"compute","cloud_name":"padawan","control_plane":"ccp"},"timestamp":1453308046000,"value":2048.0},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1469833554}')