openstack
/
monasca-transform
Code Issues Proposed changes

Added aggregation for vm.mem.used_mb and 

swiftlm.diskusage.host.val.size.  Also renamed disk.allocation to
vm.disk.allocation and resolved a problem with resource_id not
being found for certain aggregations.

Change-Id: Iad82d149e7a04ed1e0ecfe936b90acfff1dca13e
This commit is contained in:
Flint Calvin 2016-06-13 22:38:46 +00:00
parent 11e8bac2cf
commit c7aabb6927
8 changed files with 396 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
monasca_transform/data_driven_specs/pre_transform_specs/pre_transform_specs.json
View File

@ -1,11 +1,13 @@
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"mem.total_mb","metric_id_list":["mem_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"mem.usable_mb","metric_id_list":["mem_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.mem.total_mb","metric_id_list":["vm_mem_total_mb_all","vm_mem_total_mb_project"],"required_raw_fields_list":["creation_time","tenantId"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.mem.total_mb","metric_id_list":["vm_mem_total_mb_all","vm_mem_total_mb_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.mem.used_mb","metric_id_list":["vm_mem_used_mb_all","vm_mem_used_mb_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_space_mb","metric_id_list":["disk_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_used_space_mb","metric_id_list":["disk_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.disk.total_allocated_gb","metric_id_list":["nova_disk_total_allocated_gb_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.allocation","metric_id_list":["disk_allocation_all","disk_allocation_project"],"required_raw_fields_list":["creation_time","tenantId"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.disk.allocation","metric_id_list":["vm_disk_allocation_all","vm_disk_allocation_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.total_logical_cores","metric_id_list":["cpu_total_all","cpu_total_host","cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.idle_perc","metric_id_list":["cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"}

17
monasca_transform/data_driven_specs/transform_specs/transform_specs.json
View File

@ -1,16 +1,19 @@
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"mem_total_all","metric_id":"mem_total_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"mem.usable_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"mem_usable_all","metric_id":"mem_usable_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_mem_total_mb_all","metric_id":"vm_mem_total_mb_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_mem_total_mb_project","metric_id":"vm_mem_total_mb_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_mem_total_mb_all","metric_id":"vm_mem_total_mb_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_mem_total_mb_project","metric_id":"vm_mem_total_mb_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.mem.used_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_mem_used_mb_all","metric_id":"vm_mem_used_mb_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.mem.used_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_mem_used_mb_project","metric_id":"vm_mem_used_mb_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"disk.total_space_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"disk_total_all","metric_id":"disk_total_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"disk.total_used_space_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"disk_usable_all","metric_id":"disk_usable_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"nova.vm.disk.total_allocated_gb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"nova_disk_total_allocated_gb_all","metric_id":"nova_disk_total_allocated_gb_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"disk.allocation_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"disk_allocation_all","metric_id":"disk_allocation_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"disk.allocation_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"disk_allocation_project","metric_id":"disk_allocation_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.disk.allocation_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_disk_allocation_all","metric_id":"vm_disk_allocation_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.disk.allocation_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_disk_allocation_project","metric_id":"vm_disk_allocation_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"cpu.total_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"cpu_total_all","metric_id":"cpu_total_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"cpu.total_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"cpu_total_host","metric_id":"cpu_total_host"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity_util","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"cpu.utilized_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["event_type", "host"],"usage_fetch_operation": "avg","usage_fetch_util_quantity_event_type": "cpu.total_logical_cores","usage_fetch_util_idle_perc_event_type": "cpu.idle_perc","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"cpu_util_all","metric_id":"cpu_util_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity_util","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"cpu.utilized_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["event_type", "host"],"usage_fetch_operation": "avg","usage_fetch_util_quantity_event_type": "cpu.total_logical_cores","usage_fetch_util_idle_perc_event_type": "cpu.idle_perc","setter_rollup_group_by_list": ["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"cpu_util_host","metric_id":"cpu_util_host"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vcpus_all","metric_id":"vcpus_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "sum","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vcpus_all","metric_id":"vcpus_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_all"}

14
monasca_transform/driver/mon_metrics_kafka.py
View File

@ -365,8 +365,10 @@ class MonMetricsKafkaProcessor(object):
# resource_uuid
when(gen_mon_metrics_df.metric.dimensions.instanceId != '',
gen_mon_metrics_df.metric.dimensions.
instanceId).otherwise('NA').alias("resource_uuid"),
gen_mon_metrics_df.metric.dimensions.instanceId).when(
gen_mon_metrics_df.metric.dimensions.resource_id != '',
gen_mon_metrics_df.metric.dimensions.resource_id).
otherwise('NA').alias("resource_uuid"),
when(gen_mon_metrics_df.metric.dimensions.tenantId != '',
gen_mon_metrics_df.metric.dimensions.tenantId).when(
@ -376,6 +378,14 @@ class MonMetricsKafkaProcessor(object):
gen_mon_metrics_df.metric.dimensions.project_id).otherwise(
'NA').alias("tenant_id"),
when(gen_mon_metrics_df.metric.dimensions.mount != '',
gen_mon_metrics_df.metric.dimensions.mount).otherwise(
'NA').alias("mount"),
when(gen_mon_metrics_df.metric.dimensions.device != '',
gen_mon_metrics_df.metric.dimensions.device).otherwise(
'NA').alias("device"),
when(gen_mon_metrics_df.meta.userId != '',
gen_mon_metrics_df.meta.userId).otherwise('NA').alias(
"user_id"),

8
monasca_transform/transform/transform_utils.py
View File

@ -211,10 +211,10 @@ class MonMetricUtils(TransformUtils):
"""get the schema of the incoming monasca metric."""
dimensions = ["apache_host", "apache_port", "component",
"consumer_group", "device", "hostname",
"mode", "mount_point", "observer_host",
"process_name", "project_id", "service", "test_type",
"tenantId", "tenant_id", "topic", "url", "state",
"state_description", "instanceId"]
"mode", "mount", "mount_point", "observer_host",
"process_name", "project_id", "resource_id", "service",
"test_type", "tenantId", "tenant_id", "topic", "url",
"state", "state_description", "instanceId"]
dimensions_struct_fields = [
StructField(field_name, StringType(), True)

104
tests/unit/data_driven_specs/test_data_driven_specs.py
View File

@ -59,18 +59,26 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
metric_id='vm_mem_total_mb_all',
expected_agg_metric_name='vm.mem.total_mb_agg',
transform_specs_dataframe=transform_specs_data_frame)
self.check_metric(
metric_id='vm_mem_used_mb_all',
expected_agg_metric_name='vm.mem.used_mb_agg',
transform_specs_dataframe=transform_specs_data_frame)
self.check_metric(
metric_id='nova_disk_total_allocated_gb_all',
expected_agg_metric_name='nova.vm.disk.total_allocated_gb_agg',
transform_specs_dataframe=transform_specs_data_frame)
self.check_metric(
metric_id='disk_allocation_all',
expected_agg_metric_name='disk.allocation_agg',
metric_id='vm_disk_allocation_all',
expected_agg_metric_name='vm.disk.allocation_agg',
transform_specs_dataframe=transform_specs_data_frame)
self.check_metric(
metric_id='vm_cpu_util_perc_project',
expected_agg_metric_name='vm.cpu.utilization_perc_agg',
transform_specs_dataframe=transform_specs_data_frame)
self.check_metric(
metric_id='swift_usage_all',
expected_agg_metric_name='swiftlm.diskusage.val.size_agg',
transform_specs_dataframe=transform_specs_data_frame)
def check_metric(self, metric_id=None, expected_agg_metric_name=None,
transform_specs_dataframe=None):
@ -96,8 +104,10 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
u'disk.total_used_space_mb', u'disk.total_space_mb',
u'cpu.total_logical_cores',
u'cpu.idle_perc', u'vcpus',
u'vm.mem.total_mb', u'nova.vm.disk.total_allocated_gb',
u'disk.allocation', u'vm.cpu.utilization_perc']),
u'vm.mem.total_mb', u'vm.mem.used_mb',
u'nova.vm.disk.total_allocated_gb',
u'vm.disk.allocation', u'vm.cpu.utilization_perc',
u'swiftlm.diskusage.host.val.size']),
Counter([row.event_type for row in
pre_transform_specs_data_frame.collect()]))
@ -156,6 +166,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
expected_value='host_metrics'
)
# vcpus
event_type = 'vcpus'
vcpus_all_row = self.get_row_for_event_type(
event_type=event_type,
@ -169,7 +180,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
self.check_list_field_for_row(
row=vcpus_all_row,
field_name='required_raw_fields_list',
expected_list=['creation_time', 'project_id'],
expected_list=['creation_time', 'project_id', 'resource_id'],
)
self.check_dict_field_for_row(
row=vcpus_all_row,
@ -198,7 +209,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
self.check_list_field_for_row(
row=vm_mem_total_mb_all_row,
field_name='required_raw_fields_list',
expected_list=['creation_time', 'tenantId'],
expected_list=['creation_time', 'tenantId', 'resource_id'],
)
self.check_dict_field_for_row(
row=vm_mem_total_mb_all_row,
@ -213,6 +224,36 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
expected_value='host_metrics'
)
# vm.mem.used_mb
event_type = 'vm.mem.used_mb'
vm_mem_total_mb_all_row = self.get_row_for_event_type(
event_type=event_type,
pre_transform_specs_data_frame=pre_transform_specs_data_frame)
self.check_list_field_for_row(
row=vm_mem_total_mb_all_row,
field_name='metric_id_list',
expected_list=['vm_mem_used_mb_all',
'vm_mem_used_mb_project']
)
self.check_list_field_for_row(
row=vm_mem_total_mb_all_row,
field_name='required_raw_fields_list',
expected_list=['creation_time', 'tenantId', 'resource_id'],
)
self.check_dict_field_for_row(
row=vm_mem_total_mb_all_row,
field_name='event_processing_params',
expected_dict={
"set_default_zone_to": "1",
"set_default_geolocation_to": "1",
"set_default_region_to": "W"})
self.check_value_field_for_row(
row=vm_mem_total_mb_all_row,
field_name='service_id',
expected_value='host_metrics'
)
# nova.vm.disk.total_allocated_gb
event_type = 'nova.vm.disk.total_allocated_gb'
disk_total_alloc_row = self.get_row_for_event_type(
event_type=event_type,
@ -240,31 +281,31 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
expected_value='host_metrics'
)
# disk.allocation
event_type = 'disk.allocation'
disk_allocation_all_row = self.get_row_for_event_type(
# vm.disk.allocation
event_type = 'vm.disk.allocation'
vm_disk_allocation_all_row = self.get_row_for_event_type(
event_type=event_type,
pre_transform_specs_data_frame=pre_transform_specs_data_frame)
self.check_list_field_for_row(
row=disk_allocation_all_row,
row=vm_disk_allocation_all_row,
field_name='metric_id_list',
expected_list=['disk_allocation_all',
'disk_allocation_project']
expected_list=['vm_disk_allocation_all',
'vm_disk_allocation_project']
)
self.check_list_field_for_row(
row=disk_allocation_all_row,
row=vm_disk_allocation_all_row,
field_name='required_raw_fields_list',
expected_list=['creation_time', 'tenantId'],
expected_list=['creation_time', 'tenantId', 'resource_id'],
)
self.check_dict_field_for_row(
row=disk_allocation_all_row,
row=vm_disk_allocation_all_row,
field_name='event_processing_params',
expected_dict={
"set_default_zone_to": "1",
"set_default_geolocation_to": "1",
"set_default_region_to": "W"})
self.check_value_field_for_row(
row=disk_allocation_all_row,
row=vm_disk_allocation_all_row,
field_name='service_id',
expected_value='host_metrics'
)
@ -282,7 +323,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
self.check_list_field_for_row(
row=vm_cpu_util_perc_row,
field_name='required_raw_fields_list',
expected_list=['creation_time', 'tenant_id'],
expected_list=['creation_time', 'tenant_id', 'resource_id'],
)
self.check_dict_field_for_row(
row=vm_cpu_util_perc_row,
@ -297,6 +338,35 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
expected_value='host_metrics'
)
# swiftlm.diskusage.host.val.size
event_type = 'swiftlm.diskusage.host.val.size'
swiftlm_diskusage_all_row = self.get_row_for_event_type(
event_type=event_type,
pre_transform_specs_data_frame=pre_transform_specs_data_frame)
self.check_list_field_for_row(
row=swiftlm_diskusage_all_row,
field_name='metric_id_list',
expected_list=['swift_usage_all']
)
self.check_list_field_for_row(
row=swiftlm_diskusage_all_row,
field_name='required_raw_fields_list',
expected_list=['creation_time', 'hostname', 'mount',
'device'],
)
self.check_dict_field_for_row(
row=swiftlm_diskusage_all_row,
field_name='event_processing_params',
expected_dict={
"set_default_zone_to": "1",
"set_default_geolocation_to": "1",
"set_default_region_to": "W"})
self.check_value_field_for_row(
row=swiftlm_diskusage_all_row,
field_name='service_id',
expected_value='host_metrics'
)
def get_row_for_event_type(self,
event_type=None,
pre_transform_specs_data_frame=None):

323
tests/unit/driver/first_attempt_at_spark_test.py
View File

@ -114,7 +114,7 @@ class SparkTest(SparkContextTest):
result = simple_count_transform(rdd_monasca_with_offsets)
# Verify it worked
self.assertEqual(result, 324)
self.assertEqual(result, 347)
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
@ -317,7 +317,7 @@ class SparkTest(SparkContextTest):
self.assertTrue(vm_mem_total_mb_agg_metric is not None)
self.assertEqual(9728.0,
self.assertEqual(14336.0,
vm_mem_total_mb_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
@ -335,30 +335,67 @@ class SparkTest(SparkContextTest):
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(9.0,
self.assertEqual(7.0,
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:00',
self.assertEqual('2016-06-07 16:27:54',
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-01-20 16:40:46',
self.assertEqual('2016-06-07 16:30:54',
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify vm.mem.total_mb_agg metrics for
# 5f681592f7084c5fbcd4e8a20a4fef15 project
# Verify vm.mem.total_mb_agg metrics for the 1 project
vm_mem_total_mb_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'vm.mem.total_mb_agg' and
value.get('metric').get('dimensions').get('project_id') ==
'5f681592f7084c5fbcd4e8a20a4fef15'][0]
value.get('metric').get('dimensions').get('project_id') == '1'][0]
self.assertTrue(vm_mem_total_mb_agg_metric is not None)
self.assertEqual(1536.0,
self.assertEqual(6656.0,
vm_mem_total_mb_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
vm_mem_total_mb_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
vm_mem_total_mb_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
vm_mem_total_mb_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
vm_mem_total_mb_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(4.0,
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-06-07 16:27:54',
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-07 16:29:54',
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify vm.mem.total_mb_agg metrics for the 2 project
vm_mem_total_mb_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'vm.mem.total_mb_agg' and
value.get('metric').get('dimensions').get('project_id') == '2'][0]
self.assertTrue(vm_mem_total_mb_agg_metric is not None)
self.assertEqual(7680.0,
vm_mem_total_mb_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
@ -379,52 +416,11 @@ class SparkTest(SparkContextTest):
self.assertEqual(3.0,
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:00',
self.assertEqual('2016-06-07 16:28:54',
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-01-20 16:40:40',
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify vm.mem.total_mb_agg metrics for
# 6f681592f7084c5fbcd4e8a20a4fef15 project
vm_mem_total_mb_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'vm.mem.total_mb_agg' and
value.get('metric').get('dimensions').get('project_id') ==
'6f681592f7084c5fbcd4e8a20a4fef15'][0]
self.assertTrue(vm_mem_total_mb_agg_metric is not None)
self.assertEqual(8192.0,
vm_mem_total_mb_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
vm_mem_total_mb_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
vm_mem_total_mb_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
vm_mem_total_mb_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
vm_mem_total_mb_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(6.0,
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:00',
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-01-20 16:40:46',
self.assertEqual('2016-06-07 16:30:54',
vm_mem_total_mb_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
@ -463,125 +459,125 @@ class SparkTest(SparkContextTest):
total_allocated_disk_agg_metric.get('metric')
.get('value_meta').get('lastrecord_timestamp'))
# Verify disk.allocation_agg metrics for all projects
disk_allocation_agg_metric = [
# Verify vm.disk.allocation_agg metrics for all projects
vm_disk_allocation_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'disk.allocation_agg' and
'vm.disk.allocation_agg' and
value.get('metric').get('dimensions').get('project_id') ==
'all'][0]
self.assertTrue(disk_allocation_agg_metric is not None)
self.assertTrue(vm_disk_allocation_agg_metric is not None)
self.assertEqual(140.0,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(9.0,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:00',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-01-20 16:40:46',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify disk.allocation_agg metrics for
# Verify vm.disk.allocation_agg metrics for
# 5f681592f7084c5fbcd4e8a20a4fef15 project
disk_allocation_agg_metric = [
vm_disk_allocation_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'disk.allocation_agg' and
'vm.disk.allocation_agg' and
value.get('metric').get('dimensions').get('project_id') ==
'5f681592f7084c5fbcd4e8a20a4fef15'][0]
self.assertTrue(disk_allocation_agg_metric is not None)
self.assertTrue(vm_disk_allocation_agg_metric is not None)
self.assertEqual(24.0,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(3.0,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:00',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-01-20 16:40:40',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify disk.allocation_agg metrics for
# Verify vm.disk.allocation_agg metrics for
# 6f681592f7084c5fbcd4e8a20a4fef15 project
disk_allocation_agg_metric = [
vm_disk_allocation_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'disk.allocation_agg' and
'vm.disk.allocation_agg' and
value.get('metric').get('dimensions').get('project_id') ==
'6f681592f7084c5fbcd4e8a20a4fef15'][0]
self.assertTrue(disk_allocation_agg_metric is not None)
self.assertTrue(vm_disk_allocation_agg_metric is not None)
self.assertEqual(116.0,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(6.0,
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-01-20 16:40:00',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-01-20 16:40:46',
disk_allocation_agg_metric
vm_disk_allocation_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
@ -858,6 +854,155 @@ class SparkTest(SparkContextTest):
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify vm.mem.used_mb_agg metrics for all projects
vm_mem_used_mb_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'vm.mem.used_mb_agg' and
value.get('metric').get('dimensions').get('project_id') ==
'all'][0]
self.assertTrue(vm_mem_used_mb_agg_metric is not None)
self.assertEqual(6340.0,
vm_mem_used_mb_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
vm_mem_used_mb_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
vm_mem_used_mb_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
vm_mem_used_mb_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
vm_mem_used_mb_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(7.0,
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-06-07 16:27:54',
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-07 16:30:54',
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify vm.mem.used_mb_agg metrics for the 1 project
vm_mem_used_mb_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'vm.mem.used_mb_agg' and
value.get('metric').get('dimensions').get('project_id') == '1'][0]
self.assertTrue(vm_mem_used_mb_agg_metric is not None)
self.assertEqual(1840.0,
vm_mem_used_mb_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
vm_mem_used_mb_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
vm_mem_used_mb_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
vm_mem_used_mb_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
vm_mem_used_mb_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(4.0,
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-06-07 16:27:54',
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-07 16:29:54',
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify vm.mem.used_mb_agg metrics for the 2 project
vm_mem_used_mb_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'vm.mem.used_mb_agg' and
value.get('metric').get('dimensions').get('project_id') == '2'][0]
self.assertTrue(vm_mem_used_mb_agg_metric is not None)
self.assertEqual(4500.0,
vm_mem_used_mb_agg_metric
.get('metric').get('value'))
self.assertEqual('useast',
vm_mem_used_mb_agg_metric
.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
vm_mem_used_mb_agg_metric
.get('meta').get('tenantId'))
self.assertEqual('all',
vm_mem_used_mb_agg_metric
.get('metric').get('dimensions').get('host'))
self.assertEqual('hourly',
vm_mem_used_mb_agg_metric
.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(3.0,
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta').get('record_count'))
self.assertEqual('2016-06-07 16:28:54',
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-07 16:30:54',
vm_mem_used_mb_agg_metric
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify swiftlm.diskusage.val.size_agg
used_swift_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'swiftlm.diskusage.val.size_agg'][0]
self.assertEqual(360000000000342.0,
used_swift_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
used_swift_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
used_swift_agg_metric.get('meta').get('tenantId'))
self.assertEqual('all',
used_swift_agg_metric.get('metric').get('dimensions')
.get('host'))
self.assertEqual('all',
used_swift_agg_metric.get('metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
used_swift_agg_metric.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(18.0,
used_swift_agg_metric.get('metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-06-10 20:27:01',
used_swift_agg_metric.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-10 20:27:20',
used_swift_agg_metric.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
def simple_count_transform(rdd):
return rdd.count()

59
tests/unit/test_resources/kafka_data/kafka_data.txt
View File

@ -205,15 +205,6 @@
('<message id>','{"metric":{"name":"vcpus","dimensions":{"region":"useast","unit":"vcpu","resource_id":"b1a7f31d-3f85-4475-a50b-77094b62b078","source":"openstack","project_id":"9647fd5030b04a799b0411cc38c4102d","user_id":"a69aabac824c4e1688702b8eb7e22af2","type":"gauge"},"timestamp":1453308036000,"value":2.0,"value_meta":{"audit_period_beginning":"2016-05-13T23:00:00.000000","availability_zone":"None","host":"mini-mon","event_type":"compute.instance.update","launched_at":"2016-05-13T23:14:06.000000","deleted_at":"","state":"active","memory_mb":"512","root_gb":"1","vcpus":"1","audit_period_ending":"2016-05-13T23:15:29.476223","ephemeral_gb":"0","instance_flavor_id":"1","created_at":"2016-05-13 23:13:44+00:00","disk_gb":"1","image_ref_url":"http://10.241.104.4:9292/images/12990d03-8059-4052-8b93-d4c6376fea57"}},"meta":{"tenantId":"225bbc4c5c1c4dfaa910cb44b5a4d7b7","region":"unset"},"creation_time":1458166131}')
('<message id>','{"metric":{"name":"vcpus","dimensions":{"region":"useast","unit":"vcpu","resource_id":"b1a7f31d-3f85-4475-a50b-77094b62b078","source":"openstack","project_id":"9647fd5030b04a799b0411cc38c4102d","user_id":"a69aabac824c4e1688702b8eb7e22af2","type":"gauge"},"timestamp":1453308040000,"value":1.0,"value_meta":{"audit_period_beginning":"2016-05-13T23:00:00.000000","availability_zone":"None","host":"dummy001","event_type":"compute.instance.update","launched_at":"2016-05-13T23:14:06.000000","deleted_at":"","state":"active","memory_mb":"512","root_gb":"1","vcpus":"1","audit_period_ending":"2016-05-13T23:15:29.476223","ephemeral_gb":"0","instance_flavor_id":"1","created_at":"2016-05-13 23:13:44+00:00","disk_gb":"1","image_ref_url":"http://10.241.104.4:9292/images/12990d03-8059-4052-8b93-d4c6376fea57"}},"meta":{"tenantId":"225bbc4c5c1c4dfaa910cb44b5a4d7b7","region":"unset"},"creation_time":1458166131}')
('<message id>','{"metric":{"name":"vcpus","dimensions":{"region":"useast","unit":"vcpu","resource_id":"b1a7f31d-3f85-4475-a50b-77094b62b078","source":"openstack","project_id":"9647fd5030b04a799b0411cc38c4102d","user_id":"a69aabac824c4e1688702b8eb7e22af2","type":"gauge"},"timestamp":1453308046000,"value":3.0,"value_meta":{"audit_period_beginning":"2016-05-13T23:00:00.000000","availability_zone":"None","host":"mini-mon","event_type":"compute.instance.update","launched_at":"2016-05-13T23:14:06.000000","deleted_at":"","state":"active","memory_mb":"512","root_gb":"1","vcpus":"1","audit_period_ending":"2016-05-13T23:15:29.476223","ephemeral_gb":"0","instance_flavor_id":"1","created_at":"2016-05-13 23:13:44+00:00","disk_gb":"1","image_ref_url":"http://10.241.104.4:9292/images/12990d03-8059-4052-8b93-d4c6376fea57"}},"meta":{"tenantId":"225bbc4c5c1c4dfaa910cb44b5a4d7b7","region":"unset"},"creation_time":1458166131}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":1024.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":1536.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":2048.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":2048.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"dummy1","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":512.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"dummy2","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":2048.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"dummy3","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":1024.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":2048.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":9728.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"cpu.wait_perc","dimensions":{"hostname":"devstack"},"timestamp":1457366959000,"value":0.0},"meta":{"tenantId":"103e4d4d14bc4fdda4a9c73d1643e1d7","region":"useast"},"creation_time":1457366963}')
('<message id>','{"metric":{"name":"apache.performance.cpu_load_perc","dimensions":{"apache_port":"80","component":"apache","service":"apache","hostname":"devstack","apache_host":"devstack"},"timestamp":1457366959000,"value":0.143361},"meta":{"tenantId":"103e4d4d14bc4fdda4a9c73d1643e1d7","region":"useast"},"creation_time":1457366963}')
('<message id>','{"metric":{"name":"cpu.stolen_perc","dimensions":{"hostname":"devstack"},"timestamp":1457366959000,"value":0.0},"meta":{"tenantId":"103e4d4d14bc4fdda4a9c73d1643e1d7","region":"useast"},"creation_time":1457366963}')
@ -305,15 +296,15 @@
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0006-mgmt","cluster":"compute"},"timestamp":1463498061000,"value":82.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498062}')
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0003-mgmt","cluster":"compute"},"timestamp":1463498078000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498080}')
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0002-mgmt","cluster":"compute"},"timestamp":1463498084000,"value":132.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498086}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"dummy1","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":28.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"dummy2","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":30.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"dummy3","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":32.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"disk.allocation","dimensions":{"component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"c","component":"vm","service":"compute","hostname":"dummy1","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":28.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"d","component":"vm","service":"compute","hostname":"dummy2","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":30.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"e","component":"vm","service":"compute","hostname":"dummy3","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":32.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"7b805207-3c97-4ef9-8b28-3fac171f2a96","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283861000,"value":5.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"a07858d1-aa13-4259-9873-f23170bc9e1b","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283895000,"value":35.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"4cd09eac-854d-49eb-993d-914271f8e1fe","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283896000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
@ -322,3 +313,35 @@
('<message id>','{"metric":{"name":"disk.total_space_mb","dimensions":{"cloud_name":"helion-poc-hlm-004","control_plane":"ccp","service":"uncategorized","cluster":"cluster1","hostname":"hlm004-cp1-c1-m1-mgmt"},"timestamp":1464815364000,"value":1291923.0},"meta":{"tenantId":"2ee3ccf6f5ab4ab18dbc8ae24fce99e0","region":"unset"},"creation_time":1464815364}')
('<message id>','{"metric":{"name":"disk.total_used_space_mb","dimensions":{"cloud_name":"helion-poc-hlm-004","control_plane":"ccp","service":"uncategorized","cluster":"compute","hostname":"hlm004-cp1-comp0003-mgmt"},"timestamp":1464815361000,"value":2329.0},"meta":{"tenantId":"2ee3ccf6f5ab4ab18dbc8ae24fce99e0","region":"unset"},"creation_time":1464815362}')
('<message id>','{"metric":{"name":"disk.total_used_space_mb","dimensions":{"cloud_name":"helion-poc-hlm-004","control_plane":"ccp","service":"uncategorized","cluster":"cluster1","hostname":"hlm004-cp1-c1-m1-mgmt"},"timestamp":1464815364000,"value":31714.0},"meta":{"tenantId":"2ee3ccf6f5ab4ab18dbc8ae24fce99e0","region":"unset"},"creation_time":1464815364}')
('<message id>','{"metric":{"name":"vm.mem.used_mb","dimensions":{"resource_id":"a","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316874000,"value":460.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316875}')
('<message id>','{"metric":{"name":"vm.mem.used_mb","dimensions":{"resource_id":"b","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316904000,"value":920.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316905}')
('<message id>','{"metric":{"name":"vm.mem.used_mb","dimensions":{"resource_id":"c","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465316934000,"value":1000.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316935}')
('<message id>','{"metric":{"name":"vm.mem.used_mb","dimensions":{"resource_id":"a","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316964000,"value":460.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316965}')
('<message id>','{"metric":{"name":"vm.mem.used_mb","dimensions":{"resource_id":"b","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316994000,"value":1840.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316995}')
('<message id>','{"metric":{"name":"vm.mem.used_mb","dimensions":{"resource_id":"c","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317024000,"value":2000.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317025}')
('<message id>','{"metric":{"name":"vm.mem.used_mb","dimensions":{"resource_id":"d","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317054000,"value":3000.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317055}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"a","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316874000,"value":2048.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316875}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"b","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316904000,"value":4096.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316905}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"c","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465316934000,"value":1024.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316935}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"a","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316964000,"value":2048.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316965}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"b","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316994000,"value":5120.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316995}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"c","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317024000,"value":2048.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317025}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"d","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317054000,"value":6144.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317055}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000006},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000008},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000010},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000012},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000014},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000016},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000018},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000020},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000022},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000024},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000030},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000032},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000034},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000036},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')

4
tests/unit/usage/test_vm_cpu_allocated_agg.py
View File

@ -142,7 +142,7 @@ class TestVmCpuAllocatedAgg(SparkContextTest):
result = simple_count_transform(rdd_monasca_with_offsets)
# Verify it worked
self.assertEqual(result, 324)
self.assertEqual(result, 347)
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
@ -321,7 +321,7 @@ class TestVmCpuAllocatedAgg(SparkContextTest):
result = simple_count_transform(rdd_monasca_with_offsets)
# Verify it worked
self.assertEqual(result, 324)
self.assertEqual(result, 347)
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(