Added several Swift aggregations (including a new usage

component for calculating rate changes).  Also fixed some pep8
issues.

Change-Id: I46685d39ace663595aa524f04d8d35a71c9432c3
This commit is contained in:
Flint Calvin 2016-06-21 19:44:02 +00:00
parent 2e04a6e5e0
commit d8e73f3bde
16 changed files with 493 additions and 84 deletions

View File

@ -12,10 +12,10 @@
# License for the specific language governing permissions and limitations
# under the License.
from monasca_transform.driver.mon_metrics_kafka import invoke
activate_this_file = "/opt/monasca/transform/venv/bin/activate_this.py"
execfile(activate_this_file, dict(__file__=activate_this_file))
from monasca_transform.driver.mon_metrics_kafka import invoke
invoke()

View File

@ -54,7 +54,7 @@ spark_home = /opt/spark/current
spark_python_files = /opt/monasca/transform/lib/monasca-transform.zip
# How often the stream should be read (in seconds)
stream_interval = 120
stream_interval = 300
# The working directory for monasca-transform
work_dir = /var/run/monasca/transform

View File

@ -14,11 +14,12 @@
import sys
from monasca_transform.service.transform_service import main_service
activate_this_file = "/opt/monasca/transform/venv/bin/activate_this.py"
execfile(activate_this_file, dict(__file__=activate_this_file))
from monasca_transform.service.transform_service import main_service
def main():
main_service()

View File

@ -60,7 +60,7 @@ spark_home = /opt/spark/current
spark_python_files = /opt/stack/monasca-transform/dist/monasca_transform-0.0.1.egg
# How often the stream should be read (in seconds)
stream_interval = 120
stream_interval = 300
# The working directory for monasca-transform
work_dir = /opt/stack/monasca-transform

View File

@ -163,9 +163,28 @@ class RollupQuantity(SetterComponent):
transform_spec_df = transform_context.transform_spec_df_info
# get rollup operation (sum, max, avg, min)
agg_params = transform_spec_df.select(
"aggregation_params_map.setter_rollup_operation").\
collect()[0].asDict()
setter_rollup_operation = agg_params["setter_rollup_operation"]
instance_usage_trans_df = RollupQuantity.setter_by_operation(
transform_context,
instance_usage_df,
setter_rollup_operation)
return instance_usage_trans_df
@staticmethod
def setter_by_operation(transform_context, instance_usage_df,
setter_rollup_operation):
transform_spec_df = transform_context.transform_spec_df_info
# get fields we want to group by for a rollup
agg_params = transform_spec_df.select(
"aggregation_params_map.setter_rollup_group_by_list").\
"aggregation_params_map.setter_rollup_group_by_list"). \
collect()[0].asDict()
setter_rollup_group_by_list = agg_params["setter_rollup_group_by_list"]
@ -178,14 +197,8 @@ class RollupQuantity(SetterComponent):
aggregation_period)
# group by columns list
group_by_columns_list = group_by_period_list + \
setter_rollup_group_by_list
# get rollup operation (sum, max, avg, min)
agg_params = transform_spec_df.select(
"aggregation_params_map.setter_rollup_operation").\
collect()[0].asDict()
setter_rollup_operation = agg_params["setter_rollup_operation"]
group_by_columns_list = \
group_by_period_list + setter_rollup_group_by_list
# perform rollup operation
instance_usage_json_rdd = RollupQuantity._rollup_quantity(

View File

@ -0,0 +1,148 @@
# Copyright 2016 Hewlett Packard Enterprise Development Company LP
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from pyspark.sql import SQLContext
from monasca_transform.component import Component
from monasca_transform.component.setter.rollup_quantity import RollupQuantity
from monasca_transform.component.usage.fetch_quantity import FetchQuantity
from monasca_transform.component.usage import UsageComponent
from monasca_transform.transform.transform_utils import InstanceUsageUtils
import json
class CalculateRateException(Exception):
"""Exception thrown when calculating rate
Attributes:
value: string representing the error
"""
def __init__(self, value):
self.value = value
def __str__(self):
return repr(self.value)
class CalculateRate(UsageComponent):
@staticmethod
def usage(transform_context, record_store_df):
"""component which groups together record store records by
provided group by columns list,sorts within the group by event
timestamp field, calculates the rate of change between the
oldest and latest values, and returns the resultant value as an
instance usage dataframe
"""
instance_usage_data_json_list = []
transform_spec_df = transform_context.transform_spec_df_info
# get aggregated metric name
agg_params = transform_spec_df.select(
"aggregation_params_map.aggregated_metric_name"). \
collect()[0].asDict()
aggregated_metric_name = agg_params["aggregated_metric_name"]
# get aggregation period
agg_params = transform_spec_df.select(
"aggregation_params_map.aggregation_period").collect()[0].asDict()
aggregation_period = agg_params["aggregation_period"]
# Fetch the oldest quantities
latest_instance_usage_df = \
FetchQuantity().usage_by_operation(transform_context,
record_store_df,
"avg")
# Roll up the latest quantities
latest_rolled_up_instance_usage_df = \
RollupQuantity().setter_by_operation(transform_context,
latest_instance_usage_df,
"sum")
# Fetch the oldest quantities
oldest_instance_usage_df = \
FetchQuantity().usage_by_operation(transform_context,
record_store_df,
"oldest")
# Roll up the oldest quantities
oldest_rolled_up_instance_usage_df = \
RollupQuantity().setter_by_operation(transform_context,
oldest_instance_usage_df,
"sum")
# Calculate the rate change by percentage
oldest_dict = oldest_rolled_up_instance_usage_df.collect()[0].asDict()
oldest_quantity = oldest_dict['quantity']
latest_dict = latest_rolled_up_instance_usage_df.collect()[0].asDict()
latest_quantity = latest_dict['quantity']
rate_percentage = \
((latest_quantity - oldest_quantity) / oldest_quantity) * 100
# create a new instance usage dict
instance_usage_dict = {"tenant_id": "all",
"user_id": "all",
"resource_uuid": "all",
"geolocation": "all",
"region": "all",
"zone": "all",
"host": "all",
"project_id": "all",
"aggregated_metric_name":
aggregated_metric_name,
"quantity": rate_percentage,
"firstrecord_timestamp_unix":
oldest_dict["firstrecord_timestamp_unix"],
"firstrecord_timestamp_string":
oldest_dict["firstrecord_timestamp_string"],
"lastrecord_timestamp_unix":
latest_dict["lastrecord_timestamp_unix"],
"lastrecord_timestamp_string":
latest_dict["lastrecord_timestamp_string"],
"record_count": oldest_dict["record_count"] +
latest_dict["record_count"],
"service_group":
Component.DEFAULT_UNAVAILABLE_VALUE,
"service_id":
Component.DEFAULT_UNAVAILABLE_VALUE,
"usage_date": latest_dict["usage_date"],
"usage_hour": latest_dict["usage_hour"],
"usage_minute": latest_dict["usage_minute"],
"aggregation_period": aggregation_period,
"processing_meta":
{"event_type":
latest_dict.get("event_type",
Component.
DEFAULT_UNAVAILABLE_VALUE)}
}
instance_usage_data_json = json.dumps(instance_usage_dict)
instance_usage_data_json_list.append(instance_usage_data_json)
spark_context = record_store_df.rdd.context
instance_usage_rdd = \
spark_context.parallelize(instance_usage_data_json_list)
sql_context = SQLContext\
.getOrCreate(record_store_df.rdd.context)
instance_usage_df = InstanceUsageUtils.create_df_from_json_rdd(
sql_context,
instance_usage_rdd)
return instance_usage_df

View File

@ -299,8 +299,23 @@ class FetchQuantity(UsageComponent):
collect()[0].asDict()
usage_fetch_operation = agg_params["usage_fetch_operation"]
instance_usage_df = FetchQuantity.usage_by_operation(
transform_context, record_store_df, usage_fetch_operation)
return instance_usage_df
@staticmethod
def usage_by_operation(transform_context, record_store_df,
usage_fetch_operation):
"""component which groups together record store records by
provided group by columns list , sorts within the group by event
timestamp field, applies group stats udf and returns the latest
quantity as a instance usage dataframe
"""
transform_spec_df = transform_context.transform_spec_df_info
# check if operation is valid
if not FetchQuantity.\
if not FetchQuantity. \
_is_valid_fetch_operation(usage_fetch_operation):
raise FetchQuantityException(
"Operation %s is not supported" % usage_fetch_operation)
@ -314,7 +329,7 @@ class FetchQuantity(UsageComponent):
# get what we want to group by
agg_params = transform_spec_df.select(
"aggregation_params_map.aggregation_group_by_list").\
"aggregation_params_map.aggregation_group_by_list"). \
collect()[0].asDict()
aggregation_group_by_list = agg_params["aggregation_group_by_list"]
@ -323,8 +338,8 @@ class FetchQuantity(UsageComponent):
aggregation_group_by_list
instance_usage_json_rdd = None
if (usage_fetch_operation == "latest"
or usage_fetch_operation == "oldest"):
if (usage_fetch_operation == "latest" or
usage_fetch_operation == "oldest"):
grouped_rows_rdd = None
@ -342,7 +357,7 @@ class FetchQuantity(UsageComponent):
# high number is adversely affecting performance
num_of_groups = 100
grouped_rows_rdd = \
GroupSortbyTimestampPartition.\
GroupSortbyTimestampPartition. \
fetch_group_latest_oldest_quantity(
record_store_df, transform_spec_df,
group_by_columns_list,
@ -350,15 +365,15 @@ class FetchQuantity(UsageComponent):
else:
# group using key-value pair RDD's groupByKey()
grouped_rows_rdd = \
GroupSortbyTimestamp.\
GroupSortbyTimestamp. \
fetch_group_latest_oldest_quantity(
record_store_df, transform_spec_df,
group_by_columns_list)
grouped_data_rdd_with_operation = grouped_rows_rdd.map(
lambda x:
GroupedDataWithOperation(x,
str(usage_fetch_operation)))
GroupedDataWithOperation(x,
str(usage_fetch_operation)))
instance_usage_json_rdd = \
grouped_data_rdd_with_operation.map(
@ -367,10 +382,10 @@ class FetchQuantity(UsageComponent):
record_store_df_int = \
record_store_df.select(
record_store_df.event_timestamp_unix
.alias("event_timestamp_unix_for_min"),
record_store_df.event_timestamp_unix
.alias("event_timestamp_unix_for_max"),
record_store_df.event_timestamp_unix.alias(
"event_timestamp_unix_for_min"),
record_store_df.event_timestamp_unix.alias(
"event_timestamp_unix_for_max"),
"*")
# for standard sum, max, min, avg operations on grouped data
@ -385,8 +400,8 @@ class FetchQuantity(UsageComponent):
grouped_data_rdd_with_operation = grouped_record_store_df.map(
lambda x:
GroupedDataWithOperation(x,
str(usage_fetch_operation)))
GroupedDataWithOperation(x,
str(usage_fetch_operation)))
instance_usage_json_rdd = grouped_data_rdd_with_operation.map(
FetchQuantity._get_quantity)

View File

@ -251,10 +251,10 @@ class FetchQuantityUtil(UsageComponent):
#
quant_idle_perc_calc_df = quant_idle_perc_df.select(
col("quantity_df_alias.*"),
when(col("idle_perc_df_alias.quantity")
!= 0.0,
ceil(((100.0 - col("idle_perc_df_alias.quantity")))
* col("quantity_df_alias.quantity") / 100.0))
when(col("idle_perc_df_alias.quantity") != 0.0,
ceil(((100.0 - col(
"idle_perc_df_alias.quantity"))) * col(
"quantity_df_alias.quantity") / 100.0))
.otherwise(col("quantity_df_alias.quantity"))
.alias("utilized_quantity"),

View File

@ -5,9 +5,10 @@
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_space_mb","metric_id_list":["disk_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_used_space_mb","metric_id_list":["disk_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.disk.total_allocated_gb","metric_id_list":["nova_disk_total_allocated_gb_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.disk.allocation","metric_id_list":["vm_disk_allocation_all","vm_disk_allocation_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.disk.allocation","metric_id_list":["vm_disk_allocation_all","vm_disk_allocation_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.total_logical_cores","metric_id_list":["cpu_total_all","cpu_total_host","cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.idle_perc","metric_id_list":["cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_usage_all","swift_usage_rate","swift_usage_host"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"}
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.avail","metric_id_list":["swift_avail_all","swift_avail_host"],"required_raw_fields_list":["creation_time", "hostname", "mount", "device"],"service_id":"host_metrics"}

View File

@ -17,3 +17,7 @@
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_all","metric_id":"swift_usage_host"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"calculate_rate","setters":["set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.rate_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"setter_rollup_group_by_list": [],"dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_usage_rate","metric_id":"swift_usage_rate"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_avail_all","metric_id":"swift_avail_all"}
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount", "device"],"usage_fetch_operation": "avg","setter_rollup_group_by_list": ["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"]},"metric_group":"swift_avail_all","metric_id":"swift_avail_host"}

View File

@ -20,6 +20,7 @@ packages =
[entry_points]
monasca_transform.usage =
calculate_rate = monasca_transform.component.usage.calculate_rate:CalculateRate
fetch_quantity = monasca_transform.component.usage.fetch_quantity:FetchQuantity
fetch_quantity_util = monasca_transform.component.usage.fetch_quantity_util:FetchQuantityUtil

View File

@ -79,6 +79,14 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
metric_id='swift_usage_all',
expected_agg_metric_name='swiftlm.diskusage.val.size_agg',
transform_specs_dataframe=transform_specs_data_frame)
self.check_metric(
metric_id='swift_avail_all',
expected_agg_metric_name='swiftlm.diskusage.val.avail_agg',
transform_specs_dataframe=transform_specs_data_frame)
self.check_metric(
metric_id='swift_usage_rate',
expected_agg_metric_name='swiftlm.diskusage.rate_agg',
transform_specs_dataframe=transform_specs_data_frame)
def check_metric(self, metric_id=None, expected_agg_metric_name=None,
transform_specs_dataframe=None):
@ -107,7 +115,8 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
u'vm.mem.total_mb', u'vm.mem.used_mb',
u'nova.vm.disk.total_allocated_gb',
u'vm.disk.allocation', u'vm.cpu.utilization_perc',
u'swiftlm.diskusage.host.val.size']),
u'swiftlm.diskusage.host.val.size',
u'swiftlm.diskusage.host.val.avail']),
Counter([row.event_type for row in
pre_transform_specs_data_frame.collect()]))
@ -295,7 +304,7 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
self.check_list_field_for_row(
row=vm_disk_allocation_all_row,
field_name='required_raw_fields_list',
expected_list=['creation_time', 'tenantId', 'resource_id'],
expected_list=['creation_time', 'tenant_id', 'resource_id'],
)
self.check_dict_field_for_row(
row=vm_disk_allocation_all_row,
@ -346,7 +355,8 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
self.check_list_field_for_row(
row=swiftlm_diskusage_all_row,
field_name='metric_id_list',
expected_list=['swift_usage_all']
expected_list=['swift_usage_all', 'swift_usage_host',
'swift_usage_rate']
)
self.check_list_field_for_row(
row=swiftlm_diskusage_all_row,
@ -367,6 +377,35 @@ class TestDataDrivenSpecsRepo(SparkContextTest):
expected_value='host_metrics'
)
# swiftlm.diskusage.host.val.avail
event_type = 'swiftlm.diskusage.host.val.avail'
swiftlm_diskavail_all_row = self.get_row_for_event_type(
event_type=event_type,
pre_transform_specs_data_frame=pre_transform_specs_data_frame)
self.check_list_field_for_row(
row=swiftlm_diskavail_all_row,
field_name='metric_id_list',
expected_list=['swift_avail_all', 'swift_avail_host']
)
self.check_list_field_for_row(
row=swiftlm_diskavail_all_row,
field_name='required_raw_fields_list',
expected_list=['creation_time', 'hostname', 'mount',
'device'],
)
self.check_dict_field_for_row(
row=swiftlm_diskavail_all_row,
field_name='event_processing_params',
expected_dict={
"set_default_zone_to": "1",
"set_default_geolocation_to": "1",
"set_default_region_to": "W"})
self.check_value_field_for_row(
row=swiftlm_diskavail_all_row,
field_name='service_id',
expected_value='host_metrics'
)
def get_row_for_event_type(self,
event_type=None,
pre_transform_specs_data_frame=None):

View File

@ -114,7 +114,7 @@ class SparkTest(SparkContextTest):
result = simple_count_transform(rdd_monasca_with_offsets)
# Verify it worked
self.assertEqual(result, 347)
self.assertEqual(result, 363)
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
@ -972,13 +972,14 @@ class SparkTest(SparkContextTest):
.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify swiftlm.diskusage.val.size_agg
# Verify swiftlm.diskusage.val.size_agg for all hosts
used_swift_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'swiftlm.diskusage.val.size_agg'][0]
'swiftlm.diskusage.val.size_agg' and
value.get('metric').get('dimensions').get('host') == 'all'][0]
self.assertEqual(360000000000342.0,
self.assertEqual(5291.0,
used_swift_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
used_swift_agg_metric.get('meta').get('region'))
@ -986,23 +987,196 @@ class SparkTest(SparkContextTest):
used_swift_agg_metric.get('meta').get('tenantId'))
self.assertEqual('all',
used_swift_agg_metric.get('metric').get('dimensions')
.get('host'))
.get('project_id'))
self.assertEqual('hourly',
used_swift_agg_metric.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(17.0,
used_swift_agg_metric.get('metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-06-10 20:27:01',
used_swift_agg_metric.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-10 20:27:01',
used_swift_agg_metric.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify swiftlm.diskusage.val.size_agg for host a
used_swift_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'swiftlm.diskusage.val.size_agg' and
value.get('metric').get('dimensions').get('host') == 'a'][0]
self.assertEqual(2848.0,
used_swift_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
used_swift_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
used_swift_agg_metric.get('meta').get('tenantId'))
self.assertEqual('all',
used_swift_agg_metric.get('metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
used_swift_agg_metric.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(18.0,
self.assertEqual(8.0,
used_swift_agg_metric.get('metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-06-10 20:27:01',
used_swift_agg_metric.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-10 20:27:20',
self.assertEqual('2016-06-10 20:27:01',
used_swift_agg_metric.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify swiftlm.diskusage.val.size_agg for host b
used_swift_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'swiftlm.diskusage.val.size_agg' and
value.get('metric').get('dimensions').get('host') == 'b'][0]
self.assertEqual(2443.0,
used_swift_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
used_swift_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
used_swift_agg_metric.get('meta').get('tenantId'))
self.assertEqual('all',
used_swift_agg_metric.get('metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
used_swift_agg_metric.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(9.0,
used_swift_agg_metric.get('metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-06-10 20:27:01',
used_swift_agg_metric.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-10 20:27:01',
used_swift_agg_metric.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify swiftlm.diskusage.val.avail_agg for all hosts
avail_swift_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'swiftlm.diskusage.val.avail_agg' and
value.get('metric').get('dimensions').get('host') == 'all'][0]
self.assertEqual(5291.0,
avail_swift_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
avail_swift_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
avail_swift_agg_metric.get('meta').get('tenantId'))
self.assertEqual('all',
avail_swift_agg_metric.get('metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
avail_swift_agg_metric.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(17.0,
avail_swift_agg_metric.get('metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-06-10 20:27:01',
avail_swift_agg_metric.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-10 20:27:01',
avail_swift_agg_metric.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify swiftlm.diskusage.val.avail_agg for host a
avail_swift_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'swiftlm.diskusage.val.avail_agg' and
value.get('metric').get('dimensions').get('host') == 'a'][0]
self.assertEqual(2848.0,
avail_swift_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
avail_swift_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
avail_swift_agg_metric.get('meta').get('tenantId'))
self.assertEqual('all',
avail_swift_agg_metric.get('metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
avail_swift_agg_metric.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(8.0,
avail_swift_agg_metric.get('metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-06-10 20:27:01',
avail_swift_agg_metric.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-10 20:27:01',
avail_swift_agg_metric.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify swiftlm.diskusage.val.avail_agg for host b
avail_swift_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'swiftlm.diskusage.val.avail_agg' and
value.get('metric').get('dimensions').get('host') == 'b'][0]
self.assertEqual(2443.0,
avail_swift_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
avail_swift_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
avail_swift_agg_metric.get('meta').get('tenantId'))
self.assertEqual('all',
avail_swift_agg_metric.get('metric').get('dimensions')
.get('project_id'))
self.assertEqual('hourly',
avail_swift_agg_metric.get('metric').get('dimensions')
.get('aggregation_period'))
self.assertEqual(9.0,
avail_swift_agg_metric.get('metric').get('value_meta')
.get('record_count'))
self.assertEqual('2016-06-10 20:27:01',
avail_swift_agg_metric.get('metric').get('value_meta')
.get('firstrecord_timestamp'))
self.assertEqual('2016-06-10 20:27:01',
avail_swift_agg_metric.get('metric').get('value_meta')
.get('lastrecord_timestamp'))
# Verify swiftlm.diskusage.rate_agg metrics
diskusage_rate_agg_metric = [
value for value in metrics
if value.get('metric').get('name') ==
'swiftlm.diskusage.rate_agg'][0]
self.assertEqual(15.650273224043715,
diskusage_rate_agg_metric.get('metric').get('value'))
self.assertEqual('useast',
diskusage_rate_agg_metric.get('meta').get('region'))
self.assertEqual(cfg.CONF.messaging.publish_kafka_tenant_id,
diskusage_rate_agg_metric.get('meta').get('tenantId'))
self.assertEqual('all',
diskusage_rate_agg_metric.get('metric')
.get('dimensions').get('host'))
self.assertEqual('all',
diskusage_rate_agg_metric.get('metric')
.get('dimensions').get('project_id'))
self.assertEqual('hourly',
diskusage_rate_agg_metric.get('metric')
.get('dimensions').get('aggregation_period'))
self.assertEqual(34.0,
diskusage_rate_agg_metric.get('metric')
.get('value_meta').get('record_count'))
self.assertEqual('2016-06-10 20:27:01',
diskusage_rate_agg_metric.get('metric')
.get('value_meta').get('firstrecord_timestamp'))
self.assertEqual('2016-06-10 20:27:01',
diskusage_rate_agg_metric.get('metric')
.get('value_meta').get('lastrecord_timestamp'))
def simple_count_transform(rdd):
return rdd.count()

View File

@ -296,15 +296,15 @@
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0006-mgmt","cluster":"compute"},"timestamp":1463498061000,"value":82.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498062}')
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0003-mgmt","cluster":"compute"},"timestamp":1463498078000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498080}')
('<message id>','{"metric":{"name":"nova.vm.disk.total_allocated_gb","dimensions":{"cloud_name":"helion-poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","hostname":"hlm004-cp1-comp0002-mgmt","cluster":"compute"},"timestamp":1463498084000,"value":132.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1463498086}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenantId":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":24.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"c","component":"vm","service":"compute","hostname":"dummy1","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":28.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"d","component":"vm","service":"compute","hostname":"dummy2","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":30.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"e","component":"vm","service":"compute","hostname":"dummy3","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":32.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenantId":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":26.0},"meta":{"tenantId":"tenantId of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308005000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"a","component":"vm","service":"compute","hostname":"devstack","tenant_id":"5f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308040000,"value":24.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000010,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"c","component":"vm","service":"compute","hostname":"dummy1","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000020,"value":28.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"d","component":"vm","service":"compute","hostname":"dummy2","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000030,"value":30.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"e","component":"vm","service":"compute","hostname":"dummy3","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308000040,"value":32.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308006000,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.disk.allocation","dimensions":{"resource_id":"b","component":"vm","service":"compute","hostname":"mini-mon","tenant_id":"6f681592f7084c5fbcd4e8a20a4fef15"},"timestamp":1453308046000,"value":26.0},"meta":{"tenant_id":"tenant_id of metric writer","region":"useast"},"creation_time":1458578365}')
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"7b805207-3c97-4ef9-8b28-3fac171f2a96","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283861000,"value":5.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"a07858d1-aa13-4259-9873-f23170bc9e1b","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283895000,"value":35.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
('<message id>','{"metric":{"name":"vm.cpu.utilization_perc","dimensions":{"resource_id":"4cd09eac-854d-49eb-993d-914271f8e1fe","cloud_name":"poc-hlm-004","component":"vm","control_plane":"ccp","service":"compute","cluster":"compute","hostname":"004-cp1-comp0006-mgmt","tenant_id":"817331145b804dc9a7accb6edfb0674d","zone":"nova"},"timestamp":1464283896000,"value":15.0},"meta":{"tenantId":"e675b49896624464bb2e3152ef92cd11","region":"unset"},"creation_time":1464283972}')
@ -327,21 +327,37 @@
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"b","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"1","zone":"nova"},"timestamp":1465316994000,"value":5120.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465316995}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"c","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317024000,"value":2048.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317025}')
('<message id>','{"metric":{"name":"vm.mem.total_mb","dimensions":{"resource_id":"d","cloud_name":"ceilometer","component":"vm","control_plane":"control-plane-1","service":"compute","cluster":"compute","hostname":"ceil-cp1-comp0025-mgmt","tenant_id":"2","zone":"nova"},"timestamp":1465317054000,"value":6144.0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465317055}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000006},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000008},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000010},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000012},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000014},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000016},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000018},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000020},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000022},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000024},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m2-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590421000,"value":20000000000026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0003-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590421000,"value":20000000000028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0002-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590430000,"value":20000000000030},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590435}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m3-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590434000,"value":20000000000032},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590440}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swobj0001-mgmt","cluster":"swobj","label":"---NA---"},"timestamp":1465590436000,"value":20000000000034},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590442}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"ceil-cp1-swpac-m1-mgmt","cluster":"swpac","label":"---NA---"},"timestamp":1465590440000,"value":20000000000036},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590445}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":904},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":804},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":928},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421004,"value":1128},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421001,"value":916},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":901},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":602},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":926},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1126},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.size","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":904},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1004},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":804},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":928},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421002,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421003,"value":1028},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421004,"value":1128},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"a","cluster":"swobj","label":"---NA---"},"timestamp":1465590421001,"value":916},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":901},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1002},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":602},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":926},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":1026},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421003,"value":1126},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk0","control_plane":"control-plane-1","device":"/dev/sdb2","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421004,"value":1526},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421001,"value":0},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')
('<message id>','{"metric":{"name":"swiftlm.diskusage.host.val.avail","dimensions":{"cloud_name":"ceilometer","mount":"/srv/node/disk1","control_plane":"control-plane-1","device":"/dev/sdb1","service":"object-storage","hostname":"b","cluster":"swpac","label":"---NA---"},"timestamp":1465590421002,"value":914},"meta":{"tenantId":"90273cc79acc4239816d572f9397863e","region":"unset"},"creation_time":1465590426}')

View File

@ -25,6 +25,8 @@ from monasca_transform.component.setter.set_aggregated_metric_name \
import SetAggregatedMetricName
from monasca_transform.component.setter.set_aggregated_period \
import SetAggregatedPeriod
from monasca_transform.component.usage.calculate_rate \
import CalculateRate
from monasca_transform.component.usage.fetch_quantity \
import FetchQuantity
from monasca_transform.component.usage.fetch_quantity_util \
@ -49,6 +51,13 @@ class MockComponentManager(object):
'FetchQuantityUtil',
FetchQuantityUtil(),
None),
Extension(
'calculate_rate',
'monasca_transform.component.usage.'
'calculate_rate:'
'CalculateRate',
CalculateRate(),
None),
])
@staticmethod

View File

@ -138,12 +138,6 @@ class TestVmCpuAllocatedAgg(SparkContextTest):
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Do something simple with the RDD
result = simple_count_transform(rdd_monasca_with_offsets)
# Verify it worked
self.assertEqual(result, 347)
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)
@ -317,12 +311,6 @@ class TestVmCpuAllocatedAgg(SparkContextTest):
rdd_monasca_with_offsets = rdd_monasca.map(
lambda x: RddTransformContext(x, transform_context))
# Do something simple with the RDD
result = simple_count_transform(rdd_monasca_with_offsets)
# Verify it worked
self.assertEqual(result, 347)
# Call the primary method in mon_metrics_kafka
MonMetricsKafkaProcessor.rdd_to_recordstore(
rdd_monasca_with_offsets)