Merge "Started adding kubernetes metrics aggregation"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
b852f7141f
|
|
@ -149,7 +149,21 @@ class InsertComponent(Component):
|
|||
"usage_hour": row.usage_hour,
|
||||
"usage_minute": row.usage_minute,
|
||||
"aggregation_period":
|
||||
row.aggregation_period}
|
||||
row.aggregation_period,
|
||||
"namespace":
|
||||
row.namespace,
|
||||
"pod_name":
|
||||
row.pod_name,
|
||||
"app":
|
||||
row.app,
|
||||
"container_name":
|
||||
row.container_name,
|
||||
"interface":
|
||||
row.interface,
|
||||
"deployment":
|
||||
row.deployment,
|
||||
"daemon_set":
|
||||
row.daemon_set}
|
||||
metric = InsertComponent._prepare_metric(instance_usage_dict,
|
||||
agg_params)
|
||||
return metric
|
||||
|
|
@ -195,7 +209,21 @@ class InsertComponent(Component):
|
|||
"usage_minute": row.usage_minute,
|
||||
"aggregation_period":
|
||||
row.aggregation_period,
|
||||
"processing_meta": processing_meta}
|
||||
"processing_meta": processing_meta,
|
||||
"namespace":
|
||||
row.namespace,
|
||||
"pod_name":
|
||||
row.pod_name,
|
||||
"app":
|
||||
row.app,
|
||||
"container_name":
|
||||
row.container_name,
|
||||
"interface":
|
||||
row.interface,
|
||||
"deployment":
|
||||
row.deployment,
|
||||
"daemon_set":
|
||||
row.daemon_set}
|
||||
return instance_usage_dict
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -71,6 +71,20 @@ class PreHourlyCalculateRate(SetterComponent):
|
|||
latest_dict.get("user_id", "all"),
|
||||
"resource_uuid":
|
||||
latest_dict.get("resource_uuid", "all"),
|
||||
"namespace":
|
||||
latest_dict.get("namespace", "all"),
|
||||
"pod_name":
|
||||
latest_dict.get("pod_name", "all"),
|
||||
"app":
|
||||
latest_dict.get("app", "all"),
|
||||
"container_name":
|
||||
latest_dict.get("container_name", "all"),
|
||||
"interface":
|
||||
latest_dict.get("interface", "all"),
|
||||
"deployment":
|
||||
latest_dict.get("deployment", "all"),
|
||||
"daemon_set":
|
||||
latest_dict.get("daemon_set", "all"),
|
||||
"geolocation":
|
||||
latest_dict.get("geolocation", "all"),
|
||||
"region":
|
||||
|
|
|
|||
|
|
@ -116,6 +116,20 @@ class RollupQuantity(SetterComponent):
|
|||
getattr(row, "user_id", "all"),
|
||||
"resource_uuid":
|
||||
getattr(row, "resource_uuid", "all"),
|
||||
"namespace":
|
||||
getattr(row, "namespace", "all"),
|
||||
"pod_name":
|
||||
getattr(row, "pod_name", "all"),
|
||||
"app":
|
||||
getattr(row, "app", "all"),
|
||||
"container_name":
|
||||
getattr(row, "container_name", "all"),
|
||||
"interface":
|
||||
getattr(row, "interface", "all"),
|
||||
"deployment":
|
||||
getattr(row, "deployment", "all"),
|
||||
"daemon_set":
|
||||
getattr(row, "daemon_set", "all"),
|
||||
"geolocation":
|
||||
getattr(row, "geolocation", "all"),
|
||||
"region":
|
||||
|
|
|
|||
|
|
@ -41,6 +41,13 @@ class SetAggregatedMetricName(SetterComponent):
|
|||
instance_usage_dict = {"tenant_id": row.tenant_id,
|
||||
"user_id": row.user_id,
|
||||
"resource_uuid": row.resource_uuid,
|
||||
"namespace": row.namespace,
|
||||
"pod_name": row.pod_name,
|
||||
"app": row.app,
|
||||
"container_name": row.container_name,
|
||||
"interface": row.interface,
|
||||
"deployment": row.deployment,
|
||||
"daemon_set": row.daemon_set,
|
||||
"geolocation": row.geolocation,
|
||||
"region": row.region,
|
||||
"zone": row.zone,
|
||||
|
|
|
|||
|
|
@ -41,6 +41,13 @@ class SetAggregatedPeriod(SetterComponent):
|
|||
instance_usage_dict = {"tenant_id": row.tenant_id,
|
||||
"user_id": row.user_id,
|
||||
"resource_uuid": row.resource_uuid,
|
||||
"namespace": row.namespace,
|
||||
"pod_name": row.pod_name,
|
||||
"app": row.app,
|
||||
"container_name": row.container_name,
|
||||
"interface": row.interface,
|
||||
"deployment": row.deployment,
|
||||
"daemon_set": row.daemon_set,
|
||||
"geolocation": row.geolocation,
|
||||
"region": row.region,
|
||||
"zone": row.zone,
|
||||
|
|
|
|||
|
|
@ -97,6 +97,20 @@ class FetchQuantity(UsageComponent):
|
|||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
user_id = group_by_dict.get("user_id",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
namespace = group_by_dict.get("namespace",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
pod_name = group_by_dict.get("pod_name",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
app = group_by_dict.get("app",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
container_name = group_by_dict.get("container_name",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
interface = group_by_dict.get("interface",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
deployment = group_by_dict.get("deployment",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
daemon_set = group_by_dict.get("daemon_set",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
|
||||
geolocation = group_by_dict.get("geolocation",
|
||||
Component.DEFAULT_UNAVAILABLE_VALUE)
|
||||
|
|
@ -145,6 +159,13 @@ class FetchQuantity(UsageComponent):
|
|||
|
||||
instance_usage_dict = {"tenant_id": tenant_id, "user_id": user_id,
|
||||
"resource_uuid": resource_uuid,
|
||||
"namespace": namespace,
|
||||
"pod_name": pod_name,
|
||||
"app": app,
|
||||
"container_name": container_name,
|
||||
"interface": interface,
|
||||
"deployment": deployment,
|
||||
"daemon_set": daemon_set,
|
||||
"geolocation": geolocation, "region": region,
|
||||
"zone": zone, "host": host,
|
||||
"aggregated_metric_name":
|
||||
|
|
@ -239,6 +260,34 @@ class FetchQuantity(UsageComponent):
|
|||
getattr(row, "tenant_id",
|
||||
Component.
|
||||
DEFAULT_UNAVAILABLE_VALUE),
|
||||
"namespace":
|
||||
getattr(row, "namespace",
|
||||
Component.
|
||||
DEFAULT_UNAVAILABLE_VALUE),
|
||||
"pod_name":
|
||||
getattr(row, "pod_name",
|
||||
Component.
|
||||
DEFAULT_UNAVAILABLE_VALUE),
|
||||
"app":
|
||||
getattr(row, "app",
|
||||
Component.
|
||||
DEFAULT_UNAVAILABLE_VALUE),
|
||||
"container_name":
|
||||
getattr(row, "container_name",
|
||||
Component.
|
||||
DEFAULT_UNAVAILABLE_VALUE),
|
||||
"interface":
|
||||
getattr(row, "interface",
|
||||
Component.
|
||||
DEFAULT_UNAVAILABLE_VALUE),
|
||||
"deployment":
|
||||
getattr(row, "deployment",
|
||||
Component.
|
||||
DEFAULT_UNAVAILABLE_VALUE),
|
||||
"daemon_set":
|
||||
getattr(row, "daemon_set",
|
||||
Component.
|
||||
DEFAULT_UNAVAILABLE_VALUE),
|
||||
"aggregated_metric_name":
|
||||
getattr(row, "aggregated_metric_name",
|
||||
Component.
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ class ConfigInitializer(object):
|
|||
|
||||
@staticmethod
|
||||
def basic_config(default_config_files=None):
|
||||
cfg.CONF.reset()
|
||||
ConfigInitializer.load_repositories_options()
|
||||
ConfigInitializer.load_database_options()
|
||||
ConfigInitializer.load_messaging_options()
|
||||
|
|
|
|||
|
|
@ -407,6 +407,34 @@ class MonMetricsKafkaProcessor(object):
|
|||
gen_mon_metrics_df.metric.dimensions.device).otherwise(
|
||||
'NA').alias("device"),
|
||||
|
||||
when(gen_mon_metrics_df.metric.dimensions.namespace != '',
|
||||
gen_mon_metrics_df.metric.dimensions.namespace).otherwise(
|
||||
'NA').alias("namespace"),
|
||||
|
||||
when(gen_mon_metrics_df.metric.dimensions.pod_name != '',
|
||||
gen_mon_metrics_df.metric.dimensions.pod_name).otherwise(
|
||||
'NA').alias("pod_name"),
|
||||
|
||||
when(gen_mon_metrics_df.metric.dimensions.container_name != '',
|
||||
gen_mon_metrics_df.metric.dimensions
|
||||
.container_name).otherwise('NA').alias("container_name"),
|
||||
|
||||
when(gen_mon_metrics_df.metric.dimensions.app != '',
|
||||
gen_mon_metrics_df.metric.dimensions.app).otherwise(
|
||||
'NA').alias("app"),
|
||||
|
||||
when(gen_mon_metrics_df.metric.dimensions.interface != '',
|
||||
gen_mon_metrics_df.metric.dimensions.interface).otherwise(
|
||||
'NA').alias("interface"),
|
||||
|
||||
when(gen_mon_metrics_df.metric.dimensions.deployment != '',
|
||||
gen_mon_metrics_df.metric.dimensions
|
||||
.deployment).otherwise('NA').alias("deployment"),
|
||||
|
||||
when(gen_mon_metrics_df.metric.dimensions.daemon_set != '',
|
||||
gen_mon_metrics_df.metric.dimensions
|
||||
.daemon_set).otherwise('NA').alias("daemon_set"),
|
||||
|
||||
when(gen_mon_metrics_df.meta.userId != '',
|
||||
gen_mon_metrics_df.meta.userId).otherwise('NA').alias(
|
||||
"user_id"),
|
||||
|
|
|
|||
|
|
@ -13,14 +13,8 @@
|
|||
# under the License.
|
||||
|
||||
import abc
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import six
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OffsetSpec(object):
|
||||
|
||||
|
|
@ -104,147 +98,3 @@ class OffsetSpecs(object):
|
|||
"Class %s doesn't implement "
|
||||
"get_most_recent_batch_time_from_offsets()"
|
||||
% self.__class__.__name__)
|
||||
|
||||
|
||||
class JSONOffsetSpecs(OffsetSpecs):
|
||||
|
||||
def __init__(self, path=None, filename=None):
|
||||
self.kafka_offset_spec_file = os.path.join(
|
||||
(path or "/tmp/"), (filename or 'kafka_offset_specs.json'))
|
||||
|
||||
self._kafka_offsets = {}
|
||||
if os.path.exists(self.kafka_offset_spec_file):
|
||||
try:
|
||||
f = open(self.kafka_offset_spec_file)
|
||||
kafka_offset_dict = json.load(f)
|
||||
for key, value in kafka_offset_dict.items():
|
||||
log.info("Found offset %s: %s", key, value)
|
||||
self._kafka_offsets[key] = OffsetSpec(
|
||||
app_name=value.get('app_name'),
|
||||
topic=value.get('topic'),
|
||||
partition=value.get('partition'),
|
||||
from_offset=value.get('from_offset'),
|
||||
until_offset=value.get('until_offset'),
|
||||
batch_time=value.get('batch_time'),
|
||||
last_updated=value.get('last_updated'),
|
||||
revision=value.get('revision')
|
||||
)
|
||||
except Exception:
|
||||
log.info('Invalid or corrupts offsets file found at %s,'
|
||||
' starting over' % self.kafka_offset_spec_file)
|
||||
else:
|
||||
log.info('No kafka offsets found at startup')
|
||||
|
||||
def _save(self):
|
||||
"""get the specs of last run time of offset
|
||||
"""
|
||||
log.info("Saving json offsets: %s", self._kafka_offsets)
|
||||
|
||||
with open(self.kafka_offset_spec_file, 'w') as offset_file:
|
||||
offset_file.write('{')
|
||||
# json_values = []
|
||||
# for key, value in self._kafka_offsets.items():
|
||||
# json_values.append({key: })
|
||||
offset_file.write(','.join(
|
||||
['\"%s\": %s' % (key, json.dumps(self.as_dict(value)))
|
||||
for key, value in self._kafka_offsets.items()]))
|
||||
offset_file.write('}')
|
||||
|
||||
@staticmethod
|
||||
def as_dict(offset_value):
|
||||
return {"app_name": offset_value.get_app_name(),
|
||||
"topic": offset_value.get_topic(),
|
||||
"partition": offset_value.get_partition(),
|
||||
"from_offset": offset_value.get_from_offset(),
|
||||
"until_offset": offset_value.get_until_offset(),
|
||||
"batch_time": offset_value.get_batch_time(),
|
||||
"last_updated": offset_value.get_last_updated(),
|
||||
"revision": offset_value.get_revision()}
|
||||
|
||||
def add(self, app_name, topic, partition,
|
||||
from_offset, until_offset, batch_time_info):
|
||||
|
||||
# batch time
|
||||
batch_time = \
|
||||
batch_time_info.strftime(
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# last updated
|
||||
last_updated = \
|
||||
datetime.datetime.now().strftime(
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
|
||||
NEW_REVISION_NO = 1
|
||||
|
||||
key_name = "%s_%s_%s" % (
|
||||
app_name, topic, partition)
|
||||
offset = OffsetSpec(
|
||||
app_name=app_name,
|
||||
topic=topic,
|
||||
partition=partition,
|
||||
from_offset=from_offset,
|
||||
until_offset=until_offset,
|
||||
batch_time=batch_time,
|
||||
last_updated=last_updated,
|
||||
revision=NEW_REVISION_NO
|
||||
)
|
||||
log.info('Adding offset %s for key %s to current offsets: %s' %
|
||||
(offset, key_name, self._kafka_offsets))
|
||||
self._kafka_offsets[key_name] = offset
|
||||
log.info('Added so kafka offset is now %s', self._kafka_offsets)
|
||||
self._save()
|
||||
|
||||
def get_kafka_offsets(self, app_name):
|
||||
return self._kafka_offsets
|
||||
|
||||
def get_most_recent_batch_time_from_offsets(self, app_name, topic):
|
||||
try:
|
||||
# get partition 0 as a representative of all others
|
||||
key = "%s_%s_%s" % (app_name, topic, 0)
|
||||
offset = self._kafka_offsets[key]
|
||||
most_recent_batch_time = datetime.datetime.strptime(
|
||||
offset.get_batch_time(),
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
except Exception:
|
||||
most_recent_batch_time = None
|
||||
|
||||
return most_recent_batch_time
|
||||
|
||||
def delete_all_kafka_offsets(self, app_name):
|
||||
log.info("Deleting json offsets file: %s", self.kafka_offset_spec_file)
|
||||
os.remove(self.kafka_offset_spec_file)
|
||||
|
||||
def add_all_offsets(self, app_name, offsets, batch_time_info):
|
||||
|
||||
# batch time
|
||||
batch_time = \
|
||||
batch_time_info.strftime(
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# last updated
|
||||
last_updated = \
|
||||
datetime.datetime.now().strftime(
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
|
||||
NEW_REVISION_NO = -1
|
||||
|
||||
for o in offsets:
|
||||
|
||||
key_name = "%s_%s_%s" % (
|
||||
app_name, o.topic, o.partition)
|
||||
|
||||
offset = OffsetSpec(
|
||||
topic=o.topic,
|
||||
app_name=app_name,
|
||||
partition=o.partition,
|
||||
from_offset=o.fromOffset,
|
||||
until_offset=o.untilOffset,
|
||||
batch_time=batch_time,
|
||||
last_updated=last_updated,
|
||||
revision=NEW_REVISION_NO)
|
||||
|
||||
log.info('Adding offset %s for key %s to current offsets: %s' %
|
||||
(offset, key_name, self._kafka_offsets))
|
||||
self._kafka_offsets[key_name] = offset
|
||||
log.info('Added so kafka offset is now %s', self._kafka_offsets)
|
||||
self._save()
|
||||
|
|
|
|||
|
|
@ -53,7 +53,8 @@ class InstanceUsageUtils(TransformUtils):
|
|||
"lastrecord_timestamp_string",
|
||||
"service_group", "service_id",
|
||||
"usage_date", "usage_hour", "usage_minute",
|
||||
"aggregation_period"]
|
||||
"aggregation_period", "namespace", "pod_name", "app",
|
||||
"container_name", "interface", "deployment", "daemon_set"]
|
||||
|
||||
columns_struct_fields = [StructField(field_name, StringType(), True)
|
||||
for field_name in columns]
|
||||
|
|
@ -98,7 +99,9 @@ class RecordStoreUtils(TransformUtils):
|
|||
"user_id", "region", "zone",
|
||||
"host", "project_id", "service_group", "service_id",
|
||||
"event_date", "event_hour", "event_minute",
|
||||
"event_second", "metric_group", "metric_id"]
|
||||
"event_second", "metric_group", "metric_id",
|
||||
"namespace", "pod_name", "app", "container_name",
|
||||
"interface", "deployment", "daemon_set"]
|
||||
|
||||
columns_struct_fields = [StructField(field_name, StringType(), True)
|
||||
for field_name in columns]
|
||||
|
|
@ -225,7 +228,9 @@ class MonMetricUtils(TransformUtils):
|
|||
"mode", "mount", "mount_point", "observer_host",
|
||||
"process_name", "project_id", "resource_id", "service",
|
||||
"test_type", "tenantId", "tenant_id", "topic", "url",
|
||||
"state", "state_description", "instanceId"]
|
||||
"state", "state_description", "instanceId",
|
||||
"namespace", "pod_name", "app", "container_name",
|
||||
"interface", "deployment", "daemon_set"]
|
||||
|
||||
dimensions_struct_fields = [
|
||||
StructField(field_name, StringType(), True)
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ class JSONDataDrivenSpecsRepo(DataDrivenSpecsRepo):
|
|||
if data_driven_spec_type == self.transform_specs_type:
|
||||
path = (os.path.join(
|
||||
self._common_file_system_stub_path,
|
||||
"monasca_transform/data_driven_specs/"
|
||||
"tests/functional/data_driven_specs/"
|
||||
"transform_specs/transform_specs.json"
|
||||
))
|
||||
if os.path.exists(path):
|
||||
|
|
@ -41,7 +41,7 @@ class JSONDataDrivenSpecsRepo(DataDrivenSpecsRepo):
|
|||
elif data_driven_spec_type == self.pre_transform_specs_type:
|
||||
path = (os.path.join(
|
||||
self._common_file_system_stub_path,
|
||||
"monasca_transform/data_driven_specs/"
|
||||
"tests/functional/data_driven_specs/"
|
||||
"pre_transform_specs/pre_transform_specs.json"
|
||||
))
|
||||
if os.path.exists(path):
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"mem.total_mb","metric_id_list":["mem_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"mem.usable_mb","metric_id_list":["mem_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.mem.total_mb","metric_id_list":["vm_mem_total_mb_all","vm_mem_total_mb_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.mem.used_mb","metric_id_list":["vm_mem_used_mb_all","vm_mem_used_mb_project"],"required_raw_fields_list":["creation_time","tenantId","resource_id"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.mem.total_allocated_mb","metric_id_list":["nova_vm_mem_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_space_mb","metric_id_list":["disk_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"disk.total_used_space_mb","metric_id_list":["disk_usable_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.disk.total_allocated_gb","metric_id_list":["nova_disk_total_allocated_gb_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.disk.allocation","metric_id_list":["vm_disk_allocation_all","vm_disk_allocation_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.total_logical_cores","metric_id_list":["cpu_total_all","cpu_total_host","cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.idle_perc","metric_id_list":["cpu_util_all","cpu_util_host"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vcpus","metric_id_list":["vcpus_all","vcpus_project"],"required_raw_fields_list":["creation_time","project_id","resource_id"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"vm.cpu.utilization_perc","metric_id_list":["vm_cpu_util_perc_project"],"required_raw_fields_list":["creation_time","tenant_id","resource_id"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"nova.vm.cpu.total_allocated","metric_id_list":["nova_vm_cpu_total_all"],"required_raw_fields_list":["creation_time"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.size","metric_id_list":["swift_total_all","swift_total_host"],"required_raw_fields_list":["creation_time", "hostname", "mount"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"swiftlm.diskusage.host.val.avail","metric_id_list":["swift_avail_all","swift_avail_host","swift_usage_rate"],"required_raw_fields_list":["creation_time", "hostname", "mount"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"storage.objects.size","metric_id_list":["storage_objects_size_all"],"required_raw_fields_list":["creation_time", "project_id"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"pod.net.in_bytes_sec","metric_id_list":["pod_net_in_b_per_sec_total_all","pod_net_in_b_per_sec_per_namespace","pod_net_in_b_per_sec_per_app"],"required_raw_fields_list":["creation_time","tenantId","namespace","pod_name","app"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"pod.net.out_bytes_sec","metric_id_list":["pod_net_out_b_per_sec_total_all","pod_net_out_b_per_sec_per_namespace"],"required_raw_fields_list":["creation_time","tenantId","namespace","pod_name"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"pod.mem.used_bytes","metric_id_list":["pod.mem_used_b_total_all","pod.mem_used_b_per_pod_name","pod.mem_used_b_per_namespace"],"required_raw_fields_list":["creation_time","tenantId","namespace","pod_name"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"pod.cpu.total_time","metric_id_list":["pod_cpu_total_all","pod_cpu_per_namespace"],"required_raw_fields_list":["creation_time","tenantId","namespace"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"container.cpu.total_time","metric_id_list":["container_cpu_total_time_all","container_cpu_total_time_per_container"],"required_raw_fields_list":["creation_time","tenantId","container_name"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"kubernetes.node.capacity.cpu","metric_id_list":["kubernetes_node_capacity_cpu_all","kubernetes_node_capacity_cpu_per_host"],"required_raw_fields_list":["creation_time","tenantId","hostname"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"cpu.total_time_sec","metric_id_list":["cpu_total_time_sec_all","cpu_total_time_sec_per_host"],"required_raw_fields_list":["creation_time","tenantId","hostname"],"service_id":"host_metrics"}
|
||||
{"event_processing_params":{"set_default_zone_to":"1","set_default_geolocation_to":"1","set_default_region_to":"W"},"event_type":"kubernetes.node.allocatable.cpu","metric_id_list":["kubernetes_node_allocatable_cpu_all","kubernetes_node_allocatable_cpu_per_host"],"required_raw_fields_list":["creation_time","tenantId","hostname"],"service_id":"host_metrics"}
|
||||
|
|
@ -20,10 +20,10 @@ from pyspark import SQLContext
|
|||
from monasca_transform.config.config_initializer import ConfigInitializer
|
||||
from monasca_transform.data_driven_specs.data_driven_specs_repo \
|
||||
import DataDrivenSpecsRepo
|
||||
from monasca_transform.data_driven_specs.json_data_driven_specs_repo \
|
||||
import JSONDataDrivenSpecsRepo
|
||||
from monasca_transform.data_driven_specs.mysql_data_driven_specs_repo \
|
||||
import MySQLDataDrivenSpecsRepo
|
||||
from tests.functional.data_driven_specs.json_data_driven_specs_repo \
|
||||
import JSONDataDrivenSpecsRepo
|
||||
from tests.functional.spark_context_test import SparkContextTest
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,44 @@
|
|||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"mem_total_all","metric_id":"mem_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"mem.usable_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"mem_usable_all","metric_id":"mem_usable_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_mem_total_mb_all","metric_id":"vm_mem_total_mb_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.mem.total_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_mem_total_mb_project","metric_id":"vm_mem_total_mb_project"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.mem.used_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_mem_used_mb_all","metric_id":"vm_mem_used_mb_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.mem.used_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_mem_used_mb_project","metric_id":"vm_mem_used_mb_project"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"nova.vm.mem.total_allocated_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"nova_vm_mem_total_all","metric_id":"nova_vm_mem_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"disk.total_space_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"disk_total_all","metric_id":"disk_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"disk.total_used_space_mb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"disk_usable_all","metric_id":"disk_usable_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"nova.vm.disk.total_allocated_gb_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"nova_disk_total_allocated_gb_all","metric_id":"nova_disk_total_allocated_gb_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.disk.allocation_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_disk_allocation_all","metric_id":"vm_disk_allocation_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.disk.allocation_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_disk_allocation_project","metric_id":"vm_disk_allocation_project"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"cpu.total_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"cpu_total_all","metric_id":"cpu_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"cpu.total_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"cpu_total_host","metric_id":"cpu_total_host"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity_util","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"cpu.utilized_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["event_type", "host"],"usage_fetch_operation": "avg","usage_fetch_util_quantity_event_type": "cpu.total_logical_cores","usage_fetch_util_idle_perc_event_type": "cpu.idle_perc","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"cpu_util_all","metric_id":"cpu_util_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity_util","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"cpu.utilized_logical_cores_agg","aggregation_period":"hourly","aggregation_group_by_list": ["event_type", "host"],"usage_fetch_operation": "avg","usage_fetch_util_quantity_event_type": "cpu.total_logical_cores","usage_fetch_util_idle_perc_event_type": "cpu.idle_perc","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"cpu_util_host","metric_id":"cpu_util_host"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vcpus_all","metric_id":"vcpus_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vcpus_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vcpus_project","metric_id":"vcpus_project"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"vm.cpu.utilization_perc_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "tenant_id", "resource_uuid"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["tenant_id"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"vm_cpu_util_perc_project","metric_id":"vm_cpu_util_perc_project"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"nova.vm.cpu.total_allocated_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list": [],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"nova_vm_cpu_total_all","metric_id":"nova_vm_cpu_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_total_all","metric_id":"swift_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_total_host","metric_id":"swift_total_host"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_avail_all","metric_id":"swift_avail_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.val.avail_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"swift_avail_host","metric_id":"swift_avail_host"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"calculate_rate","setters":["set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"swiftlm.diskusage.rate_agg","aggregation_period":"hourly","aggregation_group_by_list": ["host", "metric_id", "mount"],"filter_by_list": [],"setter_rollup_group_by_list": [],"dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"rate","pre_hourly_group_by_list":["default"]},"metric_group":"swift_avail_rate","metric_id":"swift_usage_rate"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"storage.objects.size_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host","project_id"],"pre_hourly_operation":"sum","pre_hourly_group_by_list":["default"]},"metric_group":"storage_objects_size_all","metric_id":"storage_objects_size_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.net.in_bytes_sec_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id", "app", "namespace", "pod_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace","app","pod_name","interface","deployment"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"pod_net_in_b_per_sec_total_all","metric_id":"pod_net_in_b_per_sec_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.net.in_bytes_sec_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id", "app", "namespace", "pod_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["namespace"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace","app","pod_name","interface","deployment"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "namespace"]},"metric_group":"pod_net_in_b_per_sec_per_namespace","metric_id":"pod_net_in_b_per_sec_per_namespace"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.net.in_bytes_sec_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id", "app", "namespace", "pod_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["app"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace","app","pod_name","interface","deployment"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "app"]},"metric_group":"pod_net_in_b_per_sec_per_app","metric_id":"pod_net_in_b_per_sec_per_app"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.net.out_bytes_sec_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id", "namespace", "pod_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace","pod_name","interface","daemon_set"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"pod_net_out_b_per_sec_total_all","metric_id":"pod_net_out_b_per_sec_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.net.out_bytes_sec_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id", "namespace", "pod_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["namespace"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace","pod_name","interface","daemon_set"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "namespace"]},"metric_group":"pod_net_out_b_per_sec_per_namespace","metric_id":"pod_net_out_b_per_sec_per_namespace"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.mem.used_bytes_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id", "namespace", "pod_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace","pod_name","deployment"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"pod.mem_used_b_total_all","metric_id":"pod.mem_used_b_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.mem.used_bytes_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id", "namespace", "pod_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["pod_name"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace","pod_name","deployment"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "pod_name"]},"metric_group":"pod.mem_used_b_per_pod_name","metric_id":"pod.mem_used_b_per_pod_name"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.mem.used_bytes_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id", "namespace", "pod_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["namespace"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace","pod_name","deployment"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "namespace"]},"metric_group":"pod.mem_used_b_per_namespace","metric_id":"pod.mem_used_b_per_namespace"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.cpu.total_time_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","namespace"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"pod_cpu_total_all","metric_id":"pod_cpu_total_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"pod.cpu.total_time_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","namespace"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["namespace"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","namespace"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "namespace"]},"metric_group":"pod_cpu_per_namespace","metric_id":"pod_cpu_per_namespace"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"container.cpu.total_time_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","container_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","container_name"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"container_cpu_total_time_all","metric_id":"container_cpu_total_time_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"container.cpu.total_time_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","container_name"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["container_name"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","container_name"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "container_name"]},"metric_group":"container_cpu_total_time_per_container","metric_id":"container_cpu_total_time_per_container"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"kubernetes.node.capacity.cpu_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","host"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"kubernetes_node_capacity_cpu_all","metric_id":"kubernetes_node_capacity_cpu_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"kubernetes.node.capacity.cpu_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","host"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "host"]},"metric_group":"kubernetes_node_capacity_cpu_per_host","metric_id":"kubernetes_node_capacity_cpu_per_host"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"cpu.total_time_sec_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","host"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"cpu_total_time_sec_all","metric_id":"cpu_total_time_sec_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"cpu.total_time_sec_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","host"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "host"]},"metric_group":"cpu_total_time_sec_per_host","metric_id":"cpu_total_time_sec_per_host"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"kubernetes.node.allocatable.cpu_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","host"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":[],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["default"]},"metric_group":"kubernetes_node_allocatable_cpu_all","metric_id":"kubernetes_node_allocatable_cpu_all"}
|
||||
{"aggregation_params_map":{"aggregation_pipeline":{"source":"streaming","usage":"fetch_quantity","setters":["rollup_quantity","set_aggregated_metric_name","set_aggregated_period"],"insert":["prepare_data","insert_data_pre_hourly"]},"aggregated_metric_name":"kubernetes.node.allocatable.cpu_agg","aggregation_period":"hourly","aggregation_group_by_list": ["metric_id", "tenant_id","host"],"usage_fetch_operation": "avg","filter_by_list": [],"setter_rollup_group_by_list":["host"],"setter_rollup_operation": "sum","dimension_list":["aggregation_period","host"],"pre_hourly_operation":"avg","pre_hourly_group_by_list":["geolocation", "region", "zone", "aggregated_metric_name", "aggregation_period", "host"]},"metric_group":"kubernetes_node_allocatable_cpu_per_host","metric_id":"kubernetes_node_allocatable_cpu_per_host"}
|
||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,167 @@
|
|||
# Copyright 2016 Hewlett Packard Enterprise Development Company LP
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
from monasca_transform.offset_specs import OffsetSpec
|
||||
from monasca_transform.offset_specs import OffsetSpecs
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class JSONOffsetSpecs(OffsetSpecs):
|
||||
|
||||
def __init__(self, path=None, filename=None):
|
||||
self.kafka_offset_spec_file = os.path.join(
|
||||
(path or "/tmp/"), (filename or 'kafka_offset_specs.json'))
|
||||
|
||||
self._kafka_offsets = {}
|
||||
if os.path.exists(self.kafka_offset_spec_file):
|
||||
try:
|
||||
f = open(self.kafka_offset_spec_file)
|
||||
kafka_offset_dict = json.load(f)
|
||||
for key, value in kafka_offset_dict.items():
|
||||
log.info("Found offset %s: %s", key, value)
|
||||
self._kafka_offsets[key] = OffsetSpec(
|
||||
app_name=value.get('app_name'),
|
||||
topic=value.get('topic'),
|
||||
partition=value.get('partition'),
|
||||
from_offset=value.get('from_offset'),
|
||||
until_offset=value.get('until_offset'),
|
||||
batch_time=value.get('batch_time'),
|
||||
last_updated=value.get('last_updated'),
|
||||
revision=value.get('revision')
|
||||
)
|
||||
except Exception:
|
||||
log.info('Invalid or corrupts offsets file found at %s,'
|
||||
' starting over' % self.kafka_offset_spec_file)
|
||||
else:
|
||||
log.info('No kafka offsets found at startup')
|
||||
|
||||
def _save(self):
|
||||
"""get the specs of last run time of offset
|
||||
"""
|
||||
log.info("Saving json offsets: %s", self._kafka_offsets)
|
||||
|
||||
with open(self.kafka_offset_spec_file, 'w') as offset_file:
|
||||
offset_file.write('{')
|
||||
# json_values = []
|
||||
# for key, value in self._kafka_offsets.items():
|
||||
# json_values.append({key: })
|
||||
offset_file.write(','.join(
|
||||
['\"%s\": %s' % (key, json.dumps(self.as_dict(value)))
|
||||
for key, value in self._kafka_offsets.items()]))
|
||||
offset_file.write('}')
|
||||
|
||||
@staticmethod
|
||||
def as_dict(offset_value):
|
||||
return {"app_name": offset_value.get_app_name(),
|
||||
"topic": offset_value.get_topic(),
|
||||
"partition": offset_value.get_partition(),
|
||||
"from_offset": offset_value.get_from_offset(),
|
||||
"until_offset": offset_value.get_until_offset(),
|
||||
"batch_time": offset_value.get_batch_time(),
|
||||
"last_updated": offset_value.get_last_updated(),
|
||||
"revision": offset_value.get_revision()}
|
||||
|
||||
def add(self, app_name, topic, partition,
|
||||
from_offset, until_offset, batch_time_info):
|
||||
|
||||
# batch time
|
||||
batch_time = \
|
||||
batch_time_info.strftime(
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# last updated
|
||||
last_updated = \
|
||||
datetime.datetime.now().strftime(
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
|
||||
NEW_REVISION_NO = 1
|
||||
|
||||
key_name = "%s_%s_%s" % (
|
||||
app_name, topic, partition)
|
||||
offset = OffsetSpec(
|
||||
app_name=app_name,
|
||||
topic=topic,
|
||||
partition=partition,
|
||||
from_offset=from_offset,
|
||||
until_offset=until_offset,
|
||||
batch_time=batch_time,
|
||||
last_updated=last_updated,
|
||||
revision=NEW_REVISION_NO
|
||||
)
|
||||
log.info('Adding offset %s for key %s to current offsets: %s' %
|
||||
(offset, key_name, self._kafka_offsets))
|
||||
self._kafka_offsets[key_name] = offset
|
||||
log.info('Added so kafka offset is now %s', self._kafka_offsets)
|
||||
self._save()
|
||||
|
||||
def get_kafka_offsets(self, app_name):
|
||||
return self._kafka_offsets
|
||||
|
||||
def get_most_recent_batch_time_from_offsets(self, app_name, topic):
|
||||
try:
|
||||
# get partition 0 as a representative of all others
|
||||
key = "%s_%s_%s" % (app_name, topic, 0)
|
||||
offset = self._kafka_offsets[key]
|
||||
most_recent_batch_time = datetime.datetime.strptime(
|
||||
offset.get_batch_time(),
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
except Exception:
|
||||
most_recent_batch_time = None
|
||||
|
||||
return most_recent_batch_time
|
||||
|
||||
def delete_all_kafka_offsets(self, app_name):
|
||||
log.info("Deleting json offsets file: %s", self.kafka_offset_spec_file)
|
||||
os.remove(self.kafka_offset_spec_file)
|
||||
|
||||
def add_all_offsets(self, app_name, offsets, batch_time_info):
|
||||
|
||||
# batch time
|
||||
batch_time = \
|
||||
batch_time_info.strftime(
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
|
||||
# last updated
|
||||
last_updated = \
|
||||
datetime.datetime.now().strftime(
|
||||
'%Y-%m-%d %H:%M:%S')
|
||||
|
||||
NEW_REVISION_NO = -1
|
||||
|
||||
for o in offsets:
|
||||
|
||||
key_name = "%s_%s_%s" % (
|
||||
app_name, o.topic, o.partition)
|
||||
|
||||
offset = OffsetSpec(
|
||||
topic=o.topic,
|
||||
app_name=app_name,
|
||||
partition=o.partition,
|
||||
from_offset=o.fromOffset,
|
||||
until_offset=o.untilOffset,
|
||||
batch_time=batch_time,
|
||||
last_updated=last_updated,
|
||||
revision=NEW_REVISION_NO)
|
||||
|
||||
log.info('Adding offset %s for key %s to current offsets: %s' %
|
||||
(offset, key_name, self._kafka_offsets))
|
||||
self._kafka_offsets[key_name] = offset
|
||||
log.info('Added so kafka offset is now %s', self._kafka_offsets)
|
||||
self._save()
|
||||
|
|
@ -25,7 +25,7 @@ class DummyAdapter(MessageAdapter):
|
|||
def init():
|
||||
# object to keep track of offsets
|
||||
DummyAdapter.adapter_impl = simport.load(
|
||||
"tests.unit.messaging.adapter:DummyAdapter")()
|
||||
"tests.functional.messaging.adapter:DummyAdapter")()
|
||||
|
||||
def do_send_metric(self, metric):
|
||||
self.metric_list.append(metric)
|
||||
|
|
|
|||
|
|
@ -17,30 +17,32 @@ import sys
|
|||
import unittest
|
||||
import uuid
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
import mock
|
||||
from oslo_config import cfg
|
||||
from pyspark.streaming.kafka import OffsetRange
|
||||
from tests.unit.component.insert.dummy_insert import DummyInsert
|
||||
from tests.unit.spark_context_test import SparkContextTest
|
||||
from tests.unit.test_resources.metrics_pre_hourly_data.data_provider \
|
||||
import DataProvider
|
||||
|
||||
from monasca_transform.config.config_initializer import ConfigInitializer
|
||||
from monasca_transform.offset_specs import JSONOffsetSpecs
|
||||
from monasca_transform.processor.pre_hourly_processor import PreHourlyProcessor
|
||||
from tests.functional.component.insert.dummy_insert import DummyInsert
|
||||
from tests.functional.json_offset_specs import JSONOffsetSpecs
|
||||
from tests.functional.messaging.adapter import DummyAdapter
|
||||
from tests.functional.spark_context_test import SparkContextTest
|
||||
from tests.functional.test_resources.metrics_pre_hourly_data.data_provider \
|
||||
import DataProvider
|
||||
|
||||
|
||||
class TestPreHourlyProcessorAgg(SparkContextTest):
|
||||
|
||||
test_resources_path = 'tests/unit/test_resources'
|
||||
test_resources_path = 'tests/functional/test_resources'
|
||||
|
||||
def setUp(self):
|
||||
super(TestPreHourlyProcessorAgg, self).setUp()
|
||||
# configure the system with a dummy messaging adapter
|
||||
ConfigInitializer.basic_config(
|
||||
default_config_files=[
|
||||
'tests/unit/test_resources/config/'
|
||||
'tests/functional/test_resources/config/'
|
||||
'test_config_with_dummy_messaging_adapter.conf'])
|
||||
# reset metric_id list dummy adapter
|
||||
if not DummyAdapter.adapter_impl:
|
||||
|
|
@ -105,15 +107,22 @@ class TestPreHourlyProcessorAgg(SparkContextTest):
|
|||
metrics = DummyAdapter.adapter_impl.metric_list
|
||||
|
||||
# Verify count of instance usage data
|
||||
self.assertEqual(result, 9)
|
||||
self.assertEqual(result, 57)
|
||||
|
||||
# check aggregation result
|
||||
# make a dictionary of all metrics so that they can be directly
|
||||
# referenced in the verifications that follow
|
||||
metrics_dict = defaultdict(list)
|
||||
for metric in metrics:
|
||||
metric_name = metric.get('metric').get('name')
|
||||
metrics_dict[metric_name].append(metric)
|
||||
|
||||
# Verify mem.total_mb_agg metrics for all hosts
|
||||
metric_list = metrics_dict['mem.total_mb_agg']
|
||||
mem_total_mb_agg_metric = [
|
||||
value for value in metrics
|
||||
if value.get('metric').get('name') ==
|
||||
'mem.total_mb_agg' and
|
||||
value.get('metric').get('dimensions').get('host') ==
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('host') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(mem_total_mb_agg_metric is not None)
|
||||
self.assertEqual(16049.0,
|
||||
mem_total_mb_agg_metric
|
||||
|
|
@ -134,13 +143,14 @@ class TestPreHourlyProcessorAgg(SparkContextTest):
|
|||
.get("metric")
|
||||
.get('value_meta').get('record_count'))
|
||||
|
||||
# Verify mem.usable_mb_agg metrics for all hosts
|
||||
metric_list = metrics_dict['mem.usable_mb_agg']
|
||||
mem_usable_mb_agg_metric = [
|
||||
value for value in metrics
|
||||
if value.get('metric').get('name') ==
|
||||
'mem.usable_mb_agg' and
|
||||
value.get('metric').get('dimensions').get('host') ==
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('host') ==
|
||||
'all'][0]
|
||||
self.assertTrue(mem_usable_mb_agg_metric is not None)
|
||||
|
||||
self.assertTrue(mem_total_mb_agg_metric is not None)
|
||||
self.assertEqual(10283.1,
|
||||
mem_usable_mb_agg_metric
|
||||
.get('metric').get('value'))
|
||||
|
|
@ -160,11 +170,10 @@ class TestPreHourlyProcessorAgg(SparkContextTest):
|
|||
.get("metric")
|
||||
.get('value_meta').get('record_count'))
|
||||
|
||||
# check aggregation result for swiftlm.diskusage.rate_agg
|
||||
swift_disk_rate_agg_metric = [
|
||||
value for value in metrics
|
||||
if value.get('metric').get('name') ==
|
||||
# Verify swiftlm.diskusage.rate_agg metrics
|
||||
swift_disk_rate_agg_metric = metrics_dict[
|
||||
'swiftlm.diskusage.rate_agg'][0]
|
||||
|
||||
self.assertTrue(swift_disk_rate_agg_metric is not None)
|
||||
self.assertEqual(37.25140584281991,
|
||||
swift_disk_rate_agg_metric
|
||||
|
|
@ -198,6 +207,769 @@ class TestPreHourlyProcessorAgg(SparkContextTest):
|
|||
swift_disk_rate_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify total pod.net.in_bytes_sec_agg metrics
|
||||
metric_list = metrics_dict['pod.net.in_bytes_sec_agg']
|
||||
pod_net_in_bytes_sec_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('app') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(pod_net_in_bytes_sec_agg_metric is not None)
|
||||
self.assertEqual(75.0,
|
||||
pod_net_in_bytes_sec_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_net_in_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_net_in_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(6.0,
|
||||
pod_net_in_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_net_in_bytes_sec_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_net_in_bytes_sec_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('deployment'))
|
||||
self.assertEqual('all',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('interface'))
|
||||
self.assertEqual('all',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('namespace'))
|
||||
self.assertEqual('all',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('pod_name'))
|
||||
self.assertEqual('hourly',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify pod.net.in_bytes_sec_agg metrics for wordpress app
|
||||
metric_list = metrics_dict['pod.net.in_bytes_sec_agg']
|
||||
pod_net_in_bytes_sec_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('app') ==
|
||||
'wordpress'][0]
|
||||
|
||||
self.assertTrue(pod_net_in_bytes_sec_agg_metric is not None)
|
||||
self.assertEqual(175.0,
|
||||
pod_net_in_bytes_sec_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_net_in_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_net_in_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(14.0,
|
||||
pod_net_in_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_net_in_bytes_sec_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_net_in_bytes_sec_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('deployment'))
|
||||
self.assertEqual('all',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('interface'))
|
||||
self.assertEqual('all',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('namespace'))
|
||||
self.assertEqual('all',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('pod_name'))
|
||||
self.assertEqual('hourly',
|
||||
pod_net_in_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify total pod.cpu.total_time_agg metrics
|
||||
metric_list = metrics_dict['pod.cpu.total_time_agg']
|
||||
pod_cpu_total_time_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(pod_cpu_total_time_agg_metric is not None)
|
||||
self.assertEqual(275.0,
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(22.0,
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_cpu_total_time_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_cpu_total_time_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
pod_cpu_total_time_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify for pod.cpu.total_time_agg metrics for first_namespace
|
||||
metric_list = metrics_dict['pod.cpu.total_time_agg']
|
||||
pod_cpu_total_time_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'first_namespace'][0]
|
||||
|
||||
self.assertTrue(pod_cpu_total_time_agg_metric is not None)
|
||||
self.assertEqual(375.0,
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(30.0,
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_cpu_total_time_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_cpu_total_time_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
pod_cpu_total_time_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify pod.cpu.total_time_agg metrics second_namespace
|
||||
metric_list = metrics_dict['pod.cpu.total_time_agg']
|
||||
pod_cpu_total_time_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'second_namespace'][0]
|
||||
|
||||
self.assertTrue(pod_cpu_total_time_agg_metric is not None)
|
||||
self.assertEqual(475.0,
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(38.0,
|
||||
pod_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_cpu_total_time_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_cpu_total_time_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
pod_cpu_total_time_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify total pod.net.out_bytes_sec_agg metrics
|
||||
metric_list = metrics_dict['pod.net.out_bytes_sec_agg']
|
||||
pod_net_out_bytes_sec_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(pod_net_out_bytes_sec_agg_metric is not None)
|
||||
self.assertEqual(775.0,
|
||||
pod_net_out_bytes_sec_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_net_out_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_net_out_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(62.0,
|
||||
pod_net_out_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_net_out_bytes_sec_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_net_out_bytes_sec_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_net_out_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('interface'))
|
||||
self.assertEqual('all',
|
||||
pod_net_out_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('pod_name'))
|
||||
self.assertEqual('hourly',
|
||||
pod_net_out_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify pod.net.out_bytes_sec_agg metrics for first_namespace
|
||||
metric_list = metrics_dict['pod.net.out_bytes_sec_agg']
|
||||
pod_net_out_bytes_sec_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'first_namespace'][0]
|
||||
|
||||
self.assertTrue(pod_net_out_bytes_sec_agg_metric is not None)
|
||||
self.assertEqual(875.0,
|
||||
pod_net_out_bytes_sec_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_net_out_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_net_out_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(70.0,
|
||||
pod_net_out_bytes_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_net_out_bytes_sec_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_net_out_bytes_sec_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_net_out_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('interface'))
|
||||
self.assertEqual('all',
|
||||
pod_net_out_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('pod_name'))
|
||||
self.assertEqual('hourly',
|
||||
pod_net_out_bytes_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify total pod.mem.used_bytes_agg metrics
|
||||
metric_list = metrics_dict['pod.mem.used_bytes_agg']
|
||||
pod_mem_used_bytes_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'all' and
|
||||
value.get('metric').get('dimensions').get('pod_name') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(pod_mem_used_bytes_agg_metric is not None)
|
||||
self.assertEqual(975.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(82.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('deployment'))
|
||||
self.assertEqual('hourly',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify pod.mem.used_bytes_agg metrics for first_namespace
|
||||
metric_list = metrics_dict['pod.mem.used_bytes_agg']
|
||||
pod_mem_used_bytes_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'first_namespace' and
|
||||
value.get('metric').get('dimensions').get('pod_name') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(pod_mem_used_bytes_agg_metric is not None)
|
||||
self.assertEqual(1075.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(90.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('deployment'))
|
||||
self.assertEqual('hourly',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify pod.mem.used_bytes_agg metrics for second_namespace
|
||||
metric_list = metrics_dict['pod.mem.used_bytes_agg']
|
||||
pod_mem_used_bytes_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'second_namespace' and
|
||||
value.get('metric').get('dimensions').get('pod_name') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(pod_mem_used_bytes_agg_metric is not None)
|
||||
self.assertEqual(1175.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(98.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('deployment'))
|
||||
self.assertEqual('hourly',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify pod.mem.used_bytes_agg metrics for first_pod
|
||||
metric_list = metrics_dict['pod.mem.used_bytes_agg']
|
||||
pod_mem_used_bytes_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'all' and
|
||||
value.get('metric').get('dimensions').get('pod_name') ==
|
||||
'first_pod'][0]
|
||||
|
||||
self.assertTrue(pod_mem_used_bytes_agg_metric is not None)
|
||||
self.assertEqual(1275.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(106.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('deployment'))
|
||||
self.assertEqual('hourly',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify pod.mem.used_bytes_agg metrics for second_pod
|
||||
metric_list = metrics_dict['pod.mem.used_bytes_agg']
|
||||
pod_mem_used_bytes_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('namespace') ==
|
||||
'all' and
|
||||
value.get('metric').get('dimensions').get('pod_name') ==
|
||||
'second_pod'][0]
|
||||
|
||||
self.assertTrue(pod_mem_used_bytes_agg_metric is not None)
|
||||
self.assertEqual(1375.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(114.0,
|
||||
pod_mem_used_bytes_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
pod_mem_used_bytes_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('all',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('deployment'))
|
||||
self.assertEqual('hourly',
|
||||
pod_mem_used_bytes_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify total container.cpu.total_time_agg metrics
|
||||
metric_list = metrics_dict['container.cpu.total_time_agg']
|
||||
container_cpu_total_time_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('container_name') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(container_cpu_total_time_agg_metric is not None)
|
||||
self.assertEqual(275.0,
|
||||
container_cpu_total_time_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
container_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
container_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(22.0,
|
||||
container_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
container_cpu_total_time_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
container_cpu_total_time_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
container_cpu_total_time_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify container.cpu.total_time_agg metrics by container_name
|
||||
metric_list = metrics_dict['container.cpu.total_time_agg']
|
||||
container_cpu_total_time_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('container_name') ==
|
||||
'container_1'][0]
|
||||
|
||||
self.assertTrue(container_cpu_total_time_agg_metric is not None)
|
||||
self.assertEqual(400.0,
|
||||
container_cpu_total_time_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:29:44',
|
||||
container_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
container_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(32.0,
|
||||
container_cpu_total_time_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
container_cpu_total_time_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
container_cpu_total_time_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
container_cpu_total_time_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify total kubernetes.node.capacity.cpu_agg metrics
|
||||
metric_list = metrics_dict['kubernetes.node.capacity.cpu_agg']
|
||||
kubernetes_node_capacity_cpu_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('host') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(kubernetes_node_capacity_cpu_agg_metric is not None)
|
||||
self.assertEqual(275.0,
|
||||
kubernetes_node_capacity_cpu_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
kubernetes_node_capacity_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
kubernetes_node_capacity_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(22.0,
|
||||
kubernetes_node_capacity_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
kubernetes_node_capacity_cpu_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
kubernetes_node_capacity_cpu_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
kubernetes_node_capacity_cpu_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify kubernetes.node.capacity.cpu_agg metrics by host
|
||||
metric_list = metrics_dict['kubernetes.node.capacity.cpu_agg']
|
||||
kubernetes_node_capacity_cpu_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('host') ==
|
||||
'host1'][0]
|
||||
|
||||
self.assertTrue(kubernetes_node_capacity_cpu_agg_metric is not None)
|
||||
self.assertEqual(400.0,
|
||||
kubernetes_node_capacity_cpu_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:29:44',
|
||||
kubernetes_node_capacity_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
kubernetes_node_capacity_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(32.0,
|
||||
kubernetes_node_capacity_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
kubernetes_node_capacity_cpu_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
kubernetes_node_capacity_cpu_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
kubernetes_node_capacity_cpu_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify total cpu.total_time_sec_agg metrics
|
||||
metric_list = metrics_dict['cpu.total_time_sec_agg']
|
||||
cpu_total_time_sec_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('host') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(cpu_total_time_sec_agg_metric is not None)
|
||||
self.assertEqual(275.0,
|
||||
cpu_total_time_sec_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
cpu_total_time_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
cpu_total_time_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(22.0,
|
||||
cpu_total_time_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
cpu_total_time_sec_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
cpu_total_time_sec_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
cpu_total_time_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify cpu.total_time_sec_agg metrics by host
|
||||
metric_list = metrics_dict['cpu.total_time_sec_agg']
|
||||
cpu_total_time_sec_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('host') ==
|
||||
'host1'][0]
|
||||
|
||||
self.assertTrue(cpu_total_time_sec_agg_metric is not None)
|
||||
self.assertEqual(400.0,
|
||||
cpu_total_time_sec_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:29:44',
|
||||
cpu_total_time_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
cpu_total_time_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(32.0,
|
||||
cpu_total_time_sec_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
cpu_total_time_sec_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
cpu_total_time_sec_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
cpu_total_time_sec_agg_metric.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify total kubernetes.node.allocatable.cpu_agg metrics
|
||||
metric_list = metrics_dict['kubernetes.node.allocatable.cpu_agg']
|
||||
kubernetes_node_allocatable_cpu_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('host') ==
|
||||
'all'][0]
|
||||
|
||||
self.assertTrue(kubernetes_node_allocatable_cpu_agg_metric is not None)
|
||||
self.assertEqual(275.0,
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:39:44',
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(22.0,
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
kubernetes_node_allocatable_cpu_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
kubernetes_node_allocatable_cpu_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
# Verify kubernetes.node.allocatable.cpu_agg metrics by host
|
||||
metric_list = metrics_dict['kubernetes.node.allocatable.cpu_agg']
|
||||
kubernetes_node_allocatable_cpu_agg_metric = [
|
||||
value for value in metric_list
|
||||
if value.get('metric').get('dimensions').get('host') ==
|
||||
'host1'][0]
|
||||
|
||||
self.assertTrue(kubernetes_node_allocatable_cpu_agg_metric is not None)
|
||||
self.assertEqual(400.0,
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric').get('value'))
|
||||
self.assertEqual('2016-06-20 11:29:44',
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('lastrecord_timestamp_string'))
|
||||
self.assertEqual('2016-06-20 11:24:59',
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta')
|
||||
.get('firstrecord_timestamp_string'))
|
||||
self.assertEqual(32.0,
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('value_meta').get('record_count'))
|
||||
self.assertEqual('useast',
|
||||
kubernetes_node_allocatable_cpu_agg_metric.get('meta')
|
||||
.get('region'))
|
||||
self.assertEqual(cfg.CONF.messaging.publish_kafka_project_id,
|
||||
kubernetes_node_allocatable_cpu_agg_metric.get('meta')
|
||||
.get('tenantId'))
|
||||
self.assertEqual('hourly',
|
||||
kubernetes_node_allocatable_cpu_agg_metric
|
||||
.get('metric')
|
||||
.get('dimensions').get('aggregation_period'))
|
||||
|
||||
os.remove(file_path)
|
||||
|
||||
def simple_count_transform(self, rdd):
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
[DEFAULTS]
|
||||
|
||||
[repositories]
|
||||
offsets = tests.functional.json_offset_specs:JSONOffsetSpecs
|
||||
data_driven_specs = tests.functional.data_driven_specs.json_data_driven_specs_repo:JSONDataDrivenSpecsRepo
|
||||
|
||||
[messaging]
|
||||
adapter = tests.unit.messaging.adapter:DummyAdapter
|
||||
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import os
|
|||
|
||||
class DataProvider(object):
|
||||
|
||||
_resource_path = 'tests/unit/test_resources/kafka_data/'
|
||||
_resource_path = 'tests/functional/test_resources/kafka_data/'
|
||||
|
||||
kafka_data_path = os.path.join(_resource_path,
|
||||
"kafka_data.txt")
|
||||
|
|
|
|||
|
|
@ -389,3 +389,38 @@
|
|||
('<message id>','{"metric":{"name":"storage.objects.size","dimensions":{"cluster":"mtrmon","unit":"B","user_id":"None","project_id":"6dc15cb0a9d143859dd70fe9ca32218a","cloud_name":"mid-size","datasource":"ceilometer","resource_id":"6dc15cb0a9d143859dd70fe9ca32218a","source":"openstack","region":"None","type":"gauge","control_plane":"control-plane-1"},"timestamp":1470863052576,"value":3333.3},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1470863057}')
|
||||
('<message id>','{"metric":{"name":"storage.objects.size","dimensions":{"cluster":"mtrmon","unit":"B","user_id":"None","project_id":"942138d729e240328cc24db012c523ed","cloud_name":"mid-size","datasource":"ceilometer","resource_id":"942138d729e240328cc24db012c523ed","source":"openstack","region":"None","type":"gauge","control_plane":"control-plane-1"},"timestamp":1470863052576,"value":4444.4},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1470863057}')
|
||||
('<message id>','{"metric":{"name":"storage.objects.size","dimensions":{"cluster":"mtrmon","unit":"B","user_id":"None","project_id":"bbe076c7fe124666af13f55c35a631e0","cloud_name":"mid-size","datasource":"ceilometer","resource_id":"bbe076c7fe124666af13f55c35a631e0","source":"openstack","region":"None","type":"gauge","control_plane":"control-plane-1"},"timestamp":1470863052577,"value":5555.5},"meta":{"tenantId":"tenant_id of metric writer","region":"unset"},"creation_time":1470863057}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952510.8071, "value_meta": null, "name": "pod.cpu.total_time", "value": 13437791, "dimensions": {"replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952610.8071, "value_meta": null, "name": "pod.cpu.total_time", "value": 13437792, "dimensions": {"replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952710.8071, "value_meta": null, "name": "pod.cpu.total_time", "value": 13437793, "dimensions": {"replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947372.3062, "value_meta": null, "name": "pod.net.out_bytes_sec", "value": 9554.79, "dimensions": {"interface": "weave", "namespace": "kube-system", "pod_name": "weave-net-633vh", "daemon_set": "weave-net"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947472.3062, "value_meta": null, "name": "pod.net.out_bytes_sec", "value": 8554.79, "dimensions": {"interface": "weave", "namespace": "kube-system", "pod_name": "weave-net-633vh", "daemon_set": "weave-net"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947572.3062, "value_meta": null, "name": "pod.net.out_bytes_sec", "value": 7554.79, "dimensions": {"interface": "weave", "namespace": "kube-system", "pod_name": "weave-net-633vh", "daemon_set": "weave-net"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947372.3062, "value_meta": null, "name": "pod.net.out_bytes_sec", "value": 19554.79, "dimensions": {"interface": "weave", "namespace": "kube-system", "pod_name": "weave-net-633vh", "daemon_set": "weave-net"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947472.3062, "value_meta": null, "name": "pod.net.out_bytes_sec", "value": 18554.79, "dimensions": {"interface": "weave", "namespace": "kube-system", "pod_name": "weave-net-633vh", "daemon_set": "weave-net"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947379.724, "value_meta": null, "name": "pod.net.in_bytes_sec", "value": 22.94, "dimensions": {"interface": "eth0", "app": "wordpress", "namespace": "website", "pod_name": "wordpress-mysql-376310894-mjr8d", "deployment": "wordpress-mysql"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947479.724, "value_meta": null, "name": "pod.net.in_bytes_sec", "value": 23.94, "dimensions": {"interface": "eth0", "app": "wordpress", "namespace": "website", "pod_name": "wordpress-mysql-376310894-mjr8d", "deployment": "wordpress-mysql"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947579.724, "value_meta": null, "name": "pod.net.in_bytes_sec", "value": 21.94, "dimensions": {"interface": "eth0", "app": "wordpress", "namespace": "website", "pod_name": "wordpress-mysql-376310894-mjr8d", "deployment": "wordpress-mysql"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947379.724, "value_meta": null, "name": "pod.net.in_bytes_sec", "value": 122.94, "dimensions": {"interface": "eth0", "app": "junk", "namespace": "website2", "pod_name": "wordpress-mysql-376310894-mjr8d", "deployment": "wordpress-mysql"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947479.724, "value_meta": null, "name": "pod.net.in_bytes_sec", "value": 123.94, "dimensions": {"interface": "eth0", "app": "junk", "namespace": "website2", "pod_name": "wordpress-mysql-376310894-mjr8d", "deployment": "wordpress-mysql"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947579.724, "value_meta": null, "name": "pod.net.in_bytes_sec", "value": 121.94, "dimensions": {"interface": "eth0", "app": "junk", "namespace": "website2", "pod_name": "wordpress-mysql-376310894-mjr8d", "deployment": "wordpress-mysql"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288947572.3062, "value_meta": null, "name": "pod.net.out_bytes_sec", "value": 17554.79, "dimensions": {"interface": "weave", "namespace": "kube-system", "pod_name": "weave-net-633vh", "daemon_set": "weave-net"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288969}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288922549.4968, "value_meta": null, "name": "pod.mem.used_bytes", "value": 10000, "dimensions": {"namespace": "gluster1", "pod_name": "heketi-703226055-w13jr1", "deployment": "heketi"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288999}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288922549.4968, "value_meta": null, "name": "pod.mem.used_bytes", "value": 30000, "dimensions": {"namespace": "gluster2", "pod_name": "heketi-703226055-w13jr2", "deployment": "heketi"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288999}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288923549.4968, "value_meta": null, "name": "pod.mem.used_bytes", "value": 10000, "dimensions": {"namespace": "gluster1", "pod_name": "heketi-703226055-w13jr3", "deployment": "heketi"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288999}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288923549.4968, "value_meta": null, "name": "pod.mem.used_bytes", "value": 35000, "dimensions": {"namespace": "gluster2", "pod_name": "heketi-703226055-w13jr4", "deployment": "heketi"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288999}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288924549.4968, "value_meta": null, "name": "pod.mem.used_bytes", "value": 10000, "dimensions": {"namespace": "gluster1", "pod_name": "heketi-703226055-w13jr5", "deployment": "heketi"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288999}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288924549.4968, "value_meta": null, "name": "pod.mem.used_bytes", "value": 40000, "dimensions": {"namespace": "gluster2", "pod_name": "heketi-703226055-w13jr6", "deployment": "heketi"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288999}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288925549.4968, "value_meta": null, "name": "pod.mem.used_bytes", "value": 10000, "dimensions": {"namespace": "gluster1", "pod_name": "heketi-703226055-w13jr7", "deployment": "heketi"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288999}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288925549.4968, "value_meta": null, "name": "pod.mem.used_bytes", "value": 50000, "dimensions": {"namespace": "gluster2", "pod_name": "heketi-703226055-w13jr8", "deployment": "heketi"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485288999}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952510.8071, "value_meta": null, "name": "container.cpu.total_time", "value": 13437791, "dimensions": {"container_name": "container1", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952610.8071, "value_meta": null, "name": "container.cpu.total_time", "value": 13437792, "dimensions": {"container_name": "container2", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952710.8071, "value_meta": null, "name": "container.cpu.total_time", "value": 13437793, "dimensions": {"container_name": "container1", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952510.8071, "value_meta": null, "name": "kubernetes.node.capacity.cpu", "value": 13437791, "dimensions": {"hostname": "host1", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952610.8071, "value_meta": null, "name": "kubernetes.node.capacity.cpu", "value": 13437792, "dimensions": {"hostname": "host2", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952710.8071, "value_meta": null, "name": "kubernetes.node.capacity.cpu", "value": 13437793, "dimensions": {"hostname": "host1", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952510.8071, "value_meta": null, "name": "cpu.total_time_sec", "value": 13437791, "dimensions": {"hostname": "host1", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952610.8071, "value_meta": null, "name": "cpu.total_time_sec", "value": 13437792, "dimensions": {"hostname": "host2", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952710.8071, "value_meta": null, "name": "cpu.total_time_sec", "value": 13437793, "dimensions": {"hostname": "host1", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952510.8071, "value_meta": null, "name": "kubernetes.node.allocatable.cpu", "value": 13437791, "dimensions": {"hostname": "host1", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952610.8071, "value_meta": null, "name": "kubernetes.node.allocatable.cpu", "value": 13437792, "dimensions": {"hostname": "host2", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
('<message id>','{"metric": {"timestamp": 1485288952710.8071, "value_meta": null, "name": "kubernetes.node.allocatable.cpu", "value": 13437793, "dimensions": {"hostname": "host1", "replication_controller": "spark-worker", "namespace": "monitoring", "pod_name": "spark-worker-cxfln"}}, "meta": {"region": "useast", "tenantId": "7b1f318cb8aa400abaa9357fbc04ab83"}, "creation_time": 1485289029}')
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import os
|
|||
|
||||
class DataProvider(object):
|
||||
|
||||
_resource_path = 'tests/unit/test_resources/kafka_data_second_stage/'
|
||||
_resource_path = 'tests/functional/test_resources/kafka_data_second_stage/'
|
||||
|
||||
kafka_data_path_by_project = os.path.join(_resource_path,
|
||||
"kafka_data_by_project.txt")
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ import os
|
|||
|
||||
class DataProvider(object):
|
||||
|
||||
_resource_path = 'tests/unit/test_resources/metrics_pre_hourly_data/'
|
||||
_resource_path = 'tests/functional/test_resources/metrics_pre_hourly_data/'
|
||||
|
||||
metrics_pre_hourly_data_path = os.path.join(_resource_path,
|
||||
"metrics_pre_hourly_data.txt")
|
||||
|
|
|
|||
|
|
@ -7,3 +7,51 @@
|
|||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:27:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "7291.0", "latest_timestamp_string": "2016-06-10 20:27:02", "latest_quantity": "6575.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:27:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":15.6502})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:37:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "6291.0", "latest_timestamp_string": "2016-06-10 20:37:02", "latest_quantity": "5575.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:37:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":16.6502})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"swiftlm.diskusage.rate_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-10 20:47:01","tenant_id":"all","region":"all","usage_hour":"20","usage_date":"2016-06-10","processing_meta":{"oldest_quantity": "5291.0", "latest_timestamp_string": "2016-06-10 20:47:02", "latest_quantity": "4575.0", "metric_id":"swift_usage_rate", "oldest_timestamp_string": "2016-06-10 20:47:01"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","lastrecord_timestamp_string":"2016-06-10 20:27:02","firstrecord_timestamp_unix":1465590421.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1465590422.0,"quantity":17.6502})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.net.in_bytes_sec_agg","record_count":2.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_net_in_b_per_sec_total_all"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":50.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.net.in_bytes_sec_agg","record_count":4.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_net_in_b_per_sec_total_all"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":100.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.net.in_bytes_sec_agg","record_count":6.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_net_in_b_per_sec_per_app"},"resource_uuid":"all","aggregation_period":"hourly","app":"wordpress","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":150.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.net.in_bytes_sec_agg","record_count":8.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_net_in_b_per_sec_per_app"},"resource_uuid":"all","aggregation_period":"hourly","app":"wordpress","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":200.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.cpu.total_time_agg","record_count":10.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_cpu_total_all"},"resource_uuid":"all","aggregation_period":"hourly","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":250.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.cpu.total_time_agg","record_count":12.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_cpu_total_all"},"resource_uuid":"all","aggregation_period":"hourly","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":300.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.cpu.total_time_agg","record_count":14.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_cpu_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":350.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.cpu.total_time_agg","record_count":16.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_cpu_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":400.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.cpu.total_time_agg","record_count":18.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_cpu_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":450.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.cpu.total_time_agg","record_count":20.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_cpu_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":500.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.net.out_bytes_sec_agg","record_count":30.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_net_out_b_per_sec_total_all"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":750.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.net.out_bytes_sec_agg","record_count":32.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_net_out_b_per_sec_total_all"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":800.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.net.out_bytes_sec_agg","record_count":34.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_net_out_b_per_sec_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":850.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.net.out_bytes_sec_agg","record_count":36.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod_net_out_b_per_sec_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":900.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":40.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_total_all"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":950.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":42.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_total_all"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":1000.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":44.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":1050.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":46.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":1100.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":48.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":1150.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":50.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_per_namespace"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":1200.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":52.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_per_pod_name"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"first_pod","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":1250.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":54.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_per_pod_name"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"first_pod","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":1300.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":56.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_per_pod_name"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"second_pod","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":1350.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"pod.mem.used_bytes_agg","record_count":58.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"pod.mem_used_b_per_pod_name"},"resource_uuid":"all","aggregation_period":"hourly","app":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"second_pod","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":1400.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"container.cpu.total_time_agg","record_count":10.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"container_cpu_total_time_all"},"resource_uuid":"all","aggregation_period":"hourly","container_name":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":250.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"container.cpu.total_time_agg","record_count":12.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"container_cpu_total_time_all"},"resource_uuid":"all","aggregation_period":"hourly","container_name":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":300.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"container.cpu.total_time_agg","record_count":14.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"container_cpu_total_time_per_container"},"resource_uuid":"all","aggregation_period":"hourly","container_name":"container_1","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":350.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"container.cpu.total_time_agg","record_count":16.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"container_cpu_total_time_per_container"},"resource_uuid":"all","aggregation_period":"hourly","container_name":"container_2","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":400.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"container.cpu.total_time_agg","record_count":18.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"container_cpu_total_time_per_container"},"resource_uuid":"all","aggregation_period":"hourly","container_name":"container_1","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":450.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"container.cpu.total_time_agg","record_count":20.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"container_cpu_total_time_per_container"},"resource_uuid":"all","aggregation_period":"hourly","container_name":"container_2","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","host":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":500.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.capacity.cpu_agg","record_count":10.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_capacity_cpu_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":250.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.capacity.cpu_agg","record_count":12.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_capacity_cpu_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":300.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.capacity.cpu_agg","record_count":14.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_capacity_cpu_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host1","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":350.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.capacity.cpu_agg","record_count":16.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_capacity_cpu_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host2","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":400.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.capacity.cpu_agg","record_count":18.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_capacity_cpu_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host1","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":450.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.capacity.cpu_agg","record_count":20.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_capacity_cpu_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host2","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":500.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"cpu.total_time_sec_agg","record_count":10.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"cpu_total_time_sec_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":250.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"cpu.total_time_sec_agg","record_count":12.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"cpu_total_time_sec_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":300.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"cpu.total_time_sec_agg","record_count":14.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"cpu_total_time_sec_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host1","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":350.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"cpu.total_time_sec_agg","record_count":16.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"cpu_total_time_sec_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host2","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":400.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"cpu.total_time_sec_agg","record_count":18.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"cpu_total_time_sec_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host1","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":450.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"cpu.total_time_sec_agg","record_count":20.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"cpu_total_time_sec_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host2","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":500.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.allocatable.cpu_agg","record_count":10.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_allocatable_cpu_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":250.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.allocatable.cpu_agg","record_count":12.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_allocatable_cpu_all"},"resource_uuid":"all","aggregation_period":"hourly","host":"all","deployment":"all","interface":"all","namespace":"all","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":300.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.allocatable.cpu_agg","record_count":14.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_allocatable_cpu_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host1","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":350.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.allocatable.cpu_agg","record_count":16.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_allocatable_cpu_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host2","deployment":"all","interface":"all","namespace":"first_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":400.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.allocatable.cpu_agg","record_count":18.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:24:59","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_allocatable_cpu_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host1","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:29:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422184.0,"quantity":450.0})
|
||||
('',{"service_group":"all","geolocation":"all","aggregated_metric_name":"kubernetes.node.allocatable.cpu_agg","record_count":20.0,"user_id":"all","zone":"all","firstrecord_timestamp_string":"2016-06-20 11:29:44","tenant_id":"all","region":"all","usage_hour":"11","usage_date":"2016-06-20","processing_meta":{"metric_id":"kubernetes_node_allocatable_cpu_per_host"},"resource_uuid":"all","aggregation_period":"hourly","host":"host2","deployment":"all","interface":"all","namespace":"second_namespace","pod_name":"all","lastrecord_timestamp_string":"2016-06-20 11:39:44","firstrecord_timestamp_unix":1466421899.0,"service_id":"all","project_id":"all","usage_minute":"all","lastrecord_timestamp_unix":1466422784.0,"quantity":500.0})
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
|
||||
from stevedore.extension import Extension
|
||||
from stevedore.extension import ExtensionManager
|
||||
from tests.unit.component.insert.dummy_insert import DummyInsert
|
||||
|
||||
from monasca_transform.component.insert.prepare_data import PrepareData
|
||||
from monasca_transform.component.setter.rollup_quantity \
|
||||
|
|
@ -29,8 +28,9 @@ from monasca_transform.component.usage.fetch_quantity \
|
|||
import FetchQuantity
|
||||
from monasca_transform.component.usage.fetch_quantity_util \
|
||||
import FetchQuantityUtil
|
||||
from tests.functional.component.insert.dummy_insert_pre_hourly \
|
||||
import DummyInsertPreHourly
|
||||
from tests.functional.component.insert.dummy_insert import DummyInsert
|
||||
from tests.functional.component.insert.dummy_insert_pre_hourly import \
|
||||
DummyInsertPreHourly
|
||||
|
||||
|
||||
class MockComponentManager(object):
|
||||
|
|
@ -88,12 +88,12 @@ class MockComponentManager(object):
|
|||
PrepareData(),
|
||||
None),
|
||||
Extension('insert_data',
|
||||
'tests.unit.component.insert.dummy_insert:'
|
||||
'tests.functional.component.insert.dummy_insert:'
|
||||
'DummyInsert',
|
||||
DummyInsert(),
|
||||
None),
|
||||
Extension('insert_data_pre_hourly',
|
||||
'tests.unit.component.insert.dummy_insert:'
|
||||
'tests.functional.component.insert.dummy_insert:'
|
||||
'DummyInsert',
|
||||
DummyInsert(),
|
||||
None),
|
||||
|
|
@ -107,12 +107,12 @@ class MockComponentManager(object):
|
|||
PrepareData(),
|
||||
None),
|
||||
Extension('insert_data',
|
||||
'tests.unit.component.insert.dummy_insert:'
|
||||
'tests.functional.component.insert.dummy_insert:'
|
||||
'DummyInsert',
|
||||
DummyInsert(),
|
||||
None),
|
||||
Extension('insert_data_pre_hourly',
|
||||
'tests.unit.component.insert.'
|
||||
'tests.functional.component.insert.'
|
||||
'dummy_insert_pre_hourly:'
|
||||
'DummyInsertPreHourly',
|
||||
DummyInsertPreHourly(),
|
||||
|
|
|
|||
|
|
@ -62,6 +62,25 @@ Vagrant.configure(2) do |config|
|
|||
# # Customize the amount of memory on the VM:
|
||||
vb.memory = "16384"
|
||||
vb.cpus = "4"
|
||||
vb.customize [
|
||||
"clonehd", "#{ENV["HOME"]}/VirtualBox VMs/#{vb.name}/ubuntu-xenial-16.04-cloudimg.vmdk",
|
||||
"#{ENV["HOME"]}/VirtualBox VMs/#{vb.name}/ubuntu-xenial-16.04-cloudimg.vdi",
|
||||
"--format", "VDI"
|
||||
]
|
||||
vb.customize [
|
||||
"modifyhd", "#{ENV["HOME"]}/VirtualBox VMs/#{vb.name}/ubuntu-xenial-16.04-cloudimg.vdi",
|
||||
"--resize", 100 * 1024
|
||||
]
|
||||
vb.customize [
|
||||
"storageattach", :id,
|
||||
"--storagectl", "SCSI Controller",
|
||||
"--port", "0",
|
||||
"--device", "0",
|
||||
"--type", "hdd",
|
||||
"--nonrotational", "on",
|
||||
"--medium", "#{ENV["HOME"]}/VirtualBox VMs/#{vb.name}/ubuntu-xenial-16.04-cloudimg.vdi"
|
||||
]
|
||||
|
||||
end
|
||||
if ENV['http_proxy'] && !ENV['http_proxy'].empty?
|
||||
# we need to configure a proxy for maven also
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@ fi
|
|||
SCRIPT_HOME=$(dirname $(readlink -f $BASH_SOURCE))
|
||||
pushd $SCRIPT_HOME
|
||||
|
||||
|
||||
# TODO not sure how to stop monasca-transform from here now that
|
||||
# the control for it in DevStack is via screen -x stack
|
||||
if grep -q running <<<`sudo service monasca-transform status`; then
|
||||
sudo service monasca-transform stop
|
||||
else
|
||||
|
|
@ -48,12 +49,11 @@ sudo rm -rf /opt/monasca/transform/venv
|
|||
sudo rm -rf /opt/stack/monasca-transform
|
||||
pushd /opt/stack
|
||||
sudo git clone /home/ubuntu/monasca-transform
|
||||
sudo chown -R monasca-transform:monasca-transform /opt/stack/monasca-transform
|
||||
sudo su - monasca-transform -c "
|
||||
virtualenv /opt/monasca/transform/venv ;
|
||||
. /opt/monasca/transform/venv/bin/activate ;
|
||||
pip install -e /opt/stack/monasca-transform/ ;
|
||||
deactivate"
|
||||
sudo chown -R ubuntu:ubuntu /opt/stack/monasca-transform
|
||||
virtualenv /opt/monasca/transform/venv
|
||||
. /opt/monasca/transform/venv/bin/activate
|
||||
pip install -e /opt/stack/monasca-transform/
|
||||
deactivate
|
||||
popd
|
||||
|
||||
function get_id () {
|
||||
|
|
@ -64,6 +64,8 @@ source ~/devstack/openrc admin admin
|
|||
export ADMIN_PROJECT_ID=$(get_id openstack project show admin)
|
||||
sudo sed -i "s/publish_kafka_project_id=d2cb21079930415a9f2a33588b9f2bb6/publish_kafka_project_id=${ADMIN_PROJECT_ID}/g" /etc/monasca-transform.conf
|
||||
|
||||
# TODO not sure how to start monasca-transform from here now that
|
||||
# the control for it in DevStack is via screen -x stack
|
||||
sudo service monasca-transform start
|
||||
|
||||
popd
|
||||
Loading…
Reference in New Issue