305 lines
10 KiB
Python
305 lines
10 KiB
Python
# Copyright (C) 2014 Catalyst IT Ltd
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from distil import constants
|
|
from distil import helpers
|
|
from distil import config
|
|
import logging as log
|
|
from distil.constants import iso_time, iso_date
|
|
|
|
|
|
class Transformer(object):
|
|
def transform_usage(self, name, data, start, end):
|
|
return self._transform_usage(name, data, start, end)
|
|
|
|
def _transform_usage(self, name, data, start, end):
|
|
raise NotImplementedError
|
|
|
|
|
|
class Uptime(Transformer):
|
|
"""
|
|
Transformer to calculate uptime based on states,
|
|
which is broken apart into flavor at point in time.
|
|
This is a soon to be deprecated version that uses our state
|
|
metric.
|
|
"""
|
|
|
|
def _transform_usage(self, name, data, start, end):
|
|
# get tracked states from config
|
|
tracked = config.transformers['uptime']['tracked_states']
|
|
|
|
tracked_states = {constants.states[i] for i in tracked}
|
|
|
|
usage_dict = {}
|
|
|
|
def sort_and_clip_end(usage):
|
|
cleaned = (self._clean_entry(s) for s in usage)
|
|
clipped = [s for s in cleaned if s['timestamp'] < end]
|
|
return clipped
|
|
|
|
state = sort_and_clip_end(data)
|
|
|
|
if not len(state):
|
|
# there was no data for this period.
|
|
return usage_dict
|
|
|
|
last_state = state[0]
|
|
if last_state['timestamp'] >= start:
|
|
last_timestamp = last_state['timestamp']
|
|
seen_sample_in_window = True
|
|
else:
|
|
last_timestamp = start
|
|
seen_sample_in_window = False
|
|
|
|
def _add_usage(diff):
|
|
flav = last_state['flavor']
|
|
usage_dict[flav] = usage_dict.get(flav, 0) + diff.total_seconds()
|
|
|
|
for val in state[1:]:
|
|
if last_state["counter_volume"] in tracked_states:
|
|
diff = val["timestamp"] - last_timestamp
|
|
if val['timestamp'] > last_timestamp:
|
|
# if diff < 0 then we were looking back before the start
|
|
# of the window.
|
|
_add_usage(diff)
|
|
last_timestamp = val['timestamp']
|
|
seen_sample_in_window = True
|
|
|
|
last_state = val
|
|
|
|
# extend the last state we know about, to the end of the window,
|
|
# if we saw any actual uptime.
|
|
if (end and last_state['counter_volume'] in tracked_states
|
|
and seen_sample_in_window):
|
|
diff = end - last_timestamp
|
|
_add_usage(diff)
|
|
|
|
# map the flavors to names on the way out
|
|
return {helpers.flavor_name(f): v for f, v in usage_dict.items()}
|
|
|
|
def _clean_entry(self, entry):
|
|
result = {
|
|
'counter_volume': entry['counter_volume'],
|
|
'flavor': entry['resource_metadata'].get(
|
|
'flavor.id', entry['resource_metadata'].get(
|
|
'instance_flavor_id', 0
|
|
)
|
|
),
|
|
'timestamp': entry['timestamp']
|
|
}
|
|
return result
|
|
|
|
|
|
class InstanceUptime(Transformer):
|
|
"""
|
|
Transformer to calculate uptime based on states,
|
|
which is broken apart into flavor at point in time.
|
|
"""
|
|
|
|
def _transform_usage(self, name, data, start, end):
|
|
# get tracked states from config
|
|
tracked = config.transformers['uptime']['tracked_states']
|
|
|
|
usage_dict = {}
|
|
|
|
def sort_and_clip_end(usage):
|
|
cleaned = (self._clean_entry(s) for s in usage)
|
|
clipped = [s for s in cleaned if s['timestamp'] < end]
|
|
return clipped
|
|
|
|
state = sort_and_clip_end(data)
|
|
|
|
if not len(state):
|
|
# there was no data for this period.
|
|
return usage_dict
|
|
|
|
last_state = state[0]
|
|
if last_state['timestamp'] >= start:
|
|
last_timestamp = last_state['timestamp']
|
|
seen_sample_in_window = True
|
|
else:
|
|
last_timestamp = start
|
|
seen_sample_in_window = False
|
|
|
|
def _add_usage(diff):
|
|
flav = last_state['flavor']
|
|
usage_dict[flav] = usage_dict.get(flav, 0) + diff.total_seconds()
|
|
|
|
for val in state[1:]:
|
|
if last_state["status"] in tracked:
|
|
diff = val["timestamp"] - last_timestamp
|
|
if val['timestamp'] > last_timestamp:
|
|
# if diff < 0 then we were looking back before the start
|
|
# of the window.
|
|
_add_usage(diff)
|
|
last_timestamp = val['timestamp']
|
|
seen_sample_in_window = True
|
|
|
|
last_state = val
|
|
|
|
# extend the last state we know about, to the end of the window,
|
|
# if we saw any actual uptime.
|
|
if (end and last_state['status'] in tracked
|
|
and seen_sample_in_window):
|
|
diff = end - last_timestamp
|
|
_add_usage(diff)
|
|
|
|
# map the flavors to names on the way out
|
|
return {helpers.flavor_name(f): v for f, v in usage_dict.items()}
|
|
|
|
def _clean_entry(self, entry):
|
|
result = {
|
|
'status': entry['resource_metadata'].get(
|
|
'status', entry['resource_metadata'].get(
|
|
'state', ""
|
|
)
|
|
),
|
|
'flavor': entry['resource_metadata'].get(
|
|
'flavor.id', entry['resource_metadata'].get(
|
|
'instance_flavor_id', 0
|
|
)
|
|
),
|
|
'timestamp': entry['timestamp']
|
|
}
|
|
return result
|
|
|
|
|
|
class FromImage(Transformer):
|
|
"""
|
|
Transformer for creating Volume entries from instance metadata.
|
|
Checks if image was booted from image, and finds largest root
|
|
disk size among entries.
|
|
This relies heaviliy on instance metadata.
|
|
"""
|
|
|
|
def _transform_usage(self, name, data, start, end):
|
|
checks = config.transformers['from_image']['md_keys']
|
|
none_values = config.transformers['from_image']['none_values']
|
|
service = config.transformers['from_image']['service']
|
|
size_sources = config.transformers['from_image']['size_keys']
|
|
|
|
size = 0
|
|
for entry in data:
|
|
for source in checks:
|
|
try:
|
|
if (entry['resource_metadata'][source] in none_values):
|
|
return None
|
|
break
|
|
except KeyError:
|
|
pass
|
|
for source in size_sources:
|
|
try:
|
|
root_size = float(entry['resource_metadata'][source])
|
|
if root_size > size:
|
|
size = root_size
|
|
except KeyError:
|
|
pass
|
|
hours = (end - start).total_seconds() / 3600.0
|
|
return {service: size * hours}
|
|
|
|
|
|
class GaugeMax(Transformer):
|
|
"""
|
|
Transformer for max-integration of a gauge value over time.
|
|
If the raw unit is 'gigabytes', then the transformed unit is
|
|
'gigabyte-hours'.
|
|
"""
|
|
|
|
def _transform_usage(self, name, data, start, end):
|
|
max_vol = max([(v["counter_volume"] if v["counter_volume"] else 0)
|
|
for v in data]) if len(data) else 0
|
|
if max_vol is None:
|
|
max_vol = 0
|
|
log.warning("None max_vol value for %s in window: %s - %s " %
|
|
(name, start.strftime(iso_time),
|
|
end.strftime(iso_time)))
|
|
hours = (end - start).total_seconds() / 3600.0
|
|
return {name: max_vol * hours}
|
|
|
|
|
|
class StorageMax(Transformer):
|
|
"""
|
|
Variantion on the GaugeMax Transformer that checks for
|
|
volume_type and uses that as the service, or uses the
|
|
default service name.
|
|
"""
|
|
|
|
def _transform_usage(self, name, data, start, end):
|
|
|
|
if not data:
|
|
return None
|
|
|
|
max_vol = max([(v["counter_volume"] if v["counter_volume"] else 0)
|
|
for v in data])
|
|
|
|
if max_vol is None:
|
|
max_vol = 0
|
|
log.warning("None max_vol value for %s in window: %s - %s " %
|
|
(name, start.strftime(iso_time),
|
|
end.strftime(iso_time)))
|
|
|
|
if "volume_type" in data[-1]['resource_metadata']:
|
|
vtype = data[-1]['resource_metadata']['volume_type']
|
|
service = helpers.volume_type(vtype)
|
|
if not service:
|
|
service = name
|
|
else:
|
|
service = name
|
|
|
|
hours = (end - start).total_seconds() / 3600.0
|
|
return {service: max_vol * hours}
|
|
|
|
|
|
class GaugeSum(Transformer):
|
|
"""
|
|
Transformer for sum-integration of a gauge value for given period.
|
|
"""
|
|
def _transform_usage(self, name, data, start, end):
|
|
sum_vol = 0
|
|
for sample in data:
|
|
t = sample['timestamp']
|
|
if t >= start and t < end and sample["counter_volume"]:
|
|
sum_vol += sample["counter_volume"]
|
|
return {name: sum_vol}
|
|
|
|
|
|
class GaugeNetworkService(Transformer):
|
|
"""Transformer for Neutron network service, such as LBaaS, VPNaaS,
|
|
FWaaS, etc.
|
|
"""
|
|
|
|
def _transform_usage(self, name, data, start, end):
|
|
# NOTE(flwang): The network service pollster of Ceilometer is using
|
|
# status as the volume(see https://github.com/openstack/ceilometer/
|
|
# blob/master/ceilometer/network/services/vpnaas.py#L55), so we have
|
|
# to check the volume to make sure only the active service is
|
|
# charged(0=inactive, 1=active).
|
|
volumes = [v["counter_volume"] for v in data
|
|
if v["counter_volume"] < 2]
|
|
max_vol = max(volumes) if len(volumes) else 0
|
|
hours = (end - start).total_seconds() / 3600.0
|
|
return {name: max_vol * hours}
|
|
|
|
# Transformer dict for us with the config.
|
|
# All usable transformers need to be here.
|
|
active_transformers = {
|
|
'Uptime': Uptime,
|
|
'InstanceUptime': InstanceUptime,
|
|
'StorageMax': StorageMax,
|
|
'GaugeMax': GaugeMax,
|
|
'GaugeSum': GaugeSum,
|
|
'FromImage': FromImage,
|
|
'GaugeNetworkService': GaugeNetworkService
|
|
}
|