x
/
distil
Code Issues Proposed changes

add transformers for trove 

slightly alter how config is loaded to transformers.

Change-Id: I32bf34d270604e8c755160a7dac71e99a38f2a74
This commit is contained in:
Adrian Turjak 2020-06-30 16:59:05 +12:00
parent 07f09a1a9e
commit 0a0c7e12ef
12 changed files with 513 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
distil/collector/base.py
View File

@ -205,7 +205,9 @@ class BaseCollector(object):
service = (mapping['service'] if 'service' in mapping
else mapping['meter'])
transformer = d_transformer.get_transformer(mapping['transformer'])
transformer = d_transformer.get_transformer(
mapping['transformer'],
override_config=mapping.get('transformer_config', {}))
for res_id, entries in usage_by_resource.items():
transformed = transformer.transform_usage(

4
distil/common/general.py
View File

@ -33,7 +33,7 @@ LOG = logging.getLogger(__name__)
_TRANS_CONFIG = None
def get_transformer_config():
def get_transformer_config(name):
global _TRANS_CONFIG
if not _TRANS_CONFIG:
@ -43,7 +43,7 @@ def get_transformer_config():
except IOError as e:
raise e
return _TRANS_CONFIG
return _TRANS_CONFIG.get(name, {})
def get_windows(start, end):

49
distil/common/openstack.py
View File

@ -17,12 +17,14 @@ import re
from ceilometerclient import client as ceilometerclient
from cinderclient.v2 import client as cinderclient
from cinderclient.exceptions import NotFound as CinderNotFound
from glanceclient import client as glanceclient
from keystoneauth1.identity import v3
from keystoneauth1.exceptions import NotFound
from keystoneauth1 import session
from keystoneclient.v3 import client as ks_client
from novaclient import client as novaclient
from novaclient.exceptions import NotFound as NovaNotFound
from oslo_config import cfg
from distil.common import cache as distil_cache
@ -30,7 +32,7 @@ from distil.common import general
CONF = cfg.CONF
KS_SESSION = None
cache = defaultdict(list)
cache = defaultdict(dict)
ROOT_DEVICE_PATTERN = re.compile('^/dev/(x?v|s|h)da1?$')
@ -163,20 +165,43 @@ def get_root_volume(instance_id):
@general.disable_ssl_warnings
def get_volume_type(volume_type):
if not cache.get("volume_types"):
def get_flavor_name(flavor_id):
if flavor_id not in cache["flavors"]:
nova = get_nova_client()
try:
flavor_name = nova.flavors.get(flavor_id).name
except NovaNotFound:
return None
cache["flavors"][flavor_id] = flavor_name
return cache["flavors"][flavor_id]
@general.disable_ssl_warnings
def get_volume_type_for_volume(volume_id):
if volume_id not in cache["volume_id_to_type"]:
cinder = get_cinder_client()
for vtype in cinder.volume_types.list():
cache['volume_types'].append({'id': vtype.id, 'name': vtype.name})
try:
vol = cinder.volumes.get(volume_id)
except CinderNotFound:
return None
cache["volume_id_to_type"][volume_id] = vol.volume_type
return cache["volume_id_to_type"][volume_id]
for vtype in cache['volume_types']:
# check name first, as that will be more common
if vtype['name'] == volume_type:
return volume_type
elif vtype['id'] == volume_type:
return vtype['name']
return None
@general.disable_ssl_warnings
def get_volume_type_name(volume_type):
if volume_type not in cache["volume_types"]:
cinder = get_cinder_client()
try:
vtype = cinder.volume_types.get(volume_type)
except CinderNotFound:
try:
vtype = cinder.volume_types.find(name=volume_type)
except CinderNotFound:
return None
cache["volume_types"][vtype.id] = vtype.name
cache["volume_types"][vtype.name] = vtype.name
return cache["volume_types"][volume_type]
@general.disable_ssl_warnings

82
distil/tests/unit/transformer/test_arithmetic.py
View File

@ -20,7 +20,7 @@ from distil.common.constants import date_format
from distil.common import general
from distil.common import openstack
from distil.tests.unit import base
from distil.transformer import arithmetic
from distil.transformer import get_transformer
p = lambda t: datetime.datetime.strptime(t, date_format)
@ -53,7 +53,7 @@ class TestMaxTransformer(base.DistilTestCase):
{'timestamp': FAKE_DATA.t1.isoformat(), 'volume': 6},
]
xform = arithmetic.MaxTransformer()
xform = get_transformer('max')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -71,7 +71,7 @@ class TestMaxTransformer(base.DistilTestCase):
{'timestamp': FAKE_DATA.t1, 'volume': 25},
]
xform = arithmetic.MaxTransformer()
xform = get_transformer('max')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -86,7 +86,7 @@ class TestMaxTransformer(base.DistilTestCase):
{'timestamp': FAKE_DATA.t0, 'volume': None},
]
xform = arithmetic.MaxTransformer()
xform = get_transformer('max')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -103,7 +103,7 @@ class TestMaxTransformer(base.DistilTestCase):
{'timestamp': FAKE_DATA.t1, 'volume': 27},
]
xform = arithmetic.MaxTransformer()
xform = get_transformer('max')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -135,7 +135,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.BlockStorageMaxTransformer()
xform = get_transformer('storagemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -156,7 +156,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.BlockStorageMaxTransformer()
xform = get_transformer('storagemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -172,7 +172,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.BlockStorageMaxTransformer()
xform = get_transformer('storagemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -192,7 +192,7 @@ class TestBlockStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.BlockStorageMaxTransformer()
xform = get_transformer('storagemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -241,7 +241,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.ObjectStorageMaxTransformer()
xform = get_transformer('objectstoragemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -270,7 +270,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.ObjectStorageMaxTransformer()
xform = get_transformer('objectstoragemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -290,7 +290,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.ObjectStorageMaxTransformer()
xform = get_transformer('objectstoragemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -318,7 +318,7 @@ class TestObjectStorageMaxTransformer(base.DistilTestCase):
'metadata': {}},
]
xform = arithmetic.ObjectStorageMaxTransformer()
xform = get_transformer('objectstoragemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -339,7 +339,7 @@ class TestSumTransformer(base.DistilTestCase):
{'timestamp': '2014-01-01T01:00:00', 'volume': 1},
]
xform = arithmetic.SumTransformer()
xform = get_transformer('sum')
usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -354,7 +354,7 @@ class TestSumTransformer(base.DistilTestCase):
{'timestamp': FAKE_DATA.t0.isoformat(), 'volume': None},
]
xform = arithmetic.SumTransformer()
xform = get_transformer('sum')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -371,8 +371,58 @@ class TestSumTransformer(base.DistilTestCase):
{'timestamp': FAKE_DATA.t0_50.isoformat(), 'volume': 25},
]
xform = arithmetic.SumTransformer()
xform = get_transformer('sum')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'some_meter': 50}, usage)
@mock.patch.object(general, 'get_transformer_config', mock.Mock())
class TestDatabaseVolumeMaxTransformer(base.DistilTestCase):
@mock.patch.object(
openstack, 'get_volume_type_for_volume',
mock.Mock(return_value='b1.nvme1000'))
def test_all_different_values(self):
"""
Tests that the transformer correctly grabs the highest value,
when all values are different.
"""
data = [
{'timestamp': FAKE_DATA.t0, 'volume': 1,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {'volume.size': '24', 'volume_id': 'vol_id'}},
{'timestamp': FAKE_DATA.t0_10, 'volume': 1,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}},
{'timestamp': FAKE_DATA.t0_20, 'volume': 1,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {'volume.size': '7', 'volume_id': 'vol_id'}},
{'timestamp': FAKE_DATA.t0_30, 'volume': 1,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}},
{'timestamp': FAKE_DATA.t0_40, 'volume': 1,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {'volume.size': '3', 'volume_id': 'vol_id'}},
{'timestamp': FAKE_DATA.t0_50, 'volume': 1,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {'volume.size': '25', 'volume_id': 'vol_id'}},
{'timestamp': FAKE_DATA.t1, 'volume': 1,
'resource_id': '55d37509be3142de963caf82a9c7c447/stuff',
'project_id': '55d37509be3142de963caf82a9c7c447',
'metadata': {'volume.size': '13', 'volume_id': 'vol_id'}},
]
xform = get_transformer('databasevolumemax')
usage = xform.transform_usage('some_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'b1.nvme1000': 25}, usage)

263
distil/tests/unit/transformer/test_conversion.py
View File

@ -17,8 +17,9 @@ import mock
from distil.common.constants import date_format
from distil.common import general
from distil.common import openstack
from distil.tests.unit import base
from distil.transformer import conversion
from distil.transformer import get_transformer
p = lambda t: datetime.datetime.strptime(t, date_format)
@ -41,17 +42,28 @@ FAKE_CONFIG = {
"tracked_states": ["active", "building", "paused", "rescued",
"resized"]
},
"from_image": {
"fromimage": {
"service": "volume.size",
"md_keys": ["image_ref", "image_meta.base_image_ref"],
"none_values": ["None", ""],
"size_keys": ["root_gb"]
},
"databaseuptime": {
"tracked_states": ["ACTIVE", "BUILD"]
},
"databasemanagementuptime": {
"tracked_states": ["ACTIVE", "BUILD"],
"service_name": "d1.managment"
}
}
def fake_get_transformer_config(name):
return FAKE_CONFIG.get(name, {})
@mock.patch.object(general, 'get_transformer_config',
mock.Mock(return_value=FAKE_CONFIG))
fake_get_transformer_config)
class TestUpTimeTransformer(base.DistilTestCase):
def test_trivial_run(self):
"""
@ -59,7 +71,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
"""
state = []
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -79,7 +91,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -100,7 +112,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'status': 'stopped'}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -122,7 +134,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'status': 'stopped'}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -145,7 +157,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -168,7 +180,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -192,7 +204,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -212,7 +224,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'state': 'active'}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -230,7 +242,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'metadata': {'instance_type': FAKE_DATA.flavor}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -252,7 +264,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
'status': 'active'}}
]
xform = conversion.UpTimeTransformer()
xform = get_transformer('uptime')
result = xform.transform_usage(
'instance',
entries,
@ -265,7 +277,7 @@ class TestUpTimeTransformer(base.DistilTestCase):
@mock.patch.object(general, 'get_transformer_config',
mock.Mock(return_value=FAKE_CONFIG))
fake_get_transformer_config)
class TestFromImageTransformer(base.DistilTestCase):
"""
These tests rely on config settings for from_image,
@ -290,7 +302,7 @@ class TestFromImageTransformer(base.DistilTestCase):
'metadata': {'image_ref': "None"}}
]
xform = conversion.FromImageTransformer()
xform = get_transformer('fromimage')
usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -313,7 +325,7 @@ class TestFromImageTransformer(base.DistilTestCase):
'metadata': {'image_ref': "None"}}
]
xform = conversion.FromImageTransformer()
xform = get_transformer('fromimage')
usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -335,7 +347,7 @@ class TestFromImageTransformer(base.DistilTestCase):
'root_gb': "20"}}
]
xform = conversion.FromImageTransformer()
xform = get_transformer('fromimage')
usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -358,7 +370,7 @@ class TestFromImageTransformer(base.DistilTestCase):
'root_gb': "20"}}
]
xform = conversion.FromImageTransformer()
xform = get_transformer('fromimage')
usage = xform.transform_usage('instance', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -366,7 +378,7 @@ class TestFromImageTransformer(base.DistilTestCase):
@mock.patch.object(general, 'get_transformer_config',
mock.Mock(return_value=FAKE_CONFIG))
fake_get_transformer_config)
class TestNetworkServiceTransformer(base.DistilTestCase):
def test_basic_sum(self):
"""Tests that the transformer correctly calculate the sum value.
@ -378,7 +390,7 @@ class TestNetworkServiceTransformer(base.DistilTestCase):
{'timestamp': '2014-01-01T01:00:00', 'volume': 2},
]
xform = conversion.NetworkServiceTransformer()
xform = get_transformer('networkservice')
usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -394,7 +406,7 @@ class TestNetworkServiceTransformer(base.DistilTestCase):
{'timestamp': '2014-01-01T01:00:00', 'volume': 2},
]
xform = conversion.NetworkServiceTransformer()
xform = get_transformer('networkservice')
usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -413,7 +425,7 @@ class TestMagnumTransformer(base.DistilTestCase):
{'timestamp': '2014-01-01T01:00:00', 'volume': 2},
]
xform = conversion.MagnumTransformer()
xform = get_transformer('magnum')
usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
@ -429,8 +441,213 @@ class TestMagnumTransformer(base.DistilTestCase):
{'timestamp': '2014-01-01T01:00:00', 'volume': 18},
]
xform = conversion.MagnumTransformer()
xform = get_transformer('magnum')
usage = xform.transform_usage('fake_meter', data, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({'fake_meter': 0}, usage)
@mock.patch.object(general, 'get_transformer_config',
fake_get_transformer_config)
class TestDatabaseUpTimeTransformer(base.DistilTestCase):
@mock.patch.object(
openstack, 'get_flavor_name',
mock.Mock(return_value=FAKE_DATA.flavor))
def test_online_constant_flavor(self):
"""
Test that a machine online for a 1h period with constant
flavor works and gives 1h of uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'ACTIVE'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'ACTIVE'}}
]
xform = get_transformer('databaseuptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({FAKE_DATA.flavor: 3600}, result)
@mock.patch.object(
openstack, 'get_flavor_name',
mock.Mock(return_value=FAKE_DATA.flavor))
def test_offline_constant_flavor(self):
"""
Test that a machine offline for a 1h period with constant flavor
works and gives zero uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'stopped'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'stopped'}}
]
xform = get_transformer('databaseuptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({}, result)
@mock.patch.object(
openstack, 'get_flavor_name',
mock.Mock(return_value=FAKE_DATA.flavor))
def test_shutdown_during_period(self):
"""
Test that a machine run for 0.5 then shutdown gives 0.5h uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'ACTIVE'}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'stopped'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'stopped'}}
]
xform = get_transformer('databaseuptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({FAKE_DATA.flavor: 1800}, result)
def test_online_flavor_change(self):
"""
Test that a machine run for 0.5h as m1.tiny, resized to m1.large,
and run for a further 0.5 yields 0.5h of uptime in each class.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'ACTIVE'}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor2,
'status': 'ACTIVE'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor2,
'status': 'ACTIVE'}}
]
xform = get_transformer('databaseuptime')
def fake_get_flavor(name):
return name
with mock.patch.object(
openstack, 'get_flavor_name', fake_get_flavor):
result = xform.transform_usage(
'state', state, FAKE_DATA.t0, FAKE_DATA.t1)
self.assertDictEqual(
{FAKE_DATA.flavor: 1800, FAKE_DATA.flavor2: 1800},
result
)
@mock.patch.object(
openstack, 'get_flavor_name',
mock.Mock(return_value=FAKE_DATA.flavor))
def test_no_state_in_metedata(self):
"""
Test that the transformer doesn't fall over if there isn't one of
the two state/status key options in the metadata.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor}}
]
xform = get_transformer('databaseuptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({}, result)
@mock.patch.object(general, 'get_transformer_config',
fake_get_transformer_config)
class TestDatabaseManagementUpTimeTransformer(base.DistilTestCase):
@mock.patch.object(
openstack, 'get_flavor_name',
mock.Mock(return_value=FAKE_DATA.flavor))
def test_online_constant_flavor(self):
"""
Test that a machine online for a 1h period with constant
flavor works and gives 1h of uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'status': 'ACTIVE'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'status': 'ACTIVE'}}
]
xform = get_transformer('databasemanagementuptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({"d1.managment": 3600}, result)
@mock.patch.object(
openstack, 'get_flavor_name',
mock.Mock(return_value=FAKE_DATA.flavor))
def test_offline_constant_flavor(self):
"""
Test that a machine offline for a 1h period with constant flavor
works and gives zero uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'stopped'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'stopped'}}
]
xform = get_transformer('databasemanagementuptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({}, result)
@mock.patch.object(
openstack, 'get_flavor_name',
mock.Mock(return_value=FAKE_DATA.flavor))
def test_shutdown_during_period(self):
"""
Test that a machine run for 0.5 then shutdown gives 0.5h uptime.
"""
state = [
{'timestamp': FAKE_DATA.t0.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'ACTIVE'}},
{'timestamp': FAKE_DATA.t0_30.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'stopped'}},
{'timestamp': FAKE_DATA.t1.isoformat(),
'metadata': {'flavor.id': FAKE_DATA.flavor,
'status': 'stopped'}}
]
xform = get_transformer('databasemanagementuptime')
result = xform.transform_usage('state', state, FAKE_DATA.t0,
FAKE_DATA.t1)
self.assertEqual({"d1.managment": 1800}, result)

7
distil/transformer/__init__.py
View File

@ -20,8 +20,10 @@ from distil.common import general
class BaseTransformer(object):
def __init__(self, *args, **kwargs):
self.config = general.get_transformer_config()
def __init__(self, name, override_config=None):
self.config = general.get_transformer_config(name)
if override_config:
self.config.update(override_config)
def transform_usage(self, meter_name, raw_data, start_at, end_at):
return self._transform_usage(meter_name, raw_data, start_at, end_at)
@ -35,5 +37,6 @@ def get_transformer(name, **kwargs):
'distil.transformer',
name,
invoke_on_load=True,
invoke_args=(name,),
invoke_kwds=kwargs
).driver

26
distil/transformer/arithmetic.py
View File

@ -59,7 +59,7 @@ class BlockStorageMaxTransformer(MaxTransformer):
if "volume_type" in data[-1]['metadata']:
vtype = data[-1]['metadata']['volume_type']
service = openstack.get_volume_type(vtype)
service = openstack.get_volume_type_name(vtype)
if not service:
service = name
else:
@ -69,6 +69,30 @@ class BlockStorageMaxTransformer(MaxTransformer):
return {service: max_vol * hours}
class DatabaseVolumeMaxTransformer(BaseTransformer):
"""
Variantion on the GaugeMax Transformer that checks for
volume_type and uses that as the service, or uses the
default service name.
It also gets the actual volume size from metadata.
"""
def _transform_usage(self, name, data, start, end):
if not data:
return None
max_vol = max([int(v["metadata"]["volume.size"]) for v in data])
volume_type = openstack.get_volume_type_for_volume(
data[-1]['metadata']['volume_id'])
if not volume_type:
return None
hours = (end - start).total_seconds() / 3600.0
return {volume_type: max_vol * hours}
class ObjectStorageMaxTransformer(MaxTransformer):
"""
Variantion on the GaugeMax Transformer that checks for

61
distil/transformer/conversion.py
View File

@ -40,7 +40,7 @@ class UpTimeTransformer(BaseTransformer):
def _transform_usage(self, name, data, start, end):
# get tracked states from config
tracked = self.config['uptime']['tracked_states']
tracked = self.config['tracked_states']
usage_dict = {}
@ -118,10 +118,10 @@ class FromImageTransformer(BaseTransformer):
"""
def _transform_usage(self, name, data, start, end):
checks = self.config['from_image']['md_keys']
none_values = self.config['from_image']['none_values']
service = self.config['from_image']['service']
size_sources = self.config['from_image']['size_keys']
checks = self.config['md_keys']
none_values = self.config['none_values']
service = self.config['service']
size_sources = self.config['size_keys']
size = 0
for entry in data:
@ -183,3 +183,54 @@ class MagnumTransformer(BaseTransformer):
hours = (end - start).total_seconds() / 3600.0
return {name: max_vol * hours}
class DatabaseUpTimeTransformer(UpTimeTransformer):
"""
Transformer to calculate uptime based on states,
which is broken apart into flavor at point in time.
"""
def _clean_entry(self, entry):
try:
timestamp = datetime.strptime(
entry['timestamp'], constants.date_format)
except ValueError:
timestamp = datetime.strptime(
entry['timestamp'], constants.date_format_f)
flavor = openstack.get_flavor_name(
entry['metadata'].get('flavor.id'))
result = {
'status': entry['metadata'].get('status'),
'flavor': flavor,
'timestamp': timestamp
}
return result
class DatabaseManagementUpTimeTransformer(UpTimeTransformer):
"""
Transformer to calculate uptime based on states,
which is broken apart into flavor at point in time.
"""
def _clean_entry(self, entry):
management_service_name = self.config.get(
'service_name', 'd1.management')
try:
timestamp = datetime.strptime(
entry['timestamp'], constants.date_format)
except ValueError:
timestamp = datetime.strptime(
entry['timestamp'], constants.date_format_f)
result = {
'status': entry['metadata'].get('status'),
'flavor': management_service_name,
'timestamp': timestamp
}
return result

45
etc/meter_mappings.yml.sample
View File

@ -253,3 +253,48 @@
sources:
- name
-
meter: database.instance
type: Database Instance
transformer: databaseuptime
unit: second
metadata:
name:
sources:
- name
datastore:
sources:
- datastore
-
meter: database.instance
service: b1.standard
type: Database Volume
transformer: databasevolumemax
unit: gigabyte
res_id_template: "%s-volume"
metadata:
name:
sources:
- name
template: "%s - volume"
datastore:
sources:
- datastore
-
meter: database.instance
type: Database Management
transformer: databasemanagementuptime
transformer_config:
service_name: d1.management
unit: second
res_id_template: "%s-management"
metadata:
name:
sources:
- name
template: "%s - Management Fee"
datastore:
sources:
- datastore

29
etc/transformer.yaml.sample
View File

@ -11,7 +11,7 @@ uptime:
- suspended
- shutoff
- stopped
from_image:
fromimage:
service: b1.standard
# What metadata values to check
md_keys:
@ -23,3 +23,30 @@ from_image:
# where to get volume size from
size_keys:
- root_gb
databaseuptime:
tracked_states:
- HEALTHY
- ACTIVE
- BLOCKED
- REBOOT
- RESIZE
- BACKUP
- SHUTDOWN
- RESTART_REQUIRED
- PROMOTE
- EJECT
- UPGRADE
- DETACH
databasemanagementuptime:
- HEALTHY
- ACTIVE
- BLOCKED
- REBOOT
- RESIZE
- BACKUP
- SHUTDOWN
- RESTART_REQUIRED
- PROMOTE
- EJECT
- UPGRADE
- DETACH

4
setup.cfg
View File

@ -46,6 +46,10 @@ distil.transformer =
uptime = distil.transformer.conversion:UpTimeTransformer
fromimage = distil.transformer.conversion:FromImageTransformer
networkservice = distil.transformer.conversion:NetworkServiceTransformer
magnum = distil.transformer.conversion:MagnumTransformer
databaseuptime = distil.transformer.conversion:DatabaseUpTimeTransformer
databasevolumemax = distil.transformer.arithmetic:DatabaseVolumeMaxTransformer
databasemanagementuptime = distil.transformer.conversion:DatabaseManagementUpTimeTransformer
distil.erp =
odoo = distil.erp.drivers.odoo:OdooDriver

4
tox.ini
View File

@ -1,11 +1,11 @@
[tox]
envlist = py35,py27,pep8
envlist = py3,py27,pep8
minversion = 1.6
skipsdist = True
[testenv]
usedevelop = True
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt} {opts} {packages}
install_command = pip install -c https://opendev.org/openstack/requirements/raw/branch/stable/ussuri/upper-constraints.txt {opts} {packages}
setenv =
VIRTUAL_ENV={envdir}
DISTIL_TESTS_CONFIGS_DIR={toxinidir}/distil/tests/etc/