openstack
/
fuel-stats
Code Issues Proposed changes

Fix for export attachments from volume 

Volume contains list of attachments. Export of attachments into CSV as enumerated fields implemented.
Number of enumrations is added into config as CSV_VOLUME_ATTACHMENTS_NUM parameter.
Enumeration of keys paths implemented in export_utils.
Volume attachment skeleton added.

Change-Id: I63eb2217c3564457a8ff4a5396eb7e7c40a76321
Closes-Bug: #1434086
Closes-Bug: #1435224
This commit is contained in:
Alexander Kislitsky 2015-03-25 18:13:52 +03:00
parent 23f661b2a4
commit a6fb8e2272
7 changed files with 168 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
analytics/fuel_analytics/api/config.py
View File

@ -31,6 +31,8 @@ class Production(object):
'postgresql://collector:*****@localhost/collector'
CSV_DEFAULT_FROM_DATE_DAYS = 90
CSV_DB_YIELD_PER = 1000
# Number of attachments included into volumes CSV report
CSV_VOLUME_ATTACHMENTS_NUM = 1
class Testing(Production):

40
analytics/fuel_analytics/api/resources/utils/export_utils.py
View File

@ -131,8 +131,44 @@ def align_enumerated_field_values(values, number):
empty values positions and bool value on the first place. Bool value
is True if len(values) > number
"""
return ([len(values) > number] +
(values + [None] * (number - len(values)))[:number])
if number > 0:
return ([len(values) > number] +
(values + [None] * (number - len(values)))[:number])
else:
return []
def get_enumerated_keys_paths(resource_type, skeleton_name,
nested_data_skeleton, enum_length):
"""Gets enumerated keys paths for nested data lists or tuples in the
skeleton. For example volume contains list of attachments. Only enum_length
of them will be shown in report. The first element of result is the column
for showing if number of elements in resource greater or not than
enum_length.
:param resource_type: name of resource type. used for column names
generation
:param skeleton_name: name of skeleton. used for generation of the first
column name in result
:param nested_data_skeleton: skeleton of nested structure
:param enum_length: number of enumerated nested elements
:return: list of enumerated column names
"""
app.logger.debug("Getting additional enumerated keys paths for: "
"%s, skeleton: %s", resource_type, skeleton_name)
result = []
gt_field_name = '{}_gt_{}'.format(skeleton_name, enum_length)
result.append([resource_type, gt_field_name])
skel_keys_paths = get_keys_paths(nested_data_skeleton)
for i in six.moves.xrange(enum_length):
attachment_key_paths = [resource_type, skeleton_name,
six.text_type(i)]
for key_path in skel_keys_paths:
result.append(attachment_key_paths + key_path)
app.logger.debug("Additional enumerated keys paths for: "
"%s, skeleton: %s are: %s", resource_type,
skeleton_name, result)
return result
def flatten_data_as_csv(keys_paths, flatten_data):

55
analytics/fuel_analytics/api/resources/utils/oswl_stats_to_csv.py
View File

@ -18,6 +18,7 @@ import itertools
import six
from fuel_analytics.api.app import app
from fuel_analytics.api.common import consts
from fuel_analytics.api.resources.utils import export_utils
from fuel_analytics.api.resources.utils.skeleton import OSWL_SKELETONS
@ -26,6 +27,22 @@ class OswlStatsToCsv(object):
OSWL_INDEX_FIELDS = ('master_node_uid', 'cluster_id', 'resource_type')
def get_additional_volume_keys_paths(self):
num = app.config['CSV_VOLUME_ATTACHMENTS_NUM']
return export_utils.get_enumerated_keys_paths(
consts.OSWL_RESOURCE_TYPES.volume, 'volume_attachment',
OSWL_SKELETONS['volume_attachment'], num)
def get_additional_keys_paths(self, resource_type):
# Additional key paths for resource type info
resource_additional_key_paths = [[resource_type, 'is_added'],
[resource_type, 'is_modified'],
[resource_type, 'is_removed']]
if resource_type == consts.OSWL_RESOURCE_TYPES.volume:
resource_additional_key_paths += \
self.get_additional_volume_keys_paths()
return resource_additional_key_paths
def get_resource_keys_paths(self, resource_type):
"""Gets key paths for resource type. csv key paths is combination
of oswl, vm and additional resource type key paths
@ -33,19 +50,18 @@ class OswlStatsToCsv(object):
"""
app.logger.debug("Getting %s keys paths", resource_type)
oswl_key_paths = export_utils.get_keys_paths(OSWL_SKELETONS['general'])
vm_key_paths = export_utils.get_keys_paths(
resource_key_paths = export_utils.get_keys_paths(
{resource_type: OSWL_SKELETONS[resource_type]})
# Additional key paths for resource type info
vm_additional_key_paths = [[resource_type, 'is_added'],
[resource_type, 'is_modified'],
[resource_type, 'is_removed']]
result_key_paths = oswl_key_paths + vm_key_paths + \
vm_additional_key_paths
resource_additional_key_paths = self.get_additional_keys_paths(
resource_type)
result_key_paths = oswl_key_paths + resource_key_paths + \
resource_additional_key_paths
app.logger.debug("%s keys paths got: %s", resource_type,
result_key_paths)
return oswl_key_paths, vm_key_paths, result_key_paths
return oswl_key_paths, resource_key_paths, result_key_paths
def get_additional_resource_info(self, resource, oswl):
"""Gets additional info about operations with resource
@ -61,7 +77,23 @@ class OswlStatsToCsv(object):
is_added = id_val in set(item['id'] for item in added)
is_modified = id_val in set(item['id'] for item in modified)
is_removed = id_val in set(item['id'] for item in removed)
return [is_added, is_modified, is_removed]
result = [is_added, is_modified, is_removed]
# Handling nested lists and tuples
if oswl.resource_type == consts.OSWL_RESOURCE_TYPES.volume:
flatten_attachments = []
skeleton = OSWL_SKELETONS['volume_attachment']
enum_length = (app.config['CSV_VOLUME_ATTACHMENTS_NUM'] *
len(skeleton))
attachment_keys_paths = export_utils.get_keys_paths(skeleton)
for attachment in resource.get('attachments', []):
flatten_attachment = export_utils.get_flatten_data(
attachment_keys_paths, attachment)
flatten_attachments.extend(flatten_attachment)
result += export_utils.align_enumerated_field_values(
flatten_attachments, enum_length)
return result
def get_flatten_resources(self, resource_type, oswl_keys_paths,
resource_keys_paths, oswls):
@ -171,12 +203,13 @@ class OswlStatsToCsv(object):
def export(self, resource_type, oswls, to_date):
app.logger.info("Export oswls %s info into CSV started",
resource_type)
oswl_keys_paths, vm_keys_paths, csv_keys_paths = \
oswl_keys_paths, resource_keys_paths, csv_keys_paths = \
self.get_resource_keys_paths(resource_type)
seamless_oswls = self.fill_date_gaps(
oswls, to_date)
flatten_resources = self.get_flatten_resources(
resource_type, oswl_keys_paths, vm_keys_paths, seamless_oswls)
resource_type, oswl_keys_paths, resource_keys_paths,
seamless_oswls)
result = export_utils.flatten_data_as_csv(
csv_keys_paths, flatten_resources)
app.logger.info("Export oswls %s info into CSV finished",

6
analytics/fuel_analytics/api/resources/utils/skeleton.py
View File

@ -165,9 +165,13 @@ OSWL_SKELETONS = {
'size': None,
'host': None,
'snapshot_id': None,
'attachments': None,
'tenant_id': None
},
'volume_attachment': {
"device": None,
"server_id": None,
"id": None
},
consts.OSWL_RESOURCE_TYPES.image: {
'id': None,
'minDisk': None,

19
analytics/fuel_analytics/test/api/resources/utils/oswl_test.py
View File

@ -130,14 +130,27 @@ class OswlTest(BaseTest):
})
return result
def _generate_volume_attachments(self, num, volume_id):
result = []
for i in six.moves.range(num):
result.append({
'device': '/dev/vdb{}'.format(i),
'server_id': six.text_type(uuid.uuid4()),
'volume_id': volume_id,
'host_name': six.text_type(uuid.uuid4()),
'id': six.text_type(uuid.uuid4()),
})
return result
def generate_volumes(self, num, avail_zones=('zone_0', 'zone_1', 'zone_2'),
statuses=('available', 'error'),
volume_types=('test_0', 'test_1'),
size_range=(1, 1024),
attachments=(None, 'att_0', 'att_1')
):
attachments_range=(1, 3)):
result = []
for i in range(num):
attachments = self._generate_volume_attachments(
random.randint(*attachments_range), i)
result.append({
'id': i,
'availability_zone': random.choice(avail_zones),
@ -149,7 +162,7 @@ class OswlTest(BaseTest):
'host': six.text_type(uuid.uuid4()),
'snapshot_id': random.choice((
None, six.text_type(uuid.uuid4()))),
'attachments': random.choice(attachments),
'attachments': attachments,
'tenant_id': six.text_type(uuid.uuid4())
})
return result

22
analytics/fuel_analytics/test/api/resources/utils/test_export_utils.py
View File

@ -14,6 +14,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
from fuel_analytics.test.base import BaseTest
from fuel_analytics.api.resources.utils import export_utils
@ -142,7 +144,7 @@ class ExportUtilsTest(BaseTest):
def test_align_enumerated_field_values(self):
# Data for checks in format (source, num, expected)
checks = [
([], 0, [False]),
([], 0, []),
([], 1, [False, None]),
(['a'], 1, [False, 'a']),
(['a'], 2, [False, 'a', None]),
@ -170,3 +172,21 @@ class ExportUtilsTest(BaseTest):
]
for obj, fields, idx in checks:
self.assertTupleEqual(idx, export_utils.get_index(obj, *fields))
def test_get_enumerated_keys_paths(self):
resource_type = 'res_type'
skeleton_name = 'test_skel'
enum_num = 2
skeleton = {'id': None, 'attr': None, 'value': None}
keys_paths = export_utils.get_enumerated_keys_paths(
resource_type, skeleton_name, skeleton, enum_num)
# Checking gt field in keys paths
self.assertEqual(len(keys_paths), enum_num * len(skeleton) + 1)
self.assertEqual(keys_paths[0],
['res_type', 'test_skel_gt_{}'.format(enum_num)])
# Checking all keys paths present
for key in six.iterkeys(skeleton):
for i in six.moves.range(enum_num):
keys_path = [resource_type, skeleton_name,
six.text_type(i), key]
self.assertIn(keys_path, keys_paths)

47
analytics/fuel_analytics/test/api/resources/utils/test_oswl_stats_to_csv.py
View File

@ -106,7 +106,14 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
resource_id in modified_ids,
resource_id in removed_ids
]
actual = exporter.get_additional_resource_info(resource, oswl)
# In case of CSV_VOLUME_ATTACHMENTS_NUM > 0
# additional info of volume will be extended by attachments
# info. Attachments handling is tested in the method
# test_volumes_attachments
with mock.patch.dict(app.config,
{'CSV_VOLUME_ATTACHMENTS_NUM': 0}):
actual = exporter.get_additional_resource_info(
resource, oswl)
self.assertListEqual(expected, actual)
def test_export(self):
@ -448,7 +455,37 @@ class OswlStatsToCsvTest(OswlTest, DbTest):
oswls = self.get_saved_oswls(num, resource_type)
for oswl in oswls:
for res_data in oswl.resource_data['current']:
self.assertItemsEqual(
six.iterkeys(OSWL_SKELETONS[resource_type]),
six.iterkeys(res_data)
)
# Checking all required for report data is in resource data
for key in six.iterkeys(OSWL_SKELETONS[resource_type]):
self.assertIn(key, res_data)
def test_volumes_attachments(self):
exporter = OswlStatsToCsv()
num = 100
resource_type = consts.OSWL_RESOURCE_TYPES.volume
with app.test_request_context():
oswls_saved = self.get_saved_oswls(
num, resource_type, current_num_range=(1, 1),
removed_num_range=(0, 0))
# Saving installation structures for proper oswls filtering
self.get_saved_inst_structs(oswls_saved)
oswls = list(get_oswls(resource_type).all())
oswl_keys_paths, vm_keys_paths, csv_keys_paths = \
exporter.get_resource_keys_paths(resource_type)
flatten_volumes = exporter.get_flatten_resources(
resource_type, oswl_keys_paths, vm_keys_paths, oswls)
flatten_volumes = list(flatten_volumes)
csv_att_num = app.config['CSV_VOLUME_ATTACHMENTS_NUM']
gt_field_pos = csv_keys_paths.index([
resource_type, 'volume_attachment_gt_{}'.format(csv_att_num)])
for idx, fv in enumerate(flatten_volumes):
oswl = oswls[idx]
# Checking CSV fields alignment
self.assertEqual(len(csv_keys_paths), len(fv))
# Checking gt field calculation
volume = oswl.resource_data['current'][0]
self.assertEqual(fv[gt_field_pos],
len(volume['attachments']) > csv_att_num)