openstack
/
fuel-stats
Code Issues Proposed changes

Http hander for fetching clusters stats as CSV 

Flask application added for export clusters statstics in CSV format.
Export process streams data by the generators.

Closes-Bug: #1410262
Blueprint: export-stats-to-csv
Change-Id: I265b617e78de142f8f10f22e85f734d0df7979c2
This commit is contained in:
Alexander Kislitsky 2015-01-16 18:55:32 +03:00
parent edf8553534
commit 2f4eb79233
30 changed files with 1231 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
analytics/MANIFEST.in Normal file
View File

@ -0,0 +1,4 @@
include *.txt
graft static
prune static/bower_components
prune static/node_modules

13
analytics/fuel_analytics/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

13
analytics/fuel_analytics/api/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

22
analytics/fuel_analytics/api/app.py Normal file
View File

@ -0,0 +1,22 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from flask import Flask
app = Flask(__name__)
# Registering blueprints
from fuel_analytics.api.resources.csv_exporter import bp as csv_exporter_bp
app.register_blueprint(csv_exporter_bp, url_prefix='/api/v1/csv')

21
analytics/fuel_analytics/api/app_prod.py Normal file
View File

@ -0,0 +1,21 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuel_analytics.api.app import app
from fuel_analytics.api import log
app.config.from_object('fuel_analytics.api.config.Production')
app.config.from_envvar('ANALYTICS_SETTINGS', silent=True)
log.init_logger()

21
analytics/fuel_analytics/api/app_test.py Normal file
View File

@ -0,0 +1,21 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuel_analytics.api.app import app
from fuel_analytics.api import log
app.config.from_object('fuel_analytics.api.config.Testing')
app.config.from_envvar('ANALYTICS_SETTINGS', silent=True)
log.init_logger()

42
analytics/fuel_analytics/api/config.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import os
class Production(object):
DEBUG = False
LOG_FILE = '/var/log/fuel-stats/analytics.log'
LOG_LEVEL = logging.ERROR
LOG_ROTATION = False
LOGGER_NAME = 'analytics'
ELASTIC_HOST = 'product-stats.mirantis.com'
ELASTIC_PORT = 443
ELASTIC_USE_SSL = True
ELASTIC_INDEX_FUEL = 'fuel'
ELASTIC_DOC_TYPE_STRUCTURE = 'structure'
class Testing(Production):
DEBUG = False
LOG_FILE = os.path.realpath(os.path.join(
os.path.dirname(__file__), '..', 'test', 'logs', 'analytics.log'))
LOG_LEVEL = logging.DEBUG
LOG_ROTATION = True
LOG_FILE_SIZE = 2048000
LOG_FILES_COUNT = 5
ELASTIC_HOST = 'localhost'
ELASTIC_PORT = 9200
ELASTIC_USE_SSL = False

49
analytics/fuel_analytics/api/log.py Normal file
View File

@ -0,0 +1,49 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from logging import FileHandler
from logging import Formatter
from logging.handlers import RotatingFileHandler
import os
from fuel_analytics.api.app import app
def get_file_handler():
if app.config.get('LOG_ROTATION'):
file_handler = RotatingFileHandler(
app.config.get('LOG_FILE'),
maxBytes=app.config.get('LOG_FILE_SIZE'),
backupCount=app.config.get('LOG_FILES_COUNT')
)
else:
file_handler = FileHandler(app.config.get('LOG_FILE'))
file_handler.setLevel(app.config.get('LOG_LEVEL'))
formatter = get_formatter()
file_handler.setFormatter(formatter)
return file_handler
def get_formatter():
DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
LOG_FORMAT = "%(asctime)s.%(msecs)03d %(levelname)s " \
"[%(thread)x] (%(module)s) %(message)s"
return Formatter(fmt=LOG_FORMAT, datefmt=DATE_FORMAT)
def init_logger():
log_dir = os.path.dirname(app.config.get('LOG_FILE'))
if not os.path.exists(log_dir):
os.mkdir(log_dir, 750)
app.logger.addHandler(get_file_handler())

13
analytics/fuel_analytics/api/resources/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

36
analytics/fuel_analytics/api/resources/csv_exporter.py Normal file
View File

@ -0,0 +1,36 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from flask import Blueprint
from flask import Response
from fuel_analytics.api.app import app
from fuel_analytics.api.resources.utils.es_client import ElasticSearchClient
from fuel_analytics.api.resources.utils.stats_to_csv import StatsToCsv
bp = Blueprint('csv_exporter', __name__)
@bp.route('/clusters', methods=['GET'])
def csv_exporter():
app.logger.debug("Handling csv_exporter get request")
es_client = ElasticSearchClient()
structures = es_client.get_structures()
exporter = StatsToCsv()
result = exporter.export_clusters(structures)
# NOTE: result - is generator, but streaming can not work with some
# WSGI middlewares: http://flask.pocoo.org/docs/0.10/patterns/streaming/
return Response(result, mimetype='text/csv')

13
analytics/fuel_analytics/api/resources/utils/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

70
analytics/fuel_analytics/api/resources/utils/es_client.py Normal file
View File

@ -0,0 +1,70 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from elasticsearch import Elasticsearch
from fuel_analytics.api.app import app
class ElasticSearchClient(object):
def __init__(self):
self.es = Elasticsearch(hosts=[
{'host': app.config['ELASTIC_HOST'],
'port': app.config['ELASTIC_PORT'],
'use_ssl': app.config['ELASTIC_USE_SSL']}
])
def fetch_all_data(self, query, doc_type, show_fields=(),
sort=({"_id": {"order": "asc"}},), chunk_size=100):
"""Gets structures from the Elasticsearch by querying by chunk_size
number of structures
:param query: Elasticsearch query
:param doc_type: requested document type
:param show_fields: tuple of selected fields.
All fields will be fetched, if show_fields is not set
:param sort: tuple of fields for sorting
:param chunk_size: size of fetched structures chunk
:return: list of fetched structures
"""
received = 0
paged_query = query.copy()
paged_query["from"] = received
paged_query["size"] = chunk_size
if sort:
paged_query["sort"] = sort
if show_fields:
paged_query["_source"] = show_fields
while True:
app.logger.debug("Fetching chunk from ElasticSearch. "
"From: %d, size: %d",
paged_query["from"], chunk_size)
response = self.es.search(index=app.config['ELASTIC_INDEX_FUEL'],
doc_type=doc_type, body=paged_query)
total = response["hits"]["total"]
received += chunk_size
paged_query["from"] = received
for d in response["hits"]["hits"]:
yield d["_source"]
app.logger.debug("Chunk from ElasticSearch is fetched. "
"From: %d, size: %d",
paged_query["from"], chunk_size)
if total <= received:
break
def get_structures(self):
app.logger.debug("Fetching structures info from ElasticSearch")
query = {"query": {"match_all": {}}}
doc_type = app.config['ELASTIC_DOC_TYPE_STRUCTURE']
return self.fetch_all_data(query, doc_type)

108
analytics/fuel_analytics/api/resources/utils/skeleton.py Normal file
View File

@ -0,0 +1,108 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
INSTALLATION_INFO_SKELETON = {
'allocated_nodes_num': None,
'clusters': [
{
'attributes': {
'assign_public_to_all_nodes': None,
'ceilometer': None,
'debug_mode': None,
'ephemeral_ceph': None,
'heat': None,
'images_ceph': None,
'images_vcenter': None,
'iser': None,
'kernel_params': None,
'libvirt_type': None,
'mellanox': None,
'mellanox_vf_num': None,
'murano': None,
'nsx': None,
'nsx_replication': None,
'nsx_transport': None,
'objects_ceph': None,
'osd_pool_size': None,
'provision_method': None,
'sahara': None,
'syslog_transport': None,
'use_cow_images': None,
'vcenter': None,
'vlan_splinters': None,
'vlan_splinters_ovs': None,
'volumes_ceph': None,
'volumes_lvm': None,
'volumes_vmdk': None
},
'fuel_version': None,
'id': None,
'is_customized': None,
'mode': None,
'net_provider': None,
'node_groups': [{'id': None, 'nodes': [{}]}],
'nodes': [
{
'bond_interfaces': [
{'id': None, 'slaves': [{}]}
],
'error_type': None,
'group_id': None,
'id': None,
'manufacturer': None,
'nic_interfaces': [{'id': None}],
'online': None,
'os': None,
'pending_addition': None,
'pending_deletion': None,
'pending_roles': [{}],
'platform_name': None,
'roles': [{}],
'status': None
}
],
'nodes_num': None,
'openstack_info': {
'images': [{'size': None, 'unit': None}],
'nova_servers_count': None
},
'release': {'name': None, 'os': None, 'version': None},
'status': None
}
],
'clusters_num': None,
'creation_date': None,
'fuel_release': {
'api': None,
'astute_sha': None,
'build_id': None,
'build_number': None,
'feature_groups': [{}],
'fuellib_sha': None,
'fuelmain_sha': None,
'nailgun_sha': None,
'ostf_sha': None,
'production': None,
'release': None
},
'master_node_uid': None,
'modification_date': None,
'unallocated_nodes_num': None,
'user_information': {
'company': None,
'contact_info_provided': None,
'email': None,
'name': None
}
}

266
analytics/fuel_analytics/api/resources/utils/stats_to_csv.py Normal file
View File

@ -0,0 +1,266 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import csv
import io
import six
from fuel_analytics.api.app import app
from fuel_analytics.api.resources.utils.skeleton import \
INSTALLATION_INFO_SKELETON
class StatsToCsv(object):
MANUFACTURERS_NUM = 3
PLATFORM_NAMES_NUM = 3
def construct_skeleton(self, data):
"""Creates structure for searching all key paths in given data
:param data: fetched from ES dict
:return: skeleton of data structure
"""
if isinstance(data, dict):
result = {}
for k in sorted(data.keys()):
result[k] = self.construct_skeleton(data[k])
return result
elif isinstance(data, (list, tuple)):
list_result = []
dict_result = {}
for d in data:
if isinstance(d, dict):
dict_result.update(self.construct_skeleton(d))
elif isinstance(d, (list, tuple)):
if not list_result:
list_result.append(self.construct_skeleton(d))
else:
list_result[0].extend(self.construct_skeleton(d))
if dict_result:
list_result.append(dict_result)
return list_result
else:
return data
def get_data_skeleton(self, structures):
"""Gets skeleton by structures list
:param structures:
:return: data structure skeleton
"""
def _merge_skeletons(lh, rh):
keys_paths = self.get_keys_paths(rh)
for keys_path in keys_paths:
merge_point = lh
data_point = rh
for key in keys_path:
data_point = data_point[key]
if isinstance(data_point, dict):
if key not in merge_point:
merge_point[key] = {}
elif isinstance(data_point, list):
if key not in merge_point:
merge_point[key] = [{}]
_merge_skeletons(merge_point[key][0],
self.get_data_skeleton(data_point))
else:
merge_point[key] = None
merge_point = merge_point[key]
skeleton = {}
for structure in structures:
app.logger.debug("Constructing skeleton by data: %s", structure)
app.logger.debug("Updating skeleton by %s",
self.construct_skeleton(structure))
_merge_skeletons(skeleton, self.construct_skeleton(structure))
app.logger.debug("Result skeleton is %s", skeleton)
return skeleton
def get_keys_paths(self, skeleton):
"""Gets paths to leaf keys in the data
:param skeleton: data skeleton
:return: list of lists of dict keys
"""
def _keys_paths_helper(keys, skel):
result = []
if isinstance(skel, dict):
for k in sorted(six.iterkeys(skel)):
result.extend(_keys_paths_helper(keys + [k], skel[k]))
else:
result.append(keys)
return result
return _keys_paths_helper([], skeleton)
def flatten_data_as_csv(self, keys_paths, flatten_data):
"""Returns flatten data in CSV
:param keys_paths: list of dict keys lists for columns names
generation
:param flatten_data: list of flatten data dicts
:return: stream with data in CSV format
"""
app.logger.debug("Saving flatten data as CSV is started")
names = []
for key_path in keys_paths:
names.append('.'.join(key_path))
yield names
output = six.BytesIO()
writer = csv.writer(output)
writer.writerow(names)
def read_and_flush():
output.seek(io.SEEK_SET)
data = output.read()
output.seek(io.SEEK_SET)
output.truncate()
return data
for d in flatten_data:
writer.writerow(d)
yield read_and_flush()
app.logger.debug("Saving flatten data as CSV is finished")
def get_flatten_data(self, keys_paths, data):
"""Creates flatten data from data by keys_paths
:param keys_paths: list of dict keys lists
:param data: dict with nested structures
:return: list of flatten data dicts
"""
flatten_data = []
for key_path in keys_paths:
d = data
for key in key_path:
d = d.get(key, None)
if d is None:
break
if isinstance(d, (list, tuple)):
flatten_data.append(' '.join(d))
else:
flatten_data.append(d)
return flatten_data
def get_cluster_keys_paths(self):
app.logger.debug("Getting cluster keys paths")
structure_skeleton = INSTALLATION_INFO_SKELETON
structure_key_paths = self.get_keys_paths(structure_skeleton)
clusters = structure_skeleton.get('clusters')
if not clusters:
clusters = [{}]
cluster_skeleton = clusters[0]
# Removing lists of dicts from cluster skeleton
cluster_skeleton.pop('nodes', None)
cluster_skeleton.pop('node_groups', None)
cluster_skeleton.pop('openstack_info', None)
cluster_key_paths = self.get_keys_paths(cluster_skeleton)
result_key_paths = cluster_key_paths + structure_key_paths
def enumerated_field_keys(field_name, number):
"""Adds enumerated fields columns and property
field for showing case, when values will be cut
:param field_name: field name
:param number: number of enumerated fields
:return: list of cut fact column and enumerated columns names
"""
result = [['{}_gt{}'.format(field_name, number)]]
for i in xrange(number):
result.append(['{}_{}'.format(field_name, i)])
return result
# Handling enumeration of manufacturers names
result_key_paths.extend(enumerated_field_keys('nodes_manufacturer',
self.MANUFACTURERS_NUM))
# Handling enumeration of platform names
result_key_paths.extend(enumerated_field_keys('nodes_platform_name',
self.PLATFORM_NAMES_NUM))
app.logger.debug("Cluster keys paths got")
return structure_key_paths, cluster_key_paths, result_key_paths
@staticmethod
def align_enumerated_field_values(values, number):
"""Fills result list by the None values, if number is greater than
values len. The first element of result is bool value
len(values) > number
:param values:
:param number:
:return: aligned list to 'number' + 1 length, filled by Nones on
empty values positions and bool value on the first place. Bool value
is True if len(values) > number
"""
return ([len(values) > number] +
(values + [None] * (number - len(values)))[:number])
def get_flatten_clusters(self, structure_keys_paths, cluster_keys_paths,
structures):
"""Gets flatten clusters data
:param structure_keys_paths: list of keys paths in the
installation structure
:param cluster_keys_paths: list of keys paths in the cluster
:param structures: list of installation structures
:return: list of flatten clusters info
"""
app.logger.debug("Getting flatten clusters info is started")
def extract_nodes_fields(field, nodes):
"""Extracts fields values from nested nodes dicts
:param field: field name
:param nodes: nodes data list
:return: set of extracted fields values from nodes
"""
result = set([d.get(field) for d in nodes])
return filter(lambda x: x is not None, result)
def extract_nodes_manufacturers(nodes):
return extract_nodes_fields('manufacturer', nodes)
def extract_nodes_platform_name(nodes):
return extract_nodes_fields('platform_name', nodes)
for structure in structures:
clusters = structure.pop('clusters', [])
flatten_structure = self.get_flatten_data(structure_keys_paths,
structure)
for cluster in clusters:
flatten_cluster = self.get_flatten_data(cluster_keys_paths,
cluster)
flatten_cluster.extend(flatten_structure)
nodes = cluster.get('nodes', [])
# Adding enumerated manufacturers
manufacturers = extract_nodes_manufacturers(nodes)
flatten_cluster += StatsToCsv.align_enumerated_field_values(
manufacturers, self.MANUFACTURERS_NUM)
# Adding enumerated platforms
platform_names = extract_nodes_platform_name(nodes)
flatten_cluster += StatsToCsv.align_enumerated_field_values(
platform_names, self.PLATFORM_NAMES_NUM)
yield flatten_cluster
app.logger.debug("Flatten clusters info is got")
def export_clusters(self, structures):
app.logger.info("Export clusters info into CSV is started")
structure_keys_paths, cluster_keys_paths, csv_keys_paths = \
self.get_cluster_keys_paths()
flatten_clusters = self.get_flatten_clusters(structure_keys_paths,
cluster_keys_paths,
structures)
result = self.flatten_data_as_csv(csv_keys_paths, flatten_clusters)
app.logger.info("Export clusters info into CSV is finished")
return result

13
analytics/fuel_analytics/test/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

13
analytics/fuel_analytics/test/api/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

13
analytics/fuel_analytics/test/api/resources/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

13
analytics/fuel_analytics/test/api/resources/utils/__init__.py Normal file
View File

@ -0,0 +1,13 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

42
analytics/fuel_analytics/test/api/resources/utils/test_es_client.py Normal file
View File

@ -0,0 +1,42 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuel_analytics.test.base import ElasticTest
from fuel_analytics.api.app import app
from fuel_analytics.api.resources.utils.es_client import ElasticSearchClient
class EsClientTest(ElasticTest):
def test_fetch_all_data(self):
installations_num = 160
self.generate_data(installations_num=installations_num)
query = {"query": {"match_all": {}}}
es_client = ElasticSearchClient()
doc_type = app.config['ELASTIC_DOC_TYPE_STRUCTURE']
resp = es_client.fetch_all_data(query, doc_type,
show_fields=('master_node_uid',),
chunk_size=installations_num / 10 + 1)
mn_uids = set([row['master_node_uid'] for row in resp])
self.assertEquals(installations_num, len(mn_uids))
def test_get_structures(self):
installations_num = 100
self.generate_data(installations_num=installations_num)
es_client = ElasticSearchClient()
resp = es_client.get_structures()
mn_uids = set([row['master_node_uid'] for row in resp])
self.assertEquals(installations_num, len(mn_uids))

215
analytics/fuel_analytics/test/api/resources/utils/test_stats_to_csv.py Normal file
View File

@ -0,0 +1,215 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import csv
import six
import types
from fuel_analytics.test.base import BaseTest
from fuel_analytics.test.base import ElasticTest
from fuel_analytics.api.resources.utils.es_client import ElasticSearchClient
from fuel_analytics.api.resources.utils.stats_to_csv import StatsToCsv
class StatsToCsvTest(BaseTest):
def test_dict_construct_skeleton(self):
exporter = StatsToCsv()
data = {'a': 'b'}
skeleton = exporter.construct_skeleton(data)
self.assertDictEqual(data, skeleton)
data = {'a': 'b', 'x': None}
skeleton = exporter.construct_skeleton(data)
self.assertDictEqual(data, skeleton)
def test_list_construct_skeleton(self):
exporter = StatsToCsv()
data = ['a', 'b', 'c']
skeleton = exporter.construct_skeleton(data)
self.assertListEqual([], skeleton)
data = [{'a': None}, {'b': 'x'}, {'a': 4, 'c': 'xx'}, {}]
skeleton = exporter.construct_skeleton(data)
self.assertListEqual(
sorted(skeleton[0].keys()),
sorted(['a', 'b', 'c'])
)
data = [
'a',
['a', 'b', []],
[],
[{'x': 'z'}, 'zz', {'a': 'b'}],
['a'],
{'p': 'q'}
]
skeleton = exporter.construct_skeleton(data)
self.assertListEqual([[[], {'a': 'b', 'x': 'z'}], {'p': 'q'}],
skeleton)
def test_get_skeleton(self):
exporter = StatsToCsv()
data = [
{'ci': {'p': True, 'e': '@', 'n': 'n'}},
# reducing fields in nested dict
{'ci': {'p': False}},
# adding list values
{'c': [{'s': 'v', 'n': 2}, {'s': 'vv', 'n': 22}]},
# adding new value in the list
{'c': [{'z': 'p'}]},
# checking empty list
{'c': []},
# adding new value
{'a': 'b'},
]
skeleton = exporter.get_data_skeleton(data)
self.assertDictEqual(
{'a': None, 'c': [{'s': None, 'n': None, 'z': None}],
'ci': {'p': None, 'e': None, 'n': None}},
skeleton)
def test_get_key_paths(self):
exporter = StatsToCsv()
skeleton = {'a': 'b', 'c': 'd'}
paths = exporter.get_keys_paths(skeleton)
self.assertListEqual([['a'], ['c']], paths)
skeleton = {'a': {'e': 'f', 'g': None}}
paths = exporter.get_keys_paths(skeleton)
self.assertListEqual([['a', 'e'], ['a', 'g']], paths)
skeleton = [{'a': 'b', 'c': 'd'}]
paths = exporter.get_keys_paths(skeleton)
self.assertListEqual([[]], paths)
def test_get_flatten_data(self):
exporter = StatsToCsv()
data = [
{'a': 'b', 'c': {'e': 2.1}},
{'a': 'ee\nxx', 'c': {'e': 3.1415}, 'x': ['z', 'zz']},
]
expected_flatten_data = [
['b', 2.1, None],
['ee\nxx', 3.1415, 'z zz'],
]
skeleton = exporter.get_data_skeleton(data)
key_paths = exporter.get_keys_paths(skeleton)
for idx, expected in enumerate(expected_flatten_data):
actual = exporter.get_flatten_data(key_paths, data[idx])
self.assertListEqual(expected, actual)
def test_get_cluster_keys_paths(self):
exporter = StatsToCsv()
_, _, csv_keys_paths = exporter.get_cluster_keys_paths()
self.assertTrue(['nodes_platform_name_gt3' in csv_keys_paths])
self.assertTrue(['nodes_platform_name_0' in csv_keys_paths])
self.assertTrue(['nodes_platform_name_1' in csv_keys_paths])
self.assertTrue(['nodes_platform_name_2' in csv_keys_paths])
self.assertTrue(['manufacturer_gt3' in csv_keys_paths])
self.assertTrue(['manufacturer_0' in csv_keys_paths])
self.assertTrue(['manufacturer_1' in csv_keys_paths])
self.assertTrue(['manufacturer_2' in csv_keys_paths])
self.assertTrue(['attributes', 'heat'] in csv_keys_paths)
def test_align_enumerated_field_values(self):
# Data for checks in format (source, num, expected)
checks = [
([], 0, [False]),
([], 1, [False, None]),
(['a'], 1, [False, 'a']),
(['a'], 2, [False, 'a', None]),
(['a', 'b'], 2, [False, 'a', 'b']),
(['a', 'b'], 1, [True, 'a'])
]
for source, num, expected in checks:
self.assertListEqual(
expected,
StatsToCsv.align_enumerated_field_values(source, num)
)
class StatsToCsvExportTest(ElasticTest):
def test_new_param_handled_by_structures_skeleton(self):
installations_num = 5
self.generate_data(installations_num=installations_num)
# Mixing new pram into structures
es_client = ElasticSearchClient()
structures = es_client.get_structures()
self.assertTrue(isinstance(structures, types.GeneratorType))
structures = list(structures)
structures[-1]['mixed_param'] = 'xx'
exporter = StatsToCsv()
skeleton = exporter.get_data_skeleton(structures)
self.assertTrue('mixed_param' in skeleton)
def test_get_flatten_clusters(self):
installations_num = 200
self.generate_data(installations_num=installations_num)
es_client = ElasticSearchClient()
structures = es_client.get_structures()
exporter = StatsToCsv()
structure_paths, cluster_paths, csv_paths = \
exporter.get_cluster_keys_paths()
flatten_clusters = exporter.get_flatten_clusters(structure_paths,
cluster_paths,
structures)
self.assertTrue(isinstance(flatten_clusters, types.GeneratorType))
for flatten_cluster in flatten_clusters:
self.assertEquals(len(csv_paths), len(flatten_cluster))
def test_flatten_data_as_csv(self):
installations_num = 100
self.generate_data(installations_num=installations_num)
es_client = ElasticSearchClient()
structures = es_client.get_structures()
exporter = StatsToCsv()
structure_paths, cluster_paths, csv_paths = \
exporter.get_cluster_keys_paths()
flatten_clusters = exporter.get_flatten_clusters(structure_paths,
cluster_paths,
structures)
self.assertTrue(isinstance(flatten_clusters, types.GeneratorType))
result = exporter.flatten_data_as_csv(csv_paths, flatten_clusters)
self.assertTrue(isinstance(result, types.GeneratorType))
output = six.StringIO(list(result))
reader = csv.reader(output)
columns = reader.next()
# Checking enumerated columns are present in the output
self.assertIn('nodes_manufacturer_0', columns)
self.assertIn('nodes_manufacturer_gt3', columns)
self.assertIn('nodes_platform_name_0', columns)
self.assertIn('nodes_platform_name_gt3', columns)
# Checking reading result CSV
for _ in reader:
pass
def test_export_clusters(self):
installations_num = 100
self.generate_data(installations_num=installations_num)
es_client = ElasticSearchClient()
structures = es_client.get_structures()
exporter = StatsToCsv()
result = exporter.export_clusters(structures)
self.assertTrue(isinstance(result, types.GeneratorType))

41
analytics/fuel_analytics/test/base.py Normal file
View File

@ -0,0 +1,41 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from unittest2.case import TestCase
from fuel_analytics.api.app import app
from fuel_analytics.api.log import init_logger
# Configuring app for the test environment
app.config.from_object('fuel_analytics.api.config.Testing')
init_logger()
from migration.test.base import ElasticTest as MigrationElasticTest
class BaseTest(TestCase):
def setUp(self):
super(BaseTest, self).setUp()
self.client = app.test_client()
def check_response_ok(self, resp, codes=(200, 201)):
self.assertIn(resp.status_code, codes)
def check_response_error(self, resp, code):
self.assertEquals(code, resp.status_code)
class ElasticTest(MigrationElasticTest):
pass

22
analytics/fuel_analytics/test/test_common.py Normal file
View File

@ -0,0 +1,22 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from fuel_analytics.test.base import BaseTest
class TestCommon(BaseTest):
def test_unknown_resource(self):
resp = self.client.get('/xxx')
self.check_response_error(resp, 404)

40
analytics/manage_analytics.py Executable file
View File

@ -0,0 +1,40 @@
#!/usr/bin/env python
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from flask_script import Manager
from fuel_analytics.api import log
from fuel_analytics.api.app import app
def configure_app(mode=None):
mode_map = {
'test': 'analytics.api.config.Testing',
'prod': 'analytics.api.config.Production'
}
app.config.from_object(mode_map.get(mode))
app.config.from_envvar('ANALYTICS_SETTINGS', silent=True)
log.init_logger()
return app
manager = Manager(configure_app)
manager.add_option('--mode', help="Acceptable modes. Default: 'test'",
choices=('test', 'prod'), default='prod', dest='mode')
if __name__ == '__main__':
manager.run()

3
analytics/requirements.txt Normal file
View File

@ -0,0 +1,3 @@
elasticsearch==1.2.0
Flask==0.10.1
Flask-Script==2.0.5

55
analytics/setup.py Normal file
View File

@ -0,0 +1,55 @@
# Copyright 2014 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from setuptools import find_packages
from setuptools import setup
def parse_requirements_txt():
root = os.path.dirname(os.path.abspath(__file__))
requirements = []
with open(os.path.join(root, 'requirements.txt'), 'r') as f:
for line in f.readlines():
line = line.rstrip()
if not line or line.startswith('#'):
continue
requirements.append(line)
return requirements
setup(
name='analytics',
version='0.0.1',
description="Service of analytics reports",
long_description="""Service of analytics reports""",
license="http://www.apache.org/licenses/LICENSE-2.0",
classifiers=[
"License :: OSI Approved :: Apache Software License",
"Development Status :: 3 - Alpha",
"Programming Language :: Python",
"Topic :: Internet :: WWW/HTTP",
"Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
],
author='Mirantis Inc.',
author_email='product@mirantis.com',
url='https://mirantis.com',
keywords='fuel statistics analytics mirantis',
packages=find_packages(),
zip_safe=False,
install_requires=parse_requirements_txt(),
include_package_data=True,
scripts=['manage_analytics.py']
)

11
analytics/test-requirements.txt Normal file
View File

@ -0,0 +1,11 @@
-r requirements.txt
hacking==0.9.2
mock==1.0.1
nose==1.3.4
nose2==0.4.7
tox==1.8.0
unittest2==0.5.1
# required for use tests from migration
SQLAlchemy==0.9.8
psycopg2==2.5.4

44
analytics/tox.ini Normal file
View File

@ -0,0 +1,44 @@
[tox]
minversion = 1.6
skipsdist = True
envlist = py27,pep8
[testenv]
usedevelop = True
install_command = pip install {packages}
setenv =
VIRTUAL_ENV={envdir}
PYTHONPATH={toxinidir}/../migration
deps = -r{toxinidir}/test-requirements.txt
commands =
nosetests {posargs:fuel_analytics/test}
[tox:jenkins]
downloadcache = ~/cache/pip
[testenv:pep8]
deps = hacking==0.7
usedevelop = False
commands =
flake8 {posargs:fuel_analytics}
[testenv:cover]
setenv = NOSE_WITH_COVERAGE=1
[testenv:venv]
deps = -r{toxinidir}/test-requirements.txt
commands = {posargs:}
[testenv:devenv]
envdir = devenv
usedevelop = True
[flake8]
ignore = H234,H302,H802
exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs
show-pep8 = True
show-source = True
count = True
[hacking]
import_exceptions = testtools.matchers

7
analytics/uwsgi/analytics_example.yaml Normal file
View File

@ -0,0 +1,7 @@
uwsgi:
socket: :8082
# for production app use analytics.api.app as module
module: fuel_analytics.api.app_test
callable: app
protocol: http
env: ANALYTICS_SETTINGS=/path/to/external/config.py

3
collector/requirements.txt
View File

@ -6,4 +6,5 @@ Flask-Script==2.0.5
PyYAML==3.11
alembic==0.6.7
psycopg2==2.5.4
six>=1.8.0
six>=1.8.0
uWSGI==2.0.9

8
migration/migration/test/base.py
View File

@ -77,7 +77,10 @@ class ElasticTest(TestCase):
'zabbix', 'mongo'),
oses=('Ubuntu', 'CentOs', 'Ubuntu LTS XX'),
node_statuses = ('ready', 'discover', 'provisioning',
'provisioned', 'deploying', 'error')
'provisioned', 'deploying', 'error'),
manufacturers = ('Dell Inc.', 'VirtualBox', 'QEMU',
'VirtualBox', 'Supermicro', 'Cisco Systems Inc',
'KVM', 'VMWARE', 'HP')
):
roles = []
for _ in xrange(random.randint(*roles_range)):
@ -86,7 +89,8 @@ class ElasticTest(TestCase):
'id': self.gen_id(),
'roles': roles,
'os': random.choice(oses),
'status': random.choice(node_statuses)
'status': random.choice(node_statuses),
'manufacturer': random.choice(manufacturers)
}
return node