Http hander for fetching clusters stats as CSV
Flask application added for export clusters statstics in CSV format. Export process streams data by the generators. Closes-Bug: #1410262 Blueprint: export-stats-to-csv Change-Id: I265b617e78de142f8f10f22e85f734d0df7979c2
This commit is contained in:
parent
edf8553534
commit
2f4eb79233
|
@ -0,0 +1,4 @@
|
|||
include *.txt
|
||||
graft static
|
||||
prune static/bower_components
|
||||
prune static/node_modules
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,22 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from flask import Flask
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
# Registering blueprints
|
||||
from fuel_analytics.api.resources.csv_exporter import bp as csv_exporter_bp
|
||||
|
||||
app.register_blueprint(csv_exporter_bp, url_prefix='/api/v1/csv')
|
|
@ -0,0 +1,21 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api import log
|
||||
|
||||
|
||||
app.config.from_object('fuel_analytics.api.config.Production')
|
||||
app.config.from_envvar('ANALYTICS_SETTINGS', silent=True)
|
||||
log.init_logger()
|
|
@ -0,0 +1,21 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api import log
|
||||
|
||||
|
||||
app.config.from_object('fuel_analytics.api.config.Testing')
|
||||
app.config.from_envvar('ANALYTICS_SETTINGS', silent=True)
|
||||
log.init_logger()
|
|
@ -0,0 +1,42 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
|
||||
|
||||
class Production(object):
|
||||
DEBUG = False
|
||||
LOG_FILE = '/var/log/fuel-stats/analytics.log'
|
||||
LOG_LEVEL = logging.ERROR
|
||||
LOG_ROTATION = False
|
||||
LOGGER_NAME = 'analytics'
|
||||
ELASTIC_HOST = 'product-stats.mirantis.com'
|
||||
ELASTIC_PORT = 443
|
||||
ELASTIC_USE_SSL = True
|
||||
ELASTIC_INDEX_FUEL = 'fuel'
|
||||
ELASTIC_DOC_TYPE_STRUCTURE = 'structure'
|
||||
|
||||
|
||||
class Testing(Production):
|
||||
DEBUG = False
|
||||
LOG_FILE = os.path.realpath(os.path.join(
|
||||
os.path.dirname(__file__), '..', 'test', 'logs', 'analytics.log'))
|
||||
LOG_LEVEL = logging.DEBUG
|
||||
LOG_ROTATION = True
|
||||
LOG_FILE_SIZE = 2048000
|
||||
LOG_FILES_COUNT = 5
|
||||
ELASTIC_HOST = 'localhost'
|
||||
ELASTIC_PORT = 9200
|
||||
ELASTIC_USE_SSL = False
|
|
@ -0,0 +1,49 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from logging import FileHandler
|
||||
from logging import Formatter
|
||||
from logging.handlers import RotatingFileHandler
|
||||
import os
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
|
||||
|
||||
def get_file_handler():
|
||||
if app.config.get('LOG_ROTATION'):
|
||||
file_handler = RotatingFileHandler(
|
||||
app.config.get('LOG_FILE'),
|
||||
maxBytes=app.config.get('LOG_FILE_SIZE'),
|
||||
backupCount=app.config.get('LOG_FILES_COUNT')
|
||||
)
|
||||
else:
|
||||
file_handler = FileHandler(app.config.get('LOG_FILE'))
|
||||
file_handler.setLevel(app.config.get('LOG_LEVEL'))
|
||||
formatter = get_formatter()
|
||||
file_handler.setFormatter(formatter)
|
||||
return file_handler
|
||||
|
||||
|
||||
def get_formatter():
|
||||
DATE_FORMAT = "%Y-%m-%d %H:%M:%S"
|
||||
LOG_FORMAT = "%(asctime)s.%(msecs)03d %(levelname)s " \
|
||||
"[%(thread)x] (%(module)s) %(message)s"
|
||||
return Formatter(fmt=LOG_FORMAT, datefmt=DATE_FORMAT)
|
||||
|
||||
|
||||
def init_logger():
|
||||
log_dir = os.path.dirname(app.config.get('LOG_FILE'))
|
||||
if not os.path.exists(log_dir):
|
||||
os.mkdir(log_dir, 750)
|
||||
app.logger.addHandler(get_file_handler())
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,36 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from flask import Blueprint
|
||||
from flask import Response
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api.resources.utils.es_client import ElasticSearchClient
|
||||
from fuel_analytics.api.resources.utils.stats_to_csv import StatsToCsv
|
||||
|
||||
bp = Blueprint('csv_exporter', __name__)
|
||||
|
||||
|
||||
@bp.route('/clusters', methods=['GET'])
|
||||
def csv_exporter():
|
||||
app.logger.debug("Handling csv_exporter get request")
|
||||
es_client = ElasticSearchClient()
|
||||
structures = es_client.get_structures()
|
||||
|
||||
exporter = StatsToCsv()
|
||||
result = exporter.export_clusters(structures)
|
||||
|
||||
# NOTE: result - is generator, but streaming can not work with some
|
||||
# WSGI middlewares: http://flask.pocoo.org/docs/0.10/patterns/streaming/
|
||||
return Response(result, mimetype='text/csv')
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,70 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from elasticsearch import Elasticsearch
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
|
||||
|
||||
class ElasticSearchClient(object):
|
||||
|
||||
def __init__(self):
|
||||
self.es = Elasticsearch(hosts=[
|
||||
{'host': app.config['ELASTIC_HOST'],
|
||||
'port': app.config['ELASTIC_PORT'],
|
||||
'use_ssl': app.config['ELASTIC_USE_SSL']}
|
||||
])
|
||||
|
||||
def fetch_all_data(self, query, doc_type, show_fields=(),
|
||||
sort=({"_id": {"order": "asc"}},), chunk_size=100):
|
||||
"""Gets structures from the Elasticsearch by querying by chunk_size
|
||||
number of structures
|
||||
:param query: Elasticsearch query
|
||||
:param doc_type: requested document type
|
||||
:param show_fields: tuple of selected fields.
|
||||
All fields will be fetched, if show_fields is not set
|
||||
:param sort: tuple of fields for sorting
|
||||
:param chunk_size: size of fetched structures chunk
|
||||
:return: list of fetched structures
|
||||
"""
|
||||
received = 0
|
||||
paged_query = query.copy()
|
||||
paged_query["from"] = received
|
||||
paged_query["size"] = chunk_size
|
||||
if sort:
|
||||
paged_query["sort"] = sort
|
||||
if show_fields:
|
||||
paged_query["_source"] = show_fields
|
||||
while True:
|
||||
app.logger.debug("Fetching chunk from ElasticSearch. "
|
||||
"From: %d, size: %d",
|
||||
paged_query["from"], chunk_size)
|
||||
response = self.es.search(index=app.config['ELASTIC_INDEX_FUEL'],
|
||||
doc_type=doc_type, body=paged_query)
|
||||
total = response["hits"]["total"]
|
||||
received += chunk_size
|
||||
paged_query["from"] = received
|
||||
for d in response["hits"]["hits"]:
|
||||
yield d["_source"]
|
||||
app.logger.debug("Chunk from ElasticSearch is fetched. "
|
||||
"From: %d, size: %d",
|
||||
paged_query["from"], chunk_size)
|
||||
if total <= received:
|
||||
break
|
||||
|
||||
def get_structures(self):
|
||||
app.logger.debug("Fetching structures info from ElasticSearch")
|
||||
query = {"query": {"match_all": {}}}
|
||||
doc_type = app.config['ELASTIC_DOC_TYPE_STRUCTURE']
|
||||
return self.fetch_all_data(query, doc_type)
|
|
@ -0,0 +1,108 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
INSTALLATION_INFO_SKELETON = {
|
||||
'allocated_nodes_num': None,
|
||||
'clusters': [
|
||||
{
|
||||
'attributes': {
|
||||
'assign_public_to_all_nodes': None,
|
||||
'ceilometer': None,
|
||||
'debug_mode': None,
|
||||
'ephemeral_ceph': None,
|
||||
'heat': None,
|
||||
'images_ceph': None,
|
||||
'images_vcenter': None,
|
||||
'iser': None,
|
||||
'kernel_params': None,
|
||||
'libvirt_type': None,
|
||||
'mellanox': None,
|
||||
'mellanox_vf_num': None,
|
||||
'murano': None,
|
||||
'nsx': None,
|
||||
'nsx_replication': None,
|
||||
'nsx_transport': None,
|
||||
'objects_ceph': None,
|
||||
'osd_pool_size': None,
|
||||
'provision_method': None,
|
||||
'sahara': None,
|
||||
'syslog_transport': None,
|
||||
'use_cow_images': None,
|
||||
'vcenter': None,
|
||||
'vlan_splinters': None,
|
||||
'vlan_splinters_ovs': None,
|
||||
'volumes_ceph': None,
|
||||
'volumes_lvm': None,
|
||||
'volumes_vmdk': None
|
||||
},
|
||||
'fuel_version': None,
|
||||
'id': None,
|
||||
'is_customized': None,
|
||||
'mode': None,
|
||||
'net_provider': None,
|
||||
'node_groups': [{'id': None, 'nodes': [{}]}],
|
||||
'nodes': [
|
||||
{
|
||||
'bond_interfaces': [
|
||||
{'id': None, 'slaves': [{}]}
|
||||
],
|
||||
'error_type': None,
|
||||
'group_id': None,
|
||||
'id': None,
|
||||
'manufacturer': None,
|
||||
'nic_interfaces': [{'id': None}],
|
||||
'online': None,
|
||||
'os': None,
|
||||
'pending_addition': None,
|
||||
'pending_deletion': None,
|
||||
'pending_roles': [{}],
|
||||
'platform_name': None,
|
||||
'roles': [{}],
|
||||
'status': None
|
||||
}
|
||||
],
|
||||
'nodes_num': None,
|
||||
'openstack_info': {
|
||||
'images': [{'size': None, 'unit': None}],
|
||||
'nova_servers_count': None
|
||||
},
|
||||
'release': {'name': None, 'os': None, 'version': None},
|
||||
'status': None
|
||||
}
|
||||
],
|
||||
'clusters_num': None,
|
||||
'creation_date': None,
|
||||
'fuel_release': {
|
||||
'api': None,
|
||||
'astute_sha': None,
|
||||
'build_id': None,
|
||||
'build_number': None,
|
||||
'feature_groups': [{}],
|
||||
'fuellib_sha': None,
|
||||
'fuelmain_sha': None,
|
||||
'nailgun_sha': None,
|
||||
'ostf_sha': None,
|
||||
'production': None,
|
||||
'release': None
|
||||
},
|
||||
'master_node_uid': None,
|
||||
'modification_date': None,
|
||||
'unallocated_nodes_num': None,
|
||||
'user_information': {
|
||||
'company': None,
|
||||
'contact_info_provided': None,
|
||||
'email': None,
|
||||
'name': None
|
||||
}
|
||||
}
|
|
@ -0,0 +1,266 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import csv
|
||||
import io
|
||||
import six
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api.resources.utils.skeleton import \
|
||||
INSTALLATION_INFO_SKELETON
|
||||
|
||||
|
||||
class StatsToCsv(object):
|
||||
|
||||
MANUFACTURERS_NUM = 3
|
||||
PLATFORM_NAMES_NUM = 3
|
||||
|
||||
def construct_skeleton(self, data):
|
||||
"""Creates structure for searching all key paths in given data
|
||||
:param data: fetched from ES dict
|
||||
:return: skeleton of data structure
|
||||
"""
|
||||
if isinstance(data, dict):
|
||||
result = {}
|
||||
for k in sorted(data.keys()):
|
||||
result[k] = self.construct_skeleton(data[k])
|
||||
return result
|
||||
elif isinstance(data, (list, tuple)):
|
||||
list_result = []
|
||||
dict_result = {}
|
||||
for d in data:
|
||||
if isinstance(d, dict):
|
||||
dict_result.update(self.construct_skeleton(d))
|
||||
elif isinstance(d, (list, tuple)):
|
||||
if not list_result:
|
||||
list_result.append(self.construct_skeleton(d))
|
||||
else:
|
||||
list_result[0].extend(self.construct_skeleton(d))
|
||||
if dict_result:
|
||||
list_result.append(dict_result)
|
||||
return list_result
|
||||
else:
|
||||
return data
|
||||
|
||||
def get_data_skeleton(self, structures):
|
||||
"""Gets skeleton by structures list
|
||||
:param structures:
|
||||
:return: data structure skeleton
|
||||
"""
|
||||
def _merge_skeletons(lh, rh):
|
||||
keys_paths = self.get_keys_paths(rh)
|
||||
for keys_path in keys_paths:
|
||||
merge_point = lh
|
||||
data_point = rh
|
||||
for key in keys_path:
|
||||
data_point = data_point[key]
|
||||
if isinstance(data_point, dict):
|
||||
if key not in merge_point:
|
||||
merge_point[key] = {}
|
||||
elif isinstance(data_point, list):
|
||||
if key not in merge_point:
|
||||
merge_point[key] = [{}]
|
||||
_merge_skeletons(merge_point[key][0],
|
||||
self.get_data_skeleton(data_point))
|
||||
else:
|
||||
merge_point[key] = None
|
||||
merge_point = merge_point[key]
|
||||
|
||||
skeleton = {}
|
||||
for structure in structures:
|
||||
app.logger.debug("Constructing skeleton by data: %s", structure)
|
||||
app.logger.debug("Updating skeleton by %s",
|
||||
self.construct_skeleton(structure))
|
||||
_merge_skeletons(skeleton, self.construct_skeleton(structure))
|
||||
app.logger.debug("Result skeleton is %s", skeleton)
|
||||
return skeleton
|
||||
|
||||
def get_keys_paths(self, skeleton):
|
||||
"""Gets paths to leaf keys in the data
|
||||
:param skeleton: data skeleton
|
||||
:return: list of lists of dict keys
|
||||
"""
|
||||
|
||||
def _keys_paths_helper(keys, skel):
|
||||
result = []
|
||||
if isinstance(skel, dict):
|
||||
for k in sorted(six.iterkeys(skel)):
|
||||
result.extend(_keys_paths_helper(keys + [k], skel[k]))
|
||||
else:
|
||||
result.append(keys)
|
||||
return result
|
||||
|
||||
return _keys_paths_helper([], skeleton)
|
||||
|
||||
def flatten_data_as_csv(self, keys_paths, flatten_data):
|
||||
"""Returns flatten data in CSV
|
||||
:param keys_paths: list of dict keys lists for columns names
|
||||
generation
|
||||
:param flatten_data: list of flatten data dicts
|
||||
:return: stream with data in CSV format
|
||||
"""
|
||||
app.logger.debug("Saving flatten data as CSV is started")
|
||||
names = []
|
||||
for key_path in keys_paths:
|
||||
names.append('.'.join(key_path))
|
||||
yield names
|
||||
|
||||
output = six.BytesIO()
|
||||
writer = csv.writer(output)
|
||||
writer.writerow(names)
|
||||
|
||||
def read_and_flush():
|
||||
output.seek(io.SEEK_SET)
|
||||
data = output.read()
|
||||
output.seek(io.SEEK_SET)
|
||||
output.truncate()
|
||||
return data
|
||||
|
||||
for d in flatten_data:
|
||||
writer.writerow(d)
|
||||
yield read_and_flush()
|
||||
app.logger.debug("Saving flatten data as CSV is finished")
|
||||
|
||||
def get_flatten_data(self, keys_paths, data):
|
||||
"""Creates flatten data from data by keys_paths
|
||||
:param keys_paths: list of dict keys lists
|
||||
:param data: dict with nested structures
|
||||
:return: list of flatten data dicts
|
||||
"""
|
||||
flatten_data = []
|
||||
for key_path in keys_paths:
|
||||
d = data
|
||||
for key in key_path:
|
||||
d = d.get(key, None)
|
||||
if d is None:
|
||||
break
|
||||
if isinstance(d, (list, tuple)):
|
||||
flatten_data.append(' '.join(d))
|
||||
else:
|
||||
flatten_data.append(d)
|
||||
return flatten_data
|
||||
|
||||
def get_cluster_keys_paths(self):
|
||||
app.logger.debug("Getting cluster keys paths")
|
||||
structure_skeleton = INSTALLATION_INFO_SKELETON
|
||||
structure_key_paths = self.get_keys_paths(structure_skeleton)
|
||||
clusters = structure_skeleton.get('clusters')
|
||||
if not clusters:
|
||||
clusters = [{}]
|
||||
cluster_skeleton = clusters[0]
|
||||
|
||||
# Removing lists of dicts from cluster skeleton
|
||||
cluster_skeleton.pop('nodes', None)
|
||||
cluster_skeleton.pop('node_groups', None)
|
||||
cluster_skeleton.pop('openstack_info', None)
|
||||
cluster_key_paths = self.get_keys_paths(cluster_skeleton)
|
||||
|
||||
result_key_paths = cluster_key_paths + structure_key_paths
|
||||
|
||||
def enumerated_field_keys(field_name, number):
|
||||
"""Adds enumerated fields columns and property
|
||||
field for showing case, when values will be cut
|
||||
:param field_name: field name
|
||||
:param number: number of enumerated fields
|
||||
:return: list of cut fact column and enumerated columns names
|
||||
"""
|
||||
result = [['{}_gt{}'.format(field_name, number)]]
|
||||
for i in xrange(number):
|
||||
result.append(['{}_{}'.format(field_name, i)])
|
||||
return result
|
||||
|
||||
# Handling enumeration of manufacturers names
|
||||
result_key_paths.extend(enumerated_field_keys('nodes_manufacturer',
|
||||
self.MANUFACTURERS_NUM))
|
||||
|
||||
# Handling enumeration of platform names
|
||||
result_key_paths.extend(enumerated_field_keys('nodes_platform_name',
|
||||
self.PLATFORM_NAMES_NUM))
|
||||
|
||||
app.logger.debug("Cluster keys paths got")
|
||||
return structure_key_paths, cluster_key_paths, result_key_paths
|
||||
|
||||
@staticmethod
|
||||
def align_enumerated_field_values(values, number):
|
||||
"""Fills result list by the None values, if number is greater than
|
||||
values len. The first element of result is bool value
|
||||
len(values) > number
|
||||
:param values:
|
||||
:param number:
|
||||
:return: aligned list to 'number' + 1 length, filled by Nones on
|
||||
empty values positions and bool value on the first place. Bool value
|
||||
is True if len(values) > number
|
||||
"""
|
||||
return ([len(values) > number] +
|
||||
(values + [None] * (number - len(values)))[:number])
|
||||
|
||||
def get_flatten_clusters(self, structure_keys_paths, cluster_keys_paths,
|
||||
structures):
|
||||
"""Gets flatten clusters data
|
||||
:param structure_keys_paths: list of keys paths in the
|
||||
installation structure
|
||||
:param cluster_keys_paths: list of keys paths in the cluster
|
||||
:param structures: list of installation structures
|
||||
:return: list of flatten clusters info
|
||||
"""
|
||||
app.logger.debug("Getting flatten clusters info is started")
|
||||
|
||||
def extract_nodes_fields(field, nodes):
|
||||
"""Extracts fields values from nested nodes dicts
|
||||
:param field: field name
|
||||
:param nodes: nodes data list
|
||||
:return: set of extracted fields values from nodes
|
||||
"""
|
||||
result = set([d.get(field) for d in nodes])
|
||||
return filter(lambda x: x is not None, result)
|
||||
|
||||
def extract_nodes_manufacturers(nodes):
|
||||
return extract_nodes_fields('manufacturer', nodes)
|
||||
|
||||
def extract_nodes_platform_name(nodes):
|
||||
return extract_nodes_fields('platform_name', nodes)
|
||||
|
||||
for structure in structures:
|
||||
clusters = structure.pop('clusters', [])
|
||||
flatten_structure = self.get_flatten_data(structure_keys_paths,
|
||||
structure)
|
||||
for cluster in clusters:
|
||||
flatten_cluster = self.get_flatten_data(cluster_keys_paths,
|
||||
cluster)
|
||||
flatten_cluster.extend(flatten_structure)
|
||||
nodes = cluster.get('nodes', [])
|
||||
|
||||
# Adding enumerated manufacturers
|
||||
manufacturers = extract_nodes_manufacturers(nodes)
|
||||
flatten_cluster += StatsToCsv.align_enumerated_field_values(
|
||||
manufacturers, self.MANUFACTURERS_NUM)
|
||||
|
||||
# Adding enumerated platforms
|
||||
platform_names = extract_nodes_platform_name(nodes)
|
||||
flatten_cluster += StatsToCsv.align_enumerated_field_values(
|
||||
platform_names, self.PLATFORM_NAMES_NUM)
|
||||
yield flatten_cluster
|
||||
|
||||
app.logger.debug("Flatten clusters info is got")
|
||||
|
||||
def export_clusters(self, structures):
|
||||
app.logger.info("Export clusters info into CSV is started")
|
||||
structure_keys_paths, cluster_keys_paths, csv_keys_paths = \
|
||||
self.get_cluster_keys_paths()
|
||||
flatten_clusters = self.get_flatten_clusters(structure_keys_paths,
|
||||
cluster_keys_paths,
|
||||
structures)
|
||||
result = self.flatten_data_as_csv(csv_keys_paths, flatten_clusters)
|
||||
app.logger.info("Export clusters info into CSV is finished")
|
||||
return result
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,13 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
|
@ -0,0 +1,42 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from fuel_analytics.test.base import ElasticTest
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api.resources.utils.es_client import ElasticSearchClient
|
||||
|
||||
|
||||
class EsClientTest(ElasticTest):
|
||||
|
||||
def test_fetch_all_data(self):
|
||||
installations_num = 160
|
||||
self.generate_data(installations_num=installations_num)
|
||||
|
||||
query = {"query": {"match_all": {}}}
|
||||
es_client = ElasticSearchClient()
|
||||
doc_type = app.config['ELASTIC_DOC_TYPE_STRUCTURE']
|
||||
resp = es_client.fetch_all_data(query, doc_type,
|
||||
show_fields=('master_node_uid',),
|
||||
chunk_size=installations_num / 10 + 1)
|
||||
mn_uids = set([row['master_node_uid'] for row in resp])
|
||||
self.assertEquals(installations_num, len(mn_uids))
|
||||
|
||||
def test_get_structures(self):
|
||||
installations_num = 100
|
||||
self.generate_data(installations_num=installations_num)
|
||||
es_client = ElasticSearchClient()
|
||||
resp = es_client.get_structures()
|
||||
mn_uids = set([row['master_node_uid'] for row in resp])
|
||||
self.assertEquals(installations_num, len(mn_uids))
|
|
@ -0,0 +1,215 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import csv
|
||||
import six
|
||||
import types
|
||||
|
||||
from fuel_analytics.test.base import BaseTest
|
||||
from fuel_analytics.test.base import ElasticTest
|
||||
|
||||
from fuel_analytics.api.resources.utils.es_client import ElasticSearchClient
|
||||
from fuel_analytics.api.resources.utils.stats_to_csv import StatsToCsv
|
||||
|
||||
|
||||
class StatsToCsvTest(BaseTest):
|
||||
|
||||
def test_dict_construct_skeleton(self):
|
||||
exporter = StatsToCsv()
|
||||
data = {'a': 'b'}
|
||||
skeleton = exporter.construct_skeleton(data)
|
||||
self.assertDictEqual(data, skeleton)
|
||||
|
||||
data = {'a': 'b', 'x': None}
|
||||
skeleton = exporter.construct_skeleton(data)
|
||||
self.assertDictEqual(data, skeleton)
|
||||
|
||||
def test_list_construct_skeleton(self):
|
||||
exporter = StatsToCsv()
|
||||
data = ['a', 'b', 'c']
|
||||
skeleton = exporter.construct_skeleton(data)
|
||||
self.assertListEqual([], skeleton)
|
||||
|
||||
data = [{'a': None}, {'b': 'x'}, {'a': 4, 'c': 'xx'}, {}]
|
||||
skeleton = exporter.construct_skeleton(data)
|
||||
self.assertListEqual(
|
||||
sorted(skeleton[0].keys()),
|
||||
sorted(['a', 'b', 'c'])
|
||||
)
|
||||
|
||||
data = [
|
||||
'a',
|
||||
['a', 'b', []],
|
||||
[],
|
||||
[{'x': 'z'}, 'zz', {'a': 'b'}],
|
||||
['a'],
|
||||
{'p': 'q'}
|
||||
]
|
||||
skeleton = exporter.construct_skeleton(data)
|
||||
self.assertListEqual([[[], {'a': 'b', 'x': 'z'}], {'p': 'q'}],
|
||||
skeleton)
|
||||
|
||||
def test_get_skeleton(self):
|
||||
exporter = StatsToCsv()
|
||||
data = [
|
||||
{'ci': {'p': True, 'e': '@', 'n': 'n'}},
|
||||
# reducing fields in nested dict
|
||||
{'ci': {'p': False}},
|
||||
# adding list values
|
||||
{'c': [{'s': 'v', 'n': 2}, {'s': 'vv', 'n': 22}]},
|
||||
# adding new value in the list
|
||||
{'c': [{'z': 'p'}]},
|
||||
# checking empty list
|
||||
{'c': []},
|
||||
# adding new value
|
||||
{'a': 'b'},
|
||||
]
|
||||
skeleton = exporter.get_data_skeleton(data)
|
||||
self.assertDictEqual(
|
||||
{'a': None, 'c': [{'s': None, 'n': None, 'z': None}],
|
||||
'ci': {'p': None, 'e': None, 'n': None}},
|
||||
skeleton)
|
||||
|
||||
def test_get_key_paths(self):
|
||||
exporter = StatsToCsv()
|
||||
skeleton = {'a': 'b', 'c': 'd'}
|
||||
paths = exporter.get_keys_paths(skeleton)
|
||||
self.assertListEqual([['a'], ['c']], paths)
|
||||
|
||||
skeleton = {'a': {'e': 'f', 'g': None}}
|
||||
paths = exporter.get_keys_paths(skeleton)
|
||||
self.assertListEqual([['a', 'e'], ['a', 'g']], paths)
|
||||
|
||||
skeleton = [{'a': 'b', 'c': 'd'}]
|
||||
paths = exporter.get_keys_paths(skeleton)
|
||||
self.assertListEqual([[]], paths)
|
||||
|
||||
def test_get_flatten_data(self):
|
||||
exporter = StatsToCsv()
|
||||
data = [
|
||||
{'a': 'b', 'c': {'e': 2.1}},
|
||||
{'a': 'ee\nxx', 'c': {'e': 3.1415}, 'x': ['z', 'zz']},
|
||||
]
|
||||
expected_flatten_data = [
|
||||
['b', 2.1, None],
|
||||
['ee\nxx', 3.1415, 'z zz'],
|
||||
]
|
||||
skeleton = exporter.get_data_skeleton(data)
|
||||
key_paths = exporter.get_keys_paths(skeleton)
|
||||
|
||||
for idx, expected in enumerate(expected_flatten_data):
|
||||
actual = exporter.get_flatten_data(key_paths, data[idx])
|
||||
self.assertListEqual(expected, actual)
|
||||
|
||||
def test_get_cluster_keys_paths(self):
|
||||
exporter = StatsToCsv()
|
||||
_, _, csv_keys_paths = exporter.get_cluster_keys_paths()
|
||||
self.assertTrue(['nodes_platform_name_gt3' in csv_keys_paths])
|
||||
self.assertTrue(['nodes_platform_name_0' in csv_keys_paths])
|
||||
self.assertTrue(['nodes_platform_name_1' in csv_keys_paths])
|
||||
self.assertTrue(['nodes_platform_name_2' in csv_keys_paths])
|
||||
self.assertTrue(['manufacturer_gt3' in csv_keys_paths])
|
||||
self.assertTrue(['manufacturer_0' in csv_keys_paths])
|
||||
self.assertTrue(['manufacturer_1' in csv_keys_paths])
|
||||
self.assertTrue(['manufacturer_2' in csv_keys_paths])
|
||||
self.assertTrue(['attributes', 'heat'] in csv_keys_paths)
|
||||
|
||||
def test_align_enumerated_field_values(self):
|
||||
# Data for checks in format (source, num, expected)
|
||||
checks = [
|
||||
([], 0, [False]),
|
||||
([], 1, [False, None]),
|
||||
(['a'], 1, [False, 'a']),
|
||||
(['a'], 2, [False, 'a', None]),
|
||||
(['a', 'b'], 2, [False, 'a', 'b']),
|
||||
(['a', 'b'], 1, [True, 'a'])
|
||||
]
|
||||
for source, num, expected in checks:
|
||||
self.assertListEqual(
|
||||
expected,
|
||||
StatsToCsv.align_enumerated_field_values(source, num)
|
||||
)
|
||||
|
||||
|
||||
class StatsToCsvExportTest(ElasticTest):
|
||||
|
||||
def test_new_param_handled_by_structures_skeleton(self):
|
||||
installations_num = 5
|
||||
self.generate_data(installations_num=installations_num)
|
||||
|
||||
# Mixing new pram into structures
|
||||
es_client = ElasticSearchClient()
|
||||
structures = es_client.get_structures()
|
||||
self.assertTrue(isinstance(structures, types.GeneratorType))
|
||||
structures = list(structures)
|
||||
structures[-1]['mixed_param'] = 'xx'
|
||||
|
||||
exporter = StatsToCsv()
|
||||
skeleton = exporter.get_data_skeleton(structures)
|
||||
self.assertTrue('mixed_param' in skeleton)
|
||||
|
||||
def test_get_flatten_clusters(self):
|
||||
installations_num = 200
|
||||
self.generate_data(installations_num=installations_num)
|
||||
es_client = ElasticSearchClient()
|
||||
structures = es_client.get_structures()
|
||||
|
||||
exporter = StatsToCsv()
|
||||
structure_paths, cluster_paths, csv_paths = \
|
||||
exporter.get_cluster_keys_paths()
|
||||
flatten_clusters = exporter.get_flatten_clusters(structure_paths,
|
||||
cluster_paths,
|
||||
structures)
|
||||
self.assertTrue(isinstance(flatten_clusters, types.GeneratorType))
|
||||
for flatten_cluster in flatten_clusters:
|
||||
self.assertEquals(len(csv_paths), len(flatten_cluster))
|
||||
|
||||
def test_flatten_data_as_csv(self):
|
||||
installations_num = 100
|
||||
self.generate_data(installations_num=installations_num)
|
||||
es_client = ElasticSearchClient()
|
||||
structures = es_client.get_structures()
|
||||
|
||||
exporter = StatsToCsv()
|
||||
structure_paths, cluster_paths, csv_paths = \
|
||||
exporter.get_cluster_keys_paths()
|
||||
flatten_clusters = exporter.get_flatten_clusters(structure_paths,
|
||||
cluster_paths,
|
||||
structures)
|
||||
self.assertTrue(isinstance(flatten_clusters, types.GeneratorType))
|
||||
result = exporter.flatten_data_as_csv(csv_paths, flatten_clusters)
|
||||
self.assertTrue(isinstance(result, types.GeneratorType))
|
||||
output = six.StringIO(list(result))
|
||||
reader = csv.reader(output)
|
||||
columns = reader.next()
|
||||
|
||||
# Checking enumerated columns are present in the output
|
||||
self.assertIn('nodes_manufacturer_0', columns)
|
||||
self.assertIn('nodes_manufacturer_gt3', columns)
|
||||
self.assertIn('nodes_platform_name_0', columns)
|
||||
self.assertIn('nodes_platform_name_gt3', columns)
|
||||
|
||||
# Checking reading result CSV
|
||||
for _ in reader:
|
||||
pass
|
||||
|
||||
def test_export_clusters(self):
|
||||
installations_num = 100
|
||||
self.generate_data(installations_num=installations_num)
|
||||
|
||||
es_client = ElasticSearchClient()
|
||||
structures = es_client.get_structures()
|
||||
exporter = StatsToCsv()
|
||||
result = exporter.export_clusters(structures)
|
||||
self.assertTrue(isinstance(result, types.GeneratorType))
|
|
@ -0,0 +1,41 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from unittest2.case import TestCase
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api.log import init_logger
|
||||
|
||||
# Configuring app for the test environment
|
||||
app.config.from_object('fuel_analytics.api.config.Testing')
|
||||
init_logger()
|
||||
|
||||
from migration.test.base import ElasticTest as MigrationElasticTest
|
||||
|
||||
|
||||
class BaseTest(TestCase):
|
||||
|
||||
def setUp(self):
|
||||
super(BaseTest, self).setUp()
|
||||
self.client = app.test_client()
|
||||
|
||||
def check_response_ok(self, resp, codes=(200, 201)):
|
||||
self.assertIn(resp.status_code, codes)
|
||||
|
||||
def check_response_error(self, resp, code):
|
||||
self.assertEquals(code, resp.status_code)
|
||||
|
||||
|
||||
class ElasticTest(MigrationElasticTest):
|
||||
pass
|
|
@ -0,0 +1,22 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from fuel_analytics.test.base import BaseTest
|
||||
|
||||
|
||||
class TestCommon(BaseTest):
|
||||
|
||||
def test_unknown_resource(self):
|
||||
resp = self.client.get('/xxx')
|
||||
self.check_response_error(resp, 404)
|
|
@ -0,0 +1,40 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from flask_script import Manager
|
||||
|
||||
from fuel_analytics.api import log
|
||||
from fuel_analytics.api.app import app
|
||||
|
||||
|
||||
def configure_app(mode=None):
|
||||
mode_map = {
|
||||
'test': 'analytics.api.config.Testing',
|
||||
'prod': 'analytics.api.config.Production'
|
||||
}
|
||||
app.config.from_object(mode_map.get(mode))
|
||||
app.config.from_envvar('ANALYTICS_SETTINGS', silent=True)
|
||||
log.init_logger()
|
||||
return app
|
||||
|
||||
|
||||
manager = Manager(configure_app)
|
||||
manager.add_option('--mode', help="Acceptable modes. Default: 'test'",
|
||||
choices=('test', 'prod'), default='prod', dest='mode')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
manager.run()
|
|
@ -0,0 +1,3 @@
|
|||
elasticsearch==1.2.0
|
||||
Flask==0.10.1
|
||||
Flask-Script==2.0.5
|
|
@ -0,0 +1,55 @@
|
|||
# Copyright 2014 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import os
|
||||
|
||||
from setuptools import find_packages
|
||||
from setuptools import setup
|
||||
|
||||
|
||||
def parse_requirements_txt():
|
||||
root = os.path.dirname(os.path.abspath(__file__))
|
||||
requirements = []
|
||||
with open(os.path.join(root, 'requirements.txt'), 'r') as f:
|
||||
for line in f.readlines():
|
||||
line = line.rstrip()
|
||||
if not line or line.startswith('#'):
|
||||
continue
|
||||
requirements.append(line)
|
||||
return requirements
|
||||
|
||||
|
||||
setup(
|
||||
name='analytics',
|
||||
version='0.0.1',
|
||||
description="Service of analytics reports",
|
||||
long_description="""Service of analytics reports""",
|
||||
license="http://www.apache.org/licenses/LICENSE-2.0",
|
||||
classifiers=[
|
||||
"License :: OSI Approved :: Apache Software License",
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Programming Language :: Python",
|
||||
"Topic :: Internet :: WWW/HTTP",
|
||||
"Topic :: Internet :: WWW/HTTP :: WSGI :: Application",
|
||||
],
|
||||
author='Mirantis Inc.',
|
||||
author_email='product@mirantis.com',
|
||||
url='https://mirantis.com',
|
||||
keywords='fuel statistics analytics mirantis',
|
||||
packages=find_packages(),
|
||||
zip_safe=False,
|
||||
install_requires=parse_requirements_txt(),
|
||||
include_package_data=True,
|
||||
scripts=['manage_analytics.py']
|
||||
)
|
|
@ -0,0 +1,11 @@
|
|||
-r requirements.txt
|
||||
hacking==0.9.2
|
||||
mock==1.0.1
|
||||
nose==1.3.4
|
||||
nose2==0.4.7
|
||||
tox==1.8.0
|
||||
unittest2==0.5.1
|
||||
# required for use tests from migration
|
||||
SQLAlchemy==0.9.8
|
||||
psycopg2==2.5.4
|
||||
|
|
@ -0,0 +1,44 @@
|
|||
[tox]
|
||||
minversion = 1.6
|
||||
skipsdist = True
|
||||
envlist = py27,pep8
|
||||
|
||||
[testenv]
|
||||
usedevelop = True
|
||||
install_command = pip install {packages}
|
||||
setenv =
|
||||
VIRTUAL_ENV={envdir}
|
||||
PYTHONPATH={toxinidir}/../migration
|
||||
deps = -r{toxinidir}/test-requirements.txt
|
||||
commands =
|
||||
nosetests {posargs:fuel_analytics/test}
|
||||
|
||||
[tox:jenkins]
|
||||
downloadcache = ~/cache/pip
|
||||
|
||||
[testenv:pep8]
|
||||
deps = hacking==0.7
|
||||
usedevelop = False
|
||||
commands =
|
||||
flake8 {posargs:fuel_analytics}
|
||||
|
||||
[testenv:cover]
|
||||
setenv = NOSE_WITH_COVERAGE=1
|
||||
|
||||
[testenv:venv]
|
||||
deps = -r{toxinidir}/test-requirements.txt
|
||||
commands = {posargs:}
|
||||
|
||||
[testenv:devenv]
|
||||
envdir = devenv
|
||||
usedevelop = True
|
||||
|
||||
[flake8]
|
||||
ignore = H234,H302,H802
|
||||
exclude = .venv,.git,.tox,dist,doc,*lib/python*,*egg,build,tools,__init__.py,docs
|
||||
show-pep8 = True
|
||||
show-source = True
|
||||
count = True
|
||||
|
||||
[hacking]
|
||||
import_exceptions = testtools.matchers
|
|
@ -0,0 +1,7 @@
|
|||
uwsgi:
|
||||
socket: :8082
|
||||
# for production app use analytics.api.app as module
|
||||
module: fuel_analytics.api.app_test
|
||||
callable: app
|
||||
protocol: http
|
||||
env: ANALYTICS_SETTINGS=/path/to/external/config.py
|
|
@ -7,3 +7,4 @@ PyYAML==3.11
|
|||
alembic==0.6.7
|
||||
psycopg2==2.5.4
|
||||
six>=1.8.0
|
||||
uWSGI==2.0.9
|
|
@ -77,7 +77,10 @@ class ElasticTest(TestCase):
|
|||
'zabbix', 'mongo'),
|
||||
oses=('Ubuntu', 'CentOs', 'Ubuntu LTS XX'),
|
||||
node_statuses = ('ready', 'discover', 'provisioning',
|
||||
'provisioned', 'deploying', 'error')
|
||||
'provisioned', 'deploying', 'error'),
|
||||
manufacturers = ('Dell Inc.', 'VirtualBox', 'QEMU',
|
||||
'VirtualBox', 'Supermicro', 'Cisco Systems Inc',
|
||||
'KVM', 'VMWARE', 'HP')
|
||||
):
|
||||
roles = []
|
||||
for _ in xrange(random.randint(*roles_range)):
|
||||
|
@ -86,7 +89,8 @@ class ElasticTest(TestCase):
|
|||
'id': self.gen_id(),
|
||||
'roles': roles,
|
||||
'os': random.choice(oses),
|
||||
'status': random.choice(node_statuses)
|
||||
'status': random.choice(node_statuses),
|
||||
'manufacturer': random.choice(manufacturers)
|
||||
}
|
||||
return node
|
||||
|
||||
|
|
Loading…
Reference in New Issue