Elasticsearch removing from fuel-stats analytics
We don't use Elasticsearch for flexible reports generation on the fuel-stats web UI, only for five fixed reports. Thus using of Elasticsearch is overhead and it can be removed from the servers Instead of Elasticsearch we use fuel-stats json api calls and PostgreSQL + Memcached. Changes list: - api call added to fuel-stats json api for data required on the web UI page, - column release added to DB installation_structures table schema, - memcached is used for caching data for the web UI page, - elasticsearch client removed from js requirement, - web UI page rewritten to use fuel-stats json api instead Elaticsearch. Co-Authored-By: Kate Pimenova <kpimenova@mirantis.com> Change-Id: Ie752e0d0a3c80933888f986e2497b45adce730c9 Closes-Bug: #1595548
This commit is contained in:
Alexander Kislitsky
parent
78abf5f70c
commit
8e16219249
20
README.rst
20
README.rst
|
|
@ -203,6 +203,26 @@ Revert all migrations: ::
|
|||
python manage_collector.py --mode test db downgrade -d collector/api/db/migrations/
|
||||
|
||||
|
||||
Switching off Elasticsearch
|
||||
---------------------------
|
||||
|
||||
Elasticsearch was chosen as data storage for the dynamically generated
|
||||
statistics reports, but now only CSV reports are used for analytical purposes.
|
||||
Thus, Elasticsearch is an unnecessary complication of the infrastructure and
|
||||
data flow.
|
||||
|
||||
Without Elasticsearch, we are using memcached as cache for the web UI. Data
|
||||
expiration is configured by the parameter MEMCACHED_JSON_REPORTS_EXPIRATION
|
||||
for fuel_analytics.
|
||||
|
||||
Changes in the Nginx config: ::
|
||||
|
||||
# Add this to the block 'server'
|
||||
location /api/ {
|
||||
proxy_pass http://IP_OF_ANALYTICS_SERVICE:PORT_OF_ANALYTICS_SERVICE/api/;
|
||||
}
|
||||
|
||||
|
||||
.. _Fuel: https://docs.mirantis.com/openstack/fuel/
|
||||
.. _Elasticsearch: https://www.elastic.co/
|
||||
.. _uWSGI: https://pypi.python.org/pypi/uWSGI/
|
||||
|
|
|
|||
|
|
@ -26,9 +26,11 @@ db = flask_sqlalchemy.SQLAlchemy(app)
|
|||
# Registering blueprints
|
||||
from fuel_analytics.api.resources.csv_exporter import bp as csv_exporter_bp
|
||||
from fuel_analytics.api.resources.json_exporter import bp as json_exporter_bp
|
||||
from fuel_analytics.api.resources.json_reports import bp as json_reports_bp
|
||||
|
||||
app.register_blueprint(csv_exporter_bp, url_prefix='/api/v1/csv')
|
||||
app.register_blueprint(json_exporter_bp, url_prefix='/api/v1/json')
|
||||
app.register_blueprint(json_reports_bp, url_prefix='/api/v1/json/report')
|
||||
|
||||
|
||||
@app.errorhandler(DateExtractionError)
|
||||
|
|
|
|||
|
|
@ -32,7 +32,10 @@ class Production(object):
|
|||
CSV_DEFAULT_FROM_DATE_DAYS = 90
|
||||
CSV_DB_YIELD_PER = 100
|
||||
JSON_DB_DEFAULT_LIMIT = 1000
|
||||
JSON_DB_YIELD_PER = 100
|
||||
CSV_DEFAULT_LIST_ITEMS_NUM = 5
|
||||
MEMCACHED_HOSTS = ['localhost:11211']
|
||||
MEMCACHED_JSON_REPORTS_EXPIRATION = 3600
|
||||
|
||||
|
||||
class Testing(Production):
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ class InstallationStructure(db.Model):
|
|||
creation_date = db.Column(db.DateTime)
|
||||
modification_date = db.Column(db.DateTime)
|
||||
is_filtered = db.Column(db.Boolean)
|
||||
release = db.Column(db.Text)
|
||||
|
||||
|
||||
class ActionLog(db.Model):
|
||||
|
|
|
|||
|
|
@ -0,0 +1,151 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from collections import defaultdict
|
||||
import copy
|
||||
import json
|
||||
|
||||
from flask import Blueprint
|
||||
from flask import request
|
||||
from flask import Response
|
||||
import memcache
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api.app import db
|
||||
from fuel_analytics.api.db.model import InstallationStructure as IS
|
||||
|
||||
bp = Blueprint('reports', __name__)
|
||||
|
||||
|
||||
@bp.route('/installations', methods=['GET'])
|
||||
def get_installations_info():
|
||||
release = request.args.get('release')
|
||||
refresh = request.args.get('refresh')
|
||||
cache_key_prefix = 'fuel-stats-installations-info'
|
||||
mc = memcache.Client(app.config.get('MEMCACHED_HOSTS'))
|
||||
app.logger.debug("Fetching installations info for release: %s", release)
|
||||
|
||||
# Checking cache
|
||||
if not refresh:
|
||||
cache_key = '{0}{1}'.format(cache_key_prefix, release)
|
||||
app.logger.debug("Checking installations info by key: %s in cache",
|
||||
cache_key)
|
||||
cached_result = mc.get(cache_key)
|
||||
if cached_result:
|
||||
app.logger.debug("Installations info cache found by key: %s",
|
||||
cache_key)
|
||||
return Response(cached_result, mimetype='application/json')
|
||||
else:
|
||||
app.logger.debug("No cached installations info for key: %s",
|
||||
cache_key)
|
||||
else:
|
||||
app.logger.debug("Enforce refresh cache of installations info "
|
||||
"for release: %s", release)
|
||||
|
||||
# Fetching data from DB
|
||||
info_from_db = get_installations_info_from_db(release)
|
||||
|
||||
# Saving fetched data to cache
|
||||
for for_release, info in info_from_db.items():
|
||||
cache_key = '{0}{1}'.format(cache_key_prefix, for_release)
|
||||
app.logger.debug("Caching installations info for key: %s, data: %s",
|
||||
cache_key, info)
|
||||
mc.set(cache_key, json.dumps(info),
|
||||
app.config.get('MEMCACHED_JSON_REPORTS_EXPIRATION'))
|
||||
|
||||
return Response(json.dumps(info_from_db[release]),
|
||||
mimetype='application/json')
|
||||
|
||||
|
||||
def get_installations_info_from_db(release):
|
||||
query = db.session.query(IS.structure, IS.release).\
|
||||
filter(IS.is_filtered == bool(0))
|
||||
if release:
|
||||
query = query.filter(IS.release == release)
|
||||
|
||||
info_template = {
|
||||
'installations': {
|
||||
'count': 0,
|
||||
'environments_num': defaultdict(int)
|
||||
},
|
||||
'environments': {
|
||||
'count': 0,
|
||||
'operable_envs_count': 0,
|
||||
'statuses': defaultdict(int),
|
||||
'nodes_num': defaultdict(int),
|
||||
'hypervisors_num': defaultdict(int),
|
||||
'oses_num': defaultdict(int)
|
||||
}
|
||||
}
|
||||
|
||||
info = defaultdict(lambda: copy.deepcopy(info_template))
|
||||
|
||||
app.logger.debug("Fetching installations info from DB for release: %s",
|
||||
release)
|
||||
|
||||
yield_per = app.config['JSON_DB_YIELD_PER']
|
||||
for row in query.yield_per(yield_per):
|
||||
structure = row[0]
|
||||
extract_installation_info(structure, info[release])
|
||||
|
||||
cur_release = row[1]
|
||||
# Splitting info by release if fetching for all releases
|
||||
if not release and cur_release != release:
|
||||
extract_installation_info(structure, info[cur_release])
|
||||
|
||||
app.logger.debug("Fetched installations info from DB for release: "
|
||||
"%s, info: %s", release, info)
|
||||
|
||||
return info
|
||||
|
||||
|
||||
def extract_installation_info(source, result):
|
||||
"""Extracts installation info from structure
|
||||
|
||||
:param source: source of installation info data
|
||||
:type source: dict
|
||||
:param result: placeholder for extracted data
|
||||
:type result: dict
|
||||
"""
|
||||
|
||||
inst_info = result['installations']
|
||||
env_info = result['environments']
|
||||
|
||||
production_statuses = ('operational', 'error')
|
||||
|
||||
inst_info['count'] += 1
|
||||
envs_num = 0
|
||||
|
||||
for cluster in source.get('clusters', []):
|
||||
envs_num += 1
|
||||
env_info['count'] += 1
|
||||
|
||||
if cluster.get('status') in production_statuses:
|
||||
current_nodes_num = cluster.get('nodes_num', 0)
|
||||
env_info['nodes_num'][current_nodes_num] += 1
|
||||
env_info['operable_envs_count'] += 1
|
||||
|
||||
hypervisor = cluster.get('attributes', {}).get('libvirt_type')
|
||||
if hypervisor:
|
||||
env_info['hypervisors_num'][hypervisor.lower()] += 1
|
||||
|
||||
os = cluster.get('release', {}).get('os')
|
||||
if os:
|
||||
env_info['oses_num'][os.lower()] += 1
|
||||
|
||||
status = cluster.get('status')
|
||||
if status is not None:
|
||||
env_info['statuses'][status] += 1
|
||||
|
||||
inst_info['environments_num'][envs_num] += 1
|
||||
|
|
@ -0,0 +1,527 @@
|
|||
# Copyright 2015 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
import memcache
|
||||
import mock
|
||||
|
||||
from fuel_analytics.test.base import DbTest
|
||||
|
||||
from fuel_analytics.api.app import app
|
||||
from fuel_analytics.api.app import db
|
||||
from fuel_analytics.api.db import model
|
||||
|
||||
|
||||
class JsonReportsTest(DbTest):
|
||||
|
||||
@mock.patch.object(memcache.Client, 'get', return_value=None)
|
||||
def test_get_installations_num(self, _):
|
||||
structures = [
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x0',
|
||||
structure={},
|
||||
is_filtered=False,
|
||||
release='9.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x1',
|
||||
structure={},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x2',
|
||||
structure={},
|
||||
is_filtered=True,
|
||||
release='8.0'
|
||||
),
|
||||
]
|
||||
for structure in structures:
|
||||
db.session.add(structure)
|
||||
db.session.flush()
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(2, resp['installations']['count'])
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=8.0'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(1, resp['installations']['count'])
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=xxx'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(0, resp['installations']['count'])
|
||||
|
||||
@mock.patch.object(memcache.Client, 'get', return_value=None)
|
||||
def test_get_env_statuses(self, _):
|
||||
structures = [
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x0',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'new'},
|
||||
{'status': 'operational'},
|
||||
{'status': 'error'}
|
||||
]
|
||||
},
|
||||
is_filtered=False,
|
||||
release='9.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x1',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'deployment'},
|
||||
{'status': 'operational'},
|
||||
{'status': 'operational'},
|
||||
]
|
||||
},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x2',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'deployment'},
|
||||
{'status': 'operational'},
|
||||
]
|
||||
},
|
||||
is_filtered=True,
|
||||
release='8.0'
|
||||
),
|
||||
]
|
||||
for structure in structures:
|
||||
db.session.add(structure)
|
||||
db.session.flush()
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(
|
||||
{'new': 1, 'deployment': 1, 'error': 1, 'operational': 3},
|
||||
resp['environments']['statuses']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=8.0'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(
|
||||
{'deployment': 1, 'operational': 2},
|
||||
resp['environments']['statuses']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=xxx'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual({}, resp['environments']['statuses'])
|
||||
|
||||
@mock.patch.object(memcache.Client, 'get', return_value=None)
|
||||
def test_get_env_num(self, _):
|
||||
structures = [
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x0',
|
||||
structure={'clusters': [{}, {}, {}]},
|
||||
is_filtered=False,
|
||||
release='9.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x1',
|
||||
structure={'clusters': [{}, {}]},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x2',
|
||||
structure={'clusters': []},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x3',
|
||||
structure={'clusters': []},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x4',
|
||||
structure={'clusters': [{}, {}, {}]},
|
||||
is_filtered=True,
|
||||
release='8.0'
|
||||
),
|
||||
]
|
||||
for structure in structures:
|
||||
db.session.add(structure)
|
||||
db.session.flush()
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(5, resp['environments']['count'])
|
||||
self.assertEqual(
|
||||
{'0': 2, '2': 1, '3': 1},
|
||||
resp['installations']['environments_num']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=8.0'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(2, resp['environments']['count'])
|
||||
self.assertEqual(
|
||||
{'0': 2, '2': 1},
|
||||
resp['installations']['environments_num']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=xxx'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(0, resp['environments']['count'])
|
||||
self.assertEqual({}, resp['installations']['environments_num'])
|
||||
|
||||
@mock.patch.object(memcache.Client, 'set')
|
||||
@mock.patch.object(memcache.Client, 'get', return_value=None)
|
||||
def test_caching(self, cached_mc_get, cached_mc_set):
|
||||
structures = [
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x0',
|
||||
structure={'clusters': [{}, {}, {}]},
|
||||
is_filtered=False,
|
||||
release='9.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x1',
|
||||
structure={'clusters': [{}, {}]},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x2',
|
||||
structure={'clusters': [{}]},
|
||||
is_filtered=True,
|
||||
release='8.0'
|
||||
)
|
||||
]
|
||||
for structure in structures:
|
||||
db.session.add(structure)
|
||||
db.session.flush()
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
self.assertEqual(1, cached_mc_get.call_count)
|
||||
|
||||
# Checking that mc.set was called for each release and
|
||||
# for all releases summary info
|
||||
|
||||
# Mock call args have structure (args, kwargs)
|
||||
expected_cache_keys = ['fuel-stats-installations-info8.0',
|
||||
'fuel-stats-installations-info9.0',
|
||||
'fuel-stats-installations-infoNone']
|
||||
|
||||
actual_cache_keys = [call_args[0][0] for call_args in
|
||||
cached_mc_set.call_args_list]
|
||||
self.assertItemsEqual(expected_cache_keys, actual_cache_keys)
|
||||
|
||||
# cached_mc_set.assert_has_calls(calls, any_order=True)
|
||||
self.assertEqual(len(expected_cache_keys),
|
||||
cached_mc_set.call_count)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=8.0'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
self.assertEqual(2, cached_mc_get.call_count)
|
||||
self.assertEqual(4, cached_mc_set.call_count)
|
||||
expected_cache_key = 'fuel-stats-installations-info8.0'
|
||||
actual_cache_key = cached_mc_set.call_args[0][0]
|
||||
self.assertEqual(expected_cache_key, actual_cache_key)
|
||||
|
||||
@mock.patch.object(memcache.Client, 'set')
|
||||
@mock.patch.object(memcache.Client, 'get')
|
||||
def test_refresh_cached_data(self, cached_mc_get, cached_mc_set):
|
||||
structures = [
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x0',
|
||||
structure={'clusters': [{}, {}, {}]},
|
||||
is_filtered=False,
|
||||
release='9.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x1',
|
||||
structure={'clusters': [{}, {}]},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x2',
|
||||
structure={'clusters': [{}]},
|
||||
is_filtered=True,
|
||||
release='8.0'
|
||||
)
|
||||
]
|
||||
for structure in structures:
|
||||
db.session.add(structure)
|
||||
db.session.flush()
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?refresh=1'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
self.assertEqual(0, cached_mc_get.call_count)
|
||||
self.assertEquals(3, cached_mc_set.call_count)
|
||||
|
||||
@mock.patch.object(memcache.Client, 'get', return_value=None)
|
||||
def test_get_nodes_num(self, _):
|
||||
structures = [
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x0',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'operational', 'nodes_num': 3},
|
||||
{'status': 'new', 'nodes_num': 2},
|
||||
{'status': 'error', 'nodes_num': 1},
|
||||
]
|
||||
},
|
||||
is_filtered=False,
|
||||
release='9.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x1',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'operational', 'nodes_num': 3}
|
||||
],
|
||||
},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x2',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'operational', 'nodes_num': 5},
|
||||
{'status': 'new', 'nodes_num': 6},
|
||||
{'status': 'error', 'nodes_num': 7},
|
||||
]
|
||||
},
|
||||
is_filtered=True,
|
||||
release='8.0'
|
||||
),
|
||||
]
|
||||
for structure in structures:
|
||||
db.session.add(structure)
|
||||
db.session.flush()
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(3, resp['environments']['operable_envs_count'])
|
||||
self.assertEqual(
|
||||
{'3': 2, '1': 1},
|
||||
resp['environments']['nodes_num']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=9.0'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(2, resp['environments']['operable_envs_count'])
|
||||
self.assertEqual(
|
||||
{'3': 1, '1': 1},
|
||||
resp['environments']['nodes_num']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=xxx'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(0, resp['environments']['operable_envs_count'])
|
||||
self.assertEqual({}, resp['environments']['nodes_num'])
|
||||
|
||||
@mock.patch.object(memcache.Client, 'get', return_value=None)
|
||||
def test_get_hypervisors_num(self, _):
|
||||
structures = [
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x0',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'operational', 'attributes':
|
||||
{'libvirt_type': 'kvm'}},
|
||||
{'status': 'operational', 'attributes':
|
||||
{'libvirt_type': 'Qemu'}},
|
||||
{'status': 'operational', 'attributes':
|
||||
{'libvirt_type': 'Kvm'}},
|
||||
{'status': 'new', 'attributes':
|
||||
{'libvirt_type': 'kvm'}},
|
||||
{'status': 'error', 'attributes':
|
||||
{'libvirt_type': 'qemu'}},
|
||||
]
|
||||
},
|
||||
is_filtered=False,
|
||||
release='9.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x1',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'new', 'attributes':
|
||||
{'libvirt_type': 'qemu'}},
|
||||
{'status': 'error', 'attributes':
|
||||
{'libvirt_type': 'Kvm'}},
|
||||
{'status': 'error', 'attributes':
|
||||
{'libvirt_type': 'vcenter'}},
|
||||
],
|
||||
},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x2',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'operational', 'attributes':
|
||||
{'libvirt_type': 'kvm'}},
|
||||
{'status': 'new', 'attributes':
|
||||
{'libvirt_type': 'kvm'}},
|
||||
{'status': 'error', 'attributes':
|
||||
{'libvirt_type': 'qemu'}},
|
||||
]
|
||||
},
|
||||
is_filtered=True,
|
||||
release='8.0'
|
||||
),
|
||||
]
|
||||
for structure in structures:
|
||||
db.session.add(structure)
|
||||
db.session.flush()
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(
|
||||
{'kvm': 3, 'vcenter': 1, 'qemu': 2},
|
||||
resp['environments']['hypervisors_num']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=8.0'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(
|
||||
{'kvm': 1, 'vcenter': 1},
|
||||
resp['environments']['hypervisors_num']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=xxx'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual({}, resp['environments']['hypervisors_num'])
|
||||
|
||||
@mock.patch.object(memcache.Client, 'get', return_value=None)
|
||||
def test_get_oses_num(self, _):
|
||||
structures = [
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x0',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'operational', 'release': {'os': 'Ubuntu'}},
|
||||
{'status': 'error', 'release': {'os': 'ubuntu'}},
|
||||
{'status': 'error', 'release': {'os': 'Centos'}}
|
||||
]
|
||||
},
|
||||
is_filtered=False,
|
||||
release='9.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x1',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'new', 'release': {'os': 'Ubuntu'}},
|
||||
{'status': 'operational', 'release': {'os': 'ubuntu'}}
|
||||
],
|
||||
},
|
||||
is_filtered=False,
|
||||
release='8.0'
|
||||
),
|
||||
model.InstallationStructure(
|
||||
master_node_uid='x2',
|
||||
structure={
|
||||
'clusters': [
|
||||
{'status': 'new', 'release': {'os': 'centos'}},
|
||||
{'status': 'operational', 'release': {'os': 'centos'}},
|
||||
{'status': 'operational', 'release': {'os': 'centos'}}
|
||||
]
|
||||
},
|
||||
is_filtered=True,
|
||||
release='8.0'
|
||||
),
|
||||
]
|
||||
for structure in structures:
|
||||
db.session.add(structure)
|
||||
db.session.flush()
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual(
|
||||
{'ubuntu': 3, 'centos': 1},
|
||||
resp['environments']['oses_num']
|
||||
)
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=8.0'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual({'ubuntu': 1}, resp['environments']['oses_num'])
|
||||
|
||||
with app.test_request_context():
|
||||
url = '/api/v1/json/report/installations?release=xxx'
|
||||
resp = self.client.get(url)
|
||||
self.check_response_ok(resp)
|
||||
resp = json.loads(resp.data)
|
||||
self.assertEqual({}, resp['environments']['oses_num'])
|
||||
|
|
@ -2,5 +2,6 @@ psycopg2==2.5.4
|
|||
Flask==0.10.1
|
||||
Flask-Script==2.0.5
|
||||
Flask-SQLAlchemy==2.0
|
||||
python-memcached>=1.56
|
||||
SQLAlchemy==0.9.8
|
||||
six>=1.8.0
|
||||
|
|
|
|||
|
|
@ -8,7 +8,6 @@
|
|||
"d3-tip": "0.6.3",
|
||||
"d3pie": "0.1.4",
|
||||
"nvd3": "1.1.15-beta",
|
||||
"elasticsearch": "3.0.0",
|
||||
"requirejs": "2.1.15"
|
||||
},
|
||||
"overrides": {
|
||||
|
|
|
|||
|
|
@ -41,6 +41,48 @@ html, body {
|
|||
background-color: #f3f3f4;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
#loader {
|
||||
background-color: #415766;
|
||||
position: absolute;
|
||||
top: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
#load-error {
|
||||
margin-left: 82px;
|
||||
padding-top: 20px;
|
||||
font-size: 20px;
|
||||
color: #F1594C;
|
||||
}
|
||||
@keyframes loading {
|
||||
from {transform:rotate(0deg);}
|
||||
to {transform:rotate(360deg);}
|
||||
}
|
||||
.loading:before {
|
||||
content: "";
|
||||
display: block;
|
||||
width: 98px;
|
||||
height: 98px;
|
||||
overflow: hidden;
|
||||
margin: 200px auto 0;
|
||||
background: url(../img/loader-bg.svg);
|
||||
animation: loading 6s linear infinite;
|
||||
}
|
||||
.loading:after {
|
||||
content: "";
|
||||
display: block;
|
||||
width: 98px;
|
||||
height: 98px;
|
||||
margin: -124px auto 0;
|
||||
position: relative;
|
||||
z-index: 9999;
|
||||
background: url(../img/loader-logo.svg);
|
||||
}
|
||||
|
||||
.hidden {
|
||||
display: none;
|
||||
}
|
||||
.nav-pannel {
|
||||
width: 70px;
|
||||
height: 100%;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 16.0.4, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
width="98px" height="98px" viewBox="0 0 98 98" enable-background="new 0 0 98 98" xml:space="preserve">
|
||||
<path fill="#FFFFFF" d="M97.498,43.062l-10.425-3.683c-0.236-0.085-0.455-0.282-0.52-0.526c-0.878-3.292-2.215-6.471-3.936-9.45
|
||||
c-0.061-0.104-0.101-0.221-0.108-0.337c0-0.114,0.015-0.228,0.064-0.334l4.716-9.935c-0.004-0.005-0.01-0.01-0.014-0.016
|
||||
l0.013-0.027c-2.438-3.114-5.248-5.925-8.36-8.368l-9.925,4.723c-0.104,0.049-0.217,0.073-0.33,0.073
|
||||
c-0.133,0-0.244-0.034-0.362-0.104C65.678,13.555,63,12.343,60,11.474v0.063c0-0.111-0.758-0.237-1.133-0.337
|
||||
c-0.242-0.064-0.449-0.246-0.534-0.484l-3.686-10.36C52.688,0.12,50.7,0,48.736,0c-1.966,0-3.952,0.12-5.912,0.356l-3.681,10.361
|
||||
c-0.084,0.238-0.036,0.418-0.279,0.485C38.491,11.302,38,11.428,38,11.539v-0.064c-3,0.869-5.944,2.08-8.578,3.605
|
||||
c-0.118,0.069-0.374,0.104-0.507,0.104c-0.111,0-0.287-0.025-0.391-0.075l-9.957-4.722c-3.11,2.443-5.934,5.254-8.375,8.369
|
||||
l0.006,0.027c-0.004,0.005-0.013,0.01-0.017,0.016l4.716,9.935c0.051,0.106,0.068,0.222,0.068,0.335
|
||||
c-0.008,0.116-0.038,0.231-0.098,0.335c-1.721,2.976-3.039,6.156-3.917,9.451c-0.066,0.244-0.247,0.441-0.484,0.526l-10.23,3.684
|
||||
C-0.001,45.019,0,47.007,0,48.978C0,48.985,0,48.992,0,49s0,0.015,0,0.022c0,1.967,0,3.954,0.235,5.915l10.291,3.684
|
||||
c0.236,0.085,0.389,0.281,0.454,0.525c0.878,3.292,2.181,6.472,3.902,9.45c0.061,0.104,0.083,0.22,0.09,0.337
|
||||
c0,0.113-0.021,0.229-0.071,0.334l-4.721,9.936c0.004,0.005,0.008,0.011,0.012,0.016l-0.014,0.027
|
||||
c2.438,3.113,5.247,5.925,8.359,8.368l9.925-4.724c0.104-0.049,0.217-0.073,0.33-0.073c0.133,0,0.512,0.034,0.63,0.104
|
||||
C32.056,84.444,35,85.657,38,86.526v-0.064c0,0.112,0.491,0.238,0.865,0.338c0.243,0.064,0.316,0.246,0.401,0.483l3.618,10.361
|
||||
C44.844,97.88,46.799,98,48.763,98c1.965,0,3.936-0.12,5.896-0.355l3.672-10.362c0.084-0.237,0.299-0.418,0.542-0.484
|
||||
c0.373-0.1,1.127-0.225,1.127-0.337v0.065c3-0.869,5.677-2.081,8.311-3.605c0.118-0.069,0.24-0.104,0.374-0.104
|
||||
c0.111,0,0.22,0.025,0.323,0.075l9.924,4.723c3.11-2.443,5.916-5.255,8.357-8.369l-0.013-0.027c0.004-0.005,0.008-0.011,0.012-0.016
|
||||
l-4.718-9.936c-0.051-0.106-0.069-0.221-0.068-0.335c0.007-0.116,0.037-0.231,0.097-0.335c1.721-2.976,3.038-6.157,3.917-9.451
|
||||
c0.066-0.244,0.247-0.44,0.484-0.525l10.498-3.685C97.734,52.981,98,50.993,98,49.022c0-0.008,0-0.015,0-0.022s0-0.015,0-0.022
|
||||
C98,47.011,97.732,45.023,97.498,43.062z"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.7 KiB |
|
|
@ -0,0 +1,17 @@
|
|||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Generator: Adobe Illustrator 16.0.4, SVG Export Plug-In . SVG Version: 6.00 Build 0) -->
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
|
||||
width="98px" height="98px" viewBox="0 0 98 98" enable-background="new 0 0 98 98" xml:space="preserve">
|
||||
<path fill="#415766" d="M79.271,36.387c-0.006-0.005-0.015-0.01-0.021-0.015l1.052-0.543c0,0-7.029-5.242-15.771-8.287L70.051,20
|
||||
H27.447l5.657,7.727c-8.439,2.993-15.26,8.305-15.26,8.305l3.488,1.804l7.17,3.755c0.104,0.055,0.219,0.083,0.331,0.083
|
||||
c0.068,0,0.13-0.04,0.196-0.06c2.789-1.803,5.8-3.227,8.971-4.234v2.48v4v18.064L23.572,93.822c-0.387,0.873-0.431,1.984,0.09,2.785
|
||||
S24.941,98,25.896,98H71.57c0.953,0,1.836-0.592,2.354-1.393c0.521-0.801,0.62-1.855,0.233-2.729L60,61.925V43.86v-4v-2.37
|
||||
c0.225,0.073,0.445,0.151,0.669,0.229c0.384,0.13,0.767,0.262,1.146,0.403c2.371,0.919,4.643,2.077,6.781,3.455
|
||||
c0.104,0.054,0.213,0.097,0.324,0.097c0.114,0,0.229-0.028,0.332-0.081l7.268-3.809l1.472-0.761l1.237-0.628l0.041,0.021
|
||||
c-0.014-0.011-0.025-0.022-0.041-0.033l0.02-0.009L79.271,36.387z"/>
|
||||
<circle fill="#DA3C3B" cx="44.682" cy="25.03" r="6.987"/>
|
||||
<polygon fill="#DA3C3B" points="57.486,62.521 57.486,37.451 40.661,37.451 40.661,62.521 26.237,94.964 71.91,94.964 "/>
|
||||
<circle fill="#DA3C3B" cx="57.676" cy="17.171" r="4.527"/>
|
||||
<circle fill="#DA3C3B" cx="47.525" cy="6.342" r="3.37"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.5 KiB |
|
|
@ -36,10 +36,18 @@
|
|||
</div>
|
||||
<!-- End Left Pannel -->
|
||||
|
||||
<div id="loader">
|
||||
<div class="loading"></div>
|
||||
</div>
|
||||
<div id="load-error" class="hidden">
|
||||
Error: Service is unavailable.
|
||||
</div>
|
||||
|
||||
<!-- Start Base Layout -->
|
||||
<div class="base-box">
|
||||
<select id="release-filter"></select>
|
||||
|
||||
<div id="main" class="hidden">
|
||||
<!-- TOP BIG GRAPH -->
|
||||
<div class="container-fluid titul-graph-box">
|
||||
<div class="row top-graph">
|
||||
|
|
@ -153,6 +161,7 @@
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -5,14 +5,11 @@ define(
|
|||
'd3',
|
||||
'd3pie',
|
||||
'd3tip',
|
||||
'nv',
|
||||
'elasticsearch'
|
||||
'nv'
|
||||
],
|
||||
function($, d3, D3pie, d3tip, nv, elasticsearch) {
|
||||
function($, d3, D3pie, d3tip, nv) {
|
||||
'use strict';
|
||||
|
||||
var statuses = ['operational', 'error'];
|
||||
|
||||
var releases = [
|
||||
{name: 'All', filter: ''},
|
||||
{name: '6.0 Technical Preview', filter: '6.0-techpreview'},
|
||||
|
|
@ -34,134 +31,70 @@ function($, d3, D3pie, d3tip, nv, elasticsearch) {
|
|||
statsPage();
|
||||
});
|
||||
|
||||
var applyFilters = function(body) {
|
||||
var result = body;
|
||||
if (currentRelease) {
|
||||
result = {
|
||||
aggs: {
|
||||
releases: {
|
||||
filter: {
|
||||
terms: {
|
||||
'fuel_release.release': [currentRelease]
|
||||
}
|
||||
},
|
||||
aggs: body.aggs
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
// adding filtering by is_filtered
|
||||
result = {
|
||||
aggs: {
|
||||
is_filtered: {
|
||||
filter: {
|
||||
bool: {
|
||||
should: [
|
||||
{term: {is_filtered: false}},
|
||||
{missing: {field: 'is_filtered'}}
|
||||
]
|
||||
}
|
||||
},
|
||||
aggs: result.aggs
|
||||
}
|
||||
}
|
||||
};
|
||||
return result;
|
||||
var showLoader = function() {
|
||||
$('#main').addClass('hidden');
|
||||
$('#loader').removeClass('hidden');
|
||||
};
|
||||
|
||||
var getRootData = function(resp) {
|
||||
var root = resp.aggregations.is_filtered;
|
||||
return currentRelease ? root.releases : root;
|
||||
};
|
||||
|
||||
var elasticSearchHost = function() {
|
||||
return {
|
||||
host: {
|
||||
port: location.port || (location.protocol == 'https:' ? 443 : 80),
|
||||
protocol: location.protocol,
|
||||
host: location.hostname
|
||||
}
|
||||
};
|
||||
var hideLoader = function() {
|
||||
$('#main').removeClass('hidden');
|
||||
$('#loader').addClass('hidden');
|
||||
};
|
||||
|
||||
var statsPage = function() {
|
||||
installationsCount();
|
||||
environmentsCount();
|
||||
distributionOfInstallations();
|
||||
nodesDistributionChart();
|
||||
hypervisorDistributionChart();
|
||||
osesDistributionChart();
|
||||
};
|
||||
var url = '/api/v1/json/report/installations';
|
||||
var data = {};
|
||||
|
||||
var installationsCount = function() {
|
||||
var client = new elasticsearch.Client(elasticSearchHost());
|
||||
var request = {
|
||||
query: {
|
||||
filtered: {
|
||||
filter: {
|
||||
bool: {
|
||||
should: [
|
||||
{term: {'is_filtered': false}},
|
||||
{missing: {'field': 'is_filtered'}},
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (currentRelease) {
|
||||
request.query.filtered.filter.bool['must'] = {
|
||||
terms: {'fuel_release.release': [currentRelease]}
|
||||
}
|
||||
data['release'] = currentRelease;
|
||||
}
|
||||
$('#load-error').addClass('hidden');
|
||||
showLoader();
|
||||
|
||||
client.count({
|
||||
index: 'fuel',
|
||||
type: 'structure',
|
||||
body: request
|
||||
}).then(function(resp) {
|
||||
$('#installations-count').html(resp.count);
|
||||
$.get(url, data, function(resp) {
|
||||
installationsCount(resp);
|
||||
environmentsCount(resp);
|
||||
distributionOfInstallations(resp);
|
||||
nodesDistributionChart(resp);
|
||||
hypervisorDistributionChart(resp);
|
||||
osesDistributionChart(resp);
|
||||
})
|
||||
.done(function() {
|
||||
hideLoader();
|
||||
})
|
||||
.fail(function() {
|
||||
$('#load-error').removeClass('hidden');
|
||||
$('#loader').addClass('hidden');
|
||||
});
|
||||
|
||||
};
|
||||
|
||||
var environmentsCount = function() {
|
||||
var client = new elasticsearch.Client(elasticSearchHost());
|
||||
client.search({
|
||||
index: 'fuel',
|
||||
type: 'structure',
|
||||
body: applyFilters({
|
||||
aggs: {
|
||||
clusters: {
|
||||
nested: {
|
||||
path: 'clusters'
|
||||
},
|
||||
aggs: {
|
||||
statuses: {
|
||||
terms: {field: 'status'}
|
||||
var installationsCount = function(resp) {
|
||||
$('#installations-count').html(resp.installations.count);
|
||||
};
|
||||
|
||||
var environmentsCount = function(resp) {
|
||||
$('#environments-count').html(resp.environments.count);
|
||||
|
||||
var colors = [
|
||||
{status: 'new', code: '#999999'},
|
||||
{status: 'operational', code: '#51851A'},
|
||||
{status: 'error', code: '#FF7372'},
|
||||
{status: 'deployment', code: '#2783C0'},
|
||||
{status: 'remove', code: '#000000'},
|
||||
{status: 'stopped', code: '#FFB014'},
|
||||
{status: 'update', code: '#775575'},
|
||||
{status: 'update_error', code: '#F5007B'}
|
||||
];
|
||||
var chartData = [];
|
||||
|
||||
$.each(colors, function(index, color) {
|
||||
var in_status = resp.environments.statuses[color.status];
|
||||
if (in_status) {
|
||||
chartData.push({label: color.status, value: in_status, color: color.code});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}).then(function(resp) {
|
||||
var rootData = getRootData(resp);
|
||||
var rawData = rootData.clusters.statuses.buckets,
|
||||
total = rootData.clusters.doc_count,
|
||||
colors = {
|
||||
error: '#FF7372',
|
||||
operational: '#51851A',
|
||||
new: '#999999',
|
||||
deployment: '#2783C0',
|
||||
remove: '#000000',
|
||||
update: '#775575',
|
||||
update_error: '#F5007B',
|
||||
stopped: '#FFB014'
|
||||
},
|
||||
chartData = [];
|
||||
$.each(rawData, function(key, value) {
|
||||
chartData.push({label: value.key, value: value.doc_count, color: colors[value.key]});
|
||||
});
|
||||
$('#environments-count').html(total);
|
||||
|
||||
var data = [{
|
||||
key: 'Distribution of environments by statuses',
|
||||
values: chartData
|
||||
|
|
@ -195,30 +128,12 @@ function($, d3, D3pie, d3tip, nv, elasticsearch) {
|
|||
|
||||
return chart;
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
var distributionOfInstallations = function() {
|
||||
var client = new elasticsearch.Client(elasticSearchHost());
|
||||
client.search({
|
||||
index: 'fuel',
|
||||
size: 0,
|
||||
body: applyFilters({
|
||||
aggs: {
|
||||
envs_distribution: {
|
||||
histogram: {
|
||||
field: 'clusters_num',
|
||||
interval: 1
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}).then(function(resp) {
|
||||
var rootData = getRootData(resp);
|
||||
var rawData = rootData.envs_distribution.buckets,
|
||||
chartData = [];
|
||||
$.each(rawData, function(key, value) {
|
||||
chartData.push({label: value.key, value: value.doc_count});
|
||||
var distributionOfInstallations = function(resp) {
|
||||
var chartData = [];
|
||||
$.each(resp.installations.environments_num, function(key, value) {
|
||||
chartData.push({label: key, value: value});
|
||||
});
|
||||
var data = [{
|
||||
color: '#1DA489',
|
||||
|
|
@ -257,67 +172,36 @@ function($, d3, D3pie, d3tip, nv, elasticsearch) {
|
|||
|
||||
return chart;
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
var nodesDistributionChart = function() {
|
||||
var client = new elasticsearch.Client(elasticSearchHost()),
|
||||
ranges = [
|
||||
{from: 1, to: 5},
|
||||
{from: 5, to: 10},
|
||||
{from: 10, to: 20},
|
||||
{from: 20, to: 50},
|
||||
{from: 50, to: 100},
|
||||
{from: 100}
|
||||
var nodesDistributionChart = function(resp) {
|
||||
var total = resp.environments.operable_envs_count;
|
||||
var ranges = [
|
||||
{from: 1, to: 5, count: 0},
|
||||
{from: 5, to: 10, count: 0},
|
||||
{from: 10, to: 20, count: 0},
|
||||
{from: 20, to: 50, count: 0},
|
||||
{from: 50, to: 100, count: 0},
|
||||
{from: 100, to: null, count: 0}
|
||||
];
|
||||
var chartData = [];
|
||||
|
||||
client.search({
|
||||
index: 'fuel',
|
||||
type: 'structure',
|
||||
size: 0,
|
||||
body: applyFilters({
|
||||
aggs: {
|
||||
clusters: {
|
||||
nested: {
|
||||
path: 'clusters'
|
||||
},
|
||||
aggs: {
|
||||
statuses: {
|
||||
filter: {
|
||||
terms: {status: statuses}
|
||||
},
|
||||
aggs: {
|
||||
nodes_ranges: {
|
||||
range: {
|
||||
field: 'nodes_num',
|
||||
ranges: ranges
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}).then(function(resp) {
|
||||
var rootData = getRootData(resp);
|
||||
var rawData = rootData.clusters.statuses.nodes_ranges.buckets,
|
||||
total = rootData.clusters.statuses.doc_count,
|
||||
chartData = [];
|
||||
$('#count-nodes-distribution').html(total);
|
||||
$.each(rawData, function(key, value) {
|
||||
var labelText = '',
|
||||
labelData = value.key.split('-');
|
||||
$.each(labelData, function(key, value) {
|
||||
if (value) {
|
||||
if (key == labelData.length - 1) {
|
||||
labelText += (value == '*' ? '+' : '-' + parseInt(value));
|
||||
} else {
|
||||
labelText += parseInt(value);
|
||||
}
|
||||
$.each(resp.environments.nodes_num, function(nodes_num, count) {
|
||||
$.each(ranges, function(index, range) {
|
||||
var num = parseInt(nodes_num);
|
||||
if (
|
||||
num >= range.from &&
|
||||
(num < range.to || range.to == null)
|
||||
) {
|
||||
range.count += count;
|
||||
}
|
||||
});
|
||||
chartData.push({label: labelText, value: value.doc_count});
|
||||
});
|
||||
|
||||
$.each(ranges, function(index, range) {
|
||||
var labelText = range.from + (range.to == null ? '+' : '-' + range.to);
|
||||
chartData.push({label: labelText, value: range.count});
|
||||
});
|
||||
|
||||
var data = [{
|
||||
|
|
@ -357,54 +241,15 @@ function($, d3, D3pie, d3tip, nv, elasticsearch) {
|
|||
|
||||
return chart;
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
var hypervisorDistributionChart = function() {
|
||||
var client = new elasticsearch.Client(elasticSearchHost());
|
||||
client.search({
|
||||
size: 0,
|
||||
index: 'fuel',
|
||||
type: 'structure',
|
||||
body: applyFilters({
|
||||
aggs: {
|
||||
clusters: {
|
||||
nested: {
|
||||
path: 'clusters'
|
||||
},
|
||||
aggs: {
|
||||
statuses: {
|
||||
filter: {
|
||||
terms: {status: statuses}
|
||||
},
|
||||
aggs: {
|
||||
attributes: {
|
||||
nested: {
|
||||
path: 'clusters.attributes'
|
||||
},
|
||||
aggs: {
|
||||
libvirt_types: {
|
||||
terms: {
|
||||
field: 'libvirt_type'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}).then(function(resp) {
|
||||
var rootData = getRootData(resp);
|
||||
var rawData = rootData.clusters.statuses.attributes.libvirt_types.buckets,
|
||||
total = rootData.clusters.statuses.attributes.doc_count,
|
||||
totalСounted = 0,
|
||||
var hypervisorDistributionChart = function(resp) {
|
||||
var totalСounted = 0,
|
||||
total = resp.environments.operable_envs_count,
|
||||
chartData = [];
|
||||
$.each(rawData, function(key, value) {
|
||||
chartData.push({label: value.key, value: value.doc_count});
|
||||
totalСounted += value.doc_count;
|
||||
$.each(resp.environments.hypervisors_num, function(hypervisor, count) {
|
||||
chartData.push({label: hypervisor, value: count});
|
||||
totalСounted += count;
|
||||
});
|
||||
var unknownHypervisorsCount = total - totalСounted;
|
||||
if (unknownHypervisorsCount) {
|
||||
|
|
@ -465,54 +310,14 @@ function($, d3, D3pie, d3tip, nv, elasticsearch) {
|
|||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
var osesDistributionChart = function() {
|
||||
var client = new elasticsearch.Client(elasticSearchHost());
|
||||
client.search({
|
||||
size: 0,
|
||||
index: 'fuel',
|
||||
type: 'structure',
|
||||
body: applyFilters({
|
||||
aggs: {
|
||||
clusters: {
|
||||
nested: {
|
||||
path: 'clusters'
|
||||
},
|
||||
aggs: {
|
||||
statuses: {
|
||||
filter: {
|
||||
terms: {status: statuses}
|
||||
},
|
||||
aggs: {
|
||||
release: {
|
||||
nested: {
|
||||
path: 'clusters.release'
|
||||
},
|
||||
|
||||
aggs: {
|
||||
oses: {
|
||||
terms: {
|
||||
field: 'os'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}).then(function(resp) {
|
||||
var rootData = getRootData(resp);
|
||||
var rawData = rootData.clusters.statuses.release.oses.buckets,
|
||||
total = rootData.clusters.statuses.doc_count,
|
||||
var osesDistributionChart = function(resp) {
|
||||
var total = resp.environments.operable_envs_count,
|
||||
chartData = [];
|
||||
$('#count-distribution-of-oses').html(total);
|
||||
$.each(rawData, function(key, value) {
|
||||
chartData.push({label: value.key, value: value.doc_count});
|
||||
$.each(resp.environments.oses_num, function(os, count) {
|
||||
chartData.push({label: os, value: count});
|
||||
});
|
||||
$('#distribution-of-oses').html('');
|
||||
new D3pie("distribution-of-oses", {
|
||||
|
|
@ -568,7 +373,6 @@ function($, d3, D3pie, d3tip, nv, elasticsearch) {
|
|||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
return statsPage();
|
||||
|
|
|
|||
|
|
@ -0,0 +1,61 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
"""Release column added to installation_structures
|
||||
|
||||
Revision ID: 24081e26a283
|
||||
Revises: 2ec36f35eeaa
|
||||
Create Date: 2016-06-23 18:53:01.431773
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '24081e26a283'
|
||||
down_revision = '2ec36f35eeaa'
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
|
||||
def upgrade():
|
||||
### commands auto generated by Alembic - please adjust! ###
|
||||
op.add_column(
|
||||
'installation_structures',
|
||||
sa.Column('release', sa.Text(), nullable=True)
|
||||
)
|
||||
op.create_index(
|
||||
op.f('ix_installation_structures_release'),
|
||||
'installation_structures',
|
||||
['release'],
|
||||
unique=False
|
||||
)
|
||||
|
||||
set_release = sa.sql.text(
|
||||
"UPDATE installation_structures "
|
||||
"SET release = structure->'fuel_release'->>'release'"
|
||||
)
|
||||
connection = op.get_bind()
|
||||
connection.execute(set_release)
|
||||
### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade():
|
||||
### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(
|
||||
op.f('ix_installation_structures_release'),
|
||||
table_name='installation_structures'
|
||||
)
|
||||
op.drop_column('installation_structures', 'release')
|
||||
### end Alembic commands ###
|
||||
|
|
@ -43,6 +43,7 @@ class InstallationStructure(db.Model):
|
|||
creation_date = db.Column(db.DateTime)
|
||||
modification_date = db.Column(db.DateTime)
|
||||
is_filtered = db.Column(db.Boolean, default=False, index=True)
|
||||
release = db.Column(db.Text, index=True)
|
||||
|
||||
|
||||
class OpenStackWorkloadStats(db.Model):
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ def post():
|
|||
obj.modification_date = datetime.utcnow()
|
||||
status_code = 200
|
||||
obj.is_filtered = _is_filtered(structure)
|
||||
obj.release = get_release(structure)
|
||||
obj.structure = structure
|
||||
db.session.add(obj)
|
||||
return status_code, {'status': 'ok'}
|
||||
|
|
@ -133,3 +134,7 @@ def _is_filtered(structure):
|
|||
packages, filtered_by_packages)
|
||||
|
||||
return filtered_by_build_id or filtered_by_packages
|
||||
|
||||
|
||||
def get_release(structure):
|
||||
return structure.get('fuel_release', {}).get('release')
|
||||
|
|
|
|||
|
|
@ -507,3 +507,28 @@ class TestInstallationStructure(DbTest):
|
|||
filtering_rules = {tuple(sorted(packages)): from_dt_str}
|
||||
self.assertFalse(_is_filtered_by_build_info(
|
||||
packages, filtering_rules))
|
||||
|
||||
def test_release_column(self):
|
||||
master_node_uid = 'x'
|
||||
release = 'release'
|
||||
struct = {
|
||||
'master_node_uid': master_node_uid,
|
||||
'fuel_release': {
|
||||
'release': release,
|
||||
'feature_groups': [],
|
||||
'api': 'v1'
|
||||
},
|
||||
'allocated_nodes_num': 0,
|
||||
'unallocated_nodes_num': 0,
|
||||
'clusters_num': 0,
|
||||
'clusters': []
|
||||
}
|
||||
resp = self.post(
|
||||
'/api/v1/installation_structure/',
|
||||
{'installation_structure': struct}
|
||||
)
|
||||
self.check_response_ok(resp, codes=(201,))
|
||||
obj = db.session.query(InstallationStructure).filter(
|
||||
InstallationStructure.master_node_uid == master_node_uid).one()
|
||||
self.assertEqual(struct, obj.structure)
|
||||
self.assertEqual(release, obj.release)
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ Flask-Script==2.0.5
|
|||
Flask-SQLAlchemy==2.0
|
||||
psycopg2==2.5.4
|
||||
python-dateutil==2.2
|
||||
python-memcached>=1.56
|
||||
PyYAML==3.11
|
||||
six>=1.8.0
|
||||
SQLAlchemy==0.9.8
|
||||
|
|
|
|||
Loading…
Reference in New Issue