Status: "' + x + '"
' + '' + parseInt(y) + ' environments
'; + }); + + d3.select('#clusters-distribution svg') + .datum(data) + .call(chart); + + nv.utils.windowResize(chart.update); + + return chart; }); }; - var environmentsCount = function() { - var client = new elasticsearch.Client(elasticSearchHost()); - client.search({ - index: 'fuel', - type: 'structure', - body: applyFilters({ - aggs: { - clusters: { - nested: { - path: 'clusters' - }, - aggs: { - statuses: { - terms: {field: 'status'} - } - } - } - } - }) - }).then(function(resp) { - var rootData = getRootData(resp); - var rawData = rootData.clusters.statuses.buckets, - total = rootData.clusters.doc_count, - colors = { - error: '#FF7372', - operational: '#51851A', - new: '#999999', - deployment: '#2783C0', - remove: '#000000', - update: '#775575', - update_error: '#F5007B', - stopped: '#FFB014' - }, - chartData = []; - $.each(rawData, function(key, value) { - chartData.push({label: value.key, value: value.doc_count, color: colors[value.key]}); - }); - $('#environments-count').html(total); - var data = [{ - key: 'Distribution of environments by statuses', - values: chartData - }]; + var distributionOfInstallations = function(resp) { + var chartData = []; + $.each(resp.installations.environments_num, function(key, value) { + chartData.push({label: key, value: value}); + }); + var data = [{ + color: '#1DA489', + values: chartData + }]; - nv.addGraph(function() { - var chart = nv.models.discreteBarChart() - .x(function(d) { return d.label;}) - .y(function(d) { return d.value;}) - .margin({top: 30, bottom: 60}) - .staggerLabels(true) - .transitionDuration(350); + nv.addGraph(function() { + var chart = nv.models.multiBarChart() + .x(function(d) { return d.label;}) + .y(function(d) { return d.value;}) + .margin({top: 30, bottom: 60}) + .transitionDuration(350) + .reduceXTicks(false) //If 'false', every single x-axis tick label will be rendered. + .rotateLabels(0) //Angle to rotate x-axis labels. + .showControls(false) //Allow user to switch between 'Grouped' and 'Stacked' mode. + .showLegend(false) + .groupSpacing(0.5); //Distance between each group of bars. - chart.xAxis - .axisLabel('Statuses'); + chart.xAxis + .axisLabel('Environments count'); - chart.yAxis - .axisLabel('Environments') - .axisLabelDistance(30) - .tickFormat(d3.format('d')); + chart.yAxis + .axisLabel('Installations') + .axisLabelDistance(30) + .tickFormat(d3.format('d')); - chart.tooltipContent(function(key, x, y) { - return 'Status: "' + x + '"
' + '' + parseInt(y) + ' environments
'; - }); - - d3.select('#clusters-distribution svg') - .datum(data) - .call(chart); - - nv.utils.windowResize(chart.update); - - return chart; - }); + chart.tooltipContent(function(key, x, y) { + return '' + parseInt(y) + ' installations
' + 'with ' + x + ' environments
'; }); + + d3.select('#env-distribution svg') + .datum(data) + .call(chart); + + nv.utils.windowResize(chart.update); + + return chart; + }); }; - var distributionOfInstallations = function() { - var client = new elasticsearch.Client(elasticSearchHost()); - client.search({ - index: 'fuel', - size: 0, - body: applyFilters({ - aggs: { - envs_distribution: { - histogram: { - field: 'clusters_num', - interval: 1 - } - } + var nodesDistributionChart = function(resp) { + var total = resp.environments.operable_envs_count; + var ranges = [ + {from: 1, to: 5, count: 0}, + {from: 5, to: 10, count: 0}, + {from: 10, to: 20, count: 0}, + {from: 20, to: 50, count: 0}, + {from: 50, to: 100, count: 0}, + {from: 100, to: null, count: 0} + ]; + var chartData = []; + + $('#count-nodes-distribution').html(total); + $.each(resp.environments.nodes_num, function(nodes_num, count) { + $.each(ranges, function(index, range) { + var num = parseInt(nodes_num); + if ( + num >= range.from && + (num < range.to || range.to == null) + ) { + range.count += count; } - }) - }).then(function(resp) { - var rootData = getRootData(resp); - var rawData = rootData.envs_distribution.buckets, - chartData = []; - $.each(rawData, function(key, value) { - chartData.push({label: value.key, value: value.doc_count}); - }); - var data = [{ - color: '#1DA489', - values: chartData - }]; - - nv.addGraph(function() { - var chart = nv.models.multiBarChart() - .x(function(d) { return d.label;}) - .y(function(d) { return d.value;}) - .margin({top: 30, bottom: 60}) - .transitionDuration(350) - .reduceXTicks(false) //If 'false', every single x-axis tick label will be rendered. - .rotateLabels(0) //Angle to rotate x-axis labels. - .showControls(false) //Allow user to switch between 'Grouped' and 'Stacked' mode. - .showLegend(false) - .groupSpacing(0.5); //Distance between each group of bars. - - chart.xAxis - .axisLabel('Environments count'); - - chart.yAxis - .axisLabel('Installations') - .axisLabelDistance(30) - .tickFormat(d3.format('d')); - - chart.tooltipContent(function(key, x, y) { - return '' + parseInt(y) + ' installations
' + 'with ' + x + ' environments
'; - }); - - d3.select('#env-distribution svg') - .datum(data) - .call(chart); - - nv.utils.windowResize(chart.update); - - return chart; - }); }); + }); + + $.each(ranges, function(index, range) { + var labelText = range.from + (range.to == null ? '+' : '-' + range.to); + chartData.push({label: labelText, value: range.count}); + }); + + var data = [{ + key: 'Environment size distribution by number of nodes', + color: '#1DA489', + values: chartData + }]; + + nv.addGraph(function() { + var chart = nv.models.multiBarChart() + .x(function(d) { return d.label;}) + .y(function(d) { return d.value;}) + .margin({top: 30}) + .transitionDuration(350) + .reduceXTicks(false) //If 'false', every single x-axis tick label will be rendered. + .rotateLabels(0) //Angle to rotate x-axis labels. + .showControls(false) //Allow user to switch between 'Grouped' and 'Stacked' mode. + .groupSpacing(0.2); //Distance between each group of bars. + + chart.xAxis + .axisLabel('Number of nodes'); + + chart.yAxis + .axisLabel('Environments') + .axisLabelDistance(30) + .tickFormat(d3.format('d')); + + chart.tooltipContent(function(key, x, y) { + return '' + x + ' nodes
' + '' + parseInt(y) + '
'; + }); + + d3.select('#nodes-distribution svg') + .datum(data) + .call(chart); + + nv.utils.windowResize(chart.update); + + return chart; + }); }; - var nodesDistributionChart = function() { - var client = new elasticsearch.Client(elasticSearchHost()), - ranges = [ - {from: 1, to: 5}, - {from: 5, to: 10}, - {from: 10, to: 20}, - {from: 20, to: 50}, - {from: 50, to: 100}, - {from: 100} - ]; - - client.search({ - index: 'fuel', - type: 'structure', - size: 0, - body: applyFilters({ - aggs: { - clusters: { - nested: { - path: 'clusters' - }, - aggs: { - statuses: { - filter: { - terms: {status: statuses} - }, - aggs: { - nodes_ranges: { - range: { - field: 'nodes_num', - ranges: ranges - } - } - } - } - } - } + var hypervisorDistributionChart = function(resp) { + var totalСounted = 0, + total = resp.environments.operable_envs_count, + chartData = []; + $.each(resp.environments.hypervisors_num, function(hypervisor, count) { + chartData.push({label: hypervisor, value: count}); + totalСounted += count; + }); + var unknownHypervisorsCount = total - totalСounted; + if (unknownHypervisorsCount) { + chartData.push({label: 'unknown', value: unknownHypervisorsCount}); + } + $('#count-releases-distribution').html(total); + $('#releases-distribution').html(''); + new D3pie("releases-distribution", { + header: { + title: { + text: 'Distribution of deployed hypervisor', + fontSize: 15 + }, + location: 'top-left', + titleSubtitlePadding: 9 + }, + size: { + canvasWidth: 330, + canvasHeight: 300, + pieInnerRadius: '40%', + pieOuterRadius: '55%' + }, + labels: { + outer: { + format: 'label-value2', + pieDistance: 10 + }, + inner: { + format: "percentage", + hideWhenLessThanPercentage: 5 + }, + mainLabel: { + fontSize: 14 + }, + percentage: { + color: '#ffffff', + decimalPlaces: 2 + }, + value: { + color: '#adadad', + fontSize: 11 + }, + lines: { + enabled: true } - }) - }).then(function(resp) { - var rootData = getRootData(resp); - var rawData = rootData.clusters.statuses.nodes_ranges.buckets, - total = rootData.clusters.statuses.doc_count, - chartData = []; - $('#count-nodes-distribution').html(total); - $.each(rawData, function(key, value) { - var labelText = '', - labelData = value.key.split('-'); - $.each(labelData, function(key, value) { - if (value) { - if (key == labelData.length - 1) { - labelText += (value == '*' ? '+' : '-' + parseInt(value)); - } else { - labelText += parseInt(value); - } - } - }); - chartData.push({label: labelText, value: value.doc_count}); - }); - - var data = [{ - key: 'Environment size distribution by number of nodes', - color: '#1DA489', - values: chartData - }]; - - nv.addGraph(function() { - var chart = nv.models.multiBarChart() - .x(function(d) { return d.label;}) - .y(function(d) { return d.value;}) - .margin({top: 30}) - .transitionDuration(350) - .reduceXTicks(false) //If 'false', every single x-axis tick label will be rendered. - .rotateLabels(0) //Angle to rotate x-axis labels. - .showControls(false) //Allow user to switch between 'Grouped' and 'Stacked' mode. - .groupSpacing(0.2); //Distance between each group of bars. - - chart.xAxis - .axisLabel('Number of nodes'); - - chart.yAxis - .axisLabel('Environments') - .axisLabelDistance(30) - .tickFormat(d3.format('d')); - - chart.tooltipContent(function(key, x, y) { - return '' + x + ' nodes
' + '' + parseInt(y) + '
'; - }); - - d3.select('#nodes-distribution svg') - .datum(data) - .call(chart); - - nv.utils.windowResize(chart.update); - - return chart; - }); - }); + }, + data: { + content: chartData + }, + tooltips: { + enabled: true, + type: 'placeholder', + string: '{label}: {value} pcs, {percentage}%', + styles: { + borderRadius: 3, + fontSize: 12, + padding: 6 + } + } + }); }; - var hypervisorDistributionChart = function() { - var client = new elasticsearch.Client(elasticSearchHost()); - client.search({ - size: 0, - index: 'fuel', - type: 'structure', - body: applyFilters({ - aggs: { - clusters: { - nested: { - path: 'clusters' - }, - aggs: { - statuses: { - filter: { - terms: {status: statuses} - }, - aggs: { - attributes: { - nested: { - path: 'clusters.attributes' - }, - aggs: { - libvirt_types: { - terms: { - field: 'libvirt_type' - } - } - } - } - } - } - } - } + var osesDistributionChart = function(resp) { + var total = resp.environments.operable_envs_count, + chartData = []; + $('#count-distribution-of-oses').html(total); + $.each(resp.environments.oses_num, function(os, count) { + chartData.push({label: os, value: count}); + }); + $('#distribution-of-oses').html(''); + new D3pie("distribution-of-oses", { + header: { + title: { + text: 'Distribution of deployed operating system', + fontSize: 15 + }, + location: 'top-left', + titleSubtitlePadding: 9 + }, + size: { + canvasWidth: 330, + canvasHeight: 300, + pieInnerRadius: '40%', + pieOuterRadius: '55%' + }, + labels: { + outer: { + format: 'label-value2', + pieDistance: 10 + }, + inner: { + format: "percentage", + hideWhenLessThanPercentage: 5 + }, + mainLabel: { + fontSize: 14 + }, + percentage: { + color: '#ffffff', + decimalPlaces: 2 + }, + value: { + color: '#adadad', + fontSize: 11 + }, + lines: { + enabled: true } - }) - }).then(function(resp) { - var rootData = getRootData(resp); - var rawData = rootData.clusters.statuses.attributes.libvirt_types.buckets, - total = rootData.clusters.statuses.attributes.doc_count, - totalСounted = 0, - chartData = []; - $.each(rawData, function(key, value) { - chartData.push({label: value.key, value: value.doc_count}); - totalСounted += value.doc_count; - }); - var unknownHypervisorsCount = total - totalСounted; - if (unknownHypervisorsCount) { - chartData.push({label: 'unknown', value: unknownHypervisorsCount}); + }, + data: { + content: chartData + }, + tooltips: { + enabled: true, + type: 'placeholder', + string: '{label}: {value} pcs, {percentage}%', + styles: { + borderRadius: 3, + fontSize: 12, + padding: 6 } - $('#count-releases-distribution').html(total); - $('#releases-distribution').html(''); - new D3pie("releases-distribution", { - header: { - title: { - text: 'Distribution of deployed hypervisor', - fontSize: 15 - }, - location: 'top-left', - titleSubtitlePadding: 9 - }, - size: { - canvasWidth: 330, - canvasHeight: 300, - pieInnerRadius: '40%', - pieOuterRadius: '55%' - }, - labels: { - outer: { - format: 'label-value2', - pieDistance: 10 - }, - inner: { - format: "percentage", - hideWhenLessThanPercentage: 5 - }, - mainLabel: { - fontSize: 14 - }, - percentage: { - color: '#ffffff', - decimalPlaces: 2 - }, - value: { - color: '#adadad', - fontSize: 11 - }, - lines: { - enabled: true - } - }, - data: { - content: chartData - }, - tooltips: { - enabled: true, - type: 'placeholder', - string: '{label}: {value} pcs, {percentage}%', - styles: { - borderRadius: 3, - fontSize: 12, - padding: 6 - } - } - }); - }); - }; - - var osesDistributionChart = function() { - var client = new elasticsearch.Client(elasticSearchHost()); - client.search({ - size: 0, - index: 'fuel', - type: 'structure', - body: applyFilters({ - aggs: { - clusters: { - nested: { - path: 'clusters' - }, - aggs: { - statuses: { - filter: { - terms: {status: statuses} - }, - aggs: { - release: { - nested: { - path: 'clusters.release' - }, - - aggs: { - oses: { - terms: { - field: 'os' - } - } - } - } - } - } - } - } - } - }) - }).then(function(resp) { - var rootData = getRootData(resp); - var rawData = rootData.clusters.statuses.release.oses.buckets, - total = rootData.clusters.statuses.doc_count, - chartData = []; - $('#count-distribution-of-oses').html(total); - $.each(rawData, function(key, value) { - chartData.push({label: value.key, value: value.doc_count}); - }); - $('#distribution-of-oses').html(''); - new D3pie("distribution-of-oses", { - header: { - title: { - text: 'Distribution of deployed operating system', - fontSize: 15 - }, - location: 'top-left', - titleSubtitlePadding: 9 - }, - size: { - canvasWidth: 330, - canvasHeight: 300, - pieInnerRadius: '40%', - pieOuterRadius: '55%' - }, - labels: { - outer: { - format: 'label-value2', - pieDistance: 10 - }, - inner: { - format: "percentage", - hideWhenLessThanPercentage: 5 - }, - mainLabel: { - fontSize: 14 - }, - percentage: { - color: '#ffffff', - decimalPlaces: 2 - }, - value: { - color: '#adadad', - fontSize: 11 - }, - lines: { - enabled: true - } - }, - data: { - content: chartData - }, - tooltips: { - enabled: true, - type: 'placeholder', - string: '{label}: {value} pcs, {percentage}%', - styles: { - borderRadius: 3, - fontSize: 12, - padding: 6 - } - } - }); - }); + } + }); }; return statsPage(); diff --git a/collector/collector/api/db/migrations/versions/24081e26a283_release_column_addded_to_installation_.py b/collector/collector/api/db/migrations/versions/24081e26a283_release_column_addded_to_installation_.py new file mode 100644 index 0000000..9ca5366 --- /dev/null +++ b/collector/collector/api/db/migrations/versions/24081e26a283_release_column_addded_to_installation_.py @@ -0,0 +1,61 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +"""Release column added to installation_structures + +Revision ID: 24081e26a283 +Revises: 2ec36f35eeaa +Create Date: 2016-06-23 18:53:01.431773 + +""" + +# revision identifiers, used by Alembic. +revision = '24081e26a283' +down_revision = '2ec36f35eeaa' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.add_column( + 'installation_structures', + sa.Column('release', sa.Text(), nullable=True) + ) + op.create_index( + op.f('ix_installation_structures_release'), + 'installation_structures', + ['release'], + unique=False + ) + + set_release = sa.sql.text( + "UPDATE installation_structures " + "SET release = structure->'fuel_release'->>'release'" + ) + connection = op.get_bind() + connection.execute(set_release) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_index( + op.f('ix_installation_structures_release'), + table_name='installation_structures' + ) + op.drop_column('installation_structures', 'release') + ### end Alembic commands ### diff --git a/collector/collector/api/db/model.py b/collector/collector/api/db/model.py index 31a3204..2eced61 100644 --- a/collector/collector/api/db/model.py +++ b/collector/collector/api/db/model.py @@ -43,6 +43,7 @@ class InstallationStructure(db.Model): creation_date = db.Column(db.DateTime) modification_date = db.Column(db.DateTime) is_filtered = db.Column(db.Boolean, default=False, index=True) + release = db.Column(db.Text, index=True) class OpenStackWorkloadStats(db.Model): diff --git a/collector/collector/api/resources/installation_structure.py b/collector/collector/api/resources/installation_structure.py index bbf8b1b..3bb384e 100644 --- a/collector/collector/api/resources/installation_structure.py +++ b/collector/collector/api/resources/installation_structure.py @@ -52,6 +52,7 @@ def post(): obj.modification_date = datetime.utcnow() status_code = 200 obj.is_filtered = _is_filtered(structure) + obj.release = get_release(structure) obj.structure = structure db.session.add(obj) return status_code, {'status': 'ok'} @@ -133,3 +134,7 @@ def _is_filtered(structure): packages, filtered_by_packages) return filtered_by_build_id or filtered_by_packages + + +def get_release(structure): + return structure.get('fuel_release', {}).get('release') diff --git a/collector/collector/test/resources/test_installation_structure.py b/collector/collector/test/resources/test_installation_structure.py index 6ce1ac1..a0255e0 100644 --- a/collector/collector/test/resources/test_installation_structure.py +++ b/collector/collector/test/resources/test_installation_structure.py @@ -507,3 +507,28 @@ class TestInstallationStructure(DbTest): filtering_rules = {tuple(sorted(packages)): from_dt_str} self.assertFalse(_is_filtered_by_build_info( packages, filtering_rules)) + + def test_release_column(self): + master_node_uid = 'x' + release = 'release' + struct = { + 'master_node_uid': master_node_uid, + 'fuel_release': { + 'release': release, + 'feature_groups': [], + 'api': 'v1' + }, + 'allocated_nodes_num': 0, + 'unallocated_nodes_num': 0, + 'clusters_num': 0, + 'clusters': [] + } + resp = self.post( + '/api/v1/installation_structure/', + {'installation_structure': struct} + ) + self.check_response_ok(resp, codes=(201,)) + obj = db.session.query(InstallationStructure).filter( + InstallationStructure.master_node_uid == master_node_uid).one() + self.assertEqual(struct, obj.structure) + self.assertEqual(release, obj.release) diff --git a/requirements.txt b/requirements.txt index 2ebd159..a4f7cdf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ Flask-Script==2.0.5 Flask-SQLAlchemy==2.0 psycopg2==2.5.4 python-dateutil==2.2 +python-memcached>=1.56 PyYAML==3.11 six>=1.8.0 SQLAlchemy==0.9.8