openstack
/
tuning-box
Code Issues Proposed changes

Nailgun data pipeline implementation 

We are fetching data from the ConfigDB on the cluster and
node serialization. It should be done for ConfigDB data
propatation to the Nailgun serializers.

Change-Id: I81b56808f2e5bb233c3d570b46db176bd9eb77c9
This commit is contained in:
Alexander Kislitsky 2016-10-06 17:22:17 +03:00
parent bb6c1932a4
commit 2283d5efdb
4 changed files with 176 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
tuning_box/hiera_config.py Normal file
View File

@ -0,0 +1,31 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import yaml
# Handling Puppet configs syntax
def construct_ruby_object(loader, suffix, node):
return loader.construct_yaml_map(node)
def construct_ruby_sym(loader, node):
return loader.construct_yaml_str(node)
yaml.add_multi_constructor("!ruby/object:", construct_ruby_object)
yaml.add_constructor("!ruby/sym", construct_ruby_sym)
def load_config(path='/etc/hiera.yaml'):
with open(path, 'r') as f:
return yaml.load(f)

177
tuning_box/library/resource_values.py
View File

@ -22,6 +22,96 @@ from tuning_box.library import hierarchy_levels
from tuning_box.library import resource_keys_operation
def _calculate_effective_values(result, level_value,
resource_values_idx, show_lookup,
lookup_path):
level_value_id = getattr(level_value, 'id', None)
if level_value_id in resource_values_idx:
resource_value = resource_values_idx[level_value_id]
if show_lookup:
values = ((k, (v, lookup_path)) for k, v in
six.iteritems(resource_value.values))
overrides = ((k, (v, lookup_path)) for k, v in
six.iteritems(resource_value.overrides))
else:
values = resource_value.values
overrides = resource_value.overrides
result.update(values)
result.update(overrides)
@db.with_transaction
def get_resource_value(environment_id, resource_id_or_name, levels):
app.logger.debug("Getting resource value. Env: %s, "
"resource: %s, levels: %s", environment_id,
resource_id_or_name, levels)
environment = db.Environment.query.get_or_404(environment_id)
res_def = library.get_resource_definition(
resource_id_or_name, environment_id)
level_values = list(hierarchy_levels.iter_environment_level_values(
environment, levels))
level_values_ids = [l.id for l in level_values]
app.logger.debug("Got level values ids: %s", level_values_ids)
if 'effective' in flask.request.args:
app.logger.debug("Getting effective resource value. Env: %s, "
"resource: %s, levels: %s", environment_id,
resource_id_or_name, levels)
show_lookup = 'show_lookup' in flask.request.args
resource_values = db.ResourceValues.query.filter(
or_(
db.ResourceValues.level_value_id.in_(level_values_ids),
db.ResourceValues.level_value_id.is_(None)
),
db.ResourceValues.resource_definition == res_def,
db.ResourceValues.environment == environment
).all()
app.logger.debug("Processing values for resource: %s, env: %s. "
"Loaded resource values: %s",
res_def.id, environment.id, len(resource_values))
# Creating index of resource_values by level_value_id
resource_values_idx = {r.level_value_id: r
for r in resource_values}
app.logger.debug("Resource values index size: %s",
len(resource_values_idx))
result = {}
lookup_path = '/'
_calculate_effective_values(
result, None, resource_values_idx, show_lookup,
lookup_path)
for level_value in level_values:
name = level_value.level.name
value = level_value.value
lookup_path += name + '/' + value + '/'
_calculate_effective_values(
result, level_value, resource_values_idx, show_lookup,
lookup_path)
app.logger.debug("Effective values got for resource: "
"%s, env: %s", res_def.id, environment.id)
return result
else:
if not level_values:
level_value = None
else:
level_value = level_values[-1]
resource_values = db.ResourceValues.query.filter_by(
resource_definition=res_def,
environment=environment,
level_value=level_value,
).one_or_none()
app.logger.debug("Values got for resource: "
"%s, env: %s", res_def.id, environment.id)
if not resource_values:
return {}
return resource_values.values
class ResourceValues(flask_restful.Resource):
@db.with_transaction
@ -41,93 +131,8 @@ class ResourceValues(flask_restful.Resource):
esv.values = flask.request.json
return None, 204
def _calculate_effective_values(self, result, level_value,
resource_values_idx, show_lookup,
lookup_path):
level_value_id = getattr(level_value, 'id', None)
if level_value_id in resource_values_idx:
resource_value = resource_values_idx[level_value_id]
if show_lookup:
values = ((k, (v, lookup_path)) for k, v in
six.iteritems(resource_value.values))
overrides = ((k, (v, lookup_path)) for k, v in
six.iteritems(resource_value.overrides))
else:
values = resource_value.values
overrides = resource_value.overrides
result.update(values)
result.update(overrides)
@db.with_transaction
def get(self, environment_id, resource_id_or_name, levels):
app.logger.debug("Getting resource value. Env: %s, "
"resource: %s, levels: %s", environment_id,
resource_id_or_name, levels)
environment = db.Environment.query.get_or_404(environment_id)
res_def = library.get_resource_definition(
resource_id_or_name, environment_id)
level_values = list(hierarchy_levels.iter_environment_level_values(
environment, levels))
level_values_ids = [l.id for l in level_values]
app.logger.debug("Got level values ids: %s", level_values_ids)
if 'effective' in flask.request.args:
app.logger.debug("Getting effective resource value. Env: %s, "
"resource: %s, levels: %s", environment_id,
resource_id_or_name, levels)
show_lookup = 'show_lookup' in flask.request.args
resource_values = db.ResourceValues.query.filter(
or_(
db.ResourceValues.level_value_id.in_(level_values_ids),
db.ResourceValues.level_value_id.is_(None)
),
db.ResourceValues.resource_definition == res_def,
db.ResourceValues.environment == environment
).all()
app.logger.debug("Processing values for resource: %s, env: %s. "
"Loaded resource values: %s",
res_def.id, environment.id, len(resource_values))
# Creating index of resource_values by level_value_id
resource_values_idx = {r.level_value_id: r
for r in resource_values}
app.logger.debug("Resource values index size: %s",
len(resource_values_idx))
result = {}
lookup_path = '/'
self._calculate_effective_values(
result, None, resource_values_idx, show_lookup,
lookup_path)
for level_value in level_values:
name = level_value.level.name
value = level_value.value
lookup_path += name + '/' + value + '/'
self._calculate_effective_values(
result, level_value, resource_values_idx, show_lookup,
lookup_path)
app.logger.debug("Effective values got for resource: "
"%s, env: %s", res_def.id, environment.id)
return result
else:
if not level_values:
level_value = None
else:
level_value = level_values[-1]
resource_values = db.ResourceValues.query.filter_by(
resource_definition=res_def,
environment=environment,
level_value=level_value,
).one_or_none()
app.logger.debug("Values got for resource: "
"%s, env: %s", res_def.id, environment.id)
if not resource_values:
return {}
return resource_values.values
return get_resource_value(environment_id, resource_id_or_name, levels)
class ResourceValuesKeys(flask_restful.Resource,

30
tuning_box/nailgun.py
View File

@ -15,6 +15,7 @@ from __future__ import absolute_import
import itertools
import threading
from flask import current_app as cur_app
from nailgun.db import sqlalchemy as nailgun_sa
from nailgun import extensions
import web
@ -22,6 +23,7 @@ import web
import tuning_box
from tuning_box import app as tb_app
from tuning_box import db as tb_db
from tuning_box import hiera_config
class App2WebPy(web.application):
@ -75,13 +77,39 @@ class TB2WebPy(App2WebPy):
return app
class ConfigPipeline(extensions.BasePipeline):
def get_hierarchy(self):
config = hiera_config.load_config()
return config['hierarchy']
@classmethod
def process_deployment_for_cluster(cls, cluster, cluster_data):
"""Extend or modify deployment data for cluster.
:param cluster_data: serialized data for cluster
:param cluster: the instance of Cluster
"""
result = {}
for level in cls.get_hierarchy():
cur_app.logger.debug("Fetching info for hierarchy level: %s",
level)
return cluster_data.update(result)
class Extension(extensions.BaseExtension):
name = 'tuning_box'
version = tuning_box.__version__
description = 'Plug tuning_box endpoints into Nailgun itself'
app = TB2WebPy()
urls = [{'uri': '/config', 'handler': TB2WebPy()}]
urls = [{'uri': '/config', 'handler': app}]
@classmethod
def alembic_migrations_path(cls):
app_instance = cls.app.get_app()
with app_instance.app_context():
app_instance.logger.error("##############################")
from tuning_box.library.resource_values import execute_sql
execute_sql()
return tuning_box.get_migrations_dir()

25
tuning_box/tests/test_hiera_config.py Normal file
View File

@ -0,0 +1,25 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import mock
from tuning_box import hiera_config
from tuning_box.tests import base
class TestHieraConfig(base.TestCase):
def test_load_config(self):
data = '!ruby/sym r_key: r_val\n'
with mock.patch('__builtin__.open', mock.mock_open(read_data=data)):
result = hiera_config.load_config()
self.assertEqual({'r_key': 'r_val'}, result)