Merge "Agent for configuration file validation"
This commit is contained in:
commit
964c3d4c1f
|
@ -0,0 +1,433 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Agent is the main entry point for the configuration validator agent.
|
||||||
|
|
||||||
|
The agent is executed on the different nodes of the cloud and sends back
|
||||||
|
configuration values and metadata to the configuration validator datasource
|
||||||
|
driver.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_config import generator
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_service import service
|
||||||
|
import six
|
||||||
|
|
||||||
|
from congress.common import config
|
||||||
|
|
||||||
|
from congress.cfg_validator.agent import generator as validator_generator
|
||||||
|
from congress.cfg_validator.agent import opts as validator_opts
|
||||||
|
from congress.cfg_validator.agent import rpc
|
||||||
|
from congress.cfg_validator import parsing
|
||||||
|
from congress.cfg_validator import utils
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class Config(object):
|
||||||
|
"""Encapsulates a configuration file and its meta-data.
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
path: Path to the configuration on the local file system.
|
||||||
|
template: A Template object to use for parsing the configuration.
|
||||||
|
data: The normalized Namespace loaded by oslo-config, contains the
|
||||||
|
parsed values.
|
||||||
|
hash: Hash of the configuration file, salted with the hostname and the
|
||||||
|
template hash
|
||||||
|
service_name: The associated service name
|
||||||
|
"""
|
||||||
|
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
|
||||||
|
def __init__(self, path, template, service_name):
|
||||||
|
self.path = path
|
||||||
|
self.template = template
|
||||||
|
self.data = None
|
||||||
|
self.hash = None
|
||||||
|
self.service = service_name
|
||||||
|
|
||||||
|
def parse(self, host):
|
||||||
|
"""Parses the config at the path given. Updates data and hash.
|
||||||
|
|
||||||
|
host: the name of the host where the config is. Used for building a
|
||||||
|
unique hash.
|
||||||
|
"""
|
||||||
|
namespaces_data = [ns.data for ns in self.template.namespaces]
|
||||||
|
conf = parsing.parse_config_file(namespaces_data, self.path)
|
||||||
|
|
||||||
|
Config.sanitize_config(conf)
|
||||||
|
self.data = conf._namespace._normalized
|
||||||
|
|
||||||
|
self.hash = utils.compute_hash(host, self.template.hash,
|
||||||
|
json.dumps(self.data, sort_keys=True))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def sanitize_config(conf):
|
||||||
|
"""Sanitizes some cfg.ConfigOpts values, given its options meta-data.
|
||||||
|
|
||||||
|
:param conf: A cfg.ConfigOpts object, pre-loaded with its options
|
||||||
|
meta-data and with its configurations values.
|
||||||
|
"""
|
||||||
|
normalized = getattr(conf._namespace,
|
||||||
|
'_normalized', None)
|
||||||
|
if not normalized:
|
||||||
|
return
|
||||||
|
|
||||||
|
normalized = normalized[0]
|
||||||
|
|
||||||
|
# Obfuscate values of options declared secret
|
||||||
|
def _sanitize(opt, group_name='DEFAULT'):
|
||||||
|
if not opt.secret:
|
||||||
|
return
|
||||||
|
if group_name not in normalized:
|
||||||
|
return
|
||||||
|
if opt.name in normalized[group_name]:
|
||||||
|
normalized[group_name][opt.name] = ['*' * 4]
|
||||||
|
|
||||||
|
for option in six.itervalues(conf._opts):
|
||||||
|
_sanitize(option['opt'])
|
||||||
|
|
||||||
|
for group_name, group in six.iteritems(conf._groups):
|
||||||
|
for option in six.itervalues(group._opts):
|
||||||
|
_sanitize(option['opt'], group_name)
|
||||||
|
|
||||||
|
def get_info(self):
|
||||||
|
"""Information on the configuration file.
|
||||||
|
|
||||||
|
:return: a quadruple made of:
|
||||||
|
* the hash of the template,
|
||||||
|
* the path to the file,
|
||||||
|
* the content
|
||||||
|
* the service name.
|
||||||
|
"""
|
||||||
|
return {'template': self.template.hash, 'path': self.path,
|
||||||
|
'data': self.data, 'service': self.service}
|
||||||
|
|
||||||
|
|
||||||
|
class Namespace(object):
|
||||||
|
"""Encapsulates a namespace, as defined by oslo-config-generator.
|
||||||
|
|
||||||
|
It contains the actual meta-data of the options. The data is loaded from
|
||||||
|
the service source code, by means of oslo-config-generator.
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
name: The name, as used by oslo-config-generator.
|
||||||
|
data: The meta-data of the configuration options.
|
||||||
|
hash: Hash of the namespace.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, name):
|
||||||
|
self.name = name
|
||||||
|
self.data = None
|
||||||
|
self.hash = None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def load(name):
|
||||||
|
"""Loads a namespace from disk
|
||||||
|
|
||||||
|
:param name: the name of namespace to load.
|
||||||
|
:return: a fully configured namespace.
|
||||||
|
"""
|
||||||
|
|
||||||
|
namespace = Namespace(name)
|
||||||
|
|
||||||
|
saved_conf = cfg.CONF
|
||||||
|
cfg.CONF = cfg.ConfigOpts()
|
||||||
|
|
||||||
|
try:
|
||||||
|
json_data = validator_generator.generate_ns_data(name)
|
||||||
|
finally:
|
||||||
|
cfg.CONF = saved_conf
|
||||||
|
|
||||||
|
namespace.hash = utils.compute_hash(json_data)
|
||||||
|
namespace.data = json.loads(json_data)
|
||||||
|
|
||||||
|
return namespace
|
||||||
|
|
||||||
|
def get_info(self):
|
||||||
|
"""Information on the namespace
|
||||||
|
|
||||||
|
:return: a tuple containing
|
||||||
|
* data: the content of the namespace
|
||||||
|
* name: the name of the namespace
|
||||||
|
"""
|
||||||
|
return {'data': self.data, 'name': self.name}
|
||||||
|
|
||||||
|
|
||||||
|
class Template(object):
|
||||||
|
"""Describes a template, as defined by oslo-config-generator.
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
name: The name, as used by oslo-config-generator.
|
||||||
|
path: The path to the template configuration file, as defined by oslo-
|
||||||
|
config-generator, on the local file system.
|
||||||
|
output_file: The default output path for this template.
|
||||||
|
namespaces: A set of Namespace objects, which make up this template.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
|
||||||
|
def __init__(self, path, output_file):
|
||||||
|
self.path = path
|
||||||
|
self.output_file = output_file
|
||||||
|
self.namespaces = []
|
||||||
|
self.hash = None
|
||||||
|
|
||||||
|
name = os.path.basename(output_file)
|
||||||
|
self.name = os.path.splitext(name)[0] if name.endswith('.sample') \
|
||||||
|
else name
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_template_conf(template_path):
|
||||||
|
"""Parses a template configuration file"""
|
||||||
|
conf = cfg.ConfigOpts()
|
||||||
|
|
||||||
|
conf.register_opts(generator._generator_opts)
|
||||||
|
conf(['--config-file', template_path])
|
||||||
|
|
||||||
|
return conf.namespace, conf.output_file
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def load(template_path):
|
||||||
|
"""Loads a template configuration file
|
||||||
|
|
||||||
|
:param template_path: path to the template
|
||||||
|
:return: a fully configured Template object.
|
||||||
|
"""
|
||||||
|
namespaces, output_file = Template._parse_template_conf(template_path)
|
||||||
|
|
||||||
|
template = Template(template_path, output_file)
|
||||||
|
|
||||||
|
for namespace in namespaces:
|
||||||
|
template.namespaces.append(Namespace.load(namespace))
|
||||||
|
|
||||||
|
template.hash = utils.compute_hash(
|
||||||
|
sorted([ns.hash for ns in template.namespaces]))
|
||||||
|
|
||||||
|
return template
|
||||||
|
|
||||||
|
def get_info(self):
|
||||||
|
"""Info on the template
|
||||||
|
|
||||||
|
:return: a quadruple made of:
|
||||||
|
* path: the path to the template path
|
||||||
|
* name: the name of the template
|
||||||
|
* output_fle:
|
||||||
|
* namespaces: an array of namespace hashes.
|
||||||
|
"""
|
||||||
|
return {'path': self.path, 'name': self.name,
|
||||||
|
'output_file': self.output_file,
|
||||||
|
'namespaces': [ns.hash for ns in self.namespaces]}
|
||||||
|
|
||||||
|
|
||||||
|
class ConfigManager(object):
|
||||||
|
"""Manages the services configuration files on a node and their meta-data.
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
host: A hostname.
|
||||||
|
configs: A dict mapping config hashes to their associated Config
|
||||||
|
object.
|
||||||
|
templates: A dict mapping template hashes to their associated Template
|
||||||
|
object.
|
||||||
|
namespaces: A dict mapping namespace hashes to their associated
|
||||||
|
Namespace object.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, host, services_files):
|
||||||
|
self.host = host
|
||||||
|
|
||||||
|
self.configs = {}
|
||||||
|
self.templates = {}
|
||||||
|
self.namespaces = {}
|
||||||
|
|
||||||
|
for service_name, files in six.iteritems(services_files):
|
||||||
|
self.register_service(service_name, files)
|
||||||
|
|
||||||
|
def get_template_by_path(self, template_path):
|
||||||
|
"""Given a path finds the corresponding template if it is registered
|
||||||
|
|
||||||
|
:param template_path: the path of the searched template
|
||||||
|
:return: None or the template
|
||||||
|
"""
|
||||||
|
for template in six.itervalues(self.templates):
|
||||||
|
if template.path == template_path:
|
||||||
|
return template
|
||||||
|
|
||||||
|
def add_template(self, template_path):
|
||||||
|
"""Adds a new template (loads it from path).
|
||||||
|
|
||||||
|
:param template_path: a valid path to the template file.
|
||||||
|
"""
|
||||||
|
template = Template.load(template_path)
|
||||||
|
|
||||||
|
self.templates[template.hash] = template
|
||||||
|
self.namespaces.update({ns.hash: ns for ns in template.namespaces})
|
||||||
|
|
||||||
|
return template
|
||||||
|
|
||||||
|
def register_config(self, config_path, template_path, service_name):
|
||||||
|
"""Register a configuration file and its associated template.
|
||||||
|
|
||||||
|
Template and config are actually parsed and loaded.
|
||||||
|
|
||||||
|
:param config_path: a valid path to the config file.
|
||||||
|
:param template_path: a valid path to the template file.
|
||||||
|
"""
|
||||||
|
template = (self.get_template_by_path(template_path)
|
||||||
|
or self.add_template(template_path))
|
||||||
|
|
||||||
|
conf = Config(config_path, template, service_name)
|
||||||
|
|
||||||
|
conf.parse(self.host)
|
||||||
|
|
||||||
|
self.configs[conf.hash] = conf
|
||||||
|
|
||||||
|
LOG.info('{hash: %s, path:%s}' % (conf.hash, conf.path))
|
||||||
|
|
||||||
|
def register_service(self, service_name, files):
|
||||||
|
"""Register all configs for an identified service.
|
||||||
|
|
||||||
|
Inaccessible files are ignored and files registration pursues.
|
||||||
|
|
||||||
|
:param service_name: The name of the service
|
||||||
|
:param files: A dict, mapping a configuration path to
|
||||||
|
its associated template path
|
||||||
|
"""
|
||||||
|
for config_path, template_path in six.iteritems(files):
|
||||||
|
try:
|
||||||
|
self.register_config(config_path, template_path, service_name)
|
||||||
|
except (IOError, cfg.ConfigFilesNotFoundError):
|
||||||
|
LOG.error(('Error while registering config %s with template'
|
||||||
|
' %s for service %s') %
|
||||||
|
(config_path, template_path, service_name))
|
||||||
|
|
||||||
|
|
||||||
|
class ValidatorAgentEndpoint(object):
|
||||||
|
"""Validator Agent.
|
||||||
|
|
||||||
|
It is used as an RPC endpoint.
|
||||||
|
|
||||||
|
Attributes:
|
||||||
|
config_manager: ConfigManager object.
|
||||||
|
driver_api: RPC client to communicate with the driver.
|
||||||
|
"""
|
||||||
|
|
||||||
|
# pylint: disable=unused-argument,too-many-instance-attributes
|
||||||
|
|
||||||
|
def __init__(self, conf=None):
|
||||||
|
self.conf = conf or cfg.CONF
|
||||||
|
validator_conf = self.conf.agent
|
||||||
|
|
||||||
|
self.host = validator_conf.host
|
||||||
|
self.version = validator_conf.version
|
||||||
|
self.max_delay = validator_conf.max_delay
|
||||||
|
self.driver_api = rpc.ValidatorDriverClient()
|
||||||
|
|
||||||
|
self.services = list(validator_conf.services.keys())
|
||||||
|
|
||||||
|
service_files = validator_conf.services
|
||||||
|
self.config_manager = ConfigManager(self.host, service_files)
|
||||||
|
|
||||||
|
def publish_configs_hashes(self, context, **kwargs):
|
||||||
|
""""Sends back all configuration hashes"""
|
||||||
|
LOG.info('Sending config hashes')
|
||||||
|
conf = set(self.config_manager.configs)
|
||||||
|
self.driver_api.process_configs_hashes({}, conf, self.host)
|
||||||
|
|
||||||
|
def publish_templates_hashes(self, context, **kwargs):
|
||||||
|
""""Sends back all template hashes"""
|
||||||
|
LOG.info('Sending template hashes')
|
||||||
|
tpl = set(self.config_manager.templates)
|
||||||
|
self.driver_api.process_templates_hashes({}, tpl, self.host)
|
||||||
|
|
||||||
|
def get_namespace(self, context, **kwargs):
|
||||||
|
""""Sends back a namespace
|
||||||
|
|
||||||
|
:param context: the RPC context
|
||||||
|
:param hash: the hash of the namespace to send
|
||||||
|
:return: the namespace or None if not found
|
||||||
|
"""
|
||||||
|
ns_hash = kwargs['ns_hash']
|
||||||
|
LOG.info('Sending namespace %s' % ns_hash)
|
||||||
|
|
||||||
|
namespace = self.config_manager.namespaces.get(ns_hash, None)
|
||||||
|
if namespace is None:
|
||||||
|
return None
|
||||||
|
ret = namespace.get_info()
|
||||||
|
ret['version'] = self.version
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def get_template(self, context, **kwargs):
|
||||||
|
""""Sends back a template
|
||||||
|
|
||||||
|
:param context: the RPC context
|
||||||
|
:param hash: the hash of the template to send
|
||||||
|
:return: the template or None if not found
|
||||||
|
"""
|
||||||
|
template_hash = kwargs['tpl_hash']
|
||||||
|
|
||||||
|
LOG.info('Sending template %s' % template_hash)
|
||||||
|
|
||||||
|
template = self.config_manager.templates.get(template_hash, None)
|
||||||
|
if template is None:
|
||||||
|
return None
|
||||||
|
ret = template.get_info()
|
||||||
|
ret['version'] = self.version
|
||||||
|
return ret
|
||||||
|
|
||||||
|
def get_config(self, context, **kwargs):
|
||||||
|
""""Sends back a config
|
||||||
|
|
||||||
|
:param context: the RPC context
|
||||||
|
:param hash: the hash of the config to send
|
||||||
|
:return: the config or None if not found
|
||||||
|
"""
|
||||||
|
config_hash = kwargs['cfg_hash']
|
||||||
|
|
||||||
|
LOG.info('Sending config %s' % config_hash)
|
||||||
|
|
||||||
|
conf = self.config_manager.configs.get(config_hash, None)
|
||||||
|
if conf is None:
|
||||||
|
return None
|
||||||
|
ret = conf.get_info()
|
||||||
|
ret['version'] = self.version
|
||||||
|
return ret
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""Agent entry point"""
|
||||||
|
validator_opts.register_validator_agent_opts(cfg.CONF)
|
||||||
|
config.init(sys.argv[1:])
|
||||||
|
config.setup_logging()
|
||||||
|
|
||||||
|
if not cfg.CONF.config_file:
|
||||||
|
sys.exit("ERROR: Unable to find configuration file via default "
|
||||||
|
"search paths ~/.congress/, ~/, /etc/congress/, /etc/) and "
|
||||||
|
"the '--config-file' option!")
|
||||||
|
|
||||||
|
agent = ValidatorAgentEndpoint()
|
||||||
|
server = rpc.AgentService(utils.AGENT_TOPIC, [agent])
|
||||||
|
service.launch(agent.conf, server).wait()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
|
@ -0,0 +1,123 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
""" Generation of JSON from oslo config options (marshalling) """
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_config import generator
|
||||||
|
from oslo_config import types
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class OptionJsonEncoder(json.JSONEncoder):
|
||||||
|
"""Json encoder used to give a unique representation to namespaces"""
|
||||||
|
|
||||||
|
# pylint: disable=protected-access,method-hidden,too-many-branches
|
||||||
|
def default(self, o):
|
||||||
|
if isinstance(o, cfg.Opt):
|
||||||
|
return {
|
||||||
|
'kind': type(o).__name__,
|
||||||
|
'deprecated_for_removal': o.deprecated_for_removal,
|
||||||
|
'short': o.short,
|
||||||
|
'name': o.name,
|
||||||
|
'dest': o.dest,
|
||||||
|
'deprecated_since': o.deprecated_since,
|
||||||
|
'required': o.required,
|
||||||
|
'sample_default': o.sample_default,
|
||||||
|
'deprecated_opts': o.deprecated_opts,
|
||||||
|
'positional': o.positional,
|
||||||
|
'default': o.default,
|
||||||
|
'secret': o.secret,
|
||||||
|
'deprecated_reason': o.deprecated_reason,
|
||||||
|
'mutable': o.mutable,
|
||||||
|
'type': o.type,
|
||||||
|
'metavar': o.metavar,
|
||||||
|
'advanced': o.advanced,
|
||||||
|
'help': o.help
|
||||||
|
}
|
||||||
|
elif isinstance(o, (types.ConfigType, types.HostAddress)):
|
||||||
|
res = {
|
||||||
|
'type': type(o).__name__,
|
||||||
|
}
|
||||||
|
|
||||||
|
if isinstance(o, types.Number):
|
||||||
|
res['max'] = o.max
|
||||||
|
res['min'] = o.min
|
||||||
|
res['choices'] = o.choices
|
||||||
|
if isinstance(o, types.Range):
|
||||||
|
res['max'] = o.max
|
||||||
|
res['min'] = o.min
|
||||||
|
if isinstance(o, types.String):
|
||||||
|
if o.regex and hasattr(o.regex, 'pattern'):
|
||||||
|
res['regex'] = o.regex.pattern
|
||||||
|
else:
|
||||||
|
res['regex'] = o.regex
|
||||||
|
res['max_length'] = o.max_length
|
||||||
|
res['quotes'] = o.quotes
|
||||||
|
res['ignore_case'] = o.ignore_case
|
||||||
|
res['choices'] = o.choices
|
||||||
|
if isinstance(o, types.List):
|
||||||
|
res['item_type'] = o.item_type
|
||||||
|
res['bounds'] = o.bounds
|
||||||
|
if isinstance(o, types.Dict):
|
||||||
|
res['value_type'] = o.value_type
|
||||||
|
res['bounds'] = o.bounds
|
||||||
|
if isinstance(o, types.URI):
|
||||||
|
res['schemes'] = o.schemes
|
||||||
|
res['max_length'] = o.max_length
|
||||||
|
if isinstance(o, types.IPAddress):
|
||||||
|
if o.version_checker == o._check_ipv4:
|
||||||
|
res['version'] = 4
|
||||||
|
elif o.version_checker == o._check_ipv6:
|
||||||
|
res['version'] = 6
|
||||||
|
|
||||||
|
# Remove unused fields
|
||||||
|
remove = [k for k, v in res.items() if not v]
|
||||||
|
for k in remove:
|
||||||
|
del res[k]
|
||||||
|
|
||||||
|
return res
|
||||||
|
elif isinstance(o, cfg.DeprecatedOpt):
|
||||||
|
return {
|
||||||
|
'name': o.name,
|
||||||
|
'group': o.group
|
||||||
|
}
|
||||||
|
elif isinstance(o, cfg.OptGroup):
|
||||||
|
return {
|
||||||
|
'title': o.title,
|
||||||
|
'help': o.help
|
||||||
|
}
|
||||||
|
|
||||||
|
# TODO(vmatt): some options (auth_type, auth_section) from
|
||||||
|
# keystoneauth1, loaded by keystonemiddleware.auth,
|
||||||
|
# are not defined conventionally (stable/ocata).
|
||||||
|
elif isinstance(o, type):
|
||||||
|
return {
|
||||||
|
'type': 'String'
|
||||||
|
}
|
||||||
|
|
||||||
|
else:
|
||||||
|
return type(o).__name__
|
||||||
|
|
||||||
|
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
def generate_ns_data(namespace):
|
||||||
|
"""Generate a json string containing the namespace"""
|
||||||
|
groups = generator._get_groups(generator._list_opts([namespace]))
|
||||||
|
return OptionJsonEncoder(sort_keys=True).encode(groups)
|
|
@ -0,0 +1,58 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Options for the config validator agent"""
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_config import types
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
GROUP = cfg.OptGroup(
|
||||||
|
name='agent',
|
||||||
|
title='Congress agent options for config datasource')
|
||||||
|
|
||||||
|
AGT_OPTS = [
|
||||||
|
cfg.StrOpt('host', required=True),
|
||||||
|
cfg.StrOpt('version', required=True, help='OpenStack version'),
|
||||||
|
cfg.IntOpt('max_delay', default=10, help='The maximum delay an agent will '
|
||||||
|
'wait before sending his files. '
|
||||||
|
'The smaller the value, the more '
|
||||||
|
'likely congestion is to happen'
|
||||||
|
'.'),
|
||||||
|
|
||||||
|
cfg.Opt(
|
||||||
|
'services',
|
||||||
|
help='Services activated on this node and configuration files',
|
||||||
|
default={},
|
||||||
|
sample_default=(
|
||||||
|
'nova: { /etc/nova/nova.conf:/path1.conf }, '
|
||||||
|
'neutron: { /etc/nova/neutron.conf:/path2.conf },'),
|
||||||
|
type=types.Dict(
|
||||||
|
bounds=False,
|
||||||
|
value_type=types.Dict(bounds=True, value_type=types.String()))),
|
||||||
|
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def register_validator_agent_opts(conf):
|
||||||
|
"""Register the options of the agent in the config object"""
|
||||||
|
conf.register_group(GROUP)
|
||||||
|
conf.register_opts(AGT_OPTS, group=GROUP)
|
||||||
|
logging.register_options(conf)
|
||||||
|
|
||||||
|
|
||||||
|
def list_opts():
|
||||||
|
"""List agent options"""
|
||||||
|
return [(GROUP, AGT_OPTS)]
|
|
@ -0,0 +1,89 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Handling of RPC
|
||||||
|
|
||||||
|
Communication with the datasource driver on the config validator agent
|
||||||
|
"""
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
import oslo_messaging as messaging
|
||||||
|
from oslo_service import service
|
||||||
|
|
||||||
|
from congress.dse2 import dse_node as dse
|
||||||
|
|
||||||
|
DRIVER_TOPIC = (dse.DseNode.SERVICE_TOPIC_PREFIX + 'config' + '-'
|
||||||
|
+ cfg.CONF.dse.bus_id)
|
||||||
|
|
||||||
|
|
||||||
|
class AgentService(service.Service):
|
||||||
|
"""Definition of the agent service implemented as an RPC endpoint."""
|
||||||
|
def __init__(self, topic, endpoints, conf=None):
|
||||||
|
super(AgentService, self).__init__()
|
||||||
|
self.conf = conf or cfg.CONF
|
||||||
|
|
||||||
|
self.host = self.conf.agent.host
|
||||||
|
self.topic = topic
|
||||||
|
self.endpoints = endpoints
|
||||||
|
self.transport = messaging.get_transport(self.conf)
|
||||||
|
self.target = messaging.Target(exchange=dse.DseNode.EXCHANGE,
|
||||||
|
topic=self.topic,
|
||||||
|
version=dse.DseNode.RPC_VERSION,
|
||||||
|
server=self.host)
|
||||||
|
self.server = messaging.get_rpc_server(self.transport,
|
||||||
|
self.target,
|
||||||
|
self.endpoints,
|
||||||
|
executor='eventlet')
|
||||||
|
|
||||||
|
def start(self):
|
||||||
|
super(AgentService, self).start()
|
||||||
|
self.server.start()
|
||||||
|
|
||||||
|
def stop(self, graceful=False):
|
||||||
|
self.server.stop()
|
||||||
|
super(AgentService, self).stop(graceful)
|
||||||
|
|
||||||
|
|
||||||
|
class ValidatorDriverClient(object):
|
||||||
|
"""RPC Proxy used by the agent to access the driver."""
|
||||||
|
def __init__(self, topic=DRIVER_TOPIC):
|
||||||
|
transport = messaging.get_transport(cfg.CONF)
|
||||||
|
target = messaging.Target(exchange=dse.DseNode.EXCHANGE,
|
||||||
|
topic=topic,
|
||||||
|
version=dse.DseNode.RPC_VERSION)
|
||||||
|
self.client = messaging.RPCClient(transport, target)
|
||||||
|
|
||||||
|
# block calling thread
|
||||||
|
def process_templates_hashes(self, context, hashes, host):
|
||||||
|
"""Sends a list of template hashes to the driver for processing
|
||||||
|
|
||||||
|
:param hashes: the array of hashes
|
||||||
|
:param host: the host they come from.
|
||||||
|
"""
|
||||||
|
cctx = self.client.prepare()
|
||||||
|
return cctx.call(context, 'process_templates_hashes', hashes=hashes,
|
||||||
|
host=host)
|
||||||
|
|
||||||
|
# block calling thread
|
||||||
|
def process_configs_hashes(self, context, hashes, host):
|
||||||
|
"""Sends a list of config files hashes to the driver for processing
|
||||||
|
|
||||||
|
:param hashes: the array of hashes
|
||||||
|
:param host: the host they come from.
|
||||||
|
"""
|
||||||
|
cctx = self.client.prepare()
|
||||||
|
return cctx.call(context, 'process_configs_hashes',
|
||||||
|
hashes=hashes, host=host)
|
|
@ -0,0 +1,213 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Unmarshaling of options sent by the agent."""
|
||||||
|
import sys
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_config import types
|
||||||
|
from oslo_log import log as logging
|
||||||
|
from oslo_serialization import jsonutils as json
|
||||||
|
import six
|
||||||
|
|
||||||
|
from congress.cfg_validator import utils
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
# pylint: disable=too-few-public-methods
|
||||||
|
class IdentifiedOpt(cfg.Opt):
|
||||||
|
"""A subclass of option that adds a unique id and a namespace id
|
||||||
|
|
||||||
|
ids are based on hashes
|
||||||
|
"""
|
||||||
|
def __init__(self, id_, ns_id, **kwargs):
|
||||||
|
super(IdentifiedOpt, self).__init__(**kwargs)
|
||||||
|
self.id_ = id_
|
||||||
|
self.ns_id = ns_id
|
||||||
|
|
||||||
|
|
||||||
|
def parse_value(cfgtype, value):
|
||||||
|
"""Parse and validate a value's type, raising error if check fails.
|
||||||
|
|
||||||
|
:raises: ValueError, TypeError
|
||||||
|
"""
|
||||||
|
return cfgtype(value)
|
||||||
|
|
||||||
|
|
||||||
|
def make_type(type_descr):
|
||||||
|
"""Declares a new type
|
||||||
|
|
||||||
|
:param type_descr: a type description read from json.
|
||||||
|
:return: an oslo config type
|
||||||
|
"""
|
||||||
|
type_name = type_descr['type']
|
||||||
|
type_descr = dict(type_descr)
|
||||||
|
del type_descr['type']
|
||||||
|
|
||||||
|
if 'item_type' in type_descr:
|
||||||
|
item_type = make_type(type_descr['item_type'])
|
||||||
|
type_descr['item_type'] = item_type
|
||||||
|
|
||||||
|
if 'value_type' in type_descr:
|
||||||
|
value_type = make_type(type_descr['value_type'])
|
||||||
|
type_descr['value_type'] = value_type
|
||||||
|
|
||||||
|
return getattr(types, type_name)(**type_descr)
|
||||||
|
|
||||||
|
|
||||||
|
# This function must never fail even if the content/metadata
|
||||||
|
# of the option were weird.
|
||||||
|
# pylint: disable=broad-except
|
||||||
|
def make_opt(option, opt_hash, ns_hash):
|
||||||
|
"""Declares a new group
|
||||||
|
|
||||||
|
:param name: an option retrieved from json.
|
||||||
|
:param opt_hash: the option hash
|
||||||
|
:param ns_hash: the hash of the namespace defining it.
|
||||||
|
:return: an oslo config option representation augmented with the hashes.
|
||||||
|
"""
|
||||||
|
name = option.get('name', None)
|
||||||
|
deprecateds = []
|
||||||
|
|
||||||
|
if option.get('deprecated_opts', None):
|
||||||
|
for depr_descr in option.get('deprecated_opts', {}):
|
||||||
|
depr_name = depr_descr.get('name', None)
|
||||||
|
if depr_name is None:
|
||||||
|
depr_name = name
|
||||||
|
depr_opt = cfg.DeprecatedOpt(depr_name,
|
||||||
|
depr_descr.get('group', None))
|
||||||
|
deprecateds.append(depr_opt)
|
||||||
|
|
||||||
|
cfgtype = make_type(option.get('type', {}))
|
||||||
|
default = option.get('default', None)
|
||||||
|
if default:
|
||||||
|
try:
|
||||||
|
default = cfgtype(default)
|
||||||
|
except Exception:
|
||||||
|
_, err, _ = sys.exc_info()
|
||||||
|
LOG.error('Unvalid default value (%s, %s): %s'
|
||||||
|
% (name, default, err))
|
||||||
|
try:
|
||||||
|
cfgopt = IdentifiedOpt(
|
||||||
|
id_=opt_hash,
|
||||||
|
ns_id=ns_hash,
|
||||||
|
name=name,
|
||||||
|
type=cfgtype,
|
||||||
|
dest=option.get('dest', None),
|
||||||
|
default=default,
|
||||||
|
positional=option.get('positional', None),
|
||||||
|
help=option.get('help', None),
|
||||||
|
secret=option.get('secret', None),
|
||||||
|
required=option.get('required', None),
|
||||||
|
sample_default=option.get('sample_default', None),
|
||||||
|
deprecated_for_removal=option.get('deprecated_for_removal', None),
|
||||||
|
deprecated_reason=option.get('deprecated_reason', None),
|
||||||
|
deprecated_opts=deprecateds,
|
||||||
|
mutable=option.get('mutable', None))
|
||||||
|
except Exception:
|
||||||
|
cfgopt = None
|
||||||
|
_, err, _ = sys.exc_info()
|
||||||
|
LOG.error('Unvalid option definition (%s in %s): %s'
|
||||||
|
% (name, ns_hash, err))
|
||||||
|
return cfgopt
|
||||||
|
|
||||||
|
|
||||||
|
def make_group(name, title, help_msg):
|
||||||
|
"""Declares a new group
|
||||||
|
|
||||||
|
:param name: group name
|
||||||
|
:param title: group title
|
||||||
|
:param help_msg: descriptive help message
|
||||||
|
:return: an oslo config group representation or None for default.
|
||||||
|
"""
|
||||||
|
if name == 'DEFAULT':
|
||||||
|
return None
|
||||||
|
|
||||||
|
return cfg.OptGroup(name=name, title=title, help=help_msg)
|
||||||
|
|
||||||
|
|
||||||
|
def add_namespace(conf, ns_dict, ns_hash):
|
||||||
|
"""Add options from a kind to an already existing config"""
|
||||||
|
|
||||||
|
for group_name, group in six.iteritems(ns_dict):
|
||||||
|
|
||||||
|
try:
|
||||||
|
title = group['object'].get('title', None)
|
||||||
|
help_msg = group['object'].get('help', None)
|
||||||
|
except AttributeError:
|
||||||
|
title = help_msg = None
|
||||||
|
cfggroup = make_group(group_name, title, help_msg)
|
||||||
|
|
||||||
|
# Get back the instance already stored or register the group.
|
||||||
|
if cfggroup is not None:
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
cfggroup = conf._get_group(cfggroup, autocreate=True)
|
||||||
|
|
||||||
|
for namespace in group['namespaces']:
|
||||||
|
|
||||||
|
for option in namespace[1]:
|
||||||
|
opt_hash = utils.compute_hash(ns_hash, group_name,
|
||||||
|
option['name'])
|
||||||
|
cfgopt = make_opt(option, opt_hash, ns_hash)
|
||||||
|
conf.register_opt(cfgopt, cfggroup)
|
||||||
|
|
||||||
|
|
||||||
|
def construct_conf_manager(namespaces):
|
||||||
|
"""Construct a config manager from a list of namespaces data.
|
||||||
|
|
||||||
|
Register options of given namespaces into a cfg.ConfigOpts object.
|
||||||
|
A namespaces dict is typically cfg_validator.generator output. Options are
|
||||||
|
provided an hash as an extra field.
|
||||||
|
|
||||||
|
:param namespaces: A list of dict, containing options metadata.
|
||||||
|
:return: A cfg.ConfigOpts.
|
||||||
|
"""
|
||||||
|
conf = cfg.ConfigOpts()
|
||||||
|
|
||||||
|
for ns_dict in namespaces:
|
||||||
|
ns_hash = utils.compute_hash(json.dumps(ns_dict, sort_keys=True))
|
||||||
|
add_namespace(conf, ns_dict, ns_hash)
|
||||||
|
|
||||||
|
return conf
|
||||||
|
|
||||||
|
|
||||||
|
def add_parsed_conf(conf, normalized):
|
||||||
|
"""Add a normalized values container to a config manager.
|
||||||
|
|
||||||
|
:param conf: A cfg.ConfigOpts object.
|
||||||
|
:param normalized: A normalized values container, as introduced by oslo
|
||||||
|
cfg._Namespace.
|
||||||
|
"""
|
||||||
|
if conf:
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
conf._namespace = cfg._Namespace(conf)
|
||||||
|
conf._namespace._add_parsed_config_file(None, normalized[0])
|
||||||
|
|
||||||
|
|
||||||
|
def parse_config_file(namespaces, path):
|
||||||
|
"""Parse a config file from its pre-loaded namespaces.
|
||||||
|
|
||||||
|
:param namespaces: A list of dict, containing namespaces data.
|
||||||
|
:param path: Path to the configuration file to parse.
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
conf = construct_conf_manager(namespaces)
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
conf._namespace = cfg._Namespace(conf)
|
||||||
|
cfg.ConfigParser._parse_file(path, conf._namespace)
|
||||||
|
|
||||||
|
return conf
|
|
@ -0,0 +1,67 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Support functions for cfg_validator"""
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from congress.api import base
|
||||||
|
from congress import exception
|
||||||
|
from congress import utils
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
#: Topic for RPC between cfg validator driver (client) and the agents (server)
|
||||||
|
AGENT_TOPIC = 'congress-validator-agent'
|
||||||
|
NAMESPACE_CONGRESS = uuid.uuid3(
|
||||||
|
uuid.NAMESPACE_URL,
|
||||||
|
'http://openstack.org/congress/agent')
|
||||||
|
|
||||||
|
|
||||||
|
def compute_hash(*args):
|
||||||
|
"""computes a hash from the arguments. Not cryptographically strong."""
|
||||||
|
inputs = ''.join([str(arg) for arg in args])
|
||||||
|
return str(uuid.uuid3(NAMESPACE_CONGRESS, inputs))
|
||||||
|
|
||||||
|
|
||||||
|
def cfg_value_to_congress(value):
|
||||||
|
"""Sanitize values for congress
|
||||||
|
|
||||||
|
values of log formatting options typically contains
|
||||||
|
'%s' etc, which should not be put in datalog
|
||||||
|
"""
|
||||||
|
if isinstance(value, str):
|
||||||
|
value = value.replace('%', '')
|
||||||
|
if value is None:
|
||||||
|
return ''
|
||||||
|
return utils.value_to_congress(value)
|
||||||
|
|
||||||
|
|
||||||
|
def add_rule(bus, policy_name, rules):
|
||||||
|
"Adds a policy and rules to the engine"
|
||||||
|
try:
|
||||||
|
policy_metadata = bus.rpc(
|
||||||
|
base.ENGINE_SERVICE_ID,
|
||||||
|
'persistent_create_policy_with_rules',
|
||||||
|
{'policy_rules_obj': {
|
||||||
|
"name": policy_name,
|
||||||
|
"kind": "nonrecursive",
|
||||||
|
"rules": rules}})
|
||||||
|
return policy_metadata
|
||||||
|
except exception.CongressException as err:
|
||||||
|
LOG.error(err)
|
||||||
|
return None
|
|
@ -0,0 +1,58 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""
|
||||||
|
Utilities for testing RPC clients.
|
||||||
|
"""
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from oslotest import base
|
||||||
|
|
||||||
|
|
||||||
|
# TODO(pc): all this does not ensure that transport and target make any sense.
|
||||||
|
# But not tested in neutron ml2 use case either.
|
||||||
|
class BaseTestRpcClient(base.BaseTestCase):
|
||||||
|
"""Abstract class providing functions to test client RPC"""
|
||||||
|
|
||||||
|
def _test_rpc_api(self, rpcapi, topic, method, rpc_method, **kwargs):
|
||||||
|
"""Base function to test each call"""
|
||||||
|
ctxt = {}
|
||||||
|
expected_retval = 'foo' if rpc_method == 'call' else None
|
||||||
|
expected_version = kwargs.pop('version', None)
|
||||||
|
fanout = kwargs.pop('fanout', False)
|
||||||
|
server = kwargs.get('server', None)
|
||||||
|
|
||||||
|
with mock.patch.object(rpcapi.client, rpc_method) as rpc_mock,\
|
||||||
|
mock.patch.object(rpcapi.client, 'prepare') as prepare_mock:
|
||||||
|
prepare_mock.return_value = rpcapi.client
|
||||||
|
rpc_mock.return_value = expected_retval
|
||||||
|
retval = getattr(rpcapi, method)(ctxt, **kwargs)
|
||||||
|
|
||||||
|
prepare_args = {}
|
||||||
|
if expected_version:
|
||||||
|
prepare_args['version'] = expected_version
|
||||||
|
if fanout:
|
||||||
|
prepare_args['fanout'] = fanout
|
||||||
|
if topic:
|
||||||
|
prepare_args['topic'] = topic
|
||||||
|
if server:
|
||||||
|
prepare_args['server'] = server
|
||||||
|
prepare_mock.assert_called_once_with(**prepare_args)
|
||||||
|
|
||||||
|
self.assertEqual(retval, expected_retval)
|
||||||
|
if server:
|
||||||
|
kwargs.pop('server')
|
||||||
|
rpc_mock.assert_called_once_with(ctxt, method, **kwargs)
|
|
@ -0,0 +1,372 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
"""Tests for the config validator agent."""
|
||||||
|
|
||||||
|
from os import path
|
||||||
|
|
||||||
|
import json
|
||||||
|
import mock
|
||||||
|
import six
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from congress.cfg_validator.agent import agent
|
||||||
|
from congress.cfg_validator.agent import opts
|
||||||
|
from congress.tests import base
|
||||||
|
|
||||||
|
# pylint: disable=protected-access
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
NAMESPACE_FILE_CONTENT = """[DEFAULT]
|
||||||
|
output_file = etc/svc.conf.sample
|
||||||
|
wrap_width = 79
|
||||||
|
namespace = congress
|
||||||
|
namespace = congress-agent
|
||||||
|
"""
|
||||||
|
|
||||||
|
CONF_FILE1_CONTENT = """
|
||||||
|
[agent]
|
||||||
|
version:v0
|
||||||
|
"""
|
||||||
|
|
||||||
|
CONF_FILE2_CONTENT = """
|
||||||
|
[agent]
|
||||||
|
host:h0
|
||||||
|
"""
|
||||||
|
|
||||||
|
CONF_AGENT_CONTENT = """
|
||||||
|
[agent]
|
||||||
|
host:hhh
|
||||||
|
version:vvv
|
||||||
|
services: svc: {svc1.conf: svc.tpl}
|
||||||
|
"""
|
||||||
|
|
||||||
|
TEMPLATES = {
|
||||||
|
"tpl1": agent.Template("TPL1", "out"),
|
||||||
|
"tpl2": agent.Template("TPL2", "out")}
|
||||||
|
CONFIGS = {
|
||||||
|
"cfg1": agent.Config("CFG1", TEMPLATES["tpl1"], "svc"),
|
||||||
|
"cfg2": agent.Config("CFG2", TEMPLATES["tpl2"], "svc")}
|
||||||
|
NAMESPACES = {
|
||||||
|
"ns1": agent.Namespace("NS1"),
|
||||||
|
"ns2": agent.Namespace("NS2")}
|
||||||
|
|
||||||
|
|
||||||
|
def _gen_ns_data_fake(namespace):
|
||||||
|
return u"{\"DEFAULT\": {\"namespaces\": [[\"" + namespace + u"\", []]]}}"
|
||||||
|
|
||||||
|
|
||||||
|
class TestTemplate(base.TestCase):
|
||||||
|
|
||||||
|
"Test template loading"
|
||||||
|
|
||||||
|
@mock.patch('oslo_config.cfg.open')
|
||||||
|
def test_parse_template(self, mock_open):
|
||||||
|
"Test loading a template"
|
||||||
|
mock.mock_open(mock=mock_open, read_data=NAMESPACE_FILE_CONTENT)
|
||||||
|
# Patch the mock_open file to support iteration.
|
||||||
|
mock_open.return_value.__iter__ = lambda x: iter(x.readline, '')
|
||||||
|
|
||||||
|
tpl, out_file = agent.Template._parse_template_conf('template')
|
||||||
|
self.assertEqual(len(tpl), 2)
|
||||||
|
self.assertEqual(out_file, 'etc/svc.conf.sample')
|
||||||
|
|
||||||
|
|
||||||
|
class TestCfgConfig(base.TestCase):
|
||||||
|
|
||||||
|
"Test config handling"
|
||||||
|
|
||||||
|
def test_sanitize(self):
|
||||||
|
"test config sanitization"
|
||||||
|
conf = cfg.ConfigOpts()
|
||||||
|
conf._namespace = cfg._Namespace(conf)
|
||||||
|
conf._namespace._normalized = []
|
||||||
|
|
||||||
|
opt_1 = cfg.StrOpt(name='lorem', secret=True)
|
||||||
|
opt_2 = cfg.StrOpt(name='ipsum', secret=False)
|
||||||
|
conf.register_opts([opt_1, opt_2])
|
||||||
|
|
||||||
|
parsed = {'DEFAULT': {'lorem': ['mysecret'], 'ipsum': ['notsecret']}}
|
||||||
|
conf._namespace._normalized.append(parsed)
|
||||||
|
|
||||||
|
agent.Config.sanitize_config(conf)
|
||||||
|
|
||||||
|
self.assertFalse('mysecret' in json.dumps(conf._namespace._normalized))
|
||||||
|
self.assertTrue(
|
||||||
|
'notsecret' in json.dumps(conf._namespace._normalized))
|
||||||
|
|
||||||
|
self.assertEqual(conf.lorem, '****')
|
||||||
|
self.assertEqual(conf.ipsum, 'notsecret')
|
||||||
|
|
||||||
|
def test_get_info(self):
|
||||||
|
"test basic get_info"
|
||||||
|
cfg_mock = mock.Mock(spec=agent.Config)
|
||||||
|
tpl = mock.Mock()
|
||||||
|
tpl.hash = 'lorem'
|
||||||
|
cfg_mock.template = tpl
|
||||||
|
cfg_mock.path = 'ipsum'
|
||||||
|
cfg_mock.data = 'dolor'
|
||||||
|
cfg_mock.service = 'svc'
|
||||||
|
|
||||||
|
info = agent.Config.get_info(cfg_mock)
|
||||||
|
|
||||||
|
self.assertIn('template', info)
|
||||||
|
self.assertEqual(info['template'], 'lorem')
|
||||||
|
self.assertIn('path', info)
|
||||||
|
self.assertEqual(info['path'], 'ipsum')
|
||||||
|
self.assertIn('data', info)
|
||||||
|
self.assertEqual(info['data'], 'dolor')
|
||||||
|
|
||||||
|
|
||||||
|
class TestCfgNamespace(base.TestCase):
|
||||||
|
"Test namespace handling"
|
||||||
|
|
||||||
|
@mock.patch('congress.cfg_validator.agent.agent.'
|
||||||
|
'validator_generator.generate_ns_data')
|
||||||
|
def test_load(self, gen_ns_data_mock):
|
||||||
|
"Test load namespace"
|
||||||
|
gen_ns_data_mock.return_value = _gen_ns_data_fake('lorem')
|
||||||
|
|
||||||
|
ns_mock = agent.Namespace.load('ipsum')
|
||||||
|
|
||||||
|
self.assertEqual(ns_mock.name, 'ipsum')
|
||||||
|
self.assertEqual(json.dumps(ns_mock.data), _gen_ns_data_fake('lorem'))
|
||||||
|
self.assertIsNotNone(ns_mock.hash)
|
||||||
|
|
||||||
|
same_data_ns = agent.Namespace.load('other_ipsum')
|
||||||
|
self.assertEqual(same_data_ns.hash, ns_mock.hash)
|
||||||
|
|
||||||
|
gen_ns_data_mock.return_value = _gen_ns_data_fake('other_lorem')
|
||||||
|
other_data_ns = agent.Namespace.load('ipsum')
|
||||||
|
self.assertNotEqual(ns_mock.hash, other_data_ns.hash)
|
||||||
|
|
||||||
|
def test_get_info(self):
|
||||||
|
"Test basic info on namespace"
|
||||||
|
ns_mock = mock.Mock(spec=agent.Namespace)
|
||||||
|
ns_mock.name = 'foo'
|
||||||
|
ns_mock.data = 'bar'
|
||||||
|
|
||||||
|
info = agent.Namespace.get_info(ns_mock)
|
||||||
|
|
||||||
|
self.assertIn('name', info)
|
||||||
|
self.assertEqual(info['name'], 'foo')
|
||||||
|
self.assertIn('data', info)
|
||||||
|
self.assertEqual(info['data'], 'bar')
|
||||||
|
|
||||||
|
|
||||||
|
class TestCfgTemplate(base.TestCase):
|
||||||
|
"""Test the handling of templates"""
|
||||||
|
|
||||||
|
@mock.patch('congress.cfg_validator.agent.agent.'
|
||||||
|
'cfg.ConfigOpts._parse_config_files')
|
||||||
|
def test_template_conf(self, parse_mock):
|
||||||
|
"""Test the parsing of template"""
|
||||||
|
ns_mock = cfg._Namespace(None)
|
||||||
|
ns_mock._normalized.append({
|
||||||
|
'DEFAULT':
|
||||||
|
{
|
||||||
|
'output_file': ['etc/congress.conf.sample'],
|
||||||
|
'wrap_width': ['79'],
|
||||||
|
'namespace': ['congress', 'oslo.log'],
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
parse_mock.return_value = ns_mock
|
||||||
|
|
||||||
|
namespace, out_file = agent.Template._parse_template_conf('somewhere')
|
||||||
|
|
||||||
|
self.assertEqual(out_file, 'etc/congress.conf.sample')
|
||||||
|
self.assertIn('congress', namespace)
|
||||||
|
self.assertIn('oslo.log', namespace)
|
||||||
|
|
||||||
|
@mock.patch('congress.cfg_validator.agent.agent.Namespace.load')
|
||||||
|
@mock.patch('congress.cfg_validator.agent.agent.'
|
||||||
|
'Template._parse_template_conf')
|
||||||
|
def test_load(self, parse_tpl_conf_mock, load_ns_mock):
|
||||||
|
"""Test loading a template"""
|
||||||
|
parse_tpl_conf_mock.return_value = (['congress_h', 'oslo.log'],
|
||||||
|
'some/where.sample')
|
||||||
|
|
||||||
|
load_ns_mock.side_effect = [mock.MagicMock(hash='lorem'),
|
||||||
|
mock.MagicMock(hash='ipsum')]
|
||||||
|
|
||||||
|
tpl = agent.Template.load('path/to/template')
|
||||||
|
|
||||||
|
self.assertEqual(tpl.path, 'path/to/template')
|
||||||
|
self.assertEqual(tpl.output_file, 'some/where.sample')
|
||||||
|
self.assertEqual(tpl.name, 'where')
|
||||||
|
self.assertIsNotNone(tpl.hash)
|
||||||
|
self.assertTrue(tpl.namespaces)
|
||||||
|
self.assertIn('lorem', [ns.hash for ns in tpl.namespaces])
|
||||||
|
self.assertIn('ipsum', [ns.hash for ns in tpl.namespaces])
|
||||||
|
|
||||||
|
def test_get_info(self):
|
||||||
|
"""Test basic info on template"""
|
||||||
|
tpl = mock.Mock(spec=agent.Template)
|
||||||
|
tpl.path = 'lorem'
|
||||||
|
tpl.output_file = 'ipsum'
|
||||||
|
tpl.name = 'dolor'
|
||||||
|
ns_mock = mock.Mock()
|
||||||
|
ns_mock.hash = 'sit'
|
||||||
|
tpl.namespaces = [ns_mock]
|
||||||
|
|
||||||
|
info = agent.Template.get_info(tpl)
|
||||||
|
|
||||||
|
self.assertIn('name', info)
|
||||||
|
self.assertEqual(info['name'], 'dolor')
|
||||||
|
self.assertIn('path', info)
|
||||||
|
self.assertEqual(info['path'], 'lorem')
|
||||||
|
self.assertIn('output_file', info)
|
||||||
|
self.assertEqual(info['output_file'], 'ipsum')
|
||||||
|
self.assertIn('namespaces', info)
|
||||||
|
self.assertEqual(info['namespaces'], ['sit'])
|
||||||
|
|
||||||
|
|
||||||
|
def _file_mock(file_spec):
|
||||||
|
def _mk_content(spec):
|
||||||
|
fval = mock.mock_open(read_data=spec)
|
||||||
|
fval.return_value.__iter__ = lambda x: iter(x.readline, '')
|
||||||
|
return fval
|
||||||
|
file_map = {name: _mk_content(spec)
|
||||||
|
for name, spec in six.iteritems(file_spec)}
|
||||||
|
|
||||||
|
def _give_file(name):
|
||||||
|
basename = path.basename(name)
|
||||||
|
return file_map.get(basename, None)(name)
|
||||||
|
|
||||||
|
return mock.MagicMock(name='open', spec=open, side_effect=_give_file)
|
||||||
|
|
||||||
|
|
||||||
|
class TestCfgManager(base.TestCase):
|
||||||
|
"""Config manager tests"""
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'oslo_config.cfg.open',
|
||||||
|
_file_mock({
|
||||||
|
"svc.tpl": NAMESPACE_FILE_CONTENT,
|
||||||
|
"svc1.conf": CONF_FILE1_CONTENT,
|
||||||
|
"svc2.conf": CONF_FILE2_CONTENT}))
|
||||||
|
def test_init(self):
|
||||||
|
"""Test the creation of the config manager"""
|
||||||
|
cfg_manager = agent.ConfigManager(
|
||||||
|
'host', {"svc": {"svc1.conf": "svc.tpl", "svc2.conf": "svc.tpl"}})
|
||||||
|
self.assertEqual('host', cfg_manager.host)
|
||||||
|
self.assertEqual(2, len(cfg_manager.configs))
|
||||||
|
self.assertEqual(2, len(cfg_manager.namespaces))
|
||||||
|
self.assertEqual(1, len(cfg_manager.templates))
|
||||||
|
for conf in six.itervalues(cfg_manager.configs):
|
||||||
|
self.assertIsInstance(conf, agent.Config)
|
||||||
|
for nspc in six.itervalues(cfg_manager.namespaces):
|
||||||
|
self.assertIsInstance(nspc, agent.Namespace)
|
||||||
|
for tpl in six.itervalues(cfg_manager.templates):
|
||||||
|
self.assertIsInstance(tpl, agent.Template)
|
||||||
|
|
||||||
|
|
||||||
|
def _setup_endpoint():
|
||||||
|
with mock.patch(
|
||||||
|
'oslo_config.cfg.open',
|
||||||
|
_file_mock({"agent.conf": CONF_AGENT_CONTENT})),\
|
||||||
|
mock.patch(
|
||||||
|
'congress.cfg_validator.agent.agent.ConfigManager',
|
||||||
|
autospec=True) as mock_manager,\
|
||||||
|
mock.patch(
|
||||||
|
'congress.cfg_validator.agent.rpc.ValidatorDriverClient',
|
||||||
|
autospec=True) as mock_client:
|
||||||
|
conf = cfg.ConfigOpts()
|
||||||
|
opts.register_validator_agent_opts(conf)
|
||||||
|
conf(args=['--config-file', 'agent.conf'])
|
||||||
|
mock_manager.return_value.configs = CONFIGS
|
||||||
|
mock_manager.return_value.templates = TEMPLATES
|
||||||
|
mock_manager.return_value.namespaces = NAMESPACES
|
||||||
|
endpoint = agent.ValidatorAgentEndpoint(conf=conf)
|
||||||
|
return endpoint, mock_client, mock_manager
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidatorAgentEndpoint(base.TestCase):
|
||||||
|
"""Test the endpoint for the agent communications"""
|
||||||
|
|
||||||
|
# pylint: disable=no-self-use
|
||||||
|
|
||||||
|
@mock.patch(
|
||||||
|
'oslo_config.cfg.open',
|
||||||
|
_file_mock({
|
||||||
|
"agent.conf": CONF_AGENT_CONTENT}))
|
||||||
|
@mock.patch(
|
||||||
|
'congress.cfg_validator.agent.agent.ConfigManager',
|
||||||
|
autospec=True)
|
||||||
|
@mock.patch(
|
||||||
|
'congress.cfg_validator.agent.rpc.ValidatorDriverClient',
|
||||||
|
autospec=True)
|
||||||
|
def test_publish_template_hashes(self, mock_client, mock_manager):
|
||||||
|
"Test a request to publish hashes"
|
||||||
|
conf = cfg.ConfigOpts()
|
||||||
|
opts.register_validator_agent_opts(conf)
|
||||||
|
conf(args=['--config-file', 'agent.conf'])
|
||||||
|
templates = {"tpl1": {}, "tpl2": {}}
|
||||||
|
mock_manager.return_value.templates = templates
|
||||||
|
endpoint = agent.ValidatorAgentEndpoint(conf=conf)
|
||||||
|
endpoint.publish_templates_hashes({})
|
||||||
|
mock_client.return_value.process_templates_hashes.assert_called_with(
|
||||||
|
{}, set(templates), "hhh")
|
||||||
|
|
||||||
|
def test_publish_configs_hashes(self):
|
||||||
|
"Test a request to publish hashes"
|
||||||
|
endpoint, mock_client, _ = _setup_endpoint()
|
||||||
|
endpoint.publish_configs_hashes({})
|
||||||
|
mock_client.return_value.process_configs_hashes.assert_called_with(
|
||||||
|
{}, set(CONFIGS), "hhh")
|
||||||
|
|
||||||
|
def test_get_config(self):
|
||||||
|
"Test reply to an explicit config request"
|
||||||
|
endpoint, _, _ = _setup_endpoint()
|
||||||
|
ret = endpoint.get_config({}, cfg_hash="cfg1")
|
||||||
|
expected = {
|
||||||
|
'data': None,
|
||||||
|
'path': 'CFG1',
|
||||||
|
'service': 'svc',
|
||||||
|
'template': None,
|
||||||
|
'version': 'vvv'}
|
||||||
|
self.assertEqual(expected, ret)
|
||||||
|
ret = endpoint.get_config({}, cfg_hash="XXX")
|
||||||
|
self.assertEqual(None, ret)
|
||||||
|
|
||||||
|
def test_get_namespace(self):
|
||||||
|
"Test reply to an explicit config request"
|
||||||
|
endpoint, _, _ = _setup_endpoint()
|
||||||
|
ret = endpoint.get_namespace({}, ns_hash="ns1")
|
||||||
|
expected = {
|
||||||
|
'version': 'vvv',
|
||||||
|
'data': None,
|
||||||
|
'name': 'NS1'}
|
||||||
|
self.assertEqual(expected, ret)
|
||||||
|
ret = endpoint.get_namespace({}, ns_hash="XXX")
|
||||||
|
self.assertEqual(None, ret)
|
||||||
|
|
||||||
|
def test_get_template(self):
|
||||||
|
"Test reply to an explicit config request"
|
||||||
|
endpoint, _, _ = _setup_endpoint()
|
||||||
|
ret = endpoint.get_template({}, tpl_hash="tpl1")
|
||||||
|
expected = {
|
||||||
|
'name': 'out',
|
||||||
|
'namespaces': [],
|
||||||
|
'output_file': 'out',
|
||||||
|
'path': 'TPL1',
|
||||||
|
'version': 'vvv'}
|
||||||
|
self.assertEqual(expected, ret)
|
||||||
|
ret = endpoint.get_template({}, tpl_hash="XXX")
|
||||||
|
self.assertEqual(None, ret)
|
|
@ -0,0 +1,114 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
"""Unit test for the marshalling of options"""
|
||||||
|
import mock
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_config import types
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from congress.cfg_validator.agent import generator
|
||||||
|
from congress.tests import base
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class TestGenerator(base.TestCase):
|
||||||
|
"""Unit test for the marshalling of options"""
|
||||||
|
|
||||||
|
@mock.patch("oslo_config.generator._list_opts")
|
||||||
|
def test_encode(self, mock_list_opts):
|
||||||
|
"""Test the json encoding of options"""
|
||||||
|
|
||||||
|
opt1 = cfg.StrOpt("o1"),
|
||||||
|
opt2 = cfg.IntOpt("o2", default=10),
|
||||||
|
opt3 = cfg.StrOpt("o3", default="a"),
|
||||||
|
mock_list_opts.return_value = [
|
||||||
|
("ns", [("g1", [opt1, opt2]), ("g2", [opt3])])]
|
||||||
|
namespace = "ns"
|
||||||
|
|
||||||
|
def _mko(nam, typ, kind, defv):
|
||||||
|
return (
|
||||||
|
'{"advanced": false, "default": %s, "deprecated_for_removal":'
|
||||||
|
' false, "deprecated_opts": [], "deprecated_reason": null,'
|
||||||
|
' "deprecated_since": null, "dest": "%s", "help": null,'
|
||||||
|
' "kind": "%s", "metavar": null, "mutable": false,'
|
||||||
|
' "name": "%s", "positional": false, "required": false,'
|
||||||
|
' "sample_default": null, "secret": false, "short": null,'
|
||||||
|
' "type": {"type": "%s"}}') % (defv, nam, kind, nam, typ)
|
||||||
|
ns_string = (
|
||||||
|
('{"DEFAULT": {"namespaces": [], "object": null}, "g1": '
|
||||||
|
'{"namespaces": [["ns", [[') +
|
||||||
|
_mko("o1", "String", "StrOpt", "null") +
|
||||||
|
'], [' + _mko("o2", "Integer", "IntOpt", 10) +
|
||||||
|
']]]], "object": null}, "g2": {"namespaces": [["ns", [[' +
|
||||||
|
_mko("o3", "String", "StrOpt", "\"a\"") + ']]]], "object": null}}')
|
||||||
|
computed = generator.generate_ns_data(namespace)
|
||||||
|
self.assertEqual(
|
||||||
|
ns_string, computed,
|
||||||
|
"not the expected encoding of namespace")
|
||||||
|
|
||||||
|
@mock.patch("oslo_config.generator._list_opts")
|
||||||
|
def _test_encode_specific(self, opt, expected, mock_list_opts=None):
|
||||||
|
"""Test an option and check an expected string in result"""
|
||||||
|
mock_list_opts.return_value = [
|
||||||
|
("ns", [("g", [opt])])]
|
||||||
|
namespace = "ns"
|
||||||
|
computed = generator.generate_ns_data(namespace)
|
||||||
|
self.assertIn(expected, computed)
|
||||||
|
|
||||||
|
def test_encode_range(self):
|
||||||
|
"""Test the json encoding of range option"""
|
||||||
|
opt = cfg.Opt('o', types.Range(min=123, max=456))
|
||||||
|
expected = '"type": {"max": 456, "min": 123, "type": "Range"}'
|
||||||
|
self._test_encode_specific(opt, expected)
|
||||||
|
|
||||||
|
def test_encode_list(self):
|
||||||
|
"""Test the json encoding of list option"""
|
||||||
|
opt = cfg.ListOpt('opt', types.Boolean)
|
||||||
|
expected = '"type": {"item_type": {"type": "String"}, "type": "List"}'
|
||||||
|
self._test_encode_specific(opt, expected)
|
||||||
|
|
||||||
|
def test_encode_dict(self):
|
||||||
|
"""Test the json encoding of dict option"""
|
||||||
|
opt = cfg.DictOpt('opt')
|
||||||
|
expected = '"type": {"type": "Dict", "value_type": {"type": "String"}}'
|
||||||
|
self._test_encode_specific(opt, expected)
|
||||||
|
|
||||||
|
def test_encode_ip(self):
|
||||||
|
"""Test the json encoding of IP option"""
|
||||||
|
opt = cfg.IPOpt('opt')
|
||||||
|
expected = '"type": {"type": "IPAddress"}'
|
||||||
|
self._test_encode_specific(opt, expected)
|
||||||
|
|
||||||
|
def test_encode_uri(self):
|
||||||
|
"""Test the json encoding of a URI"""
|
||||||
|
opt = cfg.URIOpt('opt', 100)
|
||||||
|
expected = '"type": {"max_length": 100, "type": "URI"}'
|
||||||
|
self._test_encode_specific(opt, expected)
|
||||||
|
|
||||||
|
def test_encode_regex(self):
|
||||||
|
"""Test the json encoding of a string option with regex"""
|
||||||
|
opt = cfg.StrOpt('opt', regex=r'abcd')
|
||||||
|
expected = '"type": {"regex": "abcd", "type": "String"}'
|
||||||
|
self._test_encode_specific(opt, expected)
|
||||||
|
|
||||||
|
def test_encode_deprecated(self):
|
||||||
|
"""Test Deprecated Opt: this is not the way to use it"""
|
||||||
|
dep = cfg.DeprecatedOpt('o_old', 'g_old')
|
||||||
|
opt = cfg.Opt('opt', deprecated_opts=[dep])
|
||||||
|
expected = '"deprecated_opts": [{"group": "g_old", "name": "o_old"}]'
|
||||||
|
self._test_encode_specific(opt, expected)
|
|
@ -0,0 +1,50 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""
|
||||||
|
Unit Tests for Congress agent RPC
|
||||||
|
"""
|
||||||
|
|
||||||
|
from congress.cfg_validator.agent import rpc
|
||||||
|
from congress.tests import base_rpc
|
||||||
|
|
||||||
|
|
||||||
|
class TestValidatorDriverApi(base_rpc.BaseTestRpcClient):
|
||||||
|
"""Unit tests for RPC on the driver"""
|
||||||
|
|
||||||
|
def test_process_templates_hashes(self):
|
||||||
|
"unit test for process_templates_hashes"
|
||||||
|
rpcapi = rpc.ValidatorDriverClient()
|
||||||
|
self._test_rpc_api(
|
||||||
|
rpcapi,
|
||||||
|
None,
|
||||||
|
'process_templates_hashes',
|
||||||
|
rpc_method='call',
|
||||||
|
hashes='fake_hashes',
|
||||||
|
host='host'
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_process_configs_hashes(self):
|
||||||
|
"unit test for process_configs_hashes"
|
||||||
|
rpcapi = rpc.ValidatorDriverClient()
|
||||||
|
self._test_rpc_api(
|
||||||
|
rpcapi,
|
||||||
|
None,
|
||||||
|
'process_configs_hashes',
|
||||||
|
rpc_method='call',
|
||||||
|
hashes='fake_hashes',
|
||||||
|
host='host'
|
||||||
|
)
|
|
@ -0,0 +1,110 @@
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Tests for the unmarshaling of options by the driver"""
|
||||||
|
|
||||||
|
from oslo_config import cfg
|
||||||
|
from oslo_config import types
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from congress.cfg_validator import parsing
|
||||||
|
from congress.tests import base
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
OPT_TEST = {
|
||||||
|
u'positional': False, u'kind': u'BoolOpt',
|
||||||
|
u'deprecated_reason': None,
|
||||||
|
u'help': u'Enables or disables inter-process locks.',
|
||||||
|
u'default': False, u'type': {u'type': u'Boolean'},
|
||||||
|
u'required': False, u'sample_default': None,
|
||||||
|
u'deprecated_opts': [{u'group': u'DEFAULT', u'name': None}],
|
||||||
|
u'deprecated_for_removal': False,
|
||||||
|
u'dest': u'disable_process_locking',
|
||||||
|
u'secret': False, u'short': None, u'mutable': False,
|
||||||
|
u'deprecated_since': None, u'metavar': None,
|
||||||
|
u'advanced': False, u'name': u'disable_process_locking'}
|
||||||
|
DICT_NS_TEST = {
|
||||||
|
u'DEFAULT': {u'object': None, u'namespaces': []},
|
||||||
|
u'oslo_concurrency': {
|
||||||
|
u'object': None,
|
||||||
|
u'namespaces': [[u'oslo.concurrency', [OPT_TEST]]]}}
|
||||||
|
|
||||||
|
|
||||||
|
class TestParsing(base.TestCase):
|
||||||
|
"""Tests for the unmarshaling of options by the driver"""
|
||||||
|
|
||||||
|
def test_add_namespace(self):
|
||||||
|
"""Test for adding a namespace"""
|
||||||
|
conf = cfg.ConfigOpts()
|
||||||
|
parsing.add_namespace(conf, DICT_NS_TEST, 'abcde-12345')
|
||||||
|
keys = conf.keys()
|
||||||
|
self.assertEqual(1, len(keys))
|
||||||
|
self.assertIn(u'oslo_concurrency', keys)
|
||||||
|
self.assertIsNotNone(
|
||||||
|
conf.get(u'oslo_concurrency').get(u'disable_process_locking'))
|
||||||
|
|
||||||
|
def test_construct_conf_manager(self):
|
||||||
|
"""Test for building a conf manager"""
|
||||||
|
conf = parsing.construct_conf_manager([DICT_NS_TEST])
|
||||||
|
self.assertIsInstance(conf, cfg.ConfigOpts)
|
||||||
|
keys = conf.keys()
|
||||||
|
self.assertEqual(1, len(keys))
|
||||||
|
self.assertIn(u'oslo_concurrency', keys)
|
||||||
|
|
||||||
|
def test_make_group(self):
|
||||||
|
"""Test for parsing a group"""
|
||||||
|
grp = parsing.make_group('group', 'group_title', 'group help')
|
||||||
|
self.assertIsInstance(grp, cfg.OptGroup)
|
||||||
|
self.assertEqual("group", grp.name)
|
||||||
|
self.assertEqual("group_title", grp.title)
|
||||||
|
|
||||||
|
def test_make_opt(self):
|
||||||
|
"""Test for parsing an option"""
|
||||||
|
descr = {
|
||||||
|
u'positional': False,
|
||||||
|
u'kind': u'Opt',
|
||||||
|
u'deprecated_reason': None,
|
||||||
|
u'help': u'Help me',
|
||||||
|
u'default': None,
|
||||||
|
u'type': {u'type': u'String'},
|
||||||
|
u'required': False, u'sample_default': None,
|
||||||
|
u'deprecated_opts': [], u'deprecated_for_removal': False,
|
||||||
|
u'dest': u'name',
|
||||||
|
u'secret': False,
|
||||||
|
u'short': None,
|
||||||
|
u'mutable': False,
|
||||||
|
u'deprecated_since': None,
|
||||||
|
u'metavar': None,
|
||||||
|
u'advanced': False,
|
||||||
|
u'name': u'name'}
|
||||||
|
opt = parsing.make_opt(descr, 'abcd-1234', 'efgh-5678')
|
||||||
|
self.assertIsInstance(opt, parsing.IdentifiedOpt)
|
||||||
|
self.assertEqual("name", opt.name)
|
||||||
|
self.assertEqual('abcd-1234', opt.id_)
|
||||||
|
self.assertEqual('efgh-5678', opt.ns_id)
|
||||||
|
|
||||||
|
def test_make_type(self):
|
||||||
|
"""Test for parsing a type"""
|
||||||
|
typ1 = parsing.make_type({u'type': u'String'})
|
||||||
|
self.assertIsInstance(typ1, types.String)
|
||||||
|
typ2 = parsing.make_type({u'type': u'Integer'})
|
||||||
|
self.assertIsInstance(typ2, types.Integer)
|
||||||
|
typ3 = parsing.make_type(
|
||||||
|
{u'item_type': {u'type': u'Boolean'}, u'type': u'List'})
|
||||||
|
self.assertIsInstance(typ3, types.List)
|
||||||
|
self.assertIsInstance(typ3.item_type, types.Boolean)
|
|
@ -0,0 +1,66 @@
|
||||||
|
# -*- coding: utf-8
|
||||||
|
#
|
||||||
|
# Copyright (c) 2017 Orange.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
"""Test for utils"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from oslo_log import log as logging
|
||||||
|
|
||||||
|
from congress.cfg_validator import utils
|
||||||
|
from congress.tests import base
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class TestUtils(base.TestCase):
|
||||||
|
"""Test of generic utility functions"""
|
||||||
|
|
||||||
|
def test_hash(self):
|
||||||
|
"""Test shape of hash generated"""
|
||||||
|
re_hash = ('^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-'
|
||||||
|
'[0-9a-f]{4}-[0-9a-f]{8}')
|
||||||
|
self.assertTrue(re.match(re_hash, utils.compute_hash('foo')))
|
||||||
|
self.assertTrue(re.match(re_hash, utils.compute_hash('foo', 'bar')))
|
||||||
|
|
||||||
|
def test_cfg_value_to_congress(self):
|
||||||
|
"""Test sanitization of values for congress"""
|
||||||
|
self.assertEqual('aAdef', utils.cfg_value_to_congress('aA%def'))
|
||||||
|
self.assertEqual(u'aAf', utils.cfg_value_to_congress(u'aAéf'))
|
||||||
|
# Do not replace 0 with ''
|
||||||
|
self.assertEqual(0, utils.cfg_value_to_congress(0))
|
||||||
|
# Do not replace 0.0 with ''
|
||||||
|
self.assertEqual(0.0, utils.cfg_value_to_congress(0.0))
|
||||||
|
self.assertEqual('', utils.cfg_value_to_congress(None))
|
||||||
|
|
||||||
|
def test_add_rules(self):
|
||||||
|
"""Test adding a rule via the bus"""
|
||||||
|
class _bus(object):
|
||||||
|
|
||||||
|
def rpc(self, svc, command, arg):
|
||||||
|
"fake rpc"
|
||||||
|
return {"svc": svc, "command": command, "arg": arg}
|
||||||
|
|
||||||
|
res = utils.add_rule(_bus(), 'policy', ['r1', 'r2'])
|
||||||
|
expected = {
|
||||||
|
'arg': {'policy_rules_obj': {
|
||||||
|
'kind': 'nonrecursive',
|
||||||
|
'name': 'policy',
|
||||||
|
'rules': ['r1', 'r2']}},
|
||||||
|
'command': 'persistent_create_policy_with_rules',
|
||||||
|
'svc': '__engine'}
|
||||||
|
self.assertEqual(expected, res)
|
|
@ -0,0 +1,7 @@
|
||||||
|
[DEFAULT]
|
||||||
|
output_file = etc/congress-agent.conf.sample
|
||||||
|
wrap_width = 79
|
||||||
|
namespace = congress-agent
|
||||||
|
namespace = oslo.log
|
||||||
|
namespace = oslo.messaging
|
||||||
|
|
|
@ -46,6 +46,7 @@ warnerrors = True
|
||||||
[entry_points]
|
[entry_points]
|
||||||
oslo.config.opts =
|
oslo.config.opts =
|
||||||
congress = congress.opts:list_opts
|
congress = congress.opts:list_opts
|
||||||
|
congress-agent = congress.cfg_validator.agent.opts:list_opts
|
||||||
|
|
||||||
oslo.config.opts.defaults =
|
oslo.config.opts.defaults =
|
||||||
congress = congress.common.config:set_config_defaults
|
congress = congress.common.config:set_config_defaults
|
||||||
|
@ -58,6 +59,7 @@ oslo.policy.policies =
|
||||||
console_scripts =
|
console_scripts =
|
||||||
congress-server = congress.server.congress_server:main
|
congress-server = congress.server.congress_server:main
|
||||||
congress-db-manage = congress.db.migration.cli:main
|
congress-db-manage = congress.db.migration.cli:main
|
||||||
|
congress-cfg-validator-agt = congress.cfg_validator.agent.agent:main
|
||||||
|
|
||||||
tempest.test_plugins =
|
tempest.test_plugins =
|
||||||
congress_tests = congress_tempest_tests.plugin:CongressTempestPlugin
|
congress_tests = congress_tempest_tests.plugin:CongressTempestPlugin
|
||||||
|
|
1
tox.ini
1
tox.ini
|
@ -58,6 +58,7 @@ exclude=.venv,.git,.tox,dist,doc,*lib/python*,*egg,build,*thirdparty/*,CongressL
|
||||||
deps =
|
deps =
|
||||||
commands = {toxinidir}/tools/pip-install-single-req.sh requirements.txt oslo.config
|
commands = {toxinidir}/tools/pip-install-single-req.sh requirements.txt oslo.config
|
||||||
oslo-config-generator --config-file=etc/congress-config-generator.conf
|
oslo-config-generator --config-file=etc/congress-config-generator.conf
|
||||||
|
oslo-config-generator --config-file=etc/congress-agent-config-generator.conf
|
||||||
|
|
||||||
[testenv:genpolicy]
|
[testenv:genpolicy]
|
||||||
commands = oslopolicy-sample-generator --config-file etc/congress-policy-generator.conf
|
commands = oslopolicy-sample-generator --config-file etc/congress-policy-generator.conf
|
||||||
|
|
Loading…
Reference in New Issue