From 5e9a4c5d09da03cc785775fa36b06a5ab67baa9b Mon Sep 17 00:00:00 2001 From: Yuriy Taraday Date: Tue, 13 Sep 2016 20:54:34 +0300 Subject: [PATCH] Add configuration schema that dubs oslo.config definitions After we switch to cliff we'll loose all config validation otherwise. All options without defaults are allowed to be null since that's what their default will be. Change-Id: I6ceb67d1906089a22df7c641d7ac767067ec2068 --- fuel_ccp/config/__init__.py | 50 +++++++++++++++++++++++++++------ fuel_ccp/config/_yaml.py | 11 ++++++++ fuel_ccp/config/builder.py | 14 +++++++++ fuel_ccp/config/cli.py | 19 +++++++++++++ fuel_ccp/config/images.py | 14 +++++++++ fuel_ccp/config/kubernetes.py | 14 +++++++++ fuel_ccp/config/registry.py | 14 +++++++++ fuel_ccp/config/repositories.py | 21 ++++++++++++++ fuel_ccp/tests/conf_fixture.py | 2 ++ fuel_ccp/tests/test_config.py | 10 +++++++ requirements.txt | 3 +- 11 files changed, 163 insertions(+), 9 deletions(-) diff --git a/fuel_ccp/config/__init__.py b/fuel_ccp/config/__init__.py index 954e060f..e18fa2c5 100644 --- a/fuel_ccp/config/__init__.py +++ b/fuel_ccp/config/__init__.py @@ -1,23 +1,24 @@ import argparse import logging +import itertools +import jsonschema import os from oslo_config import cfg +from oslo_log import _options as log_options from oslo_log import log import six from fuel_ccp.config import _yaml +from fuel_ccp.config import builder +from fuel_ccp.config import cli +from fuel_ccp.config import images +from fuel_ccp.config import kubernetes +from fuel_ccp.config import registry +from fuel_ccp.config import repositories LOG = logging.getLogger(__name__) -cfg.CONF.import_group('builder', 'fuel_ccp.config.builder') -cfg.CONF.import_opt("action", "fuel_ccp.config.cli") -cfg.CONF.import_opt("deploy_config", "fuel_ccp.config.cli") -cfg.CONF.import_group('images', 'fuel_ccp.config.images') -cfg.CONF.import_group('kubernetes', 'fuel_ccp.config.kubernetes') -cfg.CONF.import_group('registry', 'fuel_ccp.config.registry') -cfg.CONF.import_group('repositories', 'fuel_ccp.config.repositories') - _REAL_CONF = None @@ -38,6 +39,7 @@ def setup_config(): LOG.debug('No config file loaded') yconf = _yaml.AttrDict() copy_values_from_oslo(cfg.CONF, yconf) + validate_config(yconf) global _REAL_CONF _REAL_CONF = yconf @@ -118,3 +120,35 @@ class _Wrapper(object): return _REAL_CONF[name] CONF = _Wrapper() + + +def get_config_schema(): + schema = { + '$schema': 'http://json-schema.org/draft-04/schema#', + 'additionalProperties': False, + 'properties': { + 'debug': {'type': 'boolean'}, + 'verbose': {'type': 'boolean'}, + }, + } + for module in [cli, builder, images, kubernetes, registry, repositories]: + schema['properties'].update(module.SCHEMA) + # Don't validate all options added from oslo.log and oslo.config + ignore_opts = ['config_file', 'config_dir'] + for opt in itertools.chain(log_options.logging_cli_opts, + log_options.generic_log_opts, + log_options.log_opts): + ignore_opts.append(opt.name.replace('-', '_')) + for name in ignore_opts: + schema['properties'][name] = {} + # Also for now don't validate sections that used to be in deploy config + for name in ['configs', 'nodes', 'roles', 'sources', 'versions']: + schema['properties'][name] = {'type': 'object'} + return schema + + +def validate_config(yconf=None): + if yconf is None: + yconf = _REAL_CONF + schema = get_config_schema() + jsonschema.validate(_yaml.UnwrapAttrDict(yconf), schema) diff --git a/fuel_ccp/config/_yaml.py b/fuel_ccp/config/_yaml.py index 3f7b9fef..1a1204b9 100644 --- a/fuel_ccp/config/_yaml.py +++ b/fuel_ccp/config/_yaml.py @@ -101,3 +101,14 @@ def load_with_includes(filename): else: res._merge(doc) return res + + +class UnwrapAttrDict(dict): + def __init__(self, attr_dict): + return super(UnwrapAttrDict, self).__init__(attr_dict._dict) + + def __getitem__(self, name): + res = super(UnwrapAttrDict, self).__getitem__(name) + if isinstance(res, AttrDict): + res = UnwrapAttrDict(res) + return res diff --git a/fuel_ccp/config/builder.py b/fuel_ccp/config/builder.py index 0d7488f0..5384edd6 100644 --- a/fuel_ccp/config/builder.py +++ b/fuel_ccp/config/builder.py @@ -29,3 +29,17 @@ builder_opt_group = cfg.OptGroup(name='builder', CONF.register_group(builder_opt_group) CONF.register_cli_opts(builder_opts, builder_opt_group) CONF.register_opts(builder_opts, builder_opt_group) + +SCHEMA = { + 'builder': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'workers': {'type': 'integer'}, + 'keep_image_tree_consistency': {'type': 'boolean'}, + 'build_base_images_if_not_exist': {'type': 'boolean'}, + 'push': {'type': 'boolean'}, + 'no_cache': {'type': 'boolean'}, + }, + }, +} diff --git a/fuel_ccp/config/cli.py b/fuel_ccp/config/cli.py index a6c4ddc8..7a3e8e5c 100644 --- a/fuel_ccp/config/cli.py +++ b/fuel_ccp/config/cli.py @@ -46,3 +46,22 @@ common_opts = [ cfg.StrOpt('deploy-config', help='Cluster-wide configuration overrides') ] CONF.register_cli_opts(common_opts) + +SCHEMA = { + 'deploy_config': {'anyOf': [{'type': 'string'}, {'type': 'null'}]}, + 'action': { + 'type': 'object', + 'additionalProperties': False, + 'properties': {k: {'anyOf': [v, {'type': 'null'}]} for k, v in { + 'name': {'type': 'string'}, + 'components': { + 'type': 'array', + 'items': {'type': 'string'}, + }, + 'dry_run': {'type': 'boolean'}, + 'export_dir': {'type': 'string'}, + 'auth_url': {'type': 'string'}, + 'skip_os_cleanup': {'type': 'boolean'}, + }.items()}, + }, +} diff --git a/fuel_ccp/config/images.py b/fuel_ccp/config/images.py index 8f728a9a..e76c7f57 100644 --- a/fuel_ccp/config/images.py +++ b/fuel_ccp/config/images.py @@ -23,3 +23,17 @@ images_opt_group = cfg.OptGroup(name='images', CONF.register_group(images_opt_group) CONF.register_cli_opts(images_opts, images_opt_group) CONF.register_opts(images_opts, images_opt_group) + +SCHEMA = { + 'images': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'namespace': {'type': 'string'}, + 'tag': {'type': 'string'}, + 'base_distro': {'type': 'string'}, + 'base_tag': {'type': 'string'}, + 'maintainer': {'type': 'string'}, + }, + }, +} diff --git a/fuel_ccp/config/kubernetes.py b/fuel_ccp/config/kubernetes.py index 0067966d..1517ede4 100644 --- a/fuel_ccp/config/kubernetes.py +++ b/fuel_ccp/config/kubernetes.py @@ -21,3 +21,17 @@ kubernetes_opt_group = cfg.OptGroup(name='kubernetes', CONF.register_group(kubernetes_opt_group) CONF.register_cli_opts(kubernetes_opts, kubernetes_opt_group) CONF.register_cli_opts(kubernetes_opts, kubernetes_opt_group) + +SCHEMA = { + 'kubernetes': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'server': {'type': 'string'}, + 'namespace': {'type': 'string'}, + 'ca_certs': {'anyOf': [{'type': 'string'}, {'type': 'null'}]}, + 'key_file': {'anyOf': [{'type': 'string'}, {'type': 'null'}]}, + 'cert_file': {'anyOf': [{'type': 'string'}, {'type': 'null'}]}, + }, + }, +} diff --git a/fuel_ccp/config/registry.py b/fuel_ccp/config/registry.py index a794803b..632a06b7 100644 --- a/fuel_ccp/config/registry.py +++ b/fuel_ccp/config/registry.py @@ -24,3 +24,17 @@ registry_opt_group = cfg.OptGroup(name='registry', CONF.register_group(registry_opt_group) CONF.register_cli_opts(registry_opts, registry_opt_group) CONF.register_opts(registry_opts, registry_opt_group) + +SCHEMA = { + 'registry': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'address': {'type': 'string'}, + 'insecure': {'type': 'boolean'}, + 'username': {'type': 'string'}, + 'password': {'type': 'string'}, + 'timeout': {'type': 'integer'}, + }, + }, +} diff --git a/fuel_ccp/config/repositories.py b/fuel_ccp/config/repositories.py index 304bfb3b..783de0a3 100644 --- a/fuel_ccp/config/repositories.py +++ b/fuel_ccp/config/repositories.py @@ -53,10 +53,31 @@ repositories_opts = [ help='List of repository names'), ] +SCHEMA = { + 'repositories': { + 'type': 'object', + 'additionalProperties': False, + 'properties': { + 'clone': {'type': 'boolean'}, + 'clone_concurrency': {'type': 'integer'}, + 'skip_empty': {'type': 'boolean'}, + 'path': {'type': 'string'}, + 'hostname': {'type': 'string'}, + 'port': {'type': 'integer'}, + 'protocol': {'type': 'string'}, + 'project': {'type': 'string'}, + 'username': {'type': 'string'}, + 'names': {'type': 'array', 'items': {'type': 'string'}}, + }, + }, +} + for repo in DEFAULT_REPOS: url = '$protocol://$username@$hostname:$port/$project/' option = cfg.StrOpt(repo, default=url + repo) repositories_opts.append(option) + SCHEMA['repositories']['properties'][repo.replace('-', '_')] = \ + {'type': 'string'} repositories_opt_group = cfg.OptGroup(name='repositories', title='Git repositories for components') diff --git a/fuel_ccp/tests/conf_fixture.py b/fuel_ccp/tests/conf_fixture.py index b77ade00..7d053e8e 100644 --- a/fuel_ccp/tests/conf_fixture.py +++ b/fuel_ccp/tests/conf_fixture.py @@ -1,6 +1,7 @@ import fixtures from oslo_config import cfg from oslo_config import fixture as oslo_fixture +from oslo_log import log from fuel_ccp import config from fuel_ccp.config import _yaml @@ -9,6 +10,7 @@ from fuel_ccp.config import _yaml class Config(fixtures.Fixture): def _setUp(self): self.useFixture(oslo_fixture.Config()) + log.register_options(cfg.CONF) cfg.CONF(['build'], default_config_files=[]) self.conf = _yaml.AttrDict() config.copy_values_from_oslo(cfg.CONF, self.conf) diff --git a/fuel_ccp/tests/test_config.py b/fuel_ccp/tests/test_config.py index bdf9aabd..5bd57986 100644 --- a/fuel_ccp/tests/test_config.py +++ b/fuel_ccp/tests/test_config.py @@ -2,6 +2,7 @@ import collections import functools import fixtures +import jsonschema from oslo_config import cfg import six import testscenarios @@ -137,3 +138,12 @@ class TestCopyValuesFromOslo(testscenarios.WithScenarios, base.TestCase): yconf = nested_dict_to_attrdict(self.yconf) config.copy_values_from_oslo(conf, yconf) self.assertEqual(yconf, self.expected_result) + + +class TestConfigSchema(base.TestCase): + def test_validate_config_schema(self): + schema = config.get_config_schema() + jsonschema.Draft4Validator.check_schema(schema) + + def test_validate_default_conf(self): + config.validate_config(self.conf) diff --git a/requirements.txt b/requirements.txt index c1d42cc4..92a6d78c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ futures>=3.0 # BSD docker-py>=1.6.0,<1.8.0 # Apache-2.0 GitPython>=1.0.1 # BSD License (3 clause) Jinja2>=2.8 # BSD License (3 clause) +jsonschema>=2.0.0,<3.0.0,!=2.5.0 # MIT oslo.config>=3.9.0 # Apache-2.0 oslo.log>=1.14.0 # Apache-2.0 PyYAML>=3.1.0 # MIT @@ -15,4 +16,4 @@ six>=1.9.0 # MIT python-k8sclient keystoneauth1>=2.7.0 # Apache-2.0 python-neutronclient>=4.2.0 # Apache-2.0 -python-novaclient>=2.29.0,!=2.33.0 # Apache-2.0 \ No newline at end of file +python-novaclient>=2.29.0,!=2.33.0 # Apache-2.0