Add configuration schema that dubs oslo.config definitions

After we switch to cliff we'll loose all config validation otherwise.

All options without defaults are allowed to be null since that's what
their default will be.

Change-Id: I6ceb67d1906089a22df7c641d7ac767067ec2068

fuel_ccp/config/__init__.py

@@ -1,23 +1,24 @@
 import argparse
 import logging
 
+import itertools
+import jsonschema
 import os
 from oslo_config import cfg
+from oslo_log import _options as log_options
 from oslo_log import log
 import six
 
 from fuel_ccp.config import _yaml
+from fuel_ccp.config import builder
+from fuel_ccp.config import cli
+from fuel_ccp.config import images
+from fuel_ccp.config import kubernetes
+from fuel_ccp.config import registry
+from fuel_ccp.config import repositories
 
 LOG = logging.getLogger(__name__)
 
-cfg.CONF.import_group('builder', 'fuel_ccp.config.builder')
-cfg.CONF.import_opt("action", "fuel_ccp.config.cli")
-cfg.CONF.import_opt("deploy_config", "fuel_ccp.config.cli")
-cfg.CONF.import_group('images', 'fuel_ccp.config.images')
-cfg.CONF.import_group('kubernetes', 'fuel_ccp.config.kubernetes')
-cfg.CONF.import_group('registry', 'fuel_ccp.config.registry')
-cfg.CONF.import_group('repositories', 'fuel_ccp.config.repositories')
-
 _REAL_CONF = None
 
 
@@ -38,6 +39,7 @@ def setup_config():
         LOG.debug('No config file loaded')
         yconf = _yaml.AttrDict()
     copy_values_from_oslo(cfg.CONF, yconf)
+    validate_config(yconf)
     global _REAL_CONF
     _REAL_CONF = yconf
 
@@ -118,3 +120,35 @@ class _Wrapper(object):
         return _REAL_CONF[name]
 
 CONF = _Wrapper()
+
+
+def get_config_schema():
+    schema = {
+        '$schema': 'http://json-schema.org/draft-04/schema#',
+        'additionalProperties': False,
+        'properties': {
+            'debug': {'type': 'boolean'},
+            'verbose': {'type': 'boolean'},
+        },
+    }
+    for module in [cli, builder, images, kubernetes, registry, repositories]:
+        schema['properties'].update(module.SCHEMA)
+    # Don't validate all options added from oslo.log and oslo.config
+    ignore_opts = ['config_file', 'config_dir']
+    for opt in itertools.chain(log_options.logging_cli_opts,
+                               log_options.generic_log_opts,
+                               log_options.log_opts):
+        ignore_opts.append(opt.name.replace('-', '_'))
+    for name in ignore_opts:
+        schema['properties'][name] = {}
+    # Also for now don't validate sections that used to be in deploy config
+    for name in ['configs', 'nodes', 'roles', 'sources', 'versions']:
+        schema['properties'][name] = {'type': 'object'}
+    return schema
+
+
+def validate_config(yconf=None):
+    if yconf is None:
+        yconf = _REAL_CONF
+    schema = get_config_schema()
+    jsonschema.validate(_yaml.UnwrapAttrDict(yconf), schema)

fuel_ccp/config/_yaml.py

@@ -101,3 +101,14 @@ def load_with_includes(filename):
         else:
             res._merge(doc)
     return res
+
+
+class UnwrapAttrDict(dict):
+    def __init__(self, attr_dict):
+        return super(UnwrapAttrDict, self).__init__(attr_dict._dict)
+
+    def __getitem__(self, name):
+        res = super(UnwrapAttrDict, self).__getitem__(name)
+        if isinstance(res, AttrDict):
+            res = UnwrapAttrDict(res)
+        return res

fuel_ccp/config/builder.py

@@ -29,3 +29,17 @@ builder_opt_group = cfg.OptGroup(name='builder',
 CONF.register_group(builder_opt_group)
 CONF.register_cli_opts(builder_opts, builder_opt_group)
 CONF.register_opts(builder_opts, builder_opt_group)
+
+SCHEMA = {
+    'builder': {
+        'type': 'object',
+        'additionalProperties': False,
+        'properties': {
+            'workers': {'type': 'integer'},
+            'keep_image_tree_consistency': {'type': 'boolean'},
+            'build_base_images_if_not_exist': {'type': 'boolean'},
+            'push': {'type': 'boolean'},
+            'no_cache': {'type': 'boolean'},
+        },
+    },
+}

fuel_ccp/config/cli.py

@@ -46,3 +46,22 @@ common_opts = [
     cfg.StrOpt('deploy-config', help='Cluster-wide configuration overrides')
 ]
 CONF.register_cli_opts(common_opts)
+
+SCHEMA = {
+    'deploy_config': {'anyOf': [{'type': 'string'}, {'type': 'null'}]},
+    'action': {
+        'type': 'object',
+        'additionalProperties': False,
+        'properties': {k: {'anyOf': [v, {'type': 'null'}]} for k, v in {
+            'name': {'type': 'string'},
+            'components': {
+                'type': 'array',
+                'items': {'type': 'string'},
+            },
+            'dry_run': {'type': 'boolean'},
+            'export_dir': {'type': 'string'},
+            'auth_url': {'type': 'string'},
+            'skip_os_cleanup': {'type': 'boolean'},
+        }.items()},
+    },
+}

fuel_ccp/config/images.py

@@ -23,3 +23,17 @@ images_opt_group = cfg.OptGroup(name='images',
 CONF.register_group(images_opt_group)
 CONF.register_cli_opts(images_opts, images_opt_group)
 CONF.register_opts(images_opts, images_opt_group)
+
+SCHEMA = {
+    'images': {
+        'type': 'object',
+        'additionalProperties': False,
+        'properties': {
+            'namespace': {'type': 'string'},
+            'tag': {'type': 'string'},
+            'base_distro': {'type': 'string'},
+            'base_tag': {'type': 'string'},
+            'maintainer': {'type': 'string'},
+        },
+    },
+}

fuel_ccp/config/kubernetes.py

@@ -21,3 +21,17 @@ kubernetes_opt_group = cfg.OptGroup(name='kubernetes',
 CONF.register_group(kubernetes_opt_group)
 CONF.register_cli_opts(kubernetes_opts, kubernetes_opt_group)
 CONF.register_cli_opts(kubernetes_opts, kubernetes_opt_group)
+
+SCHEMA = {
+    'kubernetes': {
+        'type': 'object',
+        'additionalProperties': False,
+        'properties': {
+            'server': {'type': 'string'},
+            'namespace': {'type': 'string'},
+            'ca_certs': {'anyOf': [{'type': 'string'}, {'type': 'null'}]},
+            'key_file': {'anyOf': [{'type': 'string'}, {'type': 'null'}]},
+            'cert_file': {'anyOf': [{'type': 'string'}, {'type': 'null'}]},
+        },
+    },
+}

fuel_ccp/config/registry.py

@@ -24,3 +24,17 @@ registry_opt_group = cfg.OptGroup(name='registry',
 CONF.register_group(registry_opt_group)
 CONF.register_cli_opts(registry_opts, registry_opt_group)
 CONF.register_opts(registry_opts, registry_opt_group)
+
+SCHEMA = {
+    'registry': {
+        'type': 'object',
+        'additionalProperties': False,
+        'properties': {
+            'address': {'type': 'string'},
+            'insecure': {'type': 'boolean'},
+            'username': {'type': 'string'},
+            'password': {'type': 'string'},
+            'timeout': {'type': 'integer'},
+        },
+    },
+}

fuel_ccp/config/repositories.py

@@ -53,10 +53,31 @@ repositories_opts = [
                 help='List of repository names'),
 ]
 
+SCHEMA = {
+    'repositories': {
+        'type': 'object',
+        'additionalProperties': False,
+        'properties': {
+            'clone': {'type': 'boolean'},
+            'clone_concurrency': {'type': 'integer'},
+            'skip_empty': {'type': 'boolean'},
+            'path': {'type': 'string'},
+            'hostname': {'type': 'string'},
+            'port': {'type': 'integer'},
+            'protocol': {'type': 'string'},
+            'project': {'type': 'string'},
+            'username': {'type': 'string'},
+            'names': {'type': 'array', 'items': {'type': 'string'}},
+        },
+    },
+}
+
 for repo in DEFAULT_REPOS:
     url = '$protocol://$username@$hostname:$port/$project/'
     option = cfg.StrOpt(repo, default=url + repo)
     repositories_opts.append(option)
+    SCHEMA['repositories']['properties'][repo.replace('-', '_')] = \
+        {'type': 'string'}
 
 repositories_opt_group = cfg.OptGroup(name='repositories',
                                       title='Git repositories for components')

fuel_ccp/tests/conf_fixture.py

@@ -1,6 +1,7 @@
 import fixtures
 from oslo_config import cfg
 from oslo_config import fixture as oslo_fixture
+from oslo_log import log
 
 from fuel_ccp import config
 from fuel_ccp.config import _yaml
@@ -9,6 +10,7 @@ from fuel_ccp.config import _yaml
 class Config(fixtures.Fixture):
     def _setUp(self):
         self.useFixture(oslo_fixture.Config())
+        log.register_options(cfg.CONF)
         cfg.CONF(['build'], default_config_files=[])
         self.conf = _yaml.AttrDict()
         config.copy_values_from_oslo(cfg.CONF, self.conf)

fuel_ccp/tests/test_config.py

@@ -2,6 +2,7 @@ import collections
 import functools
 
 import fixtures
+import jsonschema
 from oslo_config import cfg
 import six
 import testscenarios
@@ -137,3 +138,12 @@ class TestCopyValuesFromOslo(testscenarios.WithScenarios, base.TestCase):
         yconf = nested_dict_to_attrdict(self.yconf)
         config.copy_values_from_oslo(conf, yconf)
         self.assertEqual(yconf, self.expected_result)
+
+
+class TestConfigSchema(base.TestCase):
+    def test_validate_config_schema(self):
+        schema = config.get_config_schema()
+        jsonschema.Draft4Validator.check_schema(schema)
+
+    def test_validate_default_conf(self):
+        config.validate_config(self.conf)

requirements.txt

@@ -8,6 +8,7 @@ futures>=3.0 # BSD
 docker-py>=1.6.0,<1.8.0  # Apache-2.0
 GitPython>=1.0.1 # BSD License (3 clause)
 Jinja2>=2.8  # BSD License (3 clause)
+jsonschema>=2.0.0,<3.0.0,!=2.5.0  # MIT
 oslo.config>=3.9.0  # Apache-2.0
 oslo.log>=1.14.0  # Apache-2.0
 PyYAML>=3.1.0  # MIT
@@ -15,4 +16,4 @@ six>=1.9.0  # MIT
 python-k8sclient
 keystoneauth1>=2.7.0  # Apache-2.0
 python-neutronclient>=4.2.0  # Apache-2.0
-python-novaclient>=2.29.0,!=2.33.0  # Apache-2.0
+python-novaclient>=2.29.0,!=2.33.0  # Apache-2.0