Merge "Provider Config File: YAML file loading and schema validation"

This commit is contained in:
Zuul 2020-07-31 17:29:18 +00:00 committed by Gerrit Code Review
commit 8ecc29bfcc
8 changed files with 732 additions and 2 deletions

View File

@ -15,7 +15,7 @@ coverage==4.0
cryptography==2.7
cursive==0.2.1
dataclasses==0.7
ddt==1.0.1
ddt==1.2.1
debtcollector==1.19.0
decorator==3.4.0
deprecation==2.0

View File

@ -0,0 +1,278 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import jsonschema
import logging
import microversion_parse
import yaml
from nova import exception as nova_exc
from nova.i18n import _
LOG = logging.getLogger(__name__)
# A dictionary with keys for all supported major versions with lists of
# corresponding minor versions as values.
SUPPORTED_SCHEMA_VERSIONS = {
1: {0}
}
# Supported provider config file schema
SCHEMA_V1 = {
# This defintion uses JSON Schema Draft 7.
# https://json-schema.org/draft-07/json-schema-release-notes.html
'type': 'object',
'properties': {
# This property is used to track where the provider.yaml file
# originated. It is reserved for internal use and should never be
# set in a provider.yaml file supplied by an end user.
'__source_file': {'not': {}},
'meta': {
'type': 'object',
'properties': {
# Version ($Major, $minor) of the schema must successfully
# parse documents conforming to ($Major, 0..N).
# Any breaking schema change (e.g. removing fields, adding
# new required fields, imposing a stricter pattern on a value,
# etc.) must bump $Major.
'schema_version': {
'type': 'string',
'pattern': '^1.([0-9]|[1-9][0-9]+)$'
}
},
'required': ['schema_version'],
'additionalProperties': True
},
'providers': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'identification': {
'$ref': '#/$defs/providerIdentification'
},
'inventories': {
'$ref': '#/$defs/providerInventories'
},
'traits': {
'$ref': '#/$defs/providerTraits'
}
},
'required': ['identification'],
'additionalProperties': True
}
}
},
'required': ['meta'],
'additionalProperties': True,
'$defs': {
'providerIdentification': {
# Identify a single provider to configure.
# Exactly one identification method should be used. Currently
# `uuid` or `name` are supported, but future versions may
# support others. The uuid can be set to the sentinel value
# `$COMPUTE_NODE` which will cause the consuming compute service to
# apply the configuration to all compute node root providers
# it manages that are not otherwise specified using a uuid or name.
'type': 'object',
'properties': {
'uuid': {
'oneOf': [
{
# TODO(sean-k-mooney): replace this with type uuid
# when we can depend on a version of the jsonschema
# lib that implements draft 8 or later of the
# jsonschema spec.
'type': 'string',
'pattern':
'^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-'
'[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-'
'[0-9A-Fa-f]{12}$'
},
{
'type': 'string',
'const': '$COMPUTE_NODE'
}
]
},
'name': {
'type': 'string',
'minLength': 1,
'maxLength': 200
}
},
# This introduces the possibility of an unsupported key name being
# used to get by schema validation, but is necessary to support
# forward compatibility with new identification methods.
# This should be checked after schema validation.
'minProperties': 1,
'maxProperties': 1,
'additionalProperties': False
},
'providerInventories': {
# Allows the admin to specify various adjectives to create and
# manage providers' inventories. This list of adjectives can be
# extended in the future as the schema evolves to meet new use
# cases. As of v1.0, only one adjective, `additional`, is
# supported.
'type': 'object',
'properties': {
'additional': {
'type': 'array',
'items': {
'patternProperties': {
# Allows any key name matching the resource class
# pattern, check to prevent conflicts with virt
# driver owned resouces classes will be done after
# schema validation.
'^[A-Z0-9_]{1,255}$': {
'type': 'object',
'properties': {
# Any optional properties not populated
# will be given a default value by
# placement. If overriding a pre-existing
# provider values will not be preserved
# from the existing inventory.
'total': {
'type': 'integer'
},
'reserved': {
'type': 'integer'
},
'min_unit': {
'type': 'integer'
},
'max_unit': {
'type': 'integer'
},
'step_size': {
'type': 'integer'
},
'allocation_ratio': {
'type': 'number'
}
},
'required': ['total'],
# The defined properties reflect the current
# placement data model. While defining those
# in the schema and not allowing additional
# properties means we will need to bump the
# schema version if they change, that is likely
# to be part of a large change that may have
# other impacts anyway. The benefit of stricter
# validation of property names outweighs the
# (small) chance of having to bump the schema
# version as described above.
'additionalProperties': False
}
},
# This ensures only keys matching the pattern
# above are allowed.
'additionalProperties': False
}
}
},
'additionalProperties': True
},
'providerTraits': {
# Allows the admin to specify various adjectives to create and
# manage providers' traits. This list of adjectives can be extended
# in the future as the schema evolves to meet new use cases.
# As of v1.0, only one adjective, `additional`, is supported.
'type': 'object',
'properties': {
'additional': {
'type': 'array',
'items': {
# Allows any value matching the trait pattern here,
# additional validation will be done after schema
# validation.
'type': 'string',
'pattern': '^[A-Z0-9_]{1,255}$'
}
}
},
'additionalProperties': True
}
}
}
def _load_yaml_file(path):
"""Loads and parses a provider.yaml config file into a dict.
:param path: Path to the yaml file to load.
:return: Dict representing the yaml file requested.
:raise: ProviderConfigException if the path provided cannot be read
or the file is not valid yaml.
"""
try:
with open(path) as open_file:
try:
return yaml.safe_load(open_file)
except yaml.YAMLError as ex:
message = _("Unable to load yaml file: %s ") % ex
if hasattr(ex, 'problem_mark'):
pos = ex.problem_mark
message += _("File: %s ") % open_file.name
message += _("Error position: (%s:%s)") % (
pos.line + 1, pos.column + 1)
raise nova_exc.ProviderConfigException(error=message)
except OSError:
message = _("Unable to read yaml config file: %s") % path
raise nova_exc.ProviderConfigException(error=message)
def _parse_provider_yaml(path):
"""Loads schema, parses a provider.yaml file and validates the content.
:param path: File system path to the file to parse.
:return: dict representing the contents of the file.
:raise ProviderConfigException: If the specified file does
not validate against the schema, the schema version is not supported,
or if unable to read configuration or schema files.
"""
yaml_file = _load_yaml_file(path)
try:
schema_version = microversion_parse.parse_version_string(
yaml_file['meta']['schema_version'])
except (KeyError, TypeError):
message = _("Unable to detect schema version: %s") % yaml_file
raise nova_exc.ProviderConfigException(error=message)
if schema_version.major not in SUPPORTED_SCHEMA_VERSIONS:
message = _(
"Unsupported schema major version: %d") % schema_version.major
raise nova_exc.ProviderConfigException(error=message)
if schema_version.minor not in \
SUPPORTED_SCHEMA_VERSIONS[schema_version.major]:
# TODO(sean-k-mooney): We should try to provide a better
# message that identifies which fields may be ignored
# and the max minor version supported by this version of nova.
message = (
"Provider config file [%(path)s] is at schema version "
"%(schema_version)s. Nova supports the major version, "
"but not the minor. Some fields may be ignored."
% {"path": path, "schema_version": schema_version})
LOG.warning(message)
try:
jsonschema.validate(yaml_file, SCHEMA_V1)
except jsonschema.exceptions.ValidationError as e:
message = _(
"The provider config file %(path)s did not pass validation "
"for schema version %(schema_version)s: %(reason)s") % {
"path": path, "schema_version": schema_version, "reason": e}
raise nova_exc.ProviderConfigException(error=message)
return yaml_file

View File

@ -2358,3 +2358,13 @@ class MixedInstanceNotSupportByComputeService(NovaException):
class InvalidMixedInstanceDedicatedMask(Invalid):
msg_fmt = _("Mixed instance must have at least 1 pinned vCPU and 1 "
"unpinned vCPU. See 'hw:cpu_dedicated_mask'.")
class ProviderConfigException(NovaException):
"""Exception indicating an error occurred processing provider config files.
This class is used to avoid a raised exception inadvertently being caught
and mishandled by the resource tracker.
"""
msg_fmt = _("An error occurred while processing "
"a provider config file: %(error)s")

View File

@ -0,0 +1,204 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# expected_messages is a list of matches. If the test matches _all_ of the
# values in the list, it will pass.
no_metadata:
config: {}
expected_messages: ['Unable to detect schema version:']
no_schema_version:
config:
meta: {}
expected_messages: ['Unable to detect schema version:']
invalid_schema_version:
config:
meta:
schema_version: '99.99'
expected_messages: ['Unsupported schema major version: 99']
property__source_file_present_value:
config:
meta:
schema_version: '1.0'
__source_file: "present"
expected_messages:
- "{} is not allowed for"
- "validating 'not' in schema['properties']['__source_file']"
property__source_file_present_null:
config:
meta:
schema_version: '1.0'
__source_file: null
expected_messages:
- "{} is not allowed for"
- "validating 'not' in schema['properties']['__source_file']"
provider_invalid_uuid:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: not quite a uuid
expected_messages:
- "'not quite a uuid'"
- "Failed validating"
- "'^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$'"
provider_null_uuid:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: null
expected_messages:
- "The provider config file test_path did not pass validation for schema version 1.0"
- "None is not"
- "'^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$'"
- "'type': 'string'"
provider_empty_name:
config:
meta:
schema_version: '1.0'
providers:
- identification:
name: ''
expected_messages: ["'' is too short"]
provider_null_name:
config:
meta:
schema_version: '1.0'
providers:
- identification:
name: null
expected_messages: ["None is not of type 'string'"]
provider_no_name_or_uuid:
config:
meta:
schema_version: '1.0'
providers:
- identification:
expected_messages: ["Failed validating 'type' in schema['properties']['providers']['items']['properties']['identification']"]
provider_uuid_and_name:
config:
meta:
schema_version: '1.0'
providers:
- identification:
name: custom_provider
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
expected_messages:
- "'name': 'custom_provider'"
- "'uuid': 'aa884151-b4e2-4e82-9fd4-81cfcd01abb9'"
- "has too many properties"
provider_no_identification:
config:
meta:
schema_version: '1.0'
providers:
- {}
expected_messages: ["'identification' is a required property"]
inventories_additional_resource_class_no_total:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
additional:
- RESOURCE1: {}
expected_messages: ["'total' is a required property"]
inventories_additional_resource_class_invalid_total:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
additional:
- RESOURCE1:
total: invalid_total
expected_messages: ["'invalid_total' is not of type 'integer'"]
inventories_additional_resource_class_additional_property:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
additional:
- RESOURCE1:
total: 1
additional_property: 2
expected_messages: ["Additional properties are not allowed ('additional_property' was unexpected)"]
inventories_one_invalid_additional_resource_class:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
additional:
- RESOURCE1:
total: 1
- RESOURCE2: {}
expected_messages: ["'total' is a required property"]
inventories_invalid_additional_resource_class_name:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
additional:
- INVALID_RESOURCE_CLASS_NAME_!@#$%^&*()_+:
total: 1
expected_messages: ["'INVALID_RESOURCE_CLASS_NAME_!@#$%^&*()_+' does not match any of the regexes"]
traits_one_additional_trait_invalid:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
traits:
additional:
- TRAIT1: invalid_trait
expected_messages: ["{'TRAIT1': 'invalid_trait'} is not of type 'string'"]
traits_multiple_additional_traits_two_invalid:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
traits:
additional:
- TRAIT1: invalid
- TRAIT2
- TRAIT3: invalid
expected_messages: ["{'TRAIT1': 'invalid'} is not of type 'string'"]
traits_invalid_trait_name:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
traits:
additional:
- INVALID_TRAIT_NAME_!@#$%^&*()_+
expected_messages: ["'INVALID_TRAIT_NAME_!@#$%^&*()_+' does not match '^[A-Z0-9_]{1,255}$'"]

View File

@ -0,0 +1,113 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
provider_by_uuid:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
provider_magic_uuid:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: "$COMPUTE_NODE"
provider_by_name:
config:
meta:
schema_version: '1.0'
providers:
- identification:
name: custom_provider
inventories_additional_resource_class:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
additional:
- CUSTOM_RESOURCE1:
total: 1
inventories_unknown_adjective:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
invalid_adjective:
- CUSTOM_RESOURCE1:
total: 1
inventories_multiple_additional_resource_classes:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
additional:
- CUSTOM_RESOURCE1:
total: 1
- CUSTOM_RESOURCE2:
total: 1
traits_one_additional_trait:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
traits:
additional:
- CUSTOM_TRAIT1
traits_multiple_additional_traits:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
traits:
additional:
- CUSTOM_TRAIT1
- CUSTOM_TRAIT2
traits_unknown_adjective:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
traits:
invalid:
- CUSTOM_TRAIT1
inventories_and_traits_additional_resource_class_and_trait:
config:
meta:
schema_version: '1.0'
providers:
- identification:
uuid: aa884151-b4e2-4e82-9fd4-81cfcd01abb9
inventories:
additional:
- CUSTOM_RESOURCE1:
total: 1
traits:
additional:
- CUSTOM_TRAIT1

View File

@ -0,0 +1,124 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import ddt
import fixtures
import microversion_parse
from oslotest import base
from nova.compute import provider_config
from nova import exception as nova_exc
class SchemaValidationMixin(base.BaseTestCase):
"""This class provides the basic methods for running schema validation test
cases. It can be used along with ddt.file_data to test a specific schema
version using tests defined in yaml files. See SchemaValidationTestCasesV1
for an example of how this was done for schema version 1.
Because decorators can only access class properties of the class they are
defined in (even when overriding values in the subclass), the decorators
need to be placed in the subclass. This is why there are test_ functions in
the subclass that call the run_test_ methods in this class. This should
keep things simple as more schema versions are added.
"""
def setUp(self):
super(SchemaValidationMixin, self).setUp()
self.mock_load_yaml = self.useFixture(
fixtures.MockPatchObject(
provider_config, '_load_yaml_file')).mock
self.mock_LOG = self.useFixture(
fixtures.MockPatchObject(
provider_config, 'LOG')).mock
def set_config(self, config=None):
data = config or {}
self.mock_load_yaml.return_value = data
return data
def run_test_validation_errors(self, config, expected_messages):
self.set_config(config=config)
actual_msg = self.assertRaises(
nova_exc.ProviderConfigException,
provider_config._parse_provider_yaml, 'test_path').message
for msg in expected_messages:
self.assertIn(msg, actual_msg)
def run_test_validation_success(self, config):
reference = self.set_config(config=config)
actual = provider_config._parse_provider_yaml('test_path')
self.assertEqual(reference, actual)
def run_schema_version_matching(
self, min_schema_version, max_schema_version):
# note _load_yaml_file is mocked so the value is not important
# however it may appear in logs messages so changing it could
# result in tests failing unless the expected_messages field
# is updated in the test data.
path = 'test_path'
# test exactly min and max versions are supported
self.set_config(config={
'meta': {'schema_version': str(min_schema_version)}})
provider_config._parse_provider_yaml(path)
self.set_config(config={
'meta': {'schema_version': str(max_schema_version)}})
provider_config._parse_provider_yaml(path)
self.mock_LOG.warning.assert_not_called()
# test max major+1 raises
higher_major = microversion_parse.Version(
major=max_schema_version.major + 1, minor=max_schema_version.minor)
self.set_config(config={'meta': {'schema_version': str(higher_major)}})
self.assertRaises(nova_exc.ProviderConfigException,
provider_config._parse_provider_yaml, path)
# test max major with max minor+1 is logged
higher_minor = microversion_parse.Version(
major=max_schema_version.major, minor=max_schema_version.minor + 1)
expected_log_call = (
"Provider config file [%(path)s] is at schema version "
"%(schema_version)s. Nova supports the major version, but "
"not the minor. Some fields may be ignored." % {
"path": path, "schema_version": higher_minor})
self.set_config(config={'meta': {'schema_version': str(higher_minor)}})
provider_config._parse_provider_yaml(path)
self.mock_LOG.warning.assert_called_once_with(expected_log_call)
@ddt.ddt
class SchemaValidationTestCasesV1(SchemaValidationMixin):
MIN_SCHEMA_VERSION = microversion_parse.Version(1, 0)
MAX_SCHEMA_VERSION = microversion_parse.Version(1, 0)
@ddt.unpack
@ddt.file_data('provider_config_data/v1/validation_error_test_data.yaml')
def test_validation_errors(self, config, expected_messages):
self.run_test_validation_errors(config, expected_messages)
@ddt.unpack
@ddt.file_data('provider_config_data/v1/validation_success_test_data.yaml')
def test_validation_success(self, config):
self.run_test_validation_success(config)
def test_schema_version_matching(self):
self.run_schema_version_matching(self.MIN_SCHEMA_VERSION,
self.MAX_SCHEMA_VERSION)

View File

@ -71,3 +71,4 @@ zVMCloudConnector>=1.3.0;sys_platform!='win32' # Apache 2.0 License
futurist>=1.8.0 # Apache-2.0
openstacksdk>=0.35.0 # Apache-2.0
dataclasses>=0.7;python_version=='3.6' # Apache 2.0 License
PyYAML>=3.12 # MIT

View File

@ -5,7 +5,7 @@
hacking>=3.1.0,<3.2.0 # Apache-2.0
mypy>=0.761 # MIT
coverage!=4.4,>=4.0 # Apache-2.0
ddt>=1.0.1 # MIT
ddt>=1.2.1 # MIT
fixtures>=3.0.0 # Apache-2.0/BSD
mock>=3.0.0 # BSD
psycopg2>=2.7 # LGPL/ZPL