openstack
/
fuel-mirror
Code Issues Proposed changes

Added verifying input-data format 

The input data verified using json-schema.

Change-Id: If616e1d2b3b4e095ec57846a7f1e1fe592936f0c
Closes-Bug:#1529078
This commit is contained in:
Uladzimir_Niakhai 2015-12-28 15:06:02 +03:00
parent cf0d00eef6
commit d880c784b0
8 changed files with 303 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
contrib/fuel_mirror/fuel_mirror/commands/base.py
View File

@ -18,10 +18,16 @@ import os.path
from string import Template
from cliff import command
from jsonschema import validate
from jsonschema import ValidationError
import six
import yaml
from fuel_mirror.schemas.input_data_schema import SCHEMA
class BaseCommand(command.Command):
"""The Base command for fuel-mirror."""
REPO_ARCH = "x86_64"
@ -65,6 +71,26 @@ class BaseCommand(command.Command):
self.app.config['pattern_dir'], pattern + ".yaml"
)
@staticmethod
def validate_data(data, schema):
"""Validate the input data using jsonschema validation.
:param data: a data to validate represented as a dict
:param schema: a schema to validate represented as a dict;
must be in JSON Schema Draft 4 format.
"""
try:
validate(data, schema)
except ValidationError as ex:
if len(ex.path) > 0:
join_ex_path = '.'.join(six.text_type(x) for x in ex.path)
detail = ("Invalid input for field/attribute {0}."
" Value: {1}. {2}").format(join_ex_path,
ex.instance, ex.message)
else:
detail = ex.message
raise ValidationError(detail)
def load_data(self, parsed_args):
"""Load the input data.
@ -76,12 +102,13 @@ class BaseCommand(command.Command):
else:
input_file = parsed_args.input_file
# TODO(add input data validation scheme)
with open(input_file, "r") as fd:
return yaml.load(Template(fd.read()).safe_substitute(
data = yaml.load(Template(fd.read()).safe_substitute(
mos_version=self.app.config["mos_version"],
openstack_version=self.app.config["openstack_version"],
))
self.validate_data(data, SCHEMA)
return data
@classmethod
def get_groups(cls, parsed_args, data):

0
contrib/fuel_mirror/fuel_mirror/schemas/__init__.py Normal file
View File

120
contrib/fuel_mirror/fuel_mirror/schemas/input_data_schema.py Normal file
View File

@ -0,0 +1,120 @@
SCHEMA = {
"$schema": "http://json-schema.org/draft-04/schema#",
"definitions": {
"DEB_REPO_SCHEMA": {
"type": "object",
"required": [
"name",
"uri",
"suite",
"section"
],
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": ["deb"]
},
"uri": {
"type": "string"
},
"priority": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
]
},
"suite": {
"type": "string"
},
"section": {
"type": "string"
},
}
},
"RPM_REPO_SCHEMA": {
"type": "object",
"required": [
"name",
"uri",
],
"properties": {
"name": {
"type": "string"
},
"type": {
"type": "string",
"enum": ["rpm"]
},
"uri": {
"type": "string"
},
"priority": {
"anyOf": [
{
"type": "integer"
},
{
"type": "null"
}
]
},
}
},
"REPO_SCHEMA": {
"anyOf":
[
{"$ref": "#/definitions/DEB_REPO_SCHEMA"},
{"$ref": "#/definitions/RPM_REPO_SCHEMA"}
]
},
"REPOS_SCHEMA": {
"type": "array", "items": {"$ref": "#/definitions/REPO_SCHEMA"}
}
},
"type": "object",
"required": [
"groups",
],
"properties": {
"fuel_release_match": {
"type": "object",
"properties": {
"operating_system": {
"type": "string"
}
},
"required": [
"operating_system"
]
},
"requirements": {
"type": "object",
"patternProperties": {
"^[0-9a-z_-]+$": {"type": "array"}
},
"additionalProperties": False,
},
"groups": {
"type": "object",
"patternProperties": {
"^[0-9a-z_-]+$": {"$ref": "#/definitions/REPOS_SCHEMA"}
},
"additionalProperties": False,
},
"inheritance": {
"type": "object",
"patternProperties": {
"^[0-9a-z_-]+$": {"type": "string"}
},
"additionalProperties": False,
}
}
}

7
contrib/fuel_mirror/fuel_mirror/tests/base.py
View File

@ -21,4 +21,11 @@ except ImportError:
class TestCase(unittest.TestCase):
"""Test case base class for all unit tests."""
def assertNotRaises(self, exception, method, *args, **kwargs):
try:
method(*args, **kwargs)
except exception as e:
self.fail("Unexpected error: {0}".format(e))

9
contrib/fuel_mirror/fuel_mirror/tests/data/test_invalid_ubuntu.yaml Normal file
View File

@ -0,0 +1,9 @@
fuel_release_match:
operating_system: Ubuntu
inheritance:
ubuntu: mos
requirements:
ubuntu:
- "package_deb"

13
contrib/fuel_mirror/fuel_mirror/tests/test_cli_commands.py
View File

@ -18,6 +18,8 @@ import mock
import os.path
import subprocess
from jsonschema import ValidationError
# The cmd2 does not work with python3.5
# because it tries to get access to the property mswindows,
# that was removed in 3.5
@ -40,6 +42,10 @@ CENTOS_PATH = os.path.join(
os.path.dirname(__file__), "data", "test_centos.yaml"
)
INVALID_DATA_PATH = os.path.join(
os.path.dirname(__file__), "data", "test_invalid_ubuntu.yaml"
)
@mock.patch.multiple(
"fuel_mirror.app",
@ -349,3 +355,10 @@ class TestCliCommands(base.TestCase):
None,
None
)
@mock.patch("fuel_mirror.app.utils.get_fuel_settings")
def test_create_with_invalid_data(self, m_get_settings, accessors):
self.assertRaises(
ValidationError, create.debug, ["--config", CONFIG_PATH, "-G",
"mos", "-I", INVALID_DATA_PATH]
)

125
contrib/fuel_mirror/fuel_mirror/tests/test_validate_config.py
View File

@ -15,8 +15,12 @@
# under the License.
import os.path
from jsonschema import validate
from jsonschema import ValidationError
import yaml
from fuel_mirror.schemas.input_data_schema import SCHEMA
from fuel_mirror.tests import base
@ -24,10 +28,129 @@ DATA_DIR = os.path.join(os.path.dirname(__file__), "..", "..", "data")
class TestValidateConfigs(base.TestCase):
def test_validate_data_files(self):
for f in os.listdir(DATA_DIR):
with open(os.path.join(DATA_DIR, f), "r") as fd:
data = yaml.load(fd)
# TODO(add input data validation scheme)
self.assertNotRaises(ValidationError, validate, data, SCHEMA)
self.assertIn("groups", data)
self.assertIn("fuel_release_match", data)
def test_validate_fail_with_empty_data(self):
self.assertRaises(ValidationError, validate, {}, SCHEMA)
def test_validate_fail_without_groups(self):
invalid_data = {
"requirements": {
"ubuntu": ["package_deb"]
}
}
self.assertRaisesRegexp(
ValidationError, "'groups' is a required property", validate,
invalid_data, SCHEMA)
def test_invalid_requirements_in_pattern_properies(self):
invalid_data = {
"requirements": {
"ubun.tu": ["package_deb"]
},
"groups": {
}
}
self.assertRaisesRegexp(
ValidationError, "'ubun.tu' was unexpected", validate,
invalid_data, SCHEMA)
def test_invalid_requirements_type_array(self):
invalid_data = {
"requirements": {
"ubuntu": "package_deb"
},
"groups": {
}
}
self.assertRaisesRegexp(
ValidationError, "'package_deb' is not of type 'array'", validate,
invalid_data, SCHEMA)
def test_invalid_inheritens_in_pattern_properies(self):
invalid_data = {
"inheritance": {
"ubun.tu": "mos"
},
"groups": {
}
}
self.assertRaisesRegexp(
ValidationError, "'ubun.tu' was unexpected", validate,
invalid_data, SCHEMA)
def test_invalid_inheritens_type_string(self):
invalid_data = {
"inheritance": {
"ubuntu": 123
},
"groups": {
}
}
self.assertRaisesRegexp(
ValidationError, "123 is not of type 'string'", validate,
invalid_data, SCHEMA)
def test_invalid_groups_in_pattern_properies(self):
invalid_data = {
"groups": {
"mo.s": []
}
}
self.assertRaisesRegexp(
ValidationError, "'mo.s' was unexpected", validate,
invalid_data, SCHEMA)
def test_invalid_groups_type_array(self):
invalid_data = {
"groups": {
"mos": "string"
}
}
self.assertRaisesRegexp(
ValidationError, "'string' is not of type 'array'", validate,
invalid_data, SCHEMA)
def test_without_name_in_groups_array(self):
invalid_data = {
"groups": {
"mos": [
{
'type': 'deb',
'uri': 'http://localhost/mos',
'priority': None,
'suite': 'mos$mos_version',
'section': 'main restricted'
}
]
}
}
self.assertRaisesRegexp(
ValidationError, "is not valid under any of the given schemas",
validate, invalid_data, SCHEMA)
def test_with_invalid_type_in_groups_array(self):
invalid_data = {
"groups": {
"mos": [
{
'name': 'mos',
'type': 'adf',
'uri': 'http://localhost/mos',
'priority': None,
'suite': 'mos$mos_version',
'section': 'main restricted'
}
]
}
}
self.assertRaisesRegexp(
ValidationError, "is not valid under any of the given schemas",
validate, invalid_data, SCHEMA)

1
contrib/fuel_mirror/requirements.txt
View File

@ -9,3 +9,4 @@ six>=1.5.2
PyYAML>=3.10
packetary>=0.1.0
python-fuelclient>=7.0.0
jsonschema>=2.3.0