Bunch of bugfixes

Tool was run against murano-apps catalog, all found bugs was fixed

Change-Id: I7b31d6d3adff749126cbafdfbc2a45aa6d16c24a
This commit is contained in:
sslypushenko 2016-09-12 18:48:46 +03:00
parent ff9d851d7b
commit 628b30fc2c
12 changed files with 168 additions and 113 deletions

View File

@ -27,6 +27,7 @@ def check_req(check, required=True):
error.register.E203(description='Value should be string type')
error.register.E200(description='No value should be here')
error.register.E204(description='Wrong code structure/assigment')
error.register.E202(description='Not a valid yaql expression')
error.register.W202(description='Not a valid yaql expression')
error.register.E201(description='Not a valid variable name')
@ -36,12 +37,16 @@ CODE_STRUCTURE = {
'Try': {
'keywords': {
'Try': check_req('codeblock'),
'Catch': check_req('empty'),
'With': check_req('string', False),
'As': check_req('string', False),
'Do': check_req('codeblock'),
'Catch': check_req('codeblock'),
'Else': check_req('codeblock', False),
'Finally': check_req('codeblock', False)}},
'As': {
'keywords': {
'As': check_req('string'),
'With': check_req('string', False),
'Do': check_req('codeblock'),
}
},
'Parallel': {
'keywords': {
'Limit': check_req('codeblock', False),
@ -97,6 +102,11 @@ CODE_STRUCTURE = {
'Continue': check_req('empty'),
}
},
'Rethrow': {
'keywords': {
'Rethrow': check_req('empty'),
}
},
}
@ -114,20 +124,20 @@ class CheckCodeStructure(object):
def string(self, value):
if not isinstance(value, six.string_types):
yield error.report.E203('Value should be string type '
'"{0}"'.format(value), value)
yield error.report.E203('Value of "{0}" should be a string'
''.format(value), value)
def empty(self, value):
if value:
yield error.report.E200('There should be no value here '
yield error.report.E200('Statement should be empty, not a '
'"{0}"'.format(value), value)
def yaql(self, value):
if not self._yaql_checker(value):
if isinstance(value, bool):
return
yield error.report.W202('Not a valid yaql expression '
'"{0}"'.format(value), value)
yield error.report.W202('"{0}" is not valid yaql expression'
''.format(value), value)
def codeblock(self, codeblocks):
if isinstance(codeblocks, list):
@ -140,8 +150,8 @@ class CheckCodeStructure(object):
key = next(iter(block))
if not isinstance(key, six.string_types) or\
not ASSIGMENT_KEY.match(key):
yield error.report.E201('Not valid variable name '
'"{0}"'.format(key), key)
yield error.report.E201('"{0}" is not valid variable name'
''.format(key), key)
def _single_block(self, block):
if isinstance(block, dict):
@ -160,8 +170,8 @@ class CheckCodeStructure(object):
if len(block.keys()) == 1:
yield self._check_assigment(block)
else:
yield error.report.E200('Wrong code structure/assigment '
'probably typo', block)
yield error.report.E204('Wrong code structure/assigment. '
'Probably a typo', block)
return
keywords = value.get('keywords', {})
@ -169,7 +179,7 @@ class CheckCodeStructure(object):
block_keys_set = set(block.keys())
for missing in (kset - block_keys_set):
if keywords[missing]['required']:
yield error.report.E200('Missing keyword "{0}" for "{1}" '
yield error.report.E204('Missing keyword "{0}" for "{1}" '
'code structure'
.format(missing, key), block)
for unknown in (block_keys_set - kset - {key}):

View File

@ -70,7 +70,17 @@ class BaseLoader(object):
def try_load(cls, path):
loader = cls._try_load(path)
if loader is not None and loader.exists(consts.MANIFEST_PATH):
loader.try_set_format()
try:
manifest = loader.read(consts.MANIFEST_PATH).yaml()[0]
if 'FullName' not in manifest:
LOG.warning('Package does not look like Murano package',
exc_info=sys.exc_info())
return
loader.try_set_format(manifest)
except yaml.YAMLError:
LOG.warning('Unable to parse Manifest yaml',
exc_info=sys.exc_info())
return
return loader
@abc.abstractmethod
@ -95,21 +105,14 @@ class BaseLoader(object):
self._cached_files[path] = FileWrapper(self, path)
return self._cached_files[path]
def try_set_format(self):
if self.exists(consts.MANIFEST_PATH):
try:
manifest = self.read(consts.MANIFEST_PATH).yaml()
except yaml.YAMLError:
LOG.warning('Unable to parse Manifest yaml',
exc_info=sys.exc_info())
def try_set_format(self, manifest):
if manifest and 'Format' in manifest:
if '/' in six.text_type(manifest['Format']):
fmt, version = manifest['Format'].split('/', 1)
self.format = fmt
self.format_version = version
else:
if manifest and 'Format' in manifest:
if '/' in six.text_type(manifest['Format']):
fmt, version = manifest['Format'].split('/', 1)
self.format = fmt
self.format_version = version
else:
self.format_version = six.text_type(manifest['Format'])
self.format_version = six.text_type(manifest['Format'])
class DirectoryLoader(BaseLoader):

View File

@ -32,7 +32,7 @@ class CodeStructureTest(helpers.BaseValidatorTestClass):
'$a': '$.deploy()',
'$b': '$.string()'}]
self.g = self._checker.codeblock(SIMPLE_BODY)
self.assertIn('Wrong code structure/assigment probably typo',
self.assertIn('Wrong code structure/assigment. Probably a typo',
next(self.g).message)
def test_multiline(self):
@ -50,7 +50,7 @@ class CodeStructureTest(helpers.BaseValidatorTestClass):
'$.call($res)',
]
self.g = self._checker.codeblock(MULTILINE_BODY)
self.assertIn('Not valid variable name "1"', next(self.g).message)
self.assertIn('"1" is not valid variable name', next(self.g).message)
def test_bad_assigment_with_double_dollar(self):
MULTILINE_BODY = [
@ -59,7 +59,7 @@ class CodeStructureTest(helpers.BaseValidatorTestClass):
'$.call($res)',
]
self.g = self._checker.codeblock(MULTILINE_BODY)
self.assertIn('Not valid variable name "$$"', next(self.g).message)
self.assertIn('"$$" is not valid variable name', next(self.g).message)
def test_bad_assigment_case2(self):
MULTILINE_BODY = [
@ -69,7 +69,7 @@ class CodeStructureTest(helpers.BaseValidatorTestClass):
]
self.g = self._checker.codeblock(MULTILINE_BODY)
p = next(self.g)
self.assertIn('Not valid variable name "res"', p.message)
self.assertIn('"res" is not valid variable name', p.message)
def test_if(self):
MULTILINE_BODY = [
@ -128,27 +128,26 @@ class CodeStructureTest(helpers.BaseValidatorTestClass):
'$.b()']}}
]
self.g = self._checker.codeblock(MULTILINE_BODY)
self.assertIn('Not valid variable name "www"', next(self.g).message)
self.assertIn('"www" is not valid variable name',
next(self.g).message)
def test_minimal_try_block(self):
MULTILINE_BODY = [
{'Try': [
'$port.deploy()'],
'Catch': '',
'Do': ['$.string()']}]
'Catch': {}}]
self.g = self._checker.codeblock(MULTILINE_BODY)
def test_try_not_string(self):
MULTILINE_BODY = [
{'Try': [
'$port.deploy()'],
'Catch': '',
'With': 213,
'As': 'what',
'Do': [
'$.string()']}]
{'Try': ['$port.deploy()'],
'Catch': {
'With': 'exceptionName',
'As': 213,
'Do': ['$.string()']}}
]
self.g = self._checker.codeblock(MULTILINE_BODY)
self.assertIn('Value should be string type "213"',
self.assertIn('Value of "213" should be a string',
next(self.g).message)
def test_yaql_accept_bool(self):
@ -162,5 +161,5 @@ class CodeStructureTest(helpers.BaseValidatorTestClass):
{'Break': 'a'},
]
self.g = self._checker.codeblock(MULTILINE_BODY)
self.assertIn('There should be no value here "a"',
self.assertIn('Statement should be empty, not a "a"',
next(self.g).message)

View File

@ -27,6 +27,8 @@ class ManifestValidatorTests(helpers.BaseValidatorTestClass):
self.exists = self._oe_patcher.start()
self.exists.return_value = [True, True]
self.loaded_package = mock.Mock()
self.loaded_package.read.return_value.yaml.return_value = [
{'Type': 'Application'}]
self.mv = manifest.ManifestValidator(self.loaded_package)
def test_format_as_number(self):

View File

@ -128,7 +128,7 @@ class MuranoPlTests(helpers.BaseValidatorTestClass):
m_dict = deepcopy(MURANOPL_BASE['Methods'])
m_dict['foo']['Body'] = '$.deploy('
self.g = self.mpl_validator._valid_methods(m_dict)
self.assertIn('Not a valid yaql expression "$.deploy("',
self.assertIn('"$.deploy(" is not valid yaql expression',
next(self.g).message)
def test_method_body_is_return(self):
@ -141,7 +141,7 @@ class MuranoPlTests(helpers.BaseValidatorTestClass):
m_dict['foo']['Body'][0]['In'] =\
'$.deploy('
self.g = self.mpl_validator._valid_methods(m_dict)
self.assertIn('Not a valid yaql expression "$.deploy("',
self.assertIn('"$.deploy(" is not valid yaql expression',
next(self.g).message)
def test_error_in_method_for_loop_body(self):
@ -149,7 +149,7 @@ class MuranoPlTests(helpers.BaseValidatorTestClass):
m_dict['foo']['Body'][0]['Do'][1] =\
'$.deploy('
self.g = self.mpl_validator._valid_methods(m_dict)
self.assertIn('Not a valid yaql expression "$.deploy("',
self.assertIn('"$.deploy(" is not valid yaql expression',
next(self.g).message)
def test_missing_contract_in_properties(self):
@ -334,11 +334,19 @@ class MuranoPlTests(helpers.BaseValidatorTestClass):
self.assertIn('Usage "Static" is available from 1.3',
next(self.g).message)
def test_method_arguments_are_dict(self):
self.g = self.mpl_validator._valid_arguments({'a': 'b'})
self.assertIn('Methods arguments should be a list',
def test_method_arguments_invalid(self):
self.g = self.mpl_validator._valid_arguments('foobar')
self.assertIn('Methods arguments should be a list or dict',
next(self.g).message)
def test_method_arguments_are_dict(self):
self.g = self.mpl_validator._valid_arguments({'a': {'Contract': '$'}})
def test_method_arguments_are_dict_two_keys(self):
self.g = self.mpl_validator._valid_arguments({'a': 'b', 'c': 'd'})
self.assertIn('It is not safe to define methods arguments as a dict '
'with several keys', next(self.g).message)
def test_method_arguments_element_is_two_key(self):
arguments = [
{'a': {'Contract': '$.string()'},

View File

@ -14,6 +14,7 @@
import mock
import yaml
import yaml.error
from muranopkgcheck import consts
@ -23,13 +24,21 @@ from muranopkgcheck.tests import base
class FileWrapperTest(base.TestCase):
def test_file_wrapper(self):
@mock.patch('muranopkgcheck.pkg_loader.yaml')
def test_file_wrapper(self, m_yaml):
m_yaml.load_all.side_effect = yaml.load_all
fake_pkg = mock.Mock()
fake_pkg.open_file.side_effect = \
lambda f: mock.mock_open(read_data='text')()
f = pkg_loader.FileWrapper(fake_pkg, 'fake_path')
self.assertEqual('text', f.raw())
self.assertEqual(['text'], f.yaml())
m_yaml.load_all.assert_called()
self.assertEqual(['text'], f.yaml())
m_yaml.load_all.reset_mock()
m_yaml.load_all.assert_not_called()
fake_pkg.open_file.side_effect = \
lambda f: mock.mock_open(read_data='!@#$%')()
@ -60,19 +69,32 @@ class FakeLoader(pkg_loader.BaseLoader):
class BaseLoaderTest(base.TestCase):
@mock.patch.object(FakeLoader, 'read')
@mock.patch.object(FakeLoader, '_try_load')
@mock.patch.object(FakeLoader, 'try_set_format')
def test_try_load(self, m_format, m_load):
def test_try_load(self, m_format, m_load, m_read):
m_read.return_value.yaml.return_value = [{'FullName': 'fake'}]
m_load.return_value = FakeLoader('fake')
FakeLoader.try_load('fake')
fake_pkg = FakeLoader.try_load('fake')
self.assertEqual(m_load.return_value, fake_pkg)
m_load.assert_called_once_with('fake')
m_format.assert_called_once_with()
m_format.assert_called_once_with({'FullName': 'fake'})
m_format.reset_mock()
m_read.return_value.yaml.return_value = [{}]
self.assertIsNone(FakeLoader.try_load('fake'))
m_format.assert_not_called()
m_format.reset_mock()
m_read.return_value.yaml.side_effect = yaml.error.YAMLError()
self.assertIsNone(FakeLoader.try_load('fake'))
m_format.assert_not_called()
@mock.patch.object(FakeLoader, '_try_load')
def test_try_set_version(self, m_load):
m_file_wrapper = mock.Mock()
m_file = m_file_wrapper.return_value
m_file.yaml.return_value = {'Format': 'Fake/42'}
m_file.yaml.return_value = [{'Format': 'Fake/42', 'FullName': 'fake'}]
with mock.patch('muranopkgcheck.pkg_loader.FileWrapper',
m_file_wrapper):
m_load.return_value = FakeLoader('fake')
@ -82,13 +104,13 @@ class BaseLoaderTest(base.TestCase):
m_file.yaml.assert_called_once_with()
m_load.return_value = FakeLoader('fake')
m_file.yaml.return_value = {'Format': '4.2'}
m_file.yaml.return_value = [{'Format': '4.2', 'FullName': 'fake'}]
loader = FakeLoader.try_load('fake')
self.assertEqual(consts.DEFAULT_FORMAT, loader.format)
self.assertEqual('4.2', loader.format_version)
m_load.return_value = FakeLoader('fake')
m_file.yaml.return_value = {}
m_file.yaml.return_value = [{'FullName': 'fake'}]
loader = FakeLoader.try_load('fake')
self.assertEqual(consts.DEFAULT_FORMAT, loader.format)
self.assertEqual(consts.DEFAULT_FORMAT_VERSION,
@ -116,17 +138,17 @@ class BaseLoaderTest(base.TestCase):
class DirectoryLoaderTest(base.TestCase):
def _load_fake_pkg(self):
@mock.patch.object(pkg_loader.DirectoryLoader, 'read')
@mock.patch.object(pkg_loader.DirectoryLoader, 'try_set_format')
@mock.patch.object(pkg_loader.DirectoryLoader, 'exists')
def _load_fake_pkg(self, m_exists, m_try_set_format, m_read):
with mock.patch('muranopkgcheck.pkg_loader.os.path.isdir') as m_isdir:
with mock.patch.object(pkg_loader.DirectoryLoader,
'try_set_format') as m:
with mock.patch.object(pkg_loader.DirectoryLoader,
'exists') as m_exists:
m_exists.return_value = True
m_isdir.return_value = True
loader = pkg_loader.DirectoryLoader.try_load('fake')
m.assert_called_once_with()
return loader
m_read.return_value.yaml.return_value = [{'FullName': 'fake'}]
m_exists.return_value = True
m_isdir.return_value = True
loader = pkg_loader.DirectoryLoader.try_load('fake')
m_try_set_format.assert_called_once_with({'FullName': 'fake'})
return loader
def test_try_load(self):
# NOTE(sslypushenko) Using mock.patch here as decorator breaks pdb

View File

@ -26,23 +26,6 @@ LOG = log.getLogger(__name__)
FQN_REGEX = re.compile('^([a-zA-Z_$][\w$]*\.)*[a-zA-Z_$][\w$]*$')
NAME_REGEX = re.compile('^[A-Za-z_][\w]*$')
def check_version(method, version):
since = method._mpl_since
till = method._mpl_till
if (not since or version >= since) and\
(not till or version <= till):
return True
return False
def format_support(since=None, till=None):
def _func(func):
func._mpl_since = since
func._mpl_till = till
return func
return _func
error.register.E005(description='YAML multi document is not allowed')
error.register.E020(description='Missing required key')
error.register.E021(description='Unknown keyword')
@ -62,7 +45,6 @@ class BaseValidator(object):
pass
def _valid_string(self, value):
if not isinstance(value, six.string_types):
yield error.report.E040(_('Value is not a string "{}"'
'').format(value), value)

View File

@ -18,6 +18,7 @@ import os.path
import semantic_version
import six
from muranopkgcheck import consts
from muranopkgcheck import error
from muranopkgcheck.i18n import _
from muranopkgcheck.validators import base
@ -113,6 +114,10 @@ class ManifestValidator(base.YamlValidator):
def _valid_ui(self, value):
if isinstance(value, six.string_types):
pkg_type = self._loaded_pkg.read(
consts.MANIFEST_PATH).yaml()[0]['Type']
if pkg_type == 'Library':
return
if not self._loaded_pkg.exists(os.path.join('UI', value)):
yield error.report.W073(_('There is no UI file "{}"'
'').format(value), value)
@ -121,6 +126,10 @@ class ManifestValidator(base.YamlValidator):
def _valid_logo(self, value):
if isinstance(value, six.string_types):
pkg_type = self._loaded_pkg.read(
consts.MANIFEST_PATH).yaml()[0]['Type']
if pkg_type == 'Library':
return
if not self._loaded_pkg.exists(value):
yield error.report.W074(_('There is no Logo file "{}"'
'').format(value), value)

View File

@ -25,7 +25,7 @@ from muranopkgcheck.validators import base
SUPPORTED_FORMATS = frozenset(['1.0', '1.1', '1.2', '1.3', '1.4'])
METHOD_KEYWORDS = frozenset(['Body', 'Arguments', 'Usage', 'Scope', 'Meta'])
METHOD_ARGUMENTS_KEYWORDS = frozenset(['Contract', 'Usage', 'Meta'])
METHOD_ARGUMENTS_KEYWORDS = frozenset(['Contract', 'Usage', 'Meta', 'Default'])
PROPERTIES_KEYWORDS = frozenset(['Contract', 'Usage', 'Default', 'Meta'])
PROPERTIES_USAGE_VALUES = frozenset(['In', 'Out', 'InOut', 'Const', 'Static',
'Runtime', 'Config'])
@ -38,8 +38,10 @@ error.register.E026(description='Properties should be a dict')
error.register.E042(description='Not allowed usage')
error.register.E044(description='Wrong type of namespace')
error.register.E045(description='Body is not a list or scalar/yaql expression')
error.register.E046(description='Method is not a dict')
error.register.E046(description='Method is not a dict or list')
error.register.E047(description='Missing Contract in property')
error.register.E048(description='It is not safe to define methods arguments '
'as a dict with several keys')
error.register.E052(description='Arguments usage is available since 1.4')
error.register.E053(description='Usage is invalid value ')
error.register.E054(description='Invalid name of method "{}"')
@ -111,7 +113,7 @@ class MuranoPLValidator(base.YamlValidator):
if not contract:
return
for c_key, c_value in six.iteritems(contract):
yield self._valid_contract(c_key)
yield self._valid_string(c_key)
yield self._valid_contract(c_value)
elif isinstance(contract, six.string_types):
if not self.yaql_checker(contract) or \
@ -210,10 +212,17 @@ class MuranoPLValidator(base.YamlValidator):
.format(usage), usage)
def _valid_arguments(self, arguments):
if not isinstance(arguments, list):
yield error.report.E046(_('Methods arguments should be a list'),
if isinstance(arguments, dict) and len(arguments) > 1:
yield error.report.E048(_('It is not safe to define methods '
'arguments as a dict with several keys'),
arguments)
return
elif not isinstance(arguments, (list, dict)):
yield error.report.E046(_('Methods arguments should be a list or '
'dict'), arguments)
return
if isinstance(arguments, dict):
arguments = [arguments]
for argument in arguments:
if not isinstance(argument, dict) or len(argument) != 1:
yield error.report.E046(_('Methods single argument should be '

View File

@ -17,7 +17,8 @@ from muranopkgcheck import error
from muranopkgcheck.i18n import _
from muranopkgcheck.validators import base
KNOWN_FILES_DIR = frozenset(['manifest.yaml', 'UI', 'LICENSE', 'Classes'])
KNOWN_FILES_DIR = frozenset(['manifest.yaml', 'UI', 'LICENSE', 'Classes',
'images.lst', 'README.rst'])
REQUIRED_FILES_DIR = frozenset(['manifest.yaml', 'LICENSE'])
error.register.W120(description='Unknown file in the package')
@ -32,7 +33,7 @@ class PackageValidator(base.BaseValidator):
yield self._known_directories()
def _known_directories(self):
files = set(self._loaded_pkg.search_for('^\w+$'))
files = set(self._loaded_pkg.search_for('^[^/]+$'))
try:
logo_file = next(self._loaded_pkg.search_for('^manifest.yaml$'))\
.yaml()[0]['Logo']

View File

@ -15,13 +15,18 @@
import six
from muranopkgcheck import error
from muranopkgcheck.i18n import _
from muranopkgcheck.validators import base
UI_VERSION = frozenset(['1.0', '1', '2', '2.0', '2.1', '2.2', '2.3'])
FIELDS_TYPE = frozenset(['string', 'boolean', 'text', 'integer', 'password',
UI_VERSION = frozenset(('1.0', '1', '2', '2.0', '2.1', '2.2', '2.3'))
FIELDS_TYPE = frozenset(('string', 'boolean', 'text', 'integer', 'password',
'clusterip', 'floatingip', 'domain', 'databaselist',
'table', 'flavor', 'keypair', 'image', 'azone',
'psqlDatabase', 'network'])
'psqlDatabase', 'network', 'choice'))
BOOL_FIELDS = frozenset(('required', 'hidden'))
STR_FIELDS = frozenset(('name', 'label', 'description',
'descriptionTitle', 'regexpValidator', 'helpText'))
INT_FIELDS = frozenset(('minLength', 'maxLength', 'minValue', 'maxValue'))
error.register.E081(description='Value should be boolean')
error.register.E083(description='Wrong name in UI file')
@ -41,46 +46,47 @@ class UiValidator(base.YamlValidator):
def _valid_application(self, application):
if not isinstance(application, dict):
yield error.report.E084('Application is not a dict', application)
yield error.report.E084(_('Application is not a dict'),
application)
return
for name, value in six.iteritems(application):
if not self._check_name(name):
if name != '?':
yield error.report.E083('Wrong name in UI file "{0}"'
yield error.report.E083(_('Wrong name in UI file "{}"')
.format(name), name)
def _valid_version(self, version):
if str(version) not in UI_VERSION:
yield error.report.W082('Incorrect version of UI file "{0}"'
yield error.report.W082(_('Incorrect version of UI file "{}"')
.format(version), version)
def _valid_forms(self, forms):
for named_form in forms:
for name, form in six.iteritems(named_form):
yield self._valid_form(form['fields'])
yield self._valid_keywords(form.keys(), frozenset(['fields']))
yield self._valid_keywords(form.keys(),
('fields', 'validators'))
def _valid_form(self, form):
for named_params in form:
for key, value in six.iteritems(named_params):
if key == 'required':
if not isinstance(value, bool):
yield error.report.E081('Value of {0} should be '
'boolean not "{1}"'
if key in STR_FIELDS:
if not isinstance(value, six.string_types):
yield error.report.E040(_('Value of {} should be '
'string not "{}"')
.format(key, value), key)
elif key == 'hidden':
elif key in BOOL_FIELDS:
if not isinstance(value, bool):
yield error.report.E081('Value of {0} should be '
'boolean "{1}"'
yield error.report.E081(_('Value of {} should be '
'boolean not "{}"')
.format(key, value), key)
elif key in INT_FIELDS:
if not isinstance(value, int):
yield error.report.E082(_('Value of {} should be '
'int not "{}"')
.format(key, value), key)
elif key in frozenset(['requirements', 'errorMessages',
'choices', 'widgetMedia',
'validators']):
pass
elif key == 'type':
yield self._valid_field_type(value)
else:
yield self._valid_string(value)
def _valid_field_type(self, fqn, can_be_list=True):
if isinstance(fqn, list):

View File

@ -55,6 +55,10 @@ class YamlNull(YamlObject):
def __str__(self):
return 'null'
def __bool__(self):
return False
__nonzero__ = __bool__
BaseLoader = getattr(yaml, 'CSafeLoader', yaml.SafeLoader)