From 1525e5e1fa85aec3304465c187261ad3fe1737d7 Mon Sep 17 00:00:00 2001 From: Sorin Sbarnea Date: Fri, 23 Jul 2021 09:36:51 +0100 Subject: [PATCH] Enable ansible-test sanity checks (nv) Adds non-voting check jobs for ansible-test sanity and fixing many problems identified by these. We will sort the other problems in follow-up(s) and make the job voting. This approach will ease reviewing and prevent creating conflicts with other pending changes. Story: https://projects.engineering.redhat.com/browse/TRIPLEOCI-559 Change-Id: I29ce27114e0675fb5d00614df506675bfbe5bf96 --- meta/runtime.yml | 2 +- .../tripleo_repos/get_hash/constants.py | 5 +++ .../tripleo_repos/get_hash/exceptions.py | 4 ++ .../get_hash/tripleo_hash_info.py | 25 ++++++----- plugins/module_utils/tripleo_repos/main.py | 10 +++-- .../tripleo_repos/yum_config/constants.py | 4 +- .../tripleo_repos/yum_config/dnf_manager.py | 5 ++- .../tripleo_repos/yum_config/exceptions.py | 5 +++ .../tripleo_repos/yum_config/yum_config.py | 44 +++++++++++++------ plugins/modules/get_hash.py | 39 ++++++++-------- plugins/modules/yum_config.py | 38 ++++++++-------- tests/sanity/ignore-2.10.txt | 1 + tests/sanity/ignore-2.11.txt | 1 + tests/sanity/ignore-2.9.txt | 13 ++++++ tests/sanity/requirements.txt | 2 + tox.ini | 4 +- zuul.d/layout.yaml | 28 +++++++++++- 17 files changed, 157 insertions(+), 73 deletions(-) create mode 120000 tests/sanity/ignore-2.10.txt create mode 120000 tests/sanity/ignore-2.11.txt create mode 100644 tests/sanity/ignore-2.9.txt create mode 100644 tests/sanity/requirements.txt diff --git a/meta/runtime.yml b/meta/runtime.yml index 7d99f97..1a15205 100644 --- a/meta/runtime.yml +++ b/meta/runtime.yml @@ -1 +1 @@ -requires_ansible: 2.9.0 +requires_ansible: ">=2.9.0" diff --git a/plugins/module_utils/tripleo_repos/get_hash/constants.py b/plugins/module_utils/tripleo_repos/get_hash/constants.py index d05d710..db24be3 100644 --- a/plugins/module_utils/tripleo_repos/get_hash/constants.py +++ b/plugins/module_utils/tripleo_repos/get_hash/constants.py @@ -13,6 +13,11 @@ # under the License. # # +from __future__ import (absolute_import, division, print_function) + + +__metaclass__ = type + """ These are the keys we expect to find in a well-formed config.yaml diff --git a/plugins/module_utils/tripleo_repos/get_hash/exceptions.py b/plugins/module_utils/tripleo_repos/get_hash/exceptions.py index a6c4639..b144255 100644 --- a/plugins/module_utils/tripleo_repos/get_hash/exceptions.py +++ b/plugins/module_utils/tripleo_repos/get_hash/exceptions.py @@ -13,6 +13,10 @@ # under the License. # # +from __future__ import (absolute_import, division, print_function) + + +__metaclass__ = type class Base(Exception): diff --git a/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py b/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py index 21d39ea..d1db5c9 100644 --- a/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py +++ b/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py @@ -13,14 +13,18 @@ # under the License. # # +from __future__ import (absolute_import, division, print_function) import logging import sys -import yaml import os +import yaml import requests -import tripleo_repos.get_hash.constants as const -import tripleo_repos.get_hash.exceptions as exc +from .constants import CONFIG_PATH, CONFIG_KEYS +from .exceptions import TripleOHashMissingConfig, TripleOHashInvalidConfig + + +__metaclass__ = type class TripleOHashInfo: @@ -111,29 +115,29 @@ class TripleOHashInfo: config_path = '' local_config = _resolve_local_config_path() # prefer const.CONFIG_PATH then local_config - if _check_read_file(const.CONFIG_PATH): - config_path = const.CONFIG_PATH + if _check_read_file(CONFIG_PATH): + config_path = CONFIG_PATH elif local_config: config_path = local_config else: - raise exc.TripleOHashMissingConfig( + raise TripleOHashMissingConfig( "Configuration file not found at {} or {}".format( - const.CONFIG_PATH, local_config + CONFIG_PATH, local_config ) ) logging.info("Using config file at {}".format(config_path)) with open(config_path, 'r') as config_yaml: loaded_config = yaml.safe_load(config_yaml) - for k in const.CONFIG_KEYS: + for k in CONFIG_KEYS: if k not in loaded_config: error_str = ( "Malformed config file - missing {}. Expected all" "of these configuration items: {}" ).format( - k, ", ".join(const.CONFIG_KEYS) + k, ", ".join(CONFIG_KEYS) ) logging.error(error_str) - raise exc.TripleOHashInvalidConfig(error_str) + raise TripleOHashInvalidConfig(error_str) # if the passed config contains the key then use that value if passed_config.get(k): result_config[k] = passed_config[k] @@ -150,7 +154,6 @@ class TripleOHashInfo: :param tag: The Delorean server named tag e.g. current-tripleo :param config: Use an existing config dictionary and don't load it """ - config = TripleOHashInfo.load_config(config) self.os_version = os_version diff --git a/plugins/module_utils/tripleo_repos/main.py b/plugins/module_utils/tripleo_repos/main.py index 5dbd74d..2b00b01 100755 --- a/plugins/module_utils/tripleo_repos/main.py +++ b/plugins/module_utils/tripleo_repos/main.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function +from __future__ import (absolute_import, division, print_function) import argparse import os import platform @@ -22,9 +22,8 @@ import re import subprocess import sys -import requests - +__metaclass__ = type TITLE_RE = re.compile('\\[(.*)\\]') NAME_RE = re.compile('name=(.+)') PRIORITY_RE = re.compile('priority=\\d+') @@ -222,6 +221,11 @@ def _parse_args(distro_id, distro_major_version_id): def _get_repo(path, args): + + # lazy import + if 'requests' not in globals(): + import requests + r = requests.get(path) if r.status_code == 200: return _inject_mirrors(r.text, args) diff --git a/plugins/module_utils/tripleo_repos/yum_config/constants.py b/plugins/module_utils/tripleo_repos/yum_config/constants.py index e5478fa..c55e3af 100644 --- a/plugins/module_utils/tripleo_repos/yum_config/constants.py +++ b/plugins/module_utils/tripleo_repos/yum_config/constants.py @@ -13,10 +13,12 @@ # under the License. # # - +from __future__ import (absolute_import, division, print_function) """ List of options that can be updated for yum repo files. """ + +__metaclass__ = type YUM_REPO_SUPPORTED_OPTIONS = [ 'name', 'baseurl', diff --git a/plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py b/plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py index 36bcf41..ee33c52 100644 --- a/plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py +++ b/plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py @@ -11,10 +11,13 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - +from __future__ import (absolute_import, division, print_function) import logging +__metaclass__ = type + + class DnfModuleManager: """Class that manages dnf modules.""" diff --git a/plugins/module_utils/tripleo_repos/yum_config/exceptions.py b/plugins/module_utils/tripleo_repos/yum_config/exceptions.py index 96a626f..c682ab4 100644 --- a/plugins/module_utils/tripleo_repos/yum_config/exceptions.py +++ b/plugins/module_utils/tripleo_repos/yum_config/exceptions.py @@ -13,6 +13,11 @@ # under the License. # # +from __future__ import (absolute_import, division, print_function) + + +__metaclass__ = type + class Base(Exception): """Base Exception class.""" diff --git a/plugins/module_utils/tripleo_repos/yum_config/yum_config.py b/plugins/module_utils/tripleo_repos/yum_config/yum_config.py index 4c2bf5f..bc7edab 100644 --- a/plugins/module_utils/tripleo_repos/yum_config/yum_config.py +++ b/plugins/module_utils/tripleo_repos/yum_config/yum_config.py @@ -12,14 +12,30 @@ # License for the specific language governing permissions and limitations # under the License. # +from __future__ import (absolute_import, division, print_function) + import configparser import logging import os import sys -import tripleo_repos.yum_config.constants as const -import tripleo_repos.yum_config.exceptions as exc +from .constants import ( + YUM_GLOBAL_CONFIG_FILE_PATH, + YUM_REPO_DIR, + YUM_REPO_FILE_EXTENSION, + YUM_REPO_SUPPORTED_OPTIONS, +) +from .exceptions import ( + TripleOYumConfigFileParseError, + TripleOYumConfigInvalidOption, + TripleOYumConfigInvalidSection, + TripleOYumConfigNotFound, + TripleOYumConfigPermissionDenied, +) + + +__metaclass__ = type class TripleOYumConfig: @@ -81,23 +97,23 @@ class TripleOYumConfig: if not (file_path or dir_path): msg = ('A configuration file path or a directory path must be ' 'provided.') - raise exc.TripleOYumConfigNotFound(error_msg=msg) + raise TripleOYumConfigNotFound(error_msg=msg) if file_path: if not os.path.isfile(file_path): msg = ('The configuration file "{}" was not found in the ' 'provided path.').format(file_path) - raise exc.TripleOYumConfigNotFound(error_msg=msg) + raise TripleOYumConfigNotFound(error_msg=msg) if not os.access(file_path, os.W_OK): msg = ('The configuration file {} is not ' 'writable.'.format(file_path)) - raise exc.TripleOYumConfigPermissionDenied(error_msg=msg) + raise TripleOYumConfigPermissionDenied(error_msg=msg) if dir_path: if not os.path.isdir(dir_path): msg = ('The configuration dir "{}" was not found in the ' 'provided path.').format(dir_path) - raise exc.TripleOYumConfigNotFound(error_msg=msg) + raise TripleOYumConfigNotFound(error_msg=msg) def _read_config_file(self, section): """Read the configuration file associate with this object. @@ -118,13 +134,13 @@ class TripleOYumConfig: except configparser.Error: msg = 'Unable to parse configuration file {}.'.format( self.config_file_path) - raise exc.TripleOYumConfigFileParseError(error_msg=msg) + raise TripleOYumConfigFileParseError(error_msg=msg) if section not in config.sections(): msg = ('The provided section "{}" was not found in the ' 'configuration file {}.').format( section, self.config_file_path) - raise exc.TripleOYumConfigInvalidSection(error_msg=msg) + raise TripleOYumConfigInvalidSection(error_msg=msg) return config, self.config_file_path @@ -168,13 +184,13 @@ class TripleOYumConfig: if self.valid_options: if not all(key in self.valid_options for key in set_dict.keys()): msg = 'One or more provided options are not valid.' - raise exc.TripleOYumConfigInvalidOption(error_msg=msg) + raise TripleOYumConfigInvalidOption(error_msg=msg) config, config_file_path = self._read_config_file(section) if not (config and config_file_path): msg = ('The provided section "{}" was not found within any ' 'configuration file.').format(section) - raise exc.TripleOYumConfigNotFound(error_msg=msg) + raise TripleOYumConfigNotFound(error_msg=msg) # Update configuration file with dict updates config[section].update(set_dict) @@ -192,13 +208,13 @@ class TripleOYumRepoConfig(TripleOYumConfig): if file_path: logging.info( "Using '{}' as yum repo configuration file.".format(file_path)) - conf_dir_path = dir_path or const.YUM_REPO_DIR + conf_dir_path = dir_path or YUM_REPO_DIR super(TripleOYumRepoConfig, self).__init__( - valid_options=const.YUM_REPO_SUPPORTED_OPTIONS, + valid_options=YUM_REPO_SUPPORTED_OPTIONS, file_path=file_path, dir_path=conf_dir_path, - file_extension=const.YUM_REPO_FILE_EXTENSION) + file_extension=YUM_REPO_FILE_EXTENSION) def update_section(self, section, set_dict, enable=None): if enable is not None: @@ -211,7 +227,7 @@ class TripleOYumGlobalConfig(TripleOYumConfig): """Manages yum global configuration file.""" def __init__(self, file_path=None): - conf_file_path = file_path or const.YUM_GLOBAL_CONFIG_FILE_PATH + conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH logging.info("Using '{}' as yum global configuration " "file.".format(conf_file_path)) if file_path is None: diff --git a/plugins/modules/get_hash.py b/plugins/modules/get_hash.py index 543da53..e0a75ab 100644 --- a/plugins/modules/get_hash.py +++ b/plugins/modules/get_hash.py @@ -1,31 +1,22 @@ #!/usr/bin/python -# Copyright 2021 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -# +# Copyright 2021 Red Hat, Inc. +# GNU General Public License v3.0+ (see COPYING or +# https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) + + +__metaclass__ = type -from tripleo_repos.get_hash.tripleo_hash_info import TripleOHashInfo -from ansible.module_utils.basic import AnsibleModule DOCUMENTATION = r''' --- -module: tripleo_get_hash +module: get_hash short_description: Resolve rdo named tag to commit, full and distro hashes -version_added: "2.9" +version_added: "1.0.0" -description: +description: "" options: os_version: @@ -42,7 +33,6 @@ options: description: The tripleo-ci component you are interested in required: false type: str - default: None tag: description: The named tag to fetch required: false @@ -95,6 +85,8 @@ dlrn_url: sample: 'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5' # noqa E501 ''' +from ansible.module_utils.basic import AnsibleModule # noqa: E402 + def run_module(): result = dict( @@ -120,6 +112,13 @@ def run_module(): try: + try: + from ansible_collections.tripleo.repos.plugins.module_utils.\ + tripleo_repo.get_hash.tripleo_hash_info import TripleOHashInfo + except ImportError: + from tripleo_repos.get_hash.tripleo_hash_info import \ + TripleOHashInfo + os_version = module.params.get('os_version') release = module.params.get('release') component = module.params.get('component') diff --git a/plugins/modules/yum_config.py b/plugins/modules/yum_config.py index be2492c..aac6f75 100644 --- a/plugins/modules/yum_config.py +++ b/plugins/modules/yum_config.py @@ -1,30 +1,18 @@ #!/usr/bin/python -# Copyright 2021 Red Hat, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. +# Copyright 2021 Red Hat, Inc. +# GNU General Public License v3.0+ (see COPYING or +# https://www.gnu.org/licenses/gpl-3.0.txt) +from __future__ import (absolute_import, division, print_function) -from ansible.module_utils.basic import AnsibleModule - -import tripleo_repos.yum_config.dnf_manager as dnf_mgr -import tripleo_repos.yum_config.yum_config as cfg +__metaclass__ = type DOCUMENTATION = r''' --- -module: tripleo_yum_config +module: yum_config short_description: Update yum configuration files for TripleO deployments. -version_added: "2.9" +version_added: "1.0.0" description: - Update specific options for different yum configuration files like @@ -134,6 +122,8 @@ EXAMPLES = r''' keepcache: "0" ''' +from ansible.module_utils.basic import AnsibleModule # noqa: E402 + def run_module(): # define available arguments/parameters a user can pass to the module @@ -198,6 +188,16 @@ def run_module(): # Module execution try: + + try: + import ansible_collections.tripleo.repos.plugins.module_utils.\ + tripleo_repos.yum_config.dnf_manager as dnf_mgr + import ansible_collections.tripleo.repos.plugins.module_utils.\ + tripleo_repos.yum_config.yum_config as cfg + except ImportError: + import tripleo_repos.yum_config.dnf_manager as dnf_mgr + import tripleo_repos.yum_config.yum_config as cfg + if m_type == 'repo': config_obj = cfg.TripleOYumRepoConfig( file_path=m_file_path, diff --git a/tests/sanity/ignore-2.10.txt b/tests/sanity/ignore-2.10.txt new file mode 120000 index 0000000..a11c71b --- /dev/null +++ b/tests/sanity/ignore-2.10.txt @@ -0,0 +1 @@ +ignore-2.9.txt \ No newline at end of file diff --git a/tests/sanity/ignore-2.11.txt b/tests/sanity/ignore-2.11.txt new file mode 120000 index 0000000..a11c71b --- /dev/null +++ b/tests/sanity/ignore-2.11.txt @@ -0,0 +1 @@ +ignore-2.9.txt \ No newline at end of file diff --git a/tests/sanity/ignore-2.9.txt b/tests/sanity/ignore-2.9.txt new file mode 100644 index 0000000..ac03ca1 --- /dev/null +++ b/tests/sanity/ignore-2.9.txt @@ -0,0 +1,13 @@ +plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py import +plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-bad-function +plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-format-automatic-specification +plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:logging-format-interpolation +plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py replace-urlopen # we did it right +plugins/module_utils/tripleo_repos/main.py import:traceback +plugins/module_utils/tripleo_repos/main.py pylint:ansible-bad-function +plugins/module_utils/tripleo_repos/main.py pylint:ansible-format-automatic-specification +plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py pylint:ansible-format-automatic-specification +plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py pylint:logging-format-interpolation +plugins/module_utils/tripleo_repos/yum_config/yum_config.py pylint:ansible-format-automatic-specification +plugins/module_utils/tripleo_repos/yum_config/yum_config.py pylint:logging-format-interpolation +plugins/modules/yum_config.py pylint:ansible-format-automatic-specification diff --git a/tests/sanity/requirements.txt b/tests/sanity/requirements.txt new file mode 100644 index 0000000..6c9fdba --- /dev/null +++ b/tests/sanity/requirements.txt @@ -0,0 +1,2 @@ +PyYAML +requests diff --git a/tox.ini b/tox.ini index 1247351..f28686f 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,9 @@ [tox] minversion = 3.1.1 skipsdist = True -envlist = py,pep8,packaging +envlist = py,pep8,packaging,sanity +requires = + tox-ansible>=1.5.3 [testenv] usedevelop = True diff --git a/zuul.d/layout.yaml b/zuul.d/layout.yaml index 724f7eb..2f11433 100644 --- a/zuul.d/layout.yaml +++ b/zuul.d/layout.yaml @@ -1,10 +1,22 @@ +- job: + name: tox-sanity-py36 + description: Run ansible-test sanity tests on a collection + parent: openstack-tox-py36 + vars: + tox_envlist: sanity +- job: + name: tox-sanity-py39 + description: Run ansible-test sanity tests on a collection + parent: openstack-tox-py39 + vars: + tox_envlist: sanity - project: templates: - check-requirements - openstack-cover-jobs - openstack-python3-wallaby-jobs check: - jobs: &cjobs + jobs: - openstack-tox-pep8: vars: tox_envlist: pep8,packaging @@ -17,8 +29,20 @@ - openstack-tox-py39 - tripleo-build-containers-ubi-8: dependencies: *deps_unit_lint_cprovider + - tox-sanity-py36: + voting: false + - tox-sanity-py39: + voting: false gate: - jobs: *cjobs + jobs: + - openstack-tox-pep8: + vars: + tox_envlist: pep8,packaging + - openstack-tox-py39 + - tripleo-buildimage-overcloud-full-centos-8: + dependencies: *deps_unit_lint_cprovider + - tripleo-build-containers-ubi-8: + dependencies: *deps_unit_lint_cprovider post: jobs: - publish-openstack-python-branch-tarball