Enable ansible-test sanity checks (nv)

Adds non-voting check jobs for ansible-test sanity and fixing many
problems identified by these. We will sort the other problems in
follow-up(s) and make the job voting. This approach will ease
reviewing and prevent creating conflicts with other pending changes.

Story: https://projects.engineering.redhat.com/browse/TRIPLEOCI-559
Change-Id: I29ce27114e0675fb5d00614df506675bfbe5bf96

meta/runtime.yml

@@ -1 +1 @@
-requires_ansible: 2.9.0
+requires_ansible: ">=2.9.0"

plugins/module_utils/tripleo_repos/get_hash/constants.py

@@ -13,6 +13,11 @@
 #   under the License.
 #
 #
+from __future__ import (absolute_import, division, print_function)
+
+
+__metaclass__ = type
+
 
 """
 These are the keys we expect to find in a well-formed config.yaml

plugins/module_utils/tripleo_repos/get_hash/exceptions.py

@@ -13,6 +13,10 @@
 #   under the License.
 #
 #
+from __future__ import (absolute_import, division, print_function)
+
+
+__metaclass__ = type
 
 
 class Base(Exception):

plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py

@@ -13,14 +13,18 @@
 #  under the License.
 #
 #
+from __future__ import (absolute_import, division, print_function)
 
 import logging
 import sys
-import yaml
 import os
+import yaml
 import requests
-import tripleo_repos.get_hash.constants as const
+from .constants import CONFIG_PATH, CONFIG_KEYS
-import tripleo_repos.get_hash.exceptions as exc
+from .exceptions import TripleOHashMissingConfig, TripleOHashInvalidConfig
+
+
+__metaclass__ = type
 
 
 class TripleOHashInfo:
@@ -111,29 +115,29 @@ class TripleOHashInfo:
         config_path = ''
         local_config = _resolve_local_config_path()
         # prefer const.CONFIG_PATH then local_config
-        if _check_read_file(const.CONFIG_PATH):
+        if _check_read_file(CONFIG_PATH):
-            config_path = const.CONFIG_PATH
+            config_path = CONFIG_PATH
         elif local_config:
             config_path = local_config
         else:
-            raise exc.TripleOHashMissingConfig(
+            raise TripleOHashMissingConfig(
                 "Configuration file not found at {} or {}".format(
-                    const.CONFIG_PATH, local_config
+                    CONFIG_PATH, local_config
                 )
             )
         logging.info("Using config file at {}".format(config_path))
         with open(config_path, 'r') as config_yaml:
             loaded_config = yaml.safe_load(config_yaml)
-        for k in const.CONFIG_KEYS:
+        for k in CONFIG_KEYS:
             if k not in loaded_config:
                 error_str = (
                     "Malformed config file - missing {}. Expected all"
                     "of these configuration items: {}"
                 ).format(
-                    k, ", ".join(const.CONFIG_KEYS)
+                    k, ", ".join(CONFIG_KEYS)
                 )
                 logging.error(error_str)
-                raise exc.TripleOHashInvalidConfig(error_str)
+                raise TripleOHashInvalidConfig(error_str)
             # if the passed config contains the key then use that value
             if passed_config.get(k):
                 result_config[k] = passed_config[k]
@@ -150,7 +154,6 @@ class TripleOHashInfo:
         :param tag: The Delorean server named tag e.g. current-tripleo
         :param config: Use an existing config dictionary and don't load it
         """
-
         config = TripleOHashInfo.load_config(config)
 
         self.os_version = os_version

plugins/module_utils/tripleo_repos/main.py

@@ -14,7 +14,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function)
 import argparse
 import os
 import platform
@@ -22,9 +22,8 @@ import re
 import subprocess
 import sys
 
-import requests
-
 
+__metaclass__ = type
 TITLE_RE = re.compile('\\[(.*)\\]')
 NAME_RE = re.compile('name=(.+)')
 PRIORITY_RE = re.compile('priority=\\d+')
@@ -222,6 +221,11 @@ def _parse_args(distro_id, distro_major_version_id):
 
 
 def _get_repo(path, args):
+
+    # lazy import
+    if 'requests' not in globals():
+        import requests
+
     r = requests.get(path)
     if r.status_code == 200:
         return _inject_mirrors(r.text, args)

plugins/module_utils/tripleo_repos/yum_config/constants.py

@@ -13,10 +13,12 @@
 #   under the License.
 #
 #
-
+from __future__ import (absolute_import, division, print_function)
 """
 List of options that can be updated for yum repo files.
 """
+
+__metaclass__ = type
 YUM_REPO_SUPPORTED_OPTIONS = [
     'name',
     'baseurl',

plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py

@@ -11,10 +11,13 @@
 #  WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #  License for the specific language governing permissions and limitations
 #  under the License.
-
+from __future__ import (absolute_import, division, print_function)
 import logging
 
 
+__metaclass__ = type
+
+
 class DnfModuleManager:
     """Class that manages dnf modules."""
 

plugins/module_utils/tripleo_repos/yum_config/exceptions.py

@@ -13,6 +13,11 @@
 #   under the License.
 #
 #
+from __future__ import (absolute_import, division, print_function)
+
+
+__metaclass__ = type
+
 
 class Base(Exception):
     """Base Exception class."""

plugins/module_utils/tripleo_repos/yum_config/yum_config.py

@@ -12,14 +12,30 @@
 #  License for the specific language governing permissions and limitations
 #  under the License.
 #
+from __future__ import (absolute_import, division, print_function)
+
 
 import configparser
 import logging
 import os
 import sys
 
-import tripleo_repos.yum_config.constants as const
+from .constants import (
-import tripleo_repos.yum_config.exceptions as exc
+    YUM_GLOBAL_CONFIG_FILE_PATH,
+    YUM_REPO_DIR,
+    YUM_REPO_FILE_EXTENSION,
+    YUM_REPO_SUPPORTED_OPTIONS,
+)
+from .exceptions import (
+    TripleOYumConfigFileParseError,
+    TripleOYumConfigInvalidOption,
+    TripleOYumConfigInvalidSection,
+    TripleOYumConfigNotFound,
+    TripleOYumConfigPermissionDenied,
+)
+
+
+__metaclass__ = type
 
 
 class TripleOYumConfig:
@@ -81,23 +97,23 @@ class TripleOYumConfig:
         if not (file_path or dir_path):
             msg = ('A configuration file path or a directory path must be '
                    'provided.')
-            raise exc.TripleOYumConfigNotFound(error_msg=msg)
+            raise TripleOYumConfigNotFound(error_msg=msg)
 
         if file_path:
             if not os.path.isfile(file_path):
                 msg = ('The configuration file "{}" was not found in the '
                        'provided path.').format(file_path)
-                raise exc.TripleOYumConfigNotFound(error_msg=msg)
+                raise TripleOYumConfigNotFound(error_msg=msg)
             if not os.access(file_path, os.W_OK):
                 msg = ('The configuration file {} is not '
                        'writable.'.format(file_path))
-                raise exc.TripleOYumConfigPermissionDenied(error_msg=msg)
+                raise TripleOYumConfigPermissionDenied(error_msg=msg)
 
         if dir_path:
             if not os.path.isdir(dir_path):
                 msg = ('The configuration dir "{}" was not found in the '
                        'provided path.').format(dir_path)
-                raise exc.TripleOYumConfigNotFound(error_msg=msg)
+                raise TripleOYumConfigNotFound(error_msg=msg)
 
     def _read_config_file(self, section):
         """Read the configuration file associate with this object.
@@ -118,13 +134,13 @@ class TripleOYumConfig:
             except configparser.Error:
                 msg = 'Unable to parse configuration file {}.'.format(
                     self.config_file_path)
-                raise exc.TripleOYumConfigFileParseError(error_msg=msg)
+                raise TripleOYumConfigFileParseError(error_msg=msg)
 
             if section not in config.sections():
                 msg = ('The provided section "{}" was not found in the '
                        'configuration file {}.').format(
                     section, self.config_file_path)
-                raise exc.TripleOYumConfigInvalidSection(error_msg=msg)
+                raise TripleOYumConfigInvalidSection(error_msg=msg)
 
             return config, self.config_file_path
 
@@ -168,13 +184,13 @@ class TripleOYumConfig:
         if self.valid_options:
             if not all(key in self.valid_options for key in set_dict.keys()):
                 msg = 'One or more provided options are not valid.'
-                raise exc.TripleOYumConfigInvalidOption(error_msg=msg)
+                raise TripleOYumConfigInvalidOption(error_msg=msg)
 
         config, config_file_path = self._read_config_file(section)
         if not (config and config_file_path):
             msg = ('The provided section "{}" was not found within any '
                    'configuration file.').format(section)
-            raise exc.TripleOYumConfigNotFound(error_msg=msg)
+            raise TripleOYumConfigNotFound(error_msg=msg)
 
         # Update configuration file with dict updates
         config[section].update(set_dict)
@@ -192,13 +208,13 @@ class TripleOYumRepoConfig(TripleOYumConfig):
         if file_path:
             logging.info(
                 "Using '{}' as yum repo configuration file.".format(file_path))
-        conf_dir_path = dir_path or const.YUM_REPO_DIR
+        conf_dir_path = dir_path or YUM_REPO_DIR
 
         super(TripleOYumRepoConfig, self).__init__(
-            valid_options=const.YUM_REPO_SUPPORTED_OPTIONS,
+            valid_options=YUM_REPO_SUPPORTED_OPTIONS,
             file_path=file_path,
             dir_path=conf_dir_path,
-            file_extension=const.YUM_REPO_FILE_EXTENSION)
+            file_extension=YUM_REPO_FILE_EXTENSION)
 
     def update_section(self, section, set_dict, enable=None):
         if enable is not None:
@@ -211,7 +227,7 @@ class TripleOYumGlobalConfig(TripleOYumConfig):
     """Manages yum global configuration file."""
 
     def __init__(self, file_path=None):
-        conf_file_path = file_path or const.YUM_GLOBAL_CONFIG_FILE_PATH
+        conf_file_path = file_path or YUM_GLOBAL_CONFIG_FILE_PATH
         logging.info("Using '{}' as yum global configuration "
                      "file.".format(conf_file_path))
         if file_path is None:

plugins/modules/get_hash.py

@@ -1,31 +1,22 @@
 #!/usr/bin/python
-#   Copyright 2021 Red Hat, Inc.
+# Copyright 2021 Red Hat, Inc.
-#
+# GNU General Public License v3.0+ (see COPYING or
-#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+# https://www.gnu.org/licenses/gpl-3.0.txt)
-#   not use this file except in compliance with the License. You may obtain
+from __future__ import (absolute_import, division, print_function)
-#   a copy of the License at
+
-#
+
-#        http://www.apache.org/licenses/LICENSE-2.0
+__metaclass__ = type
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#   License for the specific language governing permissions and limitations
-#   under the License.
-#
 
-from tripleo_repos.get_hash.tripleo_hash_info import TripleOHashInfo
-from ansible.module_utils.basic import AnsibleModule
 
 DOCUMENTATION = r'''
 ---
-module: tripleo_get_hash
+module: get_hash
 
 short_description: Resolve rdo named tag to commit, full and distro hashes
 
-version_added: "2.9"
+version_added: "1.0.0"
 
-description:
+description: ""
 
 options:
     os_version:
@@ -42,7 +33,6 @@ options:
         description: The tripleo-ci component you are interested in
         required: false
         type: str
-        default: None
     tag:
         description: The named tag to fetch
         required: false
@@ -95,6 +85,8 @@ dlrn_url:
     sample: 'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5'  # noqa E501
 '''
 
+from ansible.module_utils.basic import AnsibleModule  # noqa: E402
+
 
 def run_module():
     result = dict(
@@ -120,6 +112,13 @@ def run_module():
 
     try:
 
+        try:
+            from ansible_collections.tripleo.repos.plugins.module_utils.\
+                tripleo_repo.get_hash.tripleo_hash_info import TripleOHashInfo
+        except ImportError:
+            from tripleo_repos.get_hash.tripleo_hash_info import \
+                TripleOHashInfo
+
         os_version = module.params.get('os_version')
         release = module.params.get('release')
         component = module.params.get('component')

plugins/modules/yum_config.py

@@ -1,30 +1,18 @@
 #!/usr/bin/python
-#   Copyright 2021 Red Hat, Inc.
+# Copyright 2021 Red Hat, Inc.
-#
+# GNU General Public License v3.0+ (see COPYING or
-#   Licensed under the Apache License, Version 2.0 (the "License"); you may
+# https://www.gnu.org/licenses/gpl-3.0.txt)
-#   not use this file except in compliance with the License. You may obtain
+from __future__ import (absolute_import, division, print_function)
-#   a copy of the License at
-#
-#        http://www.apache.org/licenses/LICENSE-2.0
-#
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#   WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#   License for the specific language governing permissions and limitations
-#   under the License.
 
-from ansible.module_utils.basic import AnsibleModule
-
-import tripleo_repos.yum_config.dnf_manager as dnf_mgr
-import tripleo_repos.yum_config.yum_config as cfg
 
+__metaclass__ = type
 DOCUMENTATION = r'''
 ---
-module: tripleo_yum_config
+module: yum_config
 
 short_description: Update yum configuration files for TripleO deployments.
 
-version_added: "2.9"
+version_added: "1.0.0"
 
 description:
     - Update specific options for different yum configuration files like
@@ -134,6 +122,8 @@ EXAMPLES = r'''
       keepcache: "0"
 '''
 
+from ansible.module_utils.basic import AnsibleModule  # noqa: E402
+
 
 def run_module():
     # define available arguments/parameters a user can pass to the module
@@ -198,6 +188,16 @@ def run_module():
 
     # Module execution
     try:
+
+        try:
+            import ansible_collections.tripleo.repos.plugins.module_utils.\
+                tripleo_repos.yum_config.dnf_manager as dnf_mgr
+            import ansible_collections.tripleo.repos.plugins.module_utils.\
+                tripleo_repos.yum_config.yum_config as cfg
+        except ImportError:
+            import tripleo_repos.yum_config.dnf_manager as dnf_mgr
+            import tripleo_repos.yum_config.yum_config as cfg
+
         if m_type == 'repo':
             config_obj = cfg.TripleOYumRepoConfig(
                 file_path=m_file_path,

tests/sanity/ignore-2.10.txt

@@ -0,0 +1 @@
+ignore-2.9.txt

tests/sanity/ignore-2.11.txt

@@ -0,0 +1 @@
+ignore-2.9.txt

tests/sanity/ignore-2.9.txt

@@ -0,0 +1,13 @@
+plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py import
+plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-bad-function
+plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:ansible-format-automatic-specification
+plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py pylint:logging-format-interpolation
+plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py replace-urlopen  # we did it right
+plugins/module_utils/tripleo_repos/main.py import:traceback
+plugins/module_utils/tripleo_repos/main.py pylint:ansible-bad-function
+plugins/module_utils/tripleo_repos/main.py pylint:ansible-format-automatic-specification
+plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py pylint:ansible-format-automatic-specification
+plugins/module_utils/tripleo_repos/yum_config/dnf_manager.py pylint:logging-format-interpolation
+plugins/module_utils/tripleo_repos/yum_config/yum_config.py pylint:ansible-format-automatic-specification
+plugins/module_utils/tripleo_repos/yum_config/yum_config.py pylint:logging-format-interpolation
+plugins/modules/yum_config.py pylint:ansible-format-automatic-specification

tests/sanity/requirements.txt

@@ -0,0 +1,2 @@
+PyYAML
+requests

tox.ini

@@ -1,7 +1,9 @@
 [tox]
 minversion = 3.1.1
 skipsdist = True
-envlist = py,pep8,packaging
+envlist = py,pep8,packaging,sanity
+requires =
+  tox-ansible>=1.5.3
 
 [testenv]
 usedevelop = True

zuul.d/layout.yaml

@@ -1,10 +1,22 @@
+- job:
+    name: tox-sanity-py36
+    description:  Run ansible-test sanity tests on a collection
+    parent: openstack-tox-py36
+    vars:
+      tox_envlist: sanity
+- job:
+    name: tox-sanity-py39
+    description:  Run ansible-test sanity tests on a collection
+    parent: openstack-tox-py39
+    vars:
+      tox_envlist: sanity
 - project:
     templates:
       - check-requirements
       - openstack-cover-jobs
       - openstack-python3-wallaby-jobs
     check:
-      jobs: &cjobs
+      jobs:
         - openstack-tox-pep8:
             vars:
               tox_envlist: pep8,packaging
@@ -17,8 +29,20 @@
               - openstack-tox-py39
         - tripleo-build-containers-ubi-8:
             dependencies: *deps_unit_lint_cprovider
+        - tox-sanity-py36:
+            voting: false
+        - tox-sanity-py39:
+            voting: false
     gate:
-      jobs: *cjobs
+      jobs:
+        - openstack-tox-pep8:
+            vars:
+              tox_envlist: pep8,packaging
+        - openstack-tox-py39
+        - tripleo-buildimage-overcloud-full-centos-8:
+            dependencies: *deps_unit_lint_cprovider
+        - tripleo-build-containers-ubi-8:
+            dependencies: *deps_unit_lint_cprovider
     post:
       jobs:
         - publish-openstack-python-branch-tarball