Fix tripleo.repos.get_hash module

- fixes import typo that prevented module_utils from importing - avoids reading config.yaml as ansiballz does not include data files - ensure module works (tox -e packaging) - replace use of requests library inside ansible module Change-Id: I86495c7da8584d5811f32178bbd985cf5f864323
2021-08-11 11:05:01 +01:00 · 2021-08-11 11:05:01 +01:00 · c31394e58c
parent 222e81c5ff
commit c31394e58c
9 changed files with 155 additions and 137 deletions
--- a/galaxy.yml
+++ b/galaxy.yml
@ -1,6 +1,6 @@
 name: repos
 namespace: tripleo
-version: 0.0.2
+version: 0.0.5
 readme: README.md
 authors:
  - Red Hat
--- a/plugins/module_utils/tripleo_repos/get_hash/constants.py
+++ b/plugins/module_utils/tripleo_repos/get_hash/constants.py
@ -36,3 +36,50 @@ This is the path that we expect to find the system installed config.yaml.
 The path is specified in [options.data_files] of the project setup.cfg.
 """
 CONFIG_PATH = '/usr/local/etc/tripleo_get_hash/config.yaml'
+
+DEFAULT_CONFIG = {
+    "tripleo_releases": [
+        "master",
+        "wallaby",
+        "victoria",
+        "ussuri",
+        "train",
+        "stein",
+        "queens",
+        "osp16-2",
+        "osp17"
+    ],
+    "dlrn_url": "https://trunk.rdoproject.org",
+    "rdo_named_tags": [
+        "current",
+        "consistent",
+        "component-ci-testing",
+        "promoted-components",
+        "tripleo-ci-testing",
+        "current-tripleo",
+        "current-tripleo-rdo"
+    ],
+    "tripleo_ci_components": [
+        "baremetal",
+        "cinder",
+        "clients",
+        "cloudops",
+        "common",
+        "compute",
+        "glance",
+        "manila",
+        "network",
+        "octavia",
+        "security",
+        "swift",
+        "tempest",
+        "tripleo",
+        "ui",
+        "validation"
+    ],
+    "os_versions": [
+        "centos7",
+        "centos8",
+        "rhel8"
+    ]
+}
--- a/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py
+++ b/plugins/module_utils/tripleo_repos/get_hash/tripleo_hash_info.py
@ -19,10 +19,28 @@ import logging
 import sys
 import os
 import yaml
-import requests
-from .constants import CONFIG_PATH, CONFIG_KEYS
-from .exceptions import TripleOHashMissingConfig, TripleOHashInvalidConfig
+from .constants import CONFIG_PATH, CONFIG_KEYS, DEFAULT_CONFIG
+from .exceptions import (
+    TripleOHashInvalidConfig)
+from typing import Tuple
+# portable http_get that uses either ansible recommended way or python native
+# urllib.
+try:
+    from ansible.module_utils.urls import open_url

+    def http_get(url: str) -> Tuple[str, int]:
+        response = open_url(url, method='GET')
+        return (response.read(), response.status)
+except ImportError:
+    import gzip
+    from urllib.request import urlopen
+
+    def http_get(url: str) -> Tuple[str, int]:
+        # https://stackoverflow.com/questions/35122232/urllib-request-urlopen-return-bytes-but-i-cannot-decode-it
+        response = urlopen(url)
+        return (
+            gzip.decompress(response.read()).decode('utf-8'),
+            int(response.status))

 __metaclass__ = type

@ -120,11 +138,8 @@ class TripleOHashInfo:
        elif local_config:
            config_path = local_config
        else:
-            raise TripleOHashMissingConfig(
-                "Configuration file not found at {0} or {1}".format(
-                    CONFIG_PATH, local_config
-                )
-            )
+            logging.info("Using embedded config file")
+            return DEFAULT_CONFIG
        logging.info("Using config file at %s", config_path)
        with open(config_path, 'r') as config_yaml:
            loaded_config = yaml.safe_load(config_yaml)
@ -164,7 +179,7 @@ class TripleOHashInfo:
        repo_url = self._resolve_repo_url(config['dlrn_url'])
        self.dlrn_url = repo_url

-        repo_url_response = requests.get(repo_url).text
+        repo_url_response, status = http_get(repo_url)
        if repo_url.endswith('commit.yaml'):
            from_commit_yaml = self._hashes_from_commit_yaml(repo_url_response)
            self.full_hash = from_commit_yaml[0]
--- a/plugins/modules/get_hash.py
+++ b/plugins/modules/get_hash.py
@ -112,12 +112,8 @@ def run_module():

    try:

-        try:
-            from ansible_collections.tripleo.repos.plugins.module_utils.\
-                tripleo_repo.get_hash.tripleo_hash_info import TripleOHashInfo
-        except ImportError:
-            from tripleo_repos.get_hash.tripleo_hash_info import \
-                TripleOHashInfo
+        from ansible_collections.tripleo.repos.plugins.module_utils.\
+            tripleo_repos.get_hash.tripleo_hash_info import TripleOHashInfo

        os_version = module.params.get('os_version')
        release = module.params.get('release')
--- a/test-requirements.txt
+++ b/test-requirements.txt
@ -13,4 +13,3 @@ testscenarios>=0.4  # Apache-2.0/BSD
 testtools>=1.4.0 # MIT
 stestr>=2.0.0 # Apache-2.0
 fixtures>=3.0.0 # Apache-2.0/BSD
-requests_mock>=1.8.0 # Apache-2.0
--- a/tests/sanity/requirements.txt
+++ b/tests/sanity/requirements.txt
@ -1,2 +1 @@
 PyYAML
-requests
--- a/tests/unit/get_hash/test_tripleo_get_hash.py
+++ b/tests/unit/get_hash/test_tripleo_get_hash.py
@ -14,11 +14,10 @@
 #
 #

-import requests_mock
 import sys
 import unittest
 from unittest import mock
-from unittest.mock import mock_open
+from unittest.mock import mock_open, MagicMock, patch
 import yaml

 import tripleo_repos.get_hash.exceptions as exc
@ -37,11 +36,10 @@ class TestGetHash(unittest.TestCase):
    """

    def test_centos_8_current_tripleo_stable(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos8-victoria/current-tripleo/delorean.repo.md5',  # noqa
-                text=test_fakes.TEST_REPO_MD5,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_REPO_MD5, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            args = ['--os-version', 'centos8', '--release', 'victoria']
            sys.argv[1:] = args
            main_res = tgh.main()
@ -57,11 +55,10 @@ class TestGetHash(unittest.TestCase):
        args = ['--verbose']
        debug_msgs = []

-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5',  # noqa
-                text=test_fakes.TEST_REPO_MD5,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_REPO_MD5, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            with self.assertLogs() as captured:
                sys.argv[1:] = args
                tgh.main()
@ -74,11 +71,12 @@ class TestGetHash(unittest.TestCase):

    def test_verbose_logging_off(self, mock_config):
        debug_msgs = []
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5',  # noqa
-                text=test_fakes.TEST_REPO_MD5,
-            )
+
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_REPO_MD5, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
+
            args = ['--tag', 'current-tripleo', '--os-version', 'centos8']
            with self.assertLogs() as captured:
                sys.argv[1:] = args
@ -101,14 +99,13 @@ class TestGetHash(unittest.TestCase):
        config_file.close()
        # interate for each of config components
        for component in config_yaml['tripleo_ci_components']:
-            with requests_mock.Mocker() as req_mock:
-                req_mock.get(
-                    "https://trunk.rdoproject.org/centos8-master/component"
-                    "/{}/current-tripleo/commit.yaml".format(
-                        component
-                    ),
-                    text=test_fakes.TEST_COMMIT_YAML_COMPONENT,
-                )
+
+            mocked = MagicMock(
+                return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200))
+            with patch(
+                    'tripleo_repos.get_hash.tripleo_hash_info.http_get',
+                    mocked):
+
                args = ['--component', "{}".format(component)]
                sys.argv[1:] = args
                main_res = tgh.main()
@ -142,14 +139,12 @@ class TestGetHash(unittest.TestCase):
        config_file.close()
        # iterate for each of config named tags
        for tag in config_yaml['rdo_named_tags']:
-            with requests_mock.Mocker() as req_mock:
-                req_mock.get(
-                    "https://trunk.rdoproject.org/centos8-master"
-                    "/{}/delorean.repo.md5".format(
-                        tag
-                    ),
-                    text=test_fakes.TEST_REPO_MD5,
-                )
+            mocked = MagicMock(
+                return_value=(test_fakes.TEST_REPO_MD5, 200))
+            with patch(
+                    'tripleo_repos.get_hash.tripleo_hash_info.http_get',
+                    mocked):
+
                args = ['--tag', "{}".format(tag)]
                sys.argv[1:] = args
                main_res = tgh.main()
@ -163,12 +158,12 @@ class TestGetHash(unittest.TestCase):
                self.assertEqual(tag, main_res.tag)

    def test_override_dlrn_url(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                "https://awoo.com/awoo/centos8-master/current-tripleo"
-                "/delorean.repo.md5",
-                text=test_fakes.TEST_REPO_MD5,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_REPO_MD5, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get',
+                mocked):
+
            args = ['--dlrn-url', 'https://awoo.com/awoo']
            sys.argv[1:] = args
            main_res = tgh.main()
@ -179,11 +174,11 @@ class TestGetHash(unittest.TestCase):
            )

    def test_override_os_version_release_rhel8(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                "https://awoo.com/awoo/rhel8-osp16-2/current-tripleo"
-                "/delorean.repo.md5", text=test_fakes.TEST_REPO_MD5,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_REPO_MD5, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get',
+                mocked):
            args = [
                '--dlrn-url',
                'https://awoo.com/awoo',
--- a/tests/unit/get_hash/test_tripleo_get_hash_info.py
+++ b/tests/unit/get_hash/test_tripleo_get_hash_info.py
@ -18,9 +18,8 @@ import unittest
 import tripleo_repos.get_hash.tripleo_hash_info as thi
 import tripleo_repos.get_hash.exceptions as exc
 from . import fakes as test_fakes
-import requests_mock
 from unittest import mock
-from unittest.mock import mock_open
+from unittest.mock import mock_open, MagicMock, patch


@mock.patch(
@ -41,11 +40,10 @@ class TestGetHashInfo(unittest.TestCase):
            '1f5a41f31db8e3eb51caa9c0e201ab0583747be8',
            'None',
        )
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos8-master/component/common/current-tripleo/commit.yaml',  # noqa
-                text=test_fakes.TEST_COMMIT_YAML_COMPONENT,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            mock_hash_info = thi.TripleOHashInfo(
                'centos8', 'master', 'common', 'current-tripleo'
            )
@ -55,12 +53,10 @@ class TestGetHashInfo(unittest.TestCase):
            self.assertEqual(expected_result, actual_result)

    def test_resolve_repo_url_component_commit_yaml(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            # test component url
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos8-master/component/common/current-tripleo/commit.yaml',  # noqa
-                text=test_fakes.TEST_COMMIT_YAML_COMPONENT,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            c8_component_hash_info = thi.TripleOHashInfo(
                'centos8', 'master', 'common', 'current-tripleo'
            )
@ -71,13 +67,10 @@ class TestGetHashInfo(unittest.TestCase):
            )

    def test_resolve_repo_url_centos8_repo_md5(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            # test vanilla centos8 url
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5',  # noqa
-
-                text=test_fakes.TEST_REPO_MD5,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_REPO_MD5, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            c8_hash_info = thi.TripleOHashInfo(
                'centos8', 'master', None, 'current-tripleo'
            )
@ -88,13 +81,10 @@ class TestGetHashInfo(unittest.TestCase):
            )

    def test_resolve_repo_url_centos7_commit_yaml(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            # test centos7 url
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos7-master/current-tripleo/commit.yaml',  # noqa
-
-                text=test_fakes.TEST_COMMIT_YAML_CENTOS_7,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_COMMIT_YAML_CENTOS_7, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            c7_hash_info = thi.TripleOHashInfo(
                'centos7', 'master', None, 'current-tripleo'
            )
@ -105,12 +95,10 @@ class TestGetHashInfo(unittest.TestCase):
            )

    def test_get_tripleo_hash_info_centos8_md5(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos8-master/current-tripleo/delorean.repo.md5',  # noqa
-
-                text=test_fakes.TEST_REPO_MD5,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_REPO_MD5, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            created_hash_info = thi.TripleOHashInfo(
                'centos8', 'master', None, 'current-tripleo'
            )
@ -126,11 +114,10 @@ class TestGetHashInfo(unittest.TestCase):
        expected_commit_hash = '476a52df13202a44336c8b01419f8b73b93d93eb'
        expected_distro_hash = '1f5a41f31db8e3eb51caa9c0e201ab0583747be8'
        expected_full_hash = '476a52df13202a44336c8b01419f8b73b93d93eb_1f5a41f3'  # noqa
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos8-victoria/component/common/tripleo-ci-testing/commit.yaml',  # noqa
-                text=test_fakes.TEST_COMMIT_YAML_COMPONENT,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            created_hash_info = thi.TripleOHashInfo(
                'centos8', 'victoria', 'common', 'tripleo-ci-testing'
            )
@ -150,11 +137,10 @@ class TestGetHashInfo(unittest.TestCase):
        expected_commit_hash = 'b5ef03c9c939db551b03e9490edc6981ff582035'
        expected_distro_hash = '76ebc4655502820b7677579349fd500eeca292e6'
        expected_full_hash = 'b5ef03c9c939db551b03e9490edc6981ff582035_76ebc465'  # noqa
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos7-master/tripleo-ci-testing/commit.yaml',  # noqa
-                text=test_fakes.TEST_COMMIT_YAML_CENTOS_7,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_COMMIT_YAML_CENTOS_7, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            created_hash_info = thi.TripleOHashInfo(
                'centos7', 'master', None, 'tripleo-ci-testing'
            )
@ -169,11 +155,10 @@ class TestGetHashInfo(unittest.TestCase):
            self.assertEqual(created_hash_info.os_version, 'centos7')

    def test_bad_config_file(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos7-master/tripleo-ci-testing/commit.yaml',  # noqa
-                text=test_fakes.TEST_COMMIT_YAML_CENTOS_7,
-            )
+        mocked = MagicMock(
+            return_value=test_fakes.TEST_COMMIT_YAML_CENTOS_7)
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            with mock.patch(
                'builtins.open',
                new_callable=mock_open,
@ -188,30 +173,12 @@ class TestGetHashInfo(unittest.TestCase):
                    'tripleo-ci-testing',
                )

-    def test_missing_config_file(self, mock_config):
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                'https://trunk.rdoproject.org/centos7-master/tripleo-ci-testing/commit.yaml',  # noqa
-                text=test_fakes.TEST_COMMIT_YAML_CENTOS_7,
-            )
-            with mock.patch('os.path.isfile') as mock_is_file:
-                mock_is_file.return_value = False
-                self.assertRaises(
-                    exc.TripleOHashMissingConfig,
-                    thi.TripleOHashInfo,
-                    'centos7',
-                    'master',
-                    None,
-                    'tripleo-ci-testing',
-                )
-
    def test_override_config_dlrn_url(self, mock_config):
        expected_dlrn_url = 'https://foo.bar.baz/centos8-master/component/common/current-tripleo/commit.yaml'  # noqa
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                expected_dlrn_url,
-                text=test_fakes.TEST_COMMIT_YAML_COMPONENT,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            mock_hash_info = thi.TripleOHashInfo(
                'centos8', 'master', 'common', 'current-tripleo',
                {'dlrn_url': 'https://foo.bar.baz'}
@ -220,11 +187,10 @@ class TestGetHashInfo(unittest.TestCase):

    def test_override_config_dlrn_url_empty_ignored(self, mock_config):
        expected_dlrn_url = 'https://trunk.rdoproject.org/centos8-master/component/common/current-tripleo/commit.yaml'  # noqa
-        with requests_mock.Mocker() as req_mock:
-            req_mock.get(
-                expected_dlrn_url,
-                text=test_fakes.TEST_COMMIT_YAML_COMPONENT,
-            )
+        mocked = MagicMock(
+            return_value=(test_fakes.TEST_COMMIT_YAML_COMPONENT, 200))
+        with patch(
+                'tripleo_repos.get_hash.tripleo_hash_info.http_get', mocked):
            mock_hash_info = thi.TripleOHashInfo(
                'centos8', 'master', 'common', 'current-tripleo',
                {'dlrn_url': ''}
--- a/tox.ini
+++ b/tox.ini
@ -76,6 +76,7 @@ commands =
  # Ensure that ansible is able to load the modules, as syntax check will fail
  # if modules cannot be loaded.
  sh -c "ansible-playbook --syntax-check  playbooks/*.yaml"
+  ansible localhost -m tripleo.repos.get_hash -a "release=master os_version=centos8"

 whitelist_externals =
  sh