From 00c22491d0cbcfec47cc0199aeb2e8bf836ec7bb Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Wed, 24 Apr 2019 14:13:53 +0100 Subject: [PATCH] Remove py_pkgs and packages_file lookups Neither of these lookups are used in OSA any more, so we can remove them. Change-Id: Ib1c19eaf5a16d59e91d5c6c5ced9b30781b1f69b --- doc/source/index.rst | 1 - doc/source/lookups.rst | 167 ---- examples/example.ini | 1 - lookup/packages_file.py | 79 -- lookup/py_pkgs.py | 736 ------------------ .../remove-py-pkgs-db29425b9e1855a6.yaml | 6 + tests/test-lookups.yml | 40 - tests/test.yml | 2 - 8 files changed, 6 insertions(+), 1026 deletions(-) delete mode 100644 doc/source/lookups.rst delete mode 100644 lookup/packages_file.py delete mode 100644 lookup/py_pkgs.py create mode 100644 releasenotes/notes/remove-py-pkgs-db29425b9e1855a6.yaml delete mode 100644 tests/test-lookups.yml diff --git a/doc/source/index.rst b/doc/source/index.rst index c341a2d0..7139a042 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -7,7 +7,6 @@ OpenStack-Ansible plugins actions.rst filters.rst - lookups.rst Example ansible.cfg file ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/lookups.rst b/doc/source/lookups.rst deleted file mode 100644 index 5f1e04a7..00000000 --- a/doc/source/lookups.rst +++ /dev/null @@ -1,167 +0,0 @@ -======= -Lookups -======= - -py_pkgs -~~~~~~~ -The ``py_pkgs`` lookup crawls a given list of directories to parse variables -and generate lists of Python packages, git repo information and Ansible group -memberships which is used within OpenStack-Ansible's repo_build role to build -wheels and virtual environments. - -Files and paths containing the following strings are evaluated: - - test-requirements.txt - - dev-requirements.txt - - requirements.txt - - global-requirements.txt - - global-requirement-pins.txt - - /defaults/ - - /vars/ - - /user_* - -Variables parsed within any evaluated files include: - - service_pip_dependencies - - pip_common_packages - - pip_container_packages - - pip_packages - -Example -------- - -.. code-block:: yaml - - - name: Load local packages - debug: - msg: "Loading Packages" - with_py_pkgs: "{{ pkg_locations }}" - register: local_packages - vars: - pkg_locations: - - "/etc/ansible/roles/os_nova" - # => { - # "packages": [ - # "httplib2", - # "keystonemiddleware", - # "libvirt-python", - # "nova", - # "nova-lxd", - # "nova-powervm", - # "pyasn1-modules", - # "pycrypto", - # "pylxd", - # "pymysql", - # "python-ironicclient", - # "python-keystoneclient", - # "python-memcached", - # "python-novaclient", - # "virtualenv", - # "websockify" - # ], - # "remote_package_parts": [ - # { - # "egg_name": "nova", - # "fragment": null, - # "name": "nova", - # "original": - # "git+https://git.openstack.org/openstack/nova@stable/newton#egg=nova&gitname=nova&projectgroup=all", - # "project_group": "all", - # "url": "https://git.openstack.org/openstack/nova", - # "version": "stable/newton" - # }, - # { - # "egg_name": "nova_lxd", - # "fragment": null, - # "name": "nova-lxd", - # "original": - # "git+https://git.openstack.org/openstack/nova-lxd@stable/newton#egg=nova_lxd&gitname=nova-lxd&projectgroup=all", - # "project_group": "all", - # "url": "https://git.openstack.org/openstack/nova-lxd", - # "version": "stable/newton" - # }, - # { - # "egg_name": "novnc", - # "fragment": null, - # "name": "novnc", - # "original": - # "git+https://github.com/kanaka/novnc@master#egg=novnc&gitname=novnc&projectgroup=all", - # "project_group": "all", - # "url": "https://github.com/kanaka/novnc", - # "version": "master" - # }, - # { - # "egg_name": "spice_html5", - # "fragment": null, - # "name": "spice-html5", - # "original": - # "git+https://gitlab.freedesktop.org/spice/spice-html5.git@master#egg=spice_html5&gitname=spice-html5&projectgroup=all", - # "project_group": "all", - # "url": "https://gitlab.freedesktop.org/spice/spice-html5.git", - # "version": "master" - # } - # ], - # "remote_packages": [ - # "git+https://git.openstack.org/openstack/nova-lxd@stable/newton#egg=nova_lxd&gitname=nova-lxd&projectgroup=all", - # "git+https://git.openstack.org/openstack/nova@stable/newton#egg=nova&gitname=nova&projectgroup=all", - # "git+https://gitlab.freedesktop.org/spice/spice-html5.git@master#egg=spice_html5&gitname=spice-html5&projectgroup=all", - # "git+https://github.com/kanaka/novnc@master#egg=novnc&gitname=novnc&projectgroup=all" - # ], - # "role_packages": { - # "os_nova": [ - # "httplib2", - # "keystonemiddleware", - # "libvirt-python", - # "nova", - # "nova-lxd", - # "nova-powervm", - # "pyasn1-modules", - # "pycrypto", - # "pylxd", - # "pymysql", - # "python-ironicclient", - # "python-keystoneclient", - # "python-memcached", - # "python-novaclient", - # "virtualenv", - # "websockify" - # ] - # }, - # "role_project_groups": { - # "os_nova": "nova_all" - # }, - # "role_requirement_files": {}, - # "role_requirements": { - # "os_nova": { - # "nova_compute_ironic_pip_packages": [ - # "python-ironicclient" - # ], - # "nova_compute_lxd_pip_packages": [ - # "nova-lxd", - # "pylxd" - # ], - # "nova_compute_pip_packages": [ - # "libvirt-python" - # ], - # "nova_compute_powervm_pip_packages": [ - # "nova-powervm", - # "pyasn1-modules" - # ], - # "nova_novnc_pip_packages": [ - # "websockify" - # ], - # "nova_pip_packages": [ - # "keystonemiddleware", - # "nova", - # "pycrypto", - # "pymysql", - # "python-keystoneclient", - # "python-memcached", - # "python-novaclient" - # ], - # "nova_requires_pip_packages": [ - # "httplib2", - # "python-keystoneclient", - # "virtualenv", - # ], - # "project_group": "nova_all" - # } - # } diff --git a/examples/example.ini b/examples/example.ini index 37115855..888ff420 100644 --- a/examples/example.ini +++ b/examples/example.ini @@ -1,5 +1,4 @@ [defaults] -lookup_plugins = /etc/ansible/plugins/lookups filter_plugins = /etc/ansible/plugins/filters action_plugins = /etc/ansible/plugins/actions library = /etc/ansible/plugins/library diff --git a/lookup/packages_file.py b/lookup/packages_file.py deleted file mode 100644 index 85345fd9..00000000 --- a/lookup/packages_file.py +++ /dev/null @@ -1,79 +0,0 @@ -# Copyright 2017, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2017, Jean-Philippe Evrard - -# Take a path to a debian mirror release file, and outputs a dict with -# package names, each of the packages holding the following: -# - package version -# - checksums -# - Relative location to pool folder -# -# example: -# get_url: -# url: http://rpc-repo.rackspace.com/apt-mirror/integrated/dists/r14.0.0rc1-trusty/main/binary-amd64/Packages -# dest: /tmp/trusty-amd64-Packages -# debug: -# var: item -# with_packages_file: -# - /tmp/trusty-amd64-Packages - -import os - -from ansible.plugins.lookup import LookupBase -from ansible.errors import AnsibleLookupError - -IMPORTANT_FIELDS = ['Version', 'Filename', 'MD5sum', 'SHA1', 'SHA256'] - -try: - from __main__ import display -except ImportError: - from ansible.utils.display import Display - display = Display() - -def parse_fields(line): - for field in IMPORTANT_FIELDS: - if line.startswith(field + ":"): - return (field, line.split(":")[1].strip()) - - -class LookupModule(LookupBase): - def run(self, terms, variables=None, **kwargs): - ret = [] - for term in terms: - pkg_details = {} - with open(term, 'r') as f: - for line in f: - #non empty line means pkg data - if line.strip(): - if line.startswith('Package:'): - currentpkg = line.split(":")[1].strip() - pkg_details[currentpkg] = {} - elif line.startswith('Provides:'): - pkg_details[line.split(":")[1].strip()] = pkg_details[currentpkg] - else: - # Now doing package data - parsed = parse_fields(line) - if parsed: - pkg_details[currentpkg][parsed[0]] = parsed[1] - else: - currentpkg="" - ret.append(pkg_details) - return ret - -# For debug purposes -if __name__ == '__main__': - import sys - import json - print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True)) diff --git a/lookup/py_pkgs.py b/lookup/py_pkgs.py deleted file mode 100644 index e1cd0d93..00000000 --- a/lookup/py_pkgs.py +++ /dev/null @@ -1,736 +0,0 @@ -# Copyright 2014, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2014, Kevin Carter - -import os -import re -import traceback -import yaml - -from ansible.errors import AnsibleError -from ansible.plugins.lookup import LookupBase - - -try: - basestring -except NameError: - basestring = str - - -# Used to keep track of git package parts as various files are processed -GIT_PACKAGE_DEFAULT_PARTS = dict() - - -# Role based package indexes -ROLE_BREAKOUT_REQUIREMENTS = dict() -ROLE_PACKAGES = dict() -ROLE_REQUIREMENTS = dict() - - -REQUIREMENTS_FILE_TYPES = [ - 'requirements.txt', - 'global-requirement-pins.txt' -] - - -# List of variable names that could be used within the yaml files that -# represent lists of python packages. -BUILT_IN_PIP_PACKAGE_VARS = [ - 'service_pip_dependencies', - 'pip_common_packages', - 'pip_container_packages', - 'pip_packages' -] - - -PACKAGE_MAPPING = { - 'packages': set(), - 'remote_packages': set(), - 'remote_package_parts': list(), - 'role_packages': dict(), - 'role_project_groups': dict(), -} - - -def map_base_and_remote_packages(package, package_map): - """Determine whether a package is a base package or a remote package - and add to the appropriate set. - - :type package: ``str`` - :type package_map: ``dict`` - """ - def check_for_ignore(p): - p_parts = GIT_PACKAGE_DEFAULT_PARTS.get(p) - if p_parts: - fragments = p_parts.get('fragments', '') or '' - if 'ignorerequirements=True' not in fragments: - package_map['packages'].add(p) - else: - package_map['packages'].add(p) - - if package.startswith(('http:', 'https:', 'git+')): - if '@' not in package: - check_for_ignore(p=package) - else: - git_parts = git_pip_link_parse(package) - package_name = git_parts[-2] - if not package_name: - package_name = git_pip_link_parse(package)[0] - - for rpkg in list(package_map['remote_packages']): - rpkg_name = git_pip_link_parse(rpkg)[-2] - if not rpkg_name: - rpkg_name = git_pip_link_parse(package)[0] - - if rpkg_name == package_name: - package_map['remote_packages'].remove(rpkg) - package_map['remote_packages'].add(package) - break - else: - package_map['remote_packages'].add(package) - else: - check_for_ignore(p=package) - - -def parse_remote_package_parts(package_map): - """Parse parts of each remote package and add them to - the remote_package_parts list. - - :type package_map: ``dict`` - """ - keys = [ - 'name', - 'version', - 'fragment', - 'url', - 'original', - 'egg_name', - 'project_group' - ] - remote_pkg_parts = [ - dict( - zip( - keys, git_pip_link_parse(i) - ) - ) for i in package_map['remote_packages'] - ] - package_map['remote_package_parts'].extend(remote_pkg_parts) - package_map['remote_package_parts'] = list( - dict( - (i['name'], i) - for i in package_map['remote_package_parts'] - ).values() - ) - - -def map_role_packages(package_map): - """Add and sort packages belonging to a role to the role_packages dict. - - :type package_map: ``dict`` - """ - for k, v in ROLE_PACKAGES.items(): - role_pkgs = package_map['role_packages'][k] = list() - package_map['role_project_groups'][k] = v.pop('project_group', 'all') - for pkg_list in v.values(): - role_pkgs.extend(pkg_list) - else: - package_map['role_packages'][k] = sorted(set(role_pkgs)) - - -def map_base_package_details(package_map): - """Parse package version and marker requirements and add to the - base packages set. - - :type package_map: ``dict`` - """ - check_pkgs = dict() - base_packages = sorted(list(package_map['packages'])) - for pkg in base_packages: - name, versions, markers = _pip_requirement_split(pkg) - if versions and markers: - versions = '%s;%s' % (versions, markers) - elif not versions and markers: - versions = ';%s' % markers - - if name in check_pkgs: - if versions and not check_pkgs[name]: - check_pkgs[name] = versions - else: - check_pkgs[name] = versions - else: - return_pkgs = list() - for k, v in check_pkgs.items(): - if v: - return_pkgs.append('%s%s' % (k, v)) - else: - return_pkgs.append(k) - package_map['packages'] = set(return_pkgs) - - -def git_pip_link_parse(repo): - """Return a tuple containing the parts of a git repository. - - Example parsing a standard git repo: - >>> git_pip_link_parse('git+https://github.com/username/repo-name@tag') - ('repo-name', - 'tag', - None, - 'https://github.com/username/repo', - 'git+https://github.com/username/repo@tag', - 'repo_name') - - Example parsing a git repo that uses an installable from a subdirectory: - >>> git_pip_link_parse( - ... 'git+https://github.com/username/repo@tag#egg=plugin.name' - ... '&subdirectory=remote_path/plugin.name' - ... ) - ('plugin.name', - 'tag', - 'remote_path/plugin.name', - 'https://github.com/username/repo', - 'git+https://github.com/username/repo@tag#egg=plugin.name&' - 'subdirectory=remote_path/plugin.name', - 'plugin.name') - - :param repo: git repo string to parse. - :type repo: ``str`` - :returns: ``tuple`` - """'meta' - - def _meta_return(meta_data, item): - """Return the value of an item in meta data.""" - - return meta_data.lstrip('#').split('%s=' % item)[-1].split('&')[0] - - _git_url = repo.split('+') - if len(_git_url) >= 2: - _git_url = _git_url[1] - else: - _git_url = _git_url[0] - - git_branch_sha = _git_url.split('@') - if len(git_branch_sha) > 2: - branch = git_branch_sha.pop() - url = '@'.join(git_branch_sha) - elif len(git_branch_sha) > 1: - url, branch = git_branch_sha - else: - url = git_branch_sha[0] - branch = 'master' - - egg_name = name = os.path.basename(url.rstrip('/')) - egg_name = egg_name.replace('-', '_') - - _branch = branch.split('#') - branch = _branch[0] - - plugin_path = None - # Determine if the package is a plugin type - if len(_branch) > 1: - if 'subdirectory=' in _branch[-1]: - plugin_path = _meta_return(_branch[-1], 'subdirectory') - name = os.path.basename(plugin_path) - - if 'egg=' in _branch[-1]: - egg_name = _meta_return(_branch[-1], 'egg') - egg_name = egg_name.replace('-', '_') - - if 'gitname=' in _branch[-1]: - name = _meta_return(_branch[-1], 'gitname') - - project_group = 'all' - if 'projectgroup=' in _branch[-1]: - project_group = _meta_return(_branch[-1], 'projectgroup') - - return name.lower(), branch, plugin_path, url, repo, egg_name, project_group - - -def _pip_requirement_split(requirement): - """Split pip versions from a given requirement. - - The method will return the package name, versions, and any markers. - - :type requirement: ``str`` - :returns: ``tuple`` - """ - version_descriptors = "(>=|<=|>|<|==|~=|!=)" - requirement = requirement.split(';') - requirement_info = re.split(r'%s\s*' % version_descriptors, requirement[0]) - name = requirement_info[0] - marker = None - if len(requirement) > 1: - marker = requirement[-1] - versions = None - if len(requirement_info) > 1: - versions = ''.join(requirement_info[1:]) - - return name, versions, marker - - -class DependencyFileProcessor(object): - def __init__(self, local_path): - """Find required files. - - :type local_path: ``str`` - :return: - """ - self.pip = dict() - self.pip['git_package'] = list() - self.pip['py_package'] = list() - self.pip['git_data'] = list() - self.git_pip_install = 'git+%s@%s' - self.file_names = self._get_files(path=local_path) - - # Process everything simply by calling the method - self._process_files() - - def _py_pkg_extend(self, packages, py_package=None): - if py_package is None: - py_package = self.pip['py_package'] - - for pkg in packages: - pkg_name = _pip_requirement_split(pkg)[0] - for py_pkg in py_package: - py_pkg_name = _pip_requirement_split(py_pkg)[0] - if pkg_name == py_pkg_name: - py_package.remove(py_pkg) - else: - norm_pkgs = [i.lower() for i in packages if not i.startswith('{{')] - py_package.extend(norm_pkgs) - return py_package - - @staticmethod - def _filter_files(file_names, ext): - """Filter the files and return a sorted list. - :type file_names: - :type ext: ``str`` or ``tuple`` - :returns: ``list`` - """ - _file_names = list() - file_name_words = ['/defaults/', '/vars/', '/user_'] - file_name_words.extend(REQUIREMENTS_FILE_TYPES) - for file_name in file_names: - if file_name.endswith(ext): - if any(i in file_name for i in file_name_words): - _file_names.append(file_name) - else: - return _file_names - - @staticmethod - def _get_files(path): - """Return a list of all files in the defaults/repo_packages directory. - - :type path: ``str`` - :returns: ``list`` - """ - paths = os.walk(os.path.abspath(path), followlinks=True) - files = list() - for fpath, _, afiles in paths: - for afile in afiles: - files.append(os.path.join(fpath, afile)) - else: - return files - - def _check_plugins(self, git_repo_plugins, git_data): - """Check if the git url is a plugin type. - - :type git_repo_plugins: ``dict`` - :type git_data: ``dict`` - """ - for repo_plugin in git_repo_plugins: - strip_plugin_path = repo_plugin['package'].lstrip('/') - plugin = '%s/%s' % ( - repo_plugin['path'].strip('/'), - strip_plugin_path - ) - - name = git_data['name'] = os.path.basename(strip_plugin_path) - git_data['egg_name'] = name.replace('-', '_') - package = self.git_pip_install % ( - git_data['repo'], git_data['branch'] - ) - package += '#egg=%s' % git_data['egg_name'] - package += '&subdirectory=%s' % plugin - package += '&gitname=%s' % name - if git_data['fragments']: - package += '&%s' % git_data['fragments'] - - self.pip['git_data'].append(git_data) - self.pip['git_package'].append(package) - - if name not in GIT_PACKAGE_DEFAULT_PARTS: - GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy() - else: - GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data.copy()) - - @staticmethod - def _check_defaults(git_data, name, item): - """Check if a default exists and use it if an item is undefined. - - :type git_data: ``dict`` - :type name: ``str`` - :type item: ``str`` - """ - if not git_data[item] and name in GIT_PACKAGE_DEFAULT_PARTS: - check_item = GIT_PACKAGE_DEFAULT_PARTS[name].get(item) - if check_item: - git_data[item] = check_item - - def _process_git(self, loaded_yaml, git_item, yaml_file_name): - """Process git repos. - - :type loaded_yaml: ``dict`` - :type git_item: ``str`` - """ - git_data = dict() - if git_item.split('_')[0] == 'git': - prefix = '' - else: - prefix = '%s_' % git_item.split('_git_repo')[0].replace('.', '_') - - # Set the various variable definitions - repo_var = prefix + 'git_repo' - name_var = prefix + 'git_package_name' - branch_var = prefix + 'git_install_branch' - fragment_var = prefix + 'git_install_fragments' - plugins_var = prefix + 'repo_plugins' - group_var = prefix + 'git_project_group' - - # get the repo definition - git_data['repo'] = loaded_yaml.get(repo_var) - group = git_data['project_group'] = loaded_yaml.get(group_var, 'all') - - # get the repo name definition - name = git_data['name'] = loaded_yaml.get(name_var) - if not name: - # NOTE: strip off trailing /, .git, or .git/ - name = git_data['name'] = os.path.basename( - re.sub(r'(\/$|\.git(\/)?$)', '', git_data['repo']) - ) - git_data['egg_name'] = name.replace('-', '_') - - # This conditional is set to ensure we're only processing git - # repos from the defaults file when those same repos are not - # being set in the repo_packages files. - if '/defaults/main' in yaml_file_name: - if name in GIT_PACKAGE_DEFAULT_PARTS: - return - - # get the repo branch definition - git_data['branch'] = loaded_yaml.get(branch_var) - self._check_defaults(git_data, name, 'branch') - if not git_data['branch']: - git_data['branch'] = 'master' - - package = self.git_pip_install % (git_data['repo'], git_data['branch']) - - # get the repo fragment definitions, if any - git_data['fragments'] = loaded_yaml.get(fragment_var) - self._check_defaults(git_data, name, 'fragments') - - package += '#egg=%s' % git_data['egg_name'] - package += '&gitname=%s' % name - package += '&projectgroup=%s' % group - if git_data['fragments']: - package += '&%s' % git_data['fragments'] - - self.pip['git_package'].append(package) - self.pip['git_data'].append(git_data.copy()) - - # Set the default package parts to track data during the run - if name not in GIT_PACKAGE_DEFAULT_PARTS: - GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy() - else: - GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data) - - # get the repo plugin definitions, if any - git_data['plugins'] = loaded_yaml.get(plugins_var) - self._check_defaults(git_data, name, 'plugins') - if git_data['plugins']: - self._check_plugins( - git_repo_plugins=git_data['plugins'], - git_data=git_data - ) - - def _package_build_index(self, packages, role_name, var_name, pkg_index, - project_group='all', var_file_name=None, - pip_packages=True): - if pip_packages: - self._py_pkg_extend(packages) - - if role_name: - if role_name in pkg_index: - role_pkgs = pkg_index[role_name] - else: - role_pkgs = pkg_index[role_name] = dict() - - role_pkgs['project_group'] = project_group - - if var_file_name: - _name = os.path.splitext(os.path.basename(var_file_name))[0] - if _name in pkg_index[role_name]: - file_name_index = pkg_index[role_name][_name] - else: - file_name_index = pkg_index[role_name][_name] = dict() - pkgs = file_name_index.get(var_name, list()) - pkgs = self._py_pkg_extend(packages, pkgs) - file_name_index[var_name] = sorted(set(pkgs)) - else: - pkgs = role_pkgs.get(var_name, list()) - pkgs.extend(packages) - if 'pip' in var_name: - pkgs = [i.lower() for i in pkgs if not i.startswith('{{')] - else: - pkgs = [i for i in pkgs if not i.startswith('{{')] - if pkgs: - pkg_index[role_name][var_name] = sorted(set(pkgs)) - else: - for k, v in pkg_index.items(): - for item_name in v.keys(): - if var_name == item_name: - pkg_index[k][item_name] = self._py_pkg_extend( - packages, - pkg_index[k][item_name] - ) - - def _process_files(self): - """Process all of the requirement files.""" - self._process_files_defaults() - self._process_files_requirements() - - def _process_files_defaults(self): - """Process files.""" - for file_name in self._filter_files(self.file_names, ('yaml', 'yml')): - with open(file_name, 'r') as f: - # If there is an exception loading the file continue - # and if the loaded_config is None continue. This makes - # no bad config gets passed to the rest of the process. - try: - loaded_config = yaml.safe_load(f.read()) - except Exception: # Broad exception so everything is caught - continue - else: - if not loaded_config or not isinstance(loaded_config, dict): - continue - - if 'roles' in file_name: - _role_name = file_name.split('roles%s' % os.sep)[-1] - role_name = _role_name.split(os.sep)[0] - else: - role_name = None - - for key, value in loaded_config.items(): - if key.endswith('role_project_group'): - project_group = value - break - else: - project_group = 'all' - - if role_name is not None: - PACKAGE_MAPPING['role_project_groups'][role_name] = project_group - for key, values in loaded_config.items(): - key = key.lower() - if key.endswith('git_repo'): - self._process_git( - loaded_yaml=loaded_config, - git_item=key, - yaml_file_name=file_name - ) - # Process pip packages - self._process_packages( - pkg_constant=BUILT_IN_PIP_PACKAGE_VARS, - pkg_breakout_index=ROLE_BREAKOUT_REQUIREMENTS, - pkg_role_index=ROLE_PACKAGES, - pkg_var_name=key, - packages=values, - role_name=role_name, - project_group=project_group - ) - - def _process_packages(self, pkg_constant, pkg_breakout_index, - pkg_role_index, pkg_var_name, packages, role_name, - project_group, role_index=True, var_file_name=None, - pip_packages=True): - """Process variables to build the package data structures. - - :param pkg_constant: CONSTANT used to validate package names - :type pkg_constant: ``list`` - :param pkg_breakout_index: CONSTANT used to store indexed packages - :type pkg_breakout_index: ``dict`` - :param pkg_role_index: CONSTANT used to store role indexed packages - :type pkg_role_index: ``dict`` - :param pkg_var_name: package variable name - :type pkg_var_name: ``str`` - :param packages: list of packages to index - :type packages: ``list`` - :param role_name: Name of the role where the packages came from - :type role_name: ``str`` - :param project_group: Name of the group being indexed - :type project_group: ``str`` - :param role_index: Enable or disable the use of the role index - :type role_index: ``bool`` - :param var_file_name: Variable file name used to index packages - :type var_file_name: ``str`` - :param pip_packages: Enable or disable pip index types - :type pip_packages: ``bool`` - """ - if [i for i in pkg_constant if i in pkg_var_name]: - if 'proprietary' in pkg_var_name: - return - - self._package_build_index( - packages=packages, - role_name=role_name, - var_name=pkg_var_name, - pkg_index=pkg_breakout_index, - project_group=project_group, - var_file_name=var_file_name, - pip_packages=pip_packages - ) - - if not role_index: - return - elif 'optional' in pkg_var_name: - return - else: - self._package_build_index( - packages=packages, - role_name=role_name, - var_name=pkg_var_name, - pkg_index=pkg_role_index, - project_group=project_group, - var_file_name=var_file_name, - pip_packages=pip_packages - ) - - def _process_files_requirements(self): - """Process requirements files.""" - return_list = self._filter_files(self.file_names, 'txt') - for file_name in return_list: - base_name = os.path.basename(file_name) - if base_name in REQUIREMENTS_FILE_TYPES: - index = REQUIREMENTS_FILE_TYPES.index(base_name) - return_list.remove(file_name) - return_list.insert(index, file_name) - else: - for file_name in return_list: - if file_name.endswith('doc/requirements.txt'): - continue - if file_name.endswith('other-requirements.txt'): - continue - elif file_name.endswith('bindep.txt'): - continue - elif 'roles' in file_name: - continue - else: - role_name = 'default' - with open(file_name, 'r') as f: - packages = [ - i.split()[0].lower() for i in f.read().splitlines() - if i - if not i.startswith('#') - ] - base_file_name = os.path.basename(file_name) - if base_file_name.endswith('test-requirements.txt'): - continue - if base_file_name.endswith('global-requirement-pins.txt'): - self._package_build_index( - packages=packages, - role_name='global_pins', - var_name='pinned_packages', - pkg_index=ROLE_REQUIREMENTS, - project_group='all' - ) - self._package_build_index( - packages=packages, - role_name=role_name, - var_name='txt_file_packages', - pkg_index=ROLE_REQUIREMENTS, - project_group='all' - ) - - -def _abs_path(path): - return os.path.abspath( - os.path.expanduser( - path - ) - ) - - -class LookupModule(LookupBase): - def __init__(self, basedir=None, **kwargs): - """Run the lookup module. - - :type basedir: - :type kwargs: - """ - - def run(self, terms, variables=None, **kwargs): - """Run the main application. - - :type terms: ``str`` - :type variables: ``str`` - :type kwargs: ``dict`` - :returns: ``list`` - """ - if isinstance(terms, str): - terms = [terms] - - return_data = PACKAGE_MAPPING - - for term in terms: - return_list = list() - try: - dfp = DependencyFileProcessor( - local_path=_abs_path(str(term)) - ) - return_list.extend(dfp.pip['py_package']) - return_list.extend(dfp.pip['git_package']) - except Exception as exp: - raise AnsibleError( - 'lookup_plugin.py_pkgs(%s) returned "%s" error "%s"' % ( - term, - str(exp), - traceback.format_exc() - ) - ) - - for item in return_list: - map_base_and_remote_packages(item, return_data) - else: - parse_remote_package_parts(return_data) - else: - map_role_packages(return_data) - map_base_package_details(return_data) - # Sort everything within the returned data - for key, value in return_data.items(): - if isinstance(value, (list, set)): - try: - if all(isinstance(item, dict) for item in value): - return_data[key] = sorted(value, key = lambda k: k['name']) - else: - return_data[key] = sorted(value) - except TypeError: - return_data[key] = value - return_data['role_requirement_files'] = ROLE_REQUIREMENTS - return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS - return [return_data] - -# Used for testing and debuging usage: `python plugins/lookups/py_pkgs.py ../` -if __name__ == '__main__': - import sys - import json - print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True)) diff --git a/releasenotes/notes/remove-py-pkgs-db29425b9e1855a6.yaml b/releasenotes/notes/remove-py-pkgs-db29425b9e1855a6.yaml new file mode 100644 index 00000000..5d4a399d --- /dev/null +++ b/releasenotes/notes/remove-py-pkgs-db29425b9e1855a6.yaml @@ -0,0 +1,6 @@ +--- +deprecations: + - | + The ``py_pkgs`` and ``packages_file`` Ansible lookups are no + longer used in OSA and have been removed from the plugins + repository. diff --git a/tests/test-lookups.yml b/tests/test-lookups.yml deleted file mode 100644 index 02de08c6..00000000 --- a/tests/test-lookups.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -# Copyright 2017, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# (c) 2017, Jean-Philippe Evrard - - -- name: Test lookups - hosts: localhost - connection: local - gather_facts: no - tasks: - - name: Ensure Package list contains the files listed in Package - set_fact: - packages_dict: "{{ lookup('packages_file', playbook_dir ~ '/files/Packages') }}" - - name: Validate that Filename, MD5sum, SHA1, SHA256 and Version are properly generated - assert: - that: - - packages_dict['0ad']['Filename'] == "pool/main/0/0ad/0ad_0.0.16-2~ubuntu14.04.1_amd64.deb" - - packages_dict['0ad']['MD5sum'] == "7e5f2ba5e1a95e47753eeb962af32e26" - - packages_dict['0ad']['SHA1'] == "3cf898d4595092daa274e6cd8d9afd0332b0afbe" - - packages_dict['0ad']['SHA256'] == "f4602a90a305abeacb4a48bbfd7d609aa7cbb3ed2ab9127ae30ef64a4be88378" - - packages_dict['0ad']['Version'] == "0.0.16-2~ubuntu14.04.1" - - name: Validate that a provides package is listed as a package too - assert: - that: - - packages_dict['aide'] is defined - - packages_dict['aide-binary'] is defined - - packages_dict['aide'] == packages_dict['aide-binary'] diff --git a/tests/test.yml b/tests/test.yml index fb12f512..8a2beca6 100644 --- a/tests/test.yml +++ b/tests/test.yml @@ -21,8 +21,6 @@ - import_playbook: test-filters.yml -- import_playbook: test-lookups.yml - - import_playbook: test-strategy-tagfilter.yml - import_playbook: test-connection-plugin.yml