# Copyright 2014, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# (c) 2014, Kevin Carter <kevin.carter@rackspace.com>

import os
import re
import traceback
import yaml

from ansible.errors import AnsibleError
from ansible.plugins.lookup import LookupBase


try:
  basestring
except NameError:
  basestring = str


# Used to keep track of git package parts as various files are processed
GIT_PACKAGE_DEFAULT_PARTS = dict()


# Role based package indexes
ROLE_BREAKOUT_REQUIREMENTS = dict()
ROLE_PACKAGES = dict()
ROLE_REQUIREMENTS = dict()


REQUIREMENTS_FILE_TYPES = [
    'requirements.txt',
    'global-requirement-pins.txt'
]


# List of variable names that could be used within the yaml files that
# represent lists of python packages.
BUILT_IN_PIP_PACKAGE_VARS = [
    'service_pip_dependencies',
    'pip_common_packages',
    'pip_container_packages',
    'pip_packages'
]


PACKAGE_MAPPING = {
    'packages': set(),
    'remote_packages': set(),
    'remote_package_parts': list(),
    'role_packages': dict(),
    'role_project_groups': dict(),
}


def map_base_and_remote_packages(package, package_map):
    """Determine whether a package is a base package or a remote package
       and add to the appropriate set.

    :type package: ``str``
    :type package_map: ``dict``
    """
    def check_for_ignore(p):
        p_parts = GIT_PACKAGE_DEFAULT_PARTS.get(p)
        if p_parts:
            fragments = p_parts.get('fragments', '') or ''
            if 'ignorerequirements=True' not in fragments:
                package_map['packages'].add(p)
        else:
            package_map['packages'].add(p)

    if package.startswith(('http:', 'https:', 'git+')):
        if '@' not in package:
            check_for_ignore(p=package)
        else:
            git_parts = git_pip_link_parse(package)
            package_name = git_parts[-2]
            if not package_name:
                package_name = git_pip_link_parse(package)[0]

            for rpkg in list(package_map['remote_packages']):
                rpkg_name = git_pip_link_parse(rpkg)[-2]
                if not rpkg_name:
                    rpkg_name = git_pip_link_parse(package)[0]

                if rpkg_name == package_name:
                    package_map['remote_packages'].remove(rpkg)
                    package_map['remote_packages'].add(package)
                    break
            else:
                package_map['remote_packages'].add(package)
    else:
        check_for_ignore(p=package)


def parse_remote_package_parts(package_map):
    """Parse parts of each remote package and add them to
       the remote_package_parts list.

    :type package_map: ``dict``
    """
    keys = [
        'name',
        'version',
        'fragment',
        'url',
        'original',
        'egg_name',
        'project_group'
    ]
    remote_pkg_parts = [
        dict(
            zip(
                keys, git_pip_link_parse(i)
            )
        ) for i in package_map['remote_packages']
    ]
    package_map['remote_package_parts'].extend(remote_pkg_parts)
    package_map['remote_package_parts'] = list(
        dict(
            (i['name'], i)
            for i in package_map['remote_package_parts']
        ).values()
    )


def map_role_packages(package_map):
    """Add and sort packages belonging to a role to the role_packages dict.

    :type package_map: ``dict``
    """
    for k, v in ROLE_PACKAGES.items():
        role_pkgs = package_map['role_packages'][k] = list()
        package_map['role_project_groups'][k] = v.pop('project_group', 'all')
        for pkg_list in v.values():
            role_pkgs.extend(pkg_list)
        else:
            package_map['role_packages'][k] = sorted(set(role_pkgs))


def map_base_package_details(package_map):
    """Parse package version and marker requirements and add to the
       base packages set.

    :type package_map: ``dict``
    """
    check_pkgs = dict()
    base_packages = sorted(list(package_map['packages']))
    for pkg in base_packages:
        name, versions, markers = _pip_requirement_split(pkg)
        if versions and markers:
            versions = '%s;%s' % (versions, markers)
        elif not versions and markers:
            versions = ';%s' % markers

        if name in check_pkgs:
            if versions and not check_pkgs[name]:
                check_pkgs[name] = versions
        else:
            check_pkgs[name] = versions
    else:
        return_pkgs = list()
        for k, v in check_pkgs.items():
            if v:
                return_pkgs.append('%s%s' % (k, v))
            else:
                return_pkgs.append(k)
        package_map['packages'] = set(return_pkgs)


def git_pip_link_parse(repo):
    """Return a tuple containing the parts of a git repository.

    Example parsing a standard git repo:
      >>> git_pip_link_parse('git+https://github.com/username/repo-name@tag')
      ('repo-name',
       'tag',
       None,
       'https://github.com/username/repo',
       'git+https://github.com/username/repo@tag',
       'repo_name')

    Example parsing a git repo that uses an installable from a subdirectory:
      >>> git_pip_link_parse(
      ...     'git+https://github.com/username/repo@tag#egg=plugin.name'
      ...     '&subdirectory=remote_path/plugin.name'
      ... )
      ('plugin.name',
       'tag',
       'remote_path/plugin.name',
       'https://github.com/username/repo',
       'git+https://github.com/username/repo@tag#egg=plugin.name&'
       'subdirectory=remote_path/plugin.name',
       'plugin.name')

    :param repo: git repo string to parse.
    :type repo: ``str``
    :returns: ``tuple``
    """'meta'

    def _meta_return(meta_data, item):
        """Return the value of an item in meta data."""

        return meta_data.lstrip('#').split('%s=' % item)[-1].split('&')[0]

    _git_url = repo.split('+')
    if len(_git_url) >= 2:
        _git_url = _git_url[1]
    else:
        _git_url = _git_url[0]

    git_branch_sha = _git_url.split('@')
    if len(git_branch_sha) > 2:
        branch = git_branch_sha.pop()
        url = '@'.join(git_branch_sha)
    elif len(git_branch_sha) > 1:
        url, branch = git_branch_sha
    else:
        url = git_branch_sha[0]
        branch = 'master'

    egg_name = name = os.path.basename(url.rstrip('/'))
    egg_name = egg_name.replace('-', '_')

    _branch = branch.split('#')
    branch = _branch[0]

    plugin_path = None
    # Determine if the package is a plugin type
    if len(_branch) > 1:
        if 'subdirectory=' in _branch[-1]:
            plugin_path = _meta_return(_branch[-1], 'subdirectory')
            name = os.path.basename(plugin_path)

        if 'egg=' in _branch[-1]:
            egg_name = _meta_return(_branch[-1], 'egg')
            egg_name = egg_name.replace('-', '_')

        if 'gitname=' in _branch[-1]:
            name = _meta_return(_branch[-1], 'gitname')

        project_group = 'all'
        if 'projectgroup=' in _branch[-1]:
            project_group = _meta_return(_branch[-1], 'projectgroup')

    return name.lower(), branch, plugin_path, url, repo, egg_name, project_group


def _pip_requirement_split(requirement):
    """Split pip versions from a given requirement.

    The method will return the package name, versions, and any markers.

    :type requirement: ``str``
    :returns: ``tuple``
    """
    version_descriptors = "(>=|<=|>|<|==|~=|!=)"
    requirement = requirement.split(';')
    requirement_info = re.split(r'%s\s*' % version_descriptors, requirement[0])
    name = requirement_info[0]
    marker = None
    if len(requirement) > 1:
        marker = requirement[-1]
    versions = None
    if len(requirement_info) > 1:
        versions = ''.join(requirement_info[1:])

    return name, versions, marker


class DependencyFileProcessor(object):
    def __init__(self, local_path):
        """Find required files.

        :type local_path: ``str``
        :return:
        """
        self.pip = dict()
        self.pip['git_package'] = list()
        self.pip['py_package'] = list()
        self.pip['git_data'] = list()
        self.git_pip_install = 'git+%s@%s'
        self.file_names = self._get_files(path=local_path)

        # Process everything simply by calling the method
        self._process_files()

    def _py_pkg_extend(self, packages, py_package=None):
        if py_package is None:
            py_package = self.pip['py_package']

        for pkg in packages:
            pkg_name = _pip_requirement_split(pkg)[0]
            for py_pkg in py_package:
                py_pkg_name = _pip_requirement_split(py_pkg)[0]
                if pkg_name == py_pkg_name:
                    py_package.remove(py_pkg)
        else:
            norm_pkgs = [i.lower() for i in packages if not i.startswith('{{')]
            py_package.extend(norm_pkgs)
        return py_package

    @staticmethod
    def _filter_files(file_names, ext):
        """Filter the files and return a sorted list.
        :type file_names:
        :type ext: ``str`` or ``tuple``
        :returns: ``list``
        """
        _file_names = list()
        file_name_words = ['/defaults/', '/vars/', '/user_']
        file_name_words.extend(REQUIREMENTS_FILE_TYPES)
        for file_name in file_names:
            if file_name.endswith(ext):
                if any(i in file_name for i in file_name_words):
                    _file_names.append(file_name)
        else:
            return _file_names

    @staticmethod
    def _get_files(path):
        """Return a list of all files in the defaults/repo_packages directory.

        :type path: ``str``
        :returns: ``list``
        """
        paths = os.walk(os.path.abspath(path), followlinks=True)
        files = list()
        for fpath, _, afiles in paths:
            for afile in afiles:
                files.append(os.path.join(fpath, afile))
        else:
            return files

    def _check_plugins(self, git_repo_plugins, git_data):
        """Check if the git url is a plugin type.

        :type git_repo_plugins: ``dict``
        :type git_data: ``dict``
        """
        for repo_plugin in git_repo_plugins:
            strip_plugin_path = repo_plugin['package'].lstrip('/')
            plugin = '%s/%s' % (
                repo_plugin['path'].strip('/'),
                strip_plugin_path
            )

            name = git_data['name'] = os.path.basename(strip_plugin_path)
            git_data['egg_name'] = name.replace('-', '_')
            package = self.git_pip_install % (
                git_data['repo'], git_data['branch']
            )
            package += '#egg=%s' % git_data['egg_name']
            package += '&subdirectory=%s' % plugin
            package += '&gitname=%s' % name
            if git_data['fragments']:
                package += '&%s' % git_data['fragments']

            self.pip['git_data'].append(git_data)
            self.pip['git_package'].append(package)

            if name not in GIT_PACKAGE_DEFAULT_PARTS:
                GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy()
            else:
                GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data.copy())

    @staticmethod
    def _check_defaults(git_data, name, item):
        """Check if a default exists and use it if an item is undefined.

        :type git_data: ``dict``
        :type name: ``str``
        :type item: ``str``
        """
        if not git_data[item] and name in GIT_PACKAGE_DEFAULT_PARTS:
            check_item = GIT_PACKAGE_DEFAULT_PARTS[name].get(item)
            if check_item:
                git_data[item] = check_item

    def _process_git(self, loaded_yaml, git_item, yaml_file_name):
        """Process git repos.

        :type loaded_yaml: ``dict``
        :type git_item: ``str``
        """
        git_data = dict()
        if git_item.split('_')[0] == 'git':
            prefix = ''
        else:
            prefix = '%s_' % git_item.split('_git_repo')[0].replace('.', '_')

        # Set the various variable definitions
        repo_var = prefix + 'git_repo'
        name_var = prefix + 'git_package_name'
        branch_var = prefix + 'git_install_branch'
        fragment_var = prefix + 'git_install_fragments'
        plugins_var = prefix + 'repo_plugins'
        group_var = prefix + 'git_project_group'

        # get the repo definition
        git_data['repo'] = loaded_yaml.get(repo_var)
        group = git_data['project_group'] = loaded_yaml.get(group_var, 'all')

        # get the repo name definition
        name = git_data['name'] = loaded_yaml.get(name_var)
        if not name:
            # NOTE: strip off trailing /, .git, or .git/
            name = git_data['name'] = os.path.basename(
                re.sub(r'(\/$|\.git(\/)?$)', '', git_data['repo'])
            )
        git_data['egg_name'] = name.replace('-', '_')

        # This conditional is set to ensure we're only processing git
        #  repos from the defaults file when those same repos are not
        #  being set in the repo_packages files.
        if '/defaults/main' in yaml_file_name:
            if name in GIT_PACKAGE_DEFAULT_PARTS:
                return

        # get the repo branch definition
        git_data['branch'] = loaded_yaml.get(branch_var)
        self._check_defaults(git_data, name, 'branch')
        if not git_data['branch']:
            git_data['branch'] = 'master'

        package = self.git_pip_install % (git_data['repo'], git_data['branch'])

        # get the repo fragment definitions, if any
        git_data['fragments'] = loaded_yaml.get(fragment_var)
        self._check_defaults(git_data, name, 'fragments')

        package += '#egg=%s' % git_data['egg_name']
        package += '&gitname=%s' % name
        package += '&projectgroup=%s' % group
        if git_data['fragments']:
            package += '&%s' % git_data['fragments']

        self.pip['git_package'].append(package)
        self.pip['git_data'].append(git_data.copy())

        # Set the default package parts to track data during the run
        if name not in GIT_PACKAGE_DEFAULT_PARTS:
            GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy()
        else:
            GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data)

        # get the repo plugin definitions, if any
        git_data['plugins'] = loaded_yaml.get(plugins_var)
        self._check_defaults(git_data, name, 'plugins')
        if git_data['plugins']:
            self._check_plugins(
                git_repo_plugins=git_data['plugins'],
                git_data=git_data
            )

    def _package_build_index(self, packages, role_name, var_name, pkg_index,
                             project_group='all', var_file_name=None,
                             pip_packages=True):
        if pip_packages:
            self._py_pkg_extend(packages)

        if role_name:
            if role_name in pkg_index:
                role_pkgs = pkg_index[role_name]
            else:
                role_pkgs = pkg_index[role_name] = dict()

            role_pkgs['project_group'] = project_group

            if var_file_name:
                _name = os.path.splitext(os.path.basename(var_file_name))[0]
                if _name in pkg_index[role_name]:
                    file_name_index = pkg_index[role_name][_name]
                else:
                    file_name_index = pkg_index[role_name][_name] = dict()
                pkgs = file_name_index.get(var_name, list())
                pkgs = self._py_pkg_extend(packages, pkgs)
                file_name_index[var_name] = sorted(set(pkgs))
            else:
                pkgs = role_pkgs.get(var_name, list())
                pkgs.extend(packages)
                if 'pip' in var_name:
                    pkgs = [i.lower() for i in pkgs if not i.startswith('{{')]
                else:
                    pkgs = [i for i in pkgs if not i.startswith('{{')]
                if pkgs:
                    pkg_index[role_name][var_name] = sorted(set(pkgs))
        else:
            for k, v in pkg_index.items():
                for item_name in v.keys():
                    if var_name == item_name:
                        pkg_index[k][item_name] = self._py_pkg_extend(
                            packages,
                            pkg_index[k][item_name]
                        )

    def _process_files(self):
        """Process all of the requirement files."""
        self._process_files_defaults()
        self._process_files_requirements()

    def _process_files_defaults(self):
        """Process files."""
        for file_name in self._filter_files(self.file_names, ('yaml', 'yml')):
            with open(file_name, 'r') as f:
                # If there is an exception loading the file continue
                #  and if the loaded_config is None continue. This makes
                #  no bad config gets passed to the rest of the process.
                try:
                    loaded_config = yaml.safe_load(f.read())
                except Exception:  # Broad exception so everything is caught
                    continue
                else:
                    if not loaded_config or not isinstance(loaded_config, dict):
                        continue

                if 'roles' in file_name:
                    _role_name = file_name.split('roles%s' % os.sep)[-1]
                    role_name = _role_name.split(os.sep)[0]
                else:
                    role_name = None

            for key, value in loaded_config.items():
                if key.endswith('role_project_group'):
                    project_group = value
                    break
            else:
                project_group = 'all'

            if role_name is not None:
                PACKAGE_MAPPING['role_project_groups'][role_name] = project_group
            for key, values in loaded_config.items():
                key = key.lower()
                if key.endswith('git_repo'):
                    self._process_git(
                        loaded_yaml=loaded_config,
                        git_item=key,
                        yaml_file_name=file_name
                    )
                # Process pip packages
                self._process_packages(
                    pkg_constant=BUILT_IN_PIP_PACKAGE_VARS,
                    pkg_breakout_index=ROLE_BREAKOUT_REQUIREMENTS,
                    pkg_role_index=ROLE_PACKAGES,
                    pkg_var_name=key,
                    packages=values,
                    role_name=role_name,
                    project_group=project_group
                )

    def _process_packages(self, pkg_constant, pkg_breakout_index,
                          pkg_role_index, pkg_var_name, packages, role_name,
                          project_group, role_index=True, var_file_name=None,
                          pip_packages=True):
        """Process variables to build the package data structures.

        :param pkg_constant: CONSTANT used to validate package names
        :type pkg_constant: ``list``
        :param pkg_breakout_index: CONSTANT used to store indexed packages
        :type pkg_breakout_index: ``dict``
        :param pkg_role_index: CONSTANT used to store role indexed packages
        :type pkg_role_index: ``dict``
        :param pkg_var_name: package variable name
        :type pkg_var_name: ``str``
        :param packages: list of packages to index
        :type packages: ``list``
        :param role_name: Name of the role where the packages came from
        :type role_name: ``str``
        :param project_group: Name of the group being indexed
        :type project_group: ``str``
        :param role_index: Enable or disable the use of the role index
        :type role_index: ``bool``
        :param var_file_name: Variable file name used to index packages
        :type var_file_name: ``str``
        :param pip_packages: Enable or disable pip index types
        :type pip_packages: ``bool``
        """
        if [i for i in pkg_constant if i in pkg_var_name]:
            if 'proprietary' in pkg_var_name:
                return

            self._package_build_index(
                packages=packages,
                role_name=role_name,
                var_name=pkg_var_name,
                pkg_index=pkg_breakout_index,
                project_group=project_group,
                var_file_name=var_file_name,
                pip_packages=pip_packages
            )

            if not role_index:
                return
            elif 'optional' in pkg_var_name:
                return
            else:
                self._package_build_index(
                    packages=packages,
                    role_name=role_name,
                    var_name=pkg_var_name,
                    pkg_index=pkg_role_index,
                    project_group=project_group,
                    var_file_name=var_file_name,
                    pip_packages=pip_packages
                )

    def _process_files_requirements(self):
        """Process requirements files."""
        return_list = self._filter_files(self.file_names, 'txt')
        for file_name in return_list:
            base_name = os.path.basename(file_name)
            if base_name in REQUIREMENTS_FILE_TYPES:
                index = REQUIREMENTS_FILE_TYPES.index(base_name)
                return_list.remove(file_name)
                return_list.insert(index, file_name)
        else:
            for file_name in return_list:
                if file_name.endswith('doc/requirements.txt'):
                    continue
                if file_name.endswith('other-requirements.txt'):
                    continue
                elif file_name.endswith('bindep.txt'):
                    continue
                elif 'roles' in file_name:
                    continue
                else:
                    role_name = 'default'
                with open(file_name, 'r') as f:
                    packages = [
                        i.split()[0].lower() for i in f.read().splitlines()
                        if i
                        if not i.startswith('#')
                    ]
                    base_file_name = os.path.basename(file_name)
                    if base_file_name.endswith('test-requirements.txt'):
                        continue
                    if base_file_name.endswith('global-requirement-pins.txt'):
                        self._package_build_index(
                            packages=packages,
                            role_name='global_pins',
                            var_name='pinned_packages',
                            pkg_index=ROLE_REQUIREMENTS,
                            project_group='all'
                        )
                    self._package_build_index(
                        packages=packages,
                        role_name=role_name,
                        var_name='txt_file_packages',
                        pkg_index=ROLE_REQUIREMENTS,
                        project_group='all'
                    )


def _abs_path(path):
    return os.path.abspath(
        os.path.expanduser(
            path
        )
    )


class LookupModule(LookupBase):
    def __init__(self, basedir=None, **kwargs):
        """Run the lookup module.

        :type basedir:
        :type kwargs:
        """

    def run(self, terms, variables=None, **kwargs):
        """Run the main application.

        :type terms: ``str``
        :type variables: ``str``
        :type kwargs: ``dict``
        :returns: ``list``
        """
        if isinstance(terms, str):
            terms = [terms]

        return_data = PACKAGE_MAPPING

        for term in terms:
            return_list = list()
            try:
                dfp = DependencyFileProcessor(
                    local_path=_abs_path(str(term))
                )
                return_list.extend(dfp.pip['py_package'])
                return_list.extend(dfp.pip['git_package'])
            except Exception as exp:
                raise AnsibleError(
                    'lookup_plugin.py_pkgs(%s) returned "%s" error "%s"' % (
                        term,
                        str(exp),
                        traceback.format_exc()
                    )
                )

            for item in return_list:
                map_base_and_remote_packages(item, return_data)
            else:
                parse_remote_package_parts(return_data)
        else:
            map_role_packages(return_data)
            map_base_package_details(return_data)
            # Sort everything within the returned data
            for key, value in return_data.items():
                if isinstance(value, (list, set)):
                    try:
                        if all(isinstance(item, dict) for item in value):
                            return_data[key] = sorted(value, key = lambda k: k['name'])
                        else:
                            return_data[key] = sorted(value)
                    except TypeError:
                        return_data[key] = value
            return_data['role_requirement_files'] = ROLE_REQUIREMENTS
            return_data['role_requirements'] = ROLE_BREAKOUT_REQUIREMENTS
            return [return_data]

# Used for testing and debuging usage: `python plugins/lookups/py_pkgs.py ../`
if __name__ == '__main__':
    import sys
    import json
    print(json.dumps(LookupModule().run(terms=sys.argv[1:]), indent=4, sort_keys=True))