Add git and branch revision support to pegleg
* Add support for URLs and directories including git clone support * Add support for http://, https://, and ssh:// git cloning * Add support for cloning behind proxy * Add support for checking out references of cloned repos * Add support for checking out references of local repos * Add support for Pegleg Git exceptions This patch set also adds support for including Pegleg source code in documentation and adds exceptions documentation. Change-Id: I417a62c815f97a70f3abc432cc342707e8ce1f54
This commit is contained in:
Felipe Monteiro
parent
f0ae58d8b9
commit
20dcaa45ae
|
|
@ -16,9 +16,9 @@
|
|||
# add these directories to sys.path here. If the directory is relative to the
|
||||
# documentation root, use os.path.abspath to make it absolute, like shown here.
|
||||
#
|
||||
# import os
|
||||
# import sys
|
||||
# sys.path.insert(0, os.path.abspath('.'))
|
||||
import os
|
||||
import sys
|
||||
sys.path.insert(0, os.path.abspath('../../src/bin/pegleg'))
|
||||
import sphinx_rtd_theme
|
||||
|
||||
|
||||
|
|
@ -52,7 +52,7 @@ master_doc = 'index'
|
|||
# General information about the project.
|
||||
project = u'pegleg'
|
||||
copyright = u'2018 AT&T Intellectual Property.'
|
||||
author = u'pegleg Authors'
|
||||
author = u'Pegleg Authors'
|
||||
|
||||
# The version info for the project you're documenting, acts as replacement for
|
||||
# |version| and |release|, also used in various other places throughout the
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
..
|
||||
Copyright 2018 AT&T Intellectual Property.
|
||||
All Rights Reserved.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
not use this file except in compliance with the License. You may obtain
|
||||
a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
License for the specific language governing permissions and limitations
|
||||
under the License.
|
||||
|
||||
Pegleg Exceptions
|
||||
==================
|
||||
|
||||
Base Exceptions
|
||||
---------------
|
||||
|
||||
.. list-table::
|
||||
:widths: 5 50
|
||||
:header-rows: 1
|
||||
|
||||
* - Exception Name
|
||||
- Description
|
||||
* - PeglegBaseException
|
||||
- .. autoexception:: pegleg.engine.exceptions.PeglegBaseException
|
||||
:members:
|
||||
:undoc-members:
|
||||
|
||||
Git Exceptions
|
||||
--------------
|
||||
|
||||
.. list-table::
|
||||
:widths: 5 50
|
||||
:header-rows: 1
|
||||
|
||||
* - Exception Name
|
||||
- Description
|
||||
* - BaseGitException
|
||||
- .. autoexception:: pegleg.engine.exceptions.BaseGitException
|
||||
:members:
|
||||
:show-inheritance:
|
||||
:undoc-members:
|
||||
* - GitException
|
||||
- .. autoexception:: pegleg.engine.exceptions.GitException
|
||||
:members:
|
||||
:show-inheritance:
|
||||
:undoc-members:
|
||||
* - GitAuthException
|
||||
- .. autoexception:: pegleg.engine.exceptions.GitAuthException
|
||||
:members:
|
||||
:show-inheritance:
|
||||
:undoc-members:
|
||||
* - GitProxyException
|
||||
- .. autoexception:: pegleg.engine.exceptions.GitProxyException
|
||||
:members:
|
||||
:show-inheritance:
|
||||
:undoc-members:
|
||||
* - GitSSHException
|
||||
- .. autoexception:: pegleg.engine.exceptions.GitSSHException
|
||||
:members:
|
||||
:show-inheritance:
|
||||
:undoc-members:
|
||||
|
|
@ -14,26 +14,32 @@
|
|||
License for the specific language governing permissions and limitations
|
||||
under the License.
|
||||
|
||||
.. tip::
|
||||
====================
|
||||
Pegleg Documentation
|
||||
====================
|
||||
|
||||
The Undercloud Platform is part of the AIC CP (AT&T Integrated Cloud
|
||||
Containerized Platform). More details may be found by using the `Treasuremap`_
|
||||
|
||||
Building this Documentation
|
||||
---------------------------
|
||||
|
||||
Use of ``tox -e docs`` will build an HTML version of this documentation that
|
||||
can be viewed using a browser at docs/build/index.html on the local filesystem.
|
||||
|
||||
Conventions and Standards
|
||||
-------------------------
|
||||
Overview
|
||||
--------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
getting_started
|
||||
authoring_strategy
|
||||
artifacts
|
||||
cli
|
||||
|
||||
.. _Treasuremap: https://github.com/att-comdev/treasuremap
|
||||
Design
|
||||
------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
artifacts
|
||||
authoring_strategy
|
||||
|
||||
Operator's Guide
|
||||
----------------
|
||||
|
||||
.. toctree::
|
||||
:maxdepth: 2
|
||||
|
||||
cli
|
||||
exceptions
|
||||
|
|
|
|||
|
|
@ -0,0 +1,79 @@
|
|||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
class PeglegBaseException(Exception):
|
||||
"""Base class for Pegleg exception and error handling."""
|
||||
|
||||
def __init__(self, message=None, **kwargs):
|
||||
self.message = message or self.message
|
||||
try: # nosec
|
||||
self.message = self.message % kwargs
|
||||
except Exception:
|
||||
pass
|
||||
super(PeglegBaseException, self).__init__(self.message)
|
||||
|
||||
|
||||
class BaseGitException(PeglegBaseException):
|
||||
"""Base class for Git exceptions and error handling."""
|
||||
|
||||
message = 'An unknown error occurred while accessing a chart source.'
|
||||
|
||||
|
||||
class GitException(BaseGitException):
|
||||
"""Exception when an error occurs cloning a Git repository."""
|
||||
|
||||
def __init__(self, location, details=None):
|
||||
self._message = ('Git exception occurred: [%s] may not be a valid git '
|
||||
'repository' % location)
|
||||
if details:
|
||||
self._message += '. Details: %s' % details
|
||||
|
||||
super(GitException, self).__init__(self._message)
|
||||
|
||||
|
||||
class GitAuthException(BaseGitException):
|
||||
"""Exception that occurs when authentication fails for cloning a repo."""
|
||||
|
||||
def __init__(self, repo_url, ssh_key_path):
|
||||
self._repo_url = repo_url
|
||||
self._ssh_key_path = ssh_key_path
|
||||
|
||||
self._message = ('Failed to authenticate for repo %s with ssh-key at '
|
||||
'path %s.' % (self._repo_url, self._ssh_key_path))
|
||||
|
||||
super(GitAuthException, self).__init__(self._message)
|
||||
|
||||
|
||||
class GitProxyException(BaseGitException):
|
||||
"""Exception when an error occurs cloning a Git repository
|
||||
through a proxy."""
|
||||
|
||||
def __init__(self, location):
|
||||
self._location = location
|
||||
self._message = ('Could not resolve proxy [%s].' % self._location)
|
||||
|
||||
super(GitProxyException, self).__init__(self._message)
|
||||
|
||||
|
||||
class GitSSHException(BaseGitException):
|
||||
"""Exception that occurs when an SSH key could not be found."""
|
||||
|
||||
def __init__(self, ssh_key_path):
|
||||
self._ssh_key_path = ssh_key_path
|
||||
|
||||
self._message = ('Failed to find specified SSH key: %s.' %
|
||||
(self._ssh_key_path))
|
||||
|
||||
super(GitSSHException, self).__init__(self._message)
|
||||
|
|
@ -16,3 +16,4 @@
|
|||
from . import definition
|
||||
from . import files
|
||||
from . import deckhand
|
||||
from . import git
|
||||
|
|
@ -0,0 +1,286 @@
|
|||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import os
|
||||
import tempfile
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from git import exc as git_exc
|
||||
from git import Git
|
||||
from git import Repo
|
||||
|
||||
from pegleg.engine import exceptions
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
__all__ = [
|
||||
'git_handler',
|
||||
]
|
||||
|
||||
|
||||
def git_handler(repo_url, ref, proxy_server=None, auth_key=None):
|
||||
"""Handle directories that are Git repositories.
|
||||
|
||||
If ``repo_url`` is a valid URL for which a local repository doesn't
|
||||
exist, then clone ``repo_url`` and checkout the given ``ref``. Otherwise,
|
||||
treat ``repo_url`` as an already-cloned repository and checkout the given
|
||||
``ref``.
|
||||
|
||||
Supported ``ref`` formats include:
|
||||
|
||||
* branch name (e.g. 'master')
|
||||
* refpath (e.g. 'refs/changes/54/457754/73')
|
||||
* hexsha (e.g. 'ff5496b9c781918fdc49d79f927323eeef2f5320')
|
||||
|
||||
:param repo_url: URL of remote Git repo or path to local Git repo. If no
|
||||
local copy exists, clone it. Afterward, check out ``ref`` in the repo.
|
||||
:param ref: branch, commit or reference in the repo to clone.
|
||||
:param proxy_server: optional, HTTP proxy to use while cloning the repo.
|
||||
:param auth_key: If supplied results in using SSH to clone the repository
|
||||
with the specified key. If the value is None, SSH is not used.
|
||||
:returns: Path to the cloned repo if a repo was cloned, else absolute
|
||||
path to ``repo_url``.
|
||||
:raises ValueError: If ``repo_url`` isn't a valid URL or doesn't begin
|
||||
with a valid protocol (http, https or ssh) for cloning.
|
||||
:raises NotADirectoryError: If ``repo_url`` isn't a valid directory path.
|
||||
|
||||
"""
|
||||
|
||||
supported_clone_protocols = ('http', 'https', 'ssh')
|
||||
|
||||
try:
|
||||
parsed_url = urlparse(repo_url)
|
||||
except Exception as e:
|
||||
raise ValueError('repo_url=%s is invalid. Details: %s' % (repo_url, e))
|
||||
|
||||
if not ref:
|
||||
raise ValueError('ref=%s must be a non-empty, valid Git ref' % ref)
|
||||
|
||||
if not os.path.exists(repo_url):
|
||||
# we need to clone the repo_url first since it doesn't exist and then
|
||||
# checkout the appropriate reference - and return the tmpdir
|
||||
if parsed_url.scheme in supported_clone_protocols:
|
||||
return _try_git_clone(repo_url, ref, proxy_server, auth_key)
|
||||
else:
|
||||
raise ValueError('repo_url=%s must use one of the following '
|
||||
'protocols: %s' %
|
||||
(repo_url, ', '.join(supported_clone_protocols)))
|
||||
|
||||
# otherwise, we're dealing with a local directory so although
|
||||
# we do not need to clone, we may need to process the reference
|
||||
# by checking that out and returning the directory they passed in
|
||||
else:
|
||||
LOG.debug('Treating repo_url=%s as an already-cloned repository. '
|
||||
'Attempting to checkout ref=%s', repo_url, ref)
|
||||
try:
|
||||
# get absolute path of what is probably a directory
|
||||
repo_url = os.path.abspath(repo_url)
|
||||
except Exception:
|
||||
msg = "The repo_url=%s is not a valid directory" % repo_url
|
||||
LOG.error(msg)
|
||||
raise NotADirectoryError(msg)
|
||||
|
||||
repo = Repo(repo_url)
|
||||
if repo.is_dirty():
|
||||
LOG.warning('The locally cloned repo_url=%s is dirty. '
|
||||
'Cleaning up untracked files.', repo_url)
|
||||
# Reset the index and working tree to match current ref.
|
||||
repo.head.reset(index=True, working_tree=True)
|
||||
|
||||
try:
|
||||
# Check whether the ref exists locally.
|
||||
LOG.info('Attempting to checkout ref=%s from repo_url=%s locally',
|
||||
ref, repo_url)
|
||||
_try_git_checkout(repo, repo_url, ref, fetch=False)
|
||||
except exceptions.GitException:
|
||||
# Otherwise, attempt to fetch and checkout the missing ref.
|
||||
LOG.info('ref=%s not found locally for repo_url=%s, fetching from '
|
||||
'remote', ref, repo_url)
|
||||
# Allow any errors to bubble up.
|
||||
_try_git_checkout(repo, repo_url, ref, fetch=True)
|
||||
|
||||
return repo_url
|
||||
|
||||
|
||||
def _try_git_clone(repo_url, ref='master', proxy_server=None, auth_key=None):
|
||||
"""Try cloning Git repo from ``repo_url`` using the reference ``ref``.
|
||||
|
||||
:param repo_url: URL of remote Git repo or path to local Git repo.
|
||||
:param ref: branch, commit or reference in the repo to clone. Default is
|
||||
'master'.
|
||||
:param proxy_server: optional, HTTP proxy to use while cloning the repo.
|
||||
:param auth_key: If supplied results in using SSH to clone the repository
|
||||
with the specified key. If the value is None, SSH is not used.
|
||||
:returns: Path to the cloned repo.
|
||||
:rtype: str
|
||||
:raises GitException: If ``repo_url`` is invalid or could not be found.
|
||||
:raises GitAuthException: If authentication with the Git repository failed.
|
||||
:raises GitProxyException: If the repo could not be cloned due to a proxy
|
||||
issue.
|
||||
|
||||
"""
|
||||
|
||||
# the name here is important as it bubbles back up to the output filename
|
||||
# and ensure we handle url/foo.git/ cases. prefix is 'tmp' by default.
|
||||
temp_dir = tempfile.mkdtemp(suffix=repo_url.rstrip('/').split('/')[-1])
|
||||
env_vars = _get_clone_env_vars(repo_url, ref, auth_key)
|
||||
ssh_cmd = env_vars.get('GIT_SSH_COMMAND')
|
||||
|
||||
try:
|
||||
if proxy_server:
|
||||
LOG.debug('Cloning [%s] with proxy [%s]', repo_url, proxy_server)
|
||||
# TODO(felipemonteiro): proxy_server can be finicky. Need a config
|
||||
# option to retry up to N times.
|
||||
repo = Repo.clone_from(
|
||||
repo_url,
|
||||
temp_dir,
|
||||
config='http.proxy=%s' % proxy_server,
|
||||
env=env_vars)
|
||||
else:
|
||||
LOG.debug('Cloning [%s]', repo_url)
|
||||
repo = Repo.clone_from(repo_url, temp_dir, env=env_vars)
|
||||
except git_exc.GitCommandError as e:
|
||||
LOG.exception('Failed to clone repo_url=%s using ref=%s.', repo_url,
|
||||
ref)
|
||||
if (ssh_cmd and ssh_cmd in e.stderr
|
||||
or 'permission denied' in e.stderr.lower()):
|
||||
raise exceptions.GitAuthException(repo_url, auth_key)
|
||||
elif 'could not resolve proxy' in e.stderr.lower():
|
||||
raise exceptions.GitProxyException(proxy_server)
|
||||
else:
|
||||
raise exceptions.GitException(repo_url, details=e)
|
||||
except Exception as e:
|
||||
msg = 'Encountered unknown Exception during clone of %s' % repo_url
|
||||
LOG.exception(msg)
|
||||
raise exceptions.GitException(repo_url, details=e)
|
||||
|
||||
_try_git_checkout(repo=repo, repo_url=repo_url, ref=ref)
|
||||
|
||||
return temp_dir
|
||||
|
||||
|
||||
def _get_clone_env_vars(repo_url, ref, auth_key):
|
||||
"""Generate environment variables include SSH command for Git clone.
|
||||
|
||||
:param repo_url: URL of remote Git repo or path to local Git repo.
|
||||
:param ref: branch, commit or reference in the repo to clone. Default is
|
||||
'master'.
|
||||
:param auth_key: If supplied results in using SSH to clone the repository
|
||||
with the specified key. If the value is None, SSH is not used.
|
||||
:returns: Dictionary of key-value pairs for Git clone.
|
||||
:rtype: dict
|
||||
:raises GitSSHException: If the SSH key specified by ``CONF.ssh_key_path``
|
||||
could not be found and ``auth_method`` is "SSH".
|
||||
|
||||
"""
|
||||
ssh_cmd = None
|
||||
env_vars = {'GIT_TERMINAL_PROMPT': '0'}
|
||||
|
||||
if auth_key:
|
||||
if os.path.exists(auth_key):
|
||||
LOG.debug('Attempting to clone the repo at %s using reference %s '
|
||||
'with SSH authentication.', repo_url, ref)
|
||||
# Ensure that host checking is ignored, to avoid unnecessary
|
||||
# required CLI input.
|
||||
ssh_cmd = (
|
||||
'ssh -i {} -o ConnectionAttempts=20 -o ConnectTimeout=10 -o '
|
||||
'StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'
|
||||
.format(os.path.expanduser(auth_key)))
|
||||
env_vars.update({'GIT_SSH_COMMAND': ssh_cmd})
|
||||
else:
|
||||
msg = "The auth_key path '%s' was not found" % auth_key
|
||||
LOG.error(msg)
|
||||
raise exceptions.GitSSHException(auth_key)
|
||||
return env_vars
|
||||
|
||||
|
||||
def _try_git_checkout(repo, repo_url, ref, fetch=True):
|
||||
"""Try to checkout a ``ref`` from ``repo``.
|
||||
|
||||
Local branches are created for multiple variations of the ``ref``,
|
||||
including its refpath and hexpath (i.e. commit ID).
|
||||
|
||||
This is to locally "memoize" references that would otherwise require
|
||||
resolution upstream. We increase performance by creating local branches
|
||||
for these other ``ref`` formats when the ``ref`` is fetched remotely for
|
||||
the first time only.
|
||||
|
||||
:param repo: Git Repo object.
|
||||
:param repo_url: URL of remote Git repo or path to local Git repo.
|
||||
:param ref: branch, commit or reference in the repo to clone. Default is
|
||||
'master'.
|
||||
:param fetch: Whether to fetch the ``ref`` from remote before checkout or
|
||||
to use the already-cloned local repo.
|
||||
:raises GitException: If ``ref`` could not be checked out.
|
||||
|
||||
"""
|
||||
try:
|
||||
g = Git(repo.working_dir)
|
||||
branches = [b.name for b in repo.branches]
|
||||
LOG.debug('Available branches for repo_url=%s: %s', repo_url, branches)
|
||||
|
||||
if fetch:
|
||||
LOG.debug('Fetching ref=%s from remote repo_url=%s', ref, repo_url)
|
||||
# fetch_info is guaranteed to be populated if ref resolves, else
|
||||
# a GitCommandError is raised.
|
||||
fetch_info = repo.remotes.origin.fetch(ref)
|
||||
hexsha = fetch_info[0].commit.hexsha.strip()
|
||||
ref_path = fetch_info[0].remote_ref_path.strip()
|
||||
|
||||
# If ``ref`` doesn't match the hexsha/refpath then create a branch
|
||||
# for each so that future checkouts can be performed using either
|
||||
# format. This way, no future processing is required to figure
|
||||
# out whether a refpath/hexsha exists within the repo.
|
||||
_create_local_ref(
|
||||
g, branches, ref=ref, newref=hexsha, reftype='hexsha')
|
||||
_create_local_ref(
|
||||
g, branches, ref=ref, newref=ref_path, reftype='refpath')
|
||||
_create_or_checkout_local_ref(g, branches, ref=ref)
|
||||
else:
|
||||
LOG.debug('Checking out ref=%s from local repo_url=%s', ref,
|
||||
repo_url)
|
||||
# Expect the reference to exist if checking out locally.
|
||||
g.checkout(ref)
|
||||
|
||||
LOG.debug('Successfully checked out ref=%s for repo_url=%s', ref,
|
||||
repo_url)
|
||||
except git_exc.GitCommandError as e:
|
||||
LOG.exception('Failed to checkout ref=%s from repo_url=%s.', ref,
|
||||
repo_url)
|
||||
raise exceptions.GitException(repo_url, details=e)
|
||||
except Exception as e:
|
||||
msg = ('Encountered unknown Exception during checkout of ref=%s for '
|
||||
'repo_url=%s' % (ref, repo_url))
|
||||
LOG.exception(msg)
|
||||
raise exceptions.GitException(repo_url, details=e)
|
||||
|
||||
|
||||
def _create_or_checkout_local_ref(g, branches, ref):
|
||||
if ref not in branches:
|
||||
LOG.debug('Creating local branch for ref=%s', ref)
|
||||
g.checkout('FETCH_HEAD', b=ref)
|
||||
branches.append(ref)
|
||||
else:
|
||||
LOG.debug('Checking out ref=%s from local repo', ref)
|
||||
g.checkout('FETCH_HEAD')
|
||||
|
||||
|
||||
def _create_local_ref(g, branches, ref, newref, reftype=None):
|
||||
if newref not in branches:
|
||||
if newref and ref != newref:
|
||||
LOG.debug('Creating local branch for ref=%s (%s for %s)', newref,
|
||||
reftype, ref)
|
||||
g.checkout('FETCH_HEAD', b=newref)
|
||||
branches.append(newref)
|
||||
|
|
@ -1,3 +1,4 @@
|
|||
gitpython
|
||||
click==6.7
|
||||
jsonschema==2.6.0
|
||||
pyyaml==3.12
|
||||
|
|
|
|||
|
|
@ -0,0 +1,432 @@
|
|||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import socket
|
||||
import requests
|
||||
|
||||
import fixtures
|
||||
import mock
|
||||
import pytest
|
||||
|
||||
from pegleg.engine import exceptions
|
||||
from pegleg.engine.util import git
|
||||
from tests.unit import test_utils
|
||||
|
||||
_REPO_DIR = None
|
||||
_PROXY_SERVERS = {
|
||||
'http':
|
||||
os.getenv('HTTP_PROXY',
|
||||
os.getenv('http_proxy', 'http://one.proxy.att.com:8888')),
|
||||
'https':
|
||||
os.getenv('HTTPS_PROXY',
|
||||
os.getenv('https_proxy', 'https://one.proxy.att.com:8888'))
|
||||
}
|
||||
|
||||
|
||||
def is_connected():
|
||||
"""Verifies whether network connectivity is up.
|
||||
|
||||
:returns: True if connected else False.
|
||||
"""
|
||||
try:
|
||||
r = requests.get("http://www.github.com/", proxies={})
|
||||
return r.ok
|
||||
except requests.exceptions.RequestException:
|
||||
return False
|
||||
|
||||
|
||||
def is_connected_behind_proxy():
|
||||
"""Verifies whether network connectivity is up behind given proxy.
|
||||
|
||||
:returns: True if connected else False.
|
||||
"""
|
||||
try:
|
||||
r = requests.get("http://www.github.com/", proxies=_PROXY_SERVERS)
|
||||
return r.ok
|
||||
except requests.exceptions.RequestException:
|
||||
return False
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def clean_git_repo():
|
||||
global _REPO_DIR
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if _REPO_DIR and os.path.exists(_REPO_DIR):
|
||||
shutil.rmtree(_REPO_DIR)
|
||||
_REPO_DIR = None
|
||||
|
||||
|
||||
def _validate_git_clone(repo_dir, fetched_ref=None, checked_out_ref=None):
|
||||
"""Validate that git clone/checkout work.
|
||||
|
||||
:param repo_dir: Path to local Git repo.
|
||||
:param fetched_ref: Reference that is stored in FETCH_HEAD following a
|
||||
remote fetch.
|
||||
:param checked_out_ref: Reference that is stored in HEAD following a local
|
||||
ref checkout.
|
||||
"""
|
||||
global _REPO_DIR
|
||||
_REPO_DIR = repo_dir
|
||||
|
||||
assert os.path.isdir(repo_dir)
|
||||
# Assert that the directory is a Git repo.
|
||||
assert os.path.isdir(os.path.join(repo_dir, '.git'))
|
||||
if fetched_ref:
|
||||
# Assert the FETCH_HEAD is at the fetched_ref ref.
|
||||
with open(os.path.join(repo_dir, '.git', 'FETCH_HEAD'), 'r') \
|
||||
as git_file:
|
||||
assert fetched_ref in git_file.read()
|
||||
if checked_out_ref:
|
||||
# Assert the HEAD is at the checked_out_ref.
|
||||
with open(os.path.join(repo_dir, '.git', 'HEAD'), 'r') \
|
||||
as git_file:
|
||||
assert checked_out_ref in git_file.read()
|
||||
|
||||
|
||||
def _assert_repo_url_was_cloned(mock_log, git_dir):
|
||||
expected_msg = ('Treating repo_url=%s as an already-cloned repository')
|
||||
assert mock_log.debug.called
|
||||
mock_calls = mock_log.debug.mock_calls
|
||||
assert any(m[1][0].startswith(expected_msg) for m in mock_calls)
|
||||
assert any(m[1][1] == git_dir for m in mock_calls)
|
||||
mock_log.debug.reset_mock()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_valid_url_http_protocol(clean_git_repo):
|
||||
url = 'http://github.com/openstack/airship-armada'
|
||||
git_dir = git.git_handler(url, ref='master')
|
||||
_validate_git_clone(git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_valid_url_https_protocol(clean_git_repo):
|
||||
url = 'https://github.com/openstack/airship-armada'
|
||||
git_dir = git.git_handler(url, ref='master')
|
||||
_validate_git_clone(git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_with_commit_reference(clean_git_repo):
|
||||
url = 'https://github.com/openstack/airship-armada'
|
||||
commit = 'cba78d1d03e4910f6ab1691bae633c5bddce893d'
|
||||
git_dir = git.git_handler(url, commit)
|
||||
_validate_git_clone(git_dir, commit)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_with_patch_ref(clean_git_repo):
|
||||
ref = 'refs/changes/54/457754/73'
|
||||
git_dir = git.git_handler('https://github.com/openstack/openstack-helm',
|
||||
ref)
|
||||
_validate_git_clone(git_dir, ref)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected_behind_proxy(),
|
||||
reason='git clone requires proxy connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_behind_proxy(mock_log, clean_git_repo):
|
||||
url = 'https://github.com/openstack/airship-armada'
|
||||
commit = 'cba78d1d03e4910f6ab1691bae633c5bddce893d'
|
||||
|
||||
for proxy_server in _PROXY_SERVERS.values():
|
||||
git_dir = git.git_handler(url, commit, proxy_server=proxy_server)
|
||||
_validate_git_clone(git_dir, commit)
|
||||
|
||||
mock_log.debug.assert_any_call('Cloning [%s] with proxy [%s]', url,
|
||||
proxy_server)
|
||||
mock_log.debug.reset_mock()
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_existing_directory_checks_out_earlier_ref_from_local(
|
||||
mock_log, clean_git_repo):
|
||||
"""Validate Git checks out an earlier patch or ref that should exist
|
||||
locally (as a later ref was already fetched which should contain that
|
||||
revision history).
|
||||
"""
|
||||
# Clone the openstack-helm repo and automatically checkout patch 34.
|
||||
ref = 'refs/changes/15/536215/35'
|
||||
repo_url = 'https://github.com/openstack/openstack-helm'
|
||||
git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(git_dir, fetched_ref=ref)
|
||||
|
||||
# Checkout ref='master' now that the repo already exists locally.
|
||||
ref = 'refs/changes/15/536215/34'
|
||||
git_dir = git.git_handler(git_dir, ref)
|
||||
_validate_git_clone(git_dir, checked_out_ref=ref)
|
||||
_assert_repo_url_was_cloned(mock_log, git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_existing_directory_checks_out_master_from_local(
|
||||
mock_log, clean_git_repo):
|
||||
"""Validate Git checks out the ref of an already cloned repo that exists
|
||||
locally.
|
||||
"""
|
||||
# Clone the openstack-helm repo and automatically checkout patch 34.
|
||||
ref = 'refs/changes/15/536215/34'
|
||||
repo_url = 'https://github.com/openstack/openstack-helm'
|
||||
git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(git_dir, fetched_ref=ref)
|
||||
|
||||
# Checkout ref='master' now that the repo already exists locally.
|
||||
ref = 'master'
|
||||
git_dir = git.git_handler(git_dir, ref)
|
||||
_validate_git_clone(git_dir, checked_out_ref=ref)
|
||||
_assert_repo_url_was_cloned(mock_log, git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_checkout_refpath_saves_references_locally(
|
||||
mock_log, clean_git_repo):
|
||||
"""Validate that refpath/hexsha branches are created in the local repo
|
||||
following clone of the repo using a refpath during initial checkout.
|
||||
"""
|
||||
# Clone the openstack-helm repo and automatically checkout patch 34.
|
||||
ref = 'refs/changes/15/536215/34'
|
||||
repo_url = 'https://github.com/openstack/openstack-helm'
|
||||
git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(git_dir, fetched_ref=ref)
|
||||
|
||||
# Now checkout patch 34 again to ensure it's still there.
|
||||
ref = 'refs/changes/15/536215/34'
|
||||
git_dir = git.git_handler(git_dir, ref)
|
||||
_validate_git_clone(git_dir, checked_out_ref=ref)
|
||||
_assert_repo_url_was_cloned(mock_log, git_dir)
|
||||
|
||||
# Verify that passing in the hexsha variation of refpath
|
||||
# 'refs/changes/15/536215/34' also works.
|
||||
hexref = '276102a115dac3c0a6e91f9047d8b086bc8d2ff0'
|
||||
git_dir = git.git_handler(git_dir, hexref)
|
||||
_validate_git_clone(git_dir, checked_out_ref=hexref)
|
||||
_assert_repo_url_was_cloned(mock_log, git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_checkout_hexsha_saves_references_locally(
|
||||
mock_log, clean_git_repo):
|
||||
"""Validate that refpath/hexsha branches are created in the local repo
|
||||
following clone of the repo using a hexsha during initial checkout.
|
||||
"""
|
||||
# Clone the openstack-helm repo and automatically checkout patch using
|
||||
# hexsha.
|
||||
# NOTE(felipemonteiro): We have to use the commit ID (hexsha) corresponding
|
||||
# to the last patch as that is what gets pushed to github. In this case,
|
||||
# this corresponds to patch 'refs/changes/15/536215/35'.
|
||||
ref = 'bf126f46b1c175a8038949a87dafb0a716e3b9b6'
|
||||
repo_url = 'https://github.com/openstack/openstack-helm'
|
||||
git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(git_dir, fetched_ref=ref)
|
||||
|
||||
# Now checkout patch using hexsha again to ensure it's still there.
|
||||
ref = 'bf126f46b1c175a8038949a87dafb0a716e3b9b6'
|
||||
git_dir = git.git_handler(git_dir, ref)
|
||||
_validate_git_clone(git_dir, checked_out_ref=ref)
|
||||
_assert_repo_url_was_cloned(mock_log, git_dir)
|
||||
|
||||
# Verify that passing in the refpath variation of hexsha also works.
|
||||
hexref = 'refs/changes/15/536215/35'
|
||||
git_dir = git.git_handler(git_dir, hexref)
|
||||
_validate_git_clone(git_dir, checked_out_ref=hexref)
|
||||
_assert_repo_url_was_cloned(mock_log, git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_existing_directory_checks_out_next_local_ref(
|
||||
mock_log, clean_git_repo):
|
||||
"""Validate Git fetches the newer ref upstream that doesn't exist locally
|
||||
in the cloned repo.
|
||||
"""
|
||||
# Clone the openstack-helm repo and automatically checkout patch 73.
|
||||
ref = 'refs/changes/54/457754/73'
|
||||
repo_url = 'https://github.com/openstack/openstack-helm'
|
||||
git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(git_dir, ref)
|
||||
|
||||
# Attempt to checkout patch 74 which requires a remote fetch even though
|
||||
# the repo has already been cloned.
|
||||
ref = 'refs/changes/54/457754/74'
|
||||
git_dir = git.git_handler(git_dir, ref)
|
||||
_validate_git_clone(git_dir, ref)
|
||||
_assert_repo_url_was_cloned(mock_log, git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_delete_repo_and_reclone(mock_log, clean_git_repo):
|
||||
"""Validate that cloning a repo, then deleting it, then recloning it works.
|
||||
"""
|
||||
# Clone the openstack-helm repo and automatically checkout patch 73.
|
||||
ref = 'refs/changes/54/457754/73'
|
||||
repo_url = 'https://github.com/openstack/openstack-helm'
|
||||
first_git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(first_git_dir, ref)
|
||||
|
||||
# Validate that the repo was cloned.
|
||||
assert mock_log.debug.called
|
||||
mock_log.debug.assert_any_call('Cloning [%s]', repo_url)
|
||||
mock_log.debug.reset_mock()
|
||||
|
||||
# Delete the just-cloned repo.
|
||||
shutil.rmtree(first_git_dir)
|
||||
|
||||
# Verify that checking out the same ref results in a re-clone.
|
||||
second_git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(second_git_dir, ref)
|
||||
|
||||
# Validate that the repo was cloned.
|
||||
assert first_git_dir != second_git_dir
|
||||
assert mock_log.debug.called
|
||||
mock_log.debug.assert_any_call('Cloning [%s]', repo_url)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_clean_dirty_local_repo(mock_log, clean_git_repo):
|
||||
"""Validate that a dirty repo is cleaned before a ref is checked out."""
|
||||
ref = 'refs/changes/54/457754/73'
|
||||
repo_url = 'https://github.com/openstack/openstack-helm'
|
||||
git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(git_dir, ref)
|
||||
|
||||
file_to_rename = os.path.join(git_dir, os.listdir(git_dir)[0])
|
||||
os.rename(file_to_rename, file_to_rename + '-renamed')
|
||||
|
||||
git_dir = git.git_handler(git_dir, ref)
|
||||
_validate_git_clone(git_dir, ref)
|
||||
|
||||
assert mock_log.warning.called
|
||||
mock_log.warning.assert_any_call(
|
||||
'The locally cloned repo_url=%s is dirty. Cleaning up untracked '
|
||||
'files.', git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch.object(git, 'LOG', autospec=True)
|
||||
def test_git_clone_existing_directory_raises_exc_for_invalid_ref(
|
||||
mock_log, clean_git_repo):
|
||||
"""Validate Git throws an error for an invalid ref when trying to checkout
|
||||
a ref for an already-cloned repo.
|
||||
"""
|
||||
# Clone the openstack-helm repo and automatically checkout patch 73.
|
||||
ref = 'refs/changes/54/457754/73'
|
||||
repo_url = 'https://github.com/openstack/openstack-helm'
|
||||
git_dir = git.git_handler(repo_url, ref)
|
||||
_validate_git_clone(git_dir, ref)
|
||||
|
||||
# Attempt to checkout patch 9000 now that the repo already exists locally.
|
||||
ref = 'refs/changes/54/457754/9000'
|
||||
with pytest.raises(exceptions.GitException):
|
||||
git_dir = git.git_handler(git_dir, ref)
|
||||
_assert_repo_url_was_cloned(mock_log, git_dir)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_empty_url_raises_value_error(clean_git_repo):
|
||||
url = ''
|
||||
with pytest.raises(ValueError):
|
||||
git.git_handler(url, ref='master')
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_invalid_url_type_raises_value_error(clean_git_repo):
|
||||
url = 5
|
||||
with pytest.raises(ValueError):
|
||||
git.git_handler(url, ref='master')
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_invalid_local_repo_url_raises_notadirectory_error(
|
||||
clean_git_repo):
|
||||
url = False
|
||||
with pytest.raises(NotADirectoryError):
|
||||
git.git_handler(url, ref='master')
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_invalid_remote_url(clean_git_repo):
|
||||
url = 'https://github.com/dummy/armada'
|
||||
with pytest.raises(exceptions.GitException):
|
||||
git.git_handler(url, ref='master')
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_invalid_remote_url_protocol(clean_git_repo):
|
||||
url = 'ftp://foo.bar'
|
||||
with pytest.raises(ValueError):
|
||||
git.git_handler(url, ref='master')
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
def test_git_clone_fake_proxy(clean_git_repo):
|
||||
url = 'https://github.com/openstack/airship-armada'
|
||||
proxy_url = test_utils.rand_name(
|
||||
'not.a.proxy.that.works.and.never.will', prefix='http://') + ":8080"
|
||||
|
||||
with pytest.raises(exceptions.GitProxyException):
|
||||
git.git_handler(url, ref='master', proxy_server=proxy_url)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch('os.path.exists', return_value=True, autospec=True)
|
||||
def test_git_clone_ssh_auth_method_fails_auth(_, clean_git_repo):
|
||||
fake_user = test_utils.rand_name('fake_user')
|
||||
url = ('ssh://%s@review.openstack.org:29418/openstack/airship-armada' %
|
||||
fake_user)
|
||||
with pytest.raises(exceptions.GitAuthException):
|
||||
git._try_git_clone(
|
||||
url, ref='refs/changes/17/388517/5', auth_key='/home/user/.ssh/')
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not is_connected(), reason='git clone requires network connectivity.')
|
||||
@mock.patch('os.path.exists', return_value=False, autospec=True)
|
||||
def test_git_clone_ssh_auth_method_missing_ssh_key(_, clean_git_repo):
|
||||
fake_user = test_utils.rand_name('fake_user')
|
||||
url = ('ssh://%s@review.openstack.org:29418/openstack/airship-armada' %
|
||||
fake_user)
|
||||
with pytest.raises(exceptions.GitSSHException):
|
||||
git.git_handler(
|
||||
url, ref='refs/changes/17/388517/5', auth_key='/home/user/.ssh/')
|
||||
|
|
@ -47,6 +47,9 @@ def _gen_document(**kwargs):
|
|||
def create_tmp_deployment_files(tmpdir):
|
||||
"""Fixture that creates a temporary directory structure."""
|
||||
sitenames = ['cicd', 'lab']
|
||||
# Used for ensuring the original global context is reset in memory
|
||||
# following each test execution.
|
||||
original_global_context = copy.deepcopy(config.GLOBAL_CONTEXT)
|
||||
|
||||
SITE_TEST_STRUCTURE = {
|
||||
'directories': {
|
||||
|
|
@ -152,8 +155,5 @@ schema: pegleg/SiteDefinition/v1
|
|||
|
||||
yield
|
||||
|
||||
config.GLOBAL_CONTEXT = {
|
||||
'primary_repo': './',
|
||||
'aux_repos': [],
|
||||
'site_path': 'site'
|
||||
}
|
||||
# Restore the global context back to blank slate status.
|
||||
config.GLOBAL_CONTEXT = original_global_context
|
||||
|
|
|
|||
|
|
@ -0,0 +1,39 @@
|
|||
# Copyright 2010 United States Government as represented by the
|
||||
# Administrator of the National Aeronautics and Space Administration.
|
||||
# Copyright 2015 Hewlett-Packard Development Company, L.P.
|
||||
# Copyright 2017 AT&T Intellectual Property.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import random
|
||||
import uuid
|
||||
|
||||
|
||||
def rand_name(name='', prefix='armada'):
|
||||
"""Generate a random name that includes a random number
|
||||
|
||||
:param str name: The name that you want to include
|
||||
:param str prefix: The prefix that you want to include
|
||||
:return: a random name. The format is
|
||||
'<prefix>-<name>-<random number>'.
|
||||
(e.g. 'prefixfoo-namebar-154876201')
|
||||
:rtype: string
|
||||
"""
|
||||
randbits = str(random.randint(1, 0x7fffffff))
|
||||
rand_name = randbits
|
||||
if name:
|
||||
rand_name = name + '-' + rand_name
|
||||
if prefix:
|
||||
rand_name = prefix + '-' + rand_name
|
||||
return rand_name
|
||||
5
tox.ini
5
tox.ini
|
|
@ -33,7 +33,10 @@ commands =
|
|||
whitelist_externals = tox
|
||||
|
||||
[testenv:docs]
|
||||
deps = -r{toxinidir}/doc/requirements.txt
|
||||
basepython = python3
|
||||
deps =
|
||||
-r{toxinidir}/src/bin/pegleg/requirements.txt
|
||||
-r{toxinidir}/doc/requirements.txt
|
||||
commands =
|
||||
rm -rf doc/build
|
||||
sphinx-build -b html doc/source doc/build -n -W -v
|
||||
|
|
|
|||
Loading…
Reference in New Issue