wrap subprocess to capture output in tests
Provide some wrappers around subprocess functions to capture stderr and stdout and divert it to the logs so that the test output is less cluttered with the output of various git commands. Change-Id: If36ef013aca498e3a0a9cc3a2b78d666775439ab Signed-off-by: Doug Hellmann <doug@doughellmann.com>
This commit is contained in:
Doug Hellmann
Sean McGinnis
parent
aa8ee5a83a
commit
bfd96f4fc0
|
|
@ -18,6 +18,7 @@ import os.path
|
|||
import subprocess
|
||||
|
||||
from openstack_releases import links
|
||||
from openstack_releases import processutils
|
||||
|
||||
# Disable warnings about insecure connections.
|
||||
from requests.packages import urllib3
|
||||
|
|
@ -31,7 +32,7 @@ CGIT_TAG_TEMPLATE = 'http://git.openstack.org/cgit/%s/tag/?h=%s'
|
|||
|
||||
def find_modified_deliverable_files():
|
||||
"Return a list of files modified by the most recent commit."
|
||||
results = subprocess.check_output(
|
||||
results = processutils.check_output(
|
||||
['git', 'diff', '--name-only', '--pretty=format:', 'HEAD^']
|
||||
).decode('utf-8')
|
||||
filenames = [
|
||||
|
|
@ -50,11 +51,11 @@ def commit_exists(workdir, repo, ref):
|
|||
|
||||
"""
|
||||
try:
|
||||
subprocess.check_output(
|
||||
processutils.check_output(
|
||||
['git', 'show', ref],
|
||||
cwd=os.path.join(workdir, repo),
|
||||
).decode('utf-8')
|
||||
except subprocess.CalledProcessError as err:
|
||||
except processutils.CalledProcessError as err:
|
||||
LOG.error('Could not find {}: {}'.format(ref, err))
|
||||
return False
|
||||
return True
|
||||
|
|
@ -80,14 +81,14 @@ def ensure_basic_git_config(workdir, repo, settings):
|
|||
for key, value in settings.items():
|
||||
LOG.info('looking for git config {}'.format(key))
|
||||
try:
|
||||
existing = subprocess.check_output(
|
||||
existing = processutils.check_output(
|
||||
['git', 'config', '--get', key],
|
||||
cwd=dest,
|
||||
).decode('utf-8').strip()
|
||||
LOG.info('using existing setting of {}: {!r}'.format(key, existing))
|
||||
except subprocess.CalledProcessError:
|
||||
except processutils.CalledProcessError:
|
||||
LOG.info('updating setting of {} to {!r}'.format(key, value))
|
||||
subprocess.check_call(
|
||||
processutils.check_call(
|
||||
['git', 'config', key, value],
|
||||
cwd=dest,
|
||||
)
|
||||
|
|
@ -106,8 +107,7 @@ def clone_repo(workdir, repo, ref=None, branch=None):
|
|||
if branch:
|
||||
cmd.extend(['--branch', branch])
|
||||
cmd.append(repo)
|
||||
LOG.info(' '.join(cmd))
|
||||
subprocess.check_call(cmd)
|
||||
processutils.check_call(cmd)
|
||||
dest = os.path.join(workdir, repo)
|
||||
return dest
|
||||
|
||||
|
|
@ -131,13 +131,13 @@ def sha_for_tag(workdir, repo, version):
|
|||
"""
|
||||
# git log 2.3.11 -n 1 --pretty=format:%H
|
||||
try:
|
||||
actual_sha = subprocess.check_output(
|
||||
actual_sha = processutils.check_output(
|
||||
['git', 'log', str(version), '-n', '1', '--pretty=format:%H'],
|
||||
cwd=os.path.join(workdir, repo),
|
||||
stderr=subprocess.STDOUT,
|
||||
).decode('utf-8')
|
||||
actual_sha = actual_sha.strip()
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.info('ERROR getting SHA for tag %r: %s [%s]',
|
||||
version, e, e.output.strip())
|
||||
actual_sha = ''
|
||||
|
|
@ -158,13 +158,13 @@ def stable_branch_exists(workdir, repo, series):
|
|||
remote_match = 'remotes/origin/stable/%s' % series
|
||||
try:
|
||||
containing_branches = _filter_branches(
|
||||
subprocess.check_output(
|
||||
processutils.check_output(
|
||||
['git', 'branch', '-a'],
|
||||
cwd=os.path.join(workdir, repo),
|
||||
).decode('utf-8')
|
||||
)
|
||||
return (remote_match in containing_branches)
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.error('failed checking for branch: %s [%s]', e, e.output.strip())
|
||||
return False
|
||||
|
||||
|
|
@ -186,7 +186,7 @@ def check_branch_sha(workdir, repo, series, sha):
|
|||
remote_match = 'remotes/origin/stable/%s' % series
|
||||
try:
|
||||
containing_branches = _filter_branches(
|
||||
subprocess.check_output(
|
||||
processutils.check_output(
|
||||
['git', 'branch', '-a', '--contains', sha],
|
||||
cwd=os.path.join(workdir, repo),
|
||||
).decode('utf-8')
|
||||
|
|
@ -203,7 +203,7 @@ def check_branch_sha(workdir, repo, series, sha):
|
|||
# that series. Allow the release, as long as it is on the
|
||||
# master branch.
|
||||
all_branches = _filter_branches(
|
||||
subprocess.check_output(
|
||||
processutils.check_output(
|
||||
['git', 'branch', '-a'],
|
||||
cwd=os.path.join(workdir, repo),
|
||||
).decode('utf-8')
|
||||
|
|
@ -223,7 +223,7 @@ def check_branch_sha(workdir, repo, series, sha):
|
|||
LOG.debug('did not find SHA on %s or master or origin/master',
|
||||
remote_match)
|
||||
return False
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.error('failed checking SHA on branch: %s [%s]' % (e, e.output.strip()))
|
||||
return False
|
||||
|
||||
|
|
@ -231,13 +231,13 @@ def check_branch_sha(workdir, repo, series, sha):
|
|||
def check_ancestry(workdir, repo, old_version, sha):
|
||||
"Check if the SHA is in the ancestry of the previous version."
|
||||
try:
|
||||
ancestors = subprocess.check_output(
|
||||
ancestors = processutils.check_output(
|
||||
['git', 'log', '--oneline', '--ancestry-path',
|
||||
'%s..%s' % (old_version, sha)],
|
||||
cwd=os.path.join(workdir, repo),
|
||||
).decode('utf-8').strip()
|
||||
return bool(ancestors)
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.error('failed checking ancestry: %s [%s]' % (e, e.output.strip()))
|
||||
return False
|
||||
|
||||
|
|
@ -247,12 +247,12 @@ def get_latest_tag(workdir, repo, sha=None):
|
|||
if sha is not None:
|
||||
cmd.append(sha)
|
||||
try:
|
||||
return subprocess.check_output(
|
||||
return processutils.check_output(
|
||||
cmd,
|
||||
cwd=os.path.join(workdir, repo),
|
||||
stderr=subprocess.STDOUT,
|
||||
).decode('utf-8').strip()
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.warning('failed to retrieve latest tag: %s [%s]',
|
||||
e, e.output.strip())
|
||||
return None
|
||||
|
|
@ -262,12 +262,12 @@ def add_tag(workdir, repo, tag, sha):
|
|||
cmd = ['git', 'tag', '-m', 'temporary tag', tag, sha]
|
||||
try:
|
||||
LOG.info(' '.join(cmd))
|
||||
return subprocess.check_output(
|
||||
return processutils.check_output(
|
||||
cmd,
|
||||
cwd=os.path.join(workdir, repo),
|
||||
stderr=subprocess.STDOUT,
|
||||
).decode('utf-8').strip()
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.warning('failed to add tag: %s [%s]',
|
||||
e, e.output.strip())
|
||||
return None
|
||||
|
|
@ -275,7 +275,7 @@ def add_tag(workdir, repo, tag, sha):
|
|||
|
||||
def get_branches(workdir, repo):
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
output = processutils.check_output(
|
||||
['git', 'branch', '-a'],
|
||||
cwd=os.path.join(workdir, repo),
|
||||
stderr=subprocess.STDOUT,
|
||||
|
|
@ -300,7 +300,7 @@ def get_branches(workdir, repo):
|
|||
continue
|
||||
results.append(branch)
|
||||
return results
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.error('failed to retrieve list of branches: %s [%s]',
|
||||
e, e.output.strip())
|
||||
return []
|
||||
|
|
@ -308,7 +308,7 @@ def get_branches(workdir, repo):
|
|||
|
||||
def branches_containing(workdir, repo, ref):
|
||||
try:
|
||||
output = subprocess.check_output(
|
||||
output = processutils.check_output(
|
||||
['git', 'branch', '-r', '--contains', ref],
|
||||
cwd=os.path.join(workdir, repo),
|
||||
stderr=subprocess.STDOUT,
|
||||
|
|
@ -319,7 +319,7 @@ def branches_containing(workdir, repo, ref):
|
|||
for line in output.splitlines():
|
||||
results.append(line.strip())
|
||||
return results
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.error('failed to retrieve list of branches containing %s: %s [%s]',
|
||||
ref, e, e.output.strip())
|
||||
return []
|
||||
|
|
@ -339,12 +339,12 @@ def get_branch_base(workdir, repo, branch):
|
|||
'master',
|
||||
]
|
||||
try:
|
||||
parents = subprocess.check_output(
|
||||
parents = processutils.check_output(
|
||||
cmd,
|
||||
cwd=os.path.join(workdir, repo),
|
||||
stderr=subprocess.STDOUT,
|
||||
).decode('utf-8').strip()
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.warning('failed to retrieve branch base: %s [%s]',
|
||||
e, e.output.strip())
|
||||
return None
|
||||
|
|
@ -356,12 +356,12 @@ def get_branch_base(workdir, repo, branch):
|
|||
'{}^^!'.format(parent),
|
||||
]
|
||||
try:
|
||||
return subprocess.check_output(
|
||||
return processutils.check_output(
|
||||
cmd,
|
||||
cwd=os.path.join(workdir, repo),
|
||||
stderr=subprocess.STDOUT,
|
||||
).decode('utf-8').strip()
|
||||
except subprocess.CalledProcessError as e:
|
||||
except processutils.CalledProcessError as e:
|
||||
LOG.warning('failed to retrieve branch base: %s [%s]',
|
||||
e, e.output.strip())
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -0,0 +1,82 @@
|
|||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import logging
|
||||
import subprocess
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
from subprocess import CalledProcessError # noqa
|
||||
|
||||
|
||||
def _multi_line_log(level, msg):
|
||||
for line in msg.splitlines():
|
||||
LOG.log(level, line)
|
||||
|
||||
|
||||
def check_call(*popenargs, timeout=None, **kwargs):
|
||||
# A variation of subprocess.check_call that captures and then
|
||||
# logs the output of the command which makes it easier for tests
|
||||
# to capture it.
|
||||
|
||||
kwargs['stdout'] = subprocess.PIPE
|
||||
kwargs['stderr'] = subprocess.STDOUT
|
||||
|
||||
cmd = kwargs.get("args")
|
||||
if cmd is None:
|
||||
cmd = popenargs[0]
|
||||
LOG.info('$ {}'.format(' '.join(cmd)))
|
||||
|
||||
completed = subprocess.run(*popenargs, **kwargs)
|
||||
_multi_line_log(logging.INFO, completed.stdout.decode('utf-8'))
|
||||
|
||||
if completed.returncode:
|
||||
raise subprocess.CalledProcessError(completed.returncode, cmd)
|
||||
return 0
|
||||
|
||||
|
||||
def check_output(*popenargs, timeout=None, **kwargs):
|
||||
# A variation of subprocess.check_output that captures stderr and
|
||||
# logs it instead of letting it go to the console directly to make
|
||||
# it easier for tests to capture it.
|
||||
|
||||
# NOTE(dhellmann): copied from subprocess.py vv
|
||||
if 'stdout' in kwargs:
|
||||
raise ValueError('stdout argument not allowed, it will be overridden.')
|
||||
if 'input' in kwargs and kwargs['input'] is None:
|
||||
# Explicitly passing input=None was previously equivalent to passing an
|
||||
# empty string. That is maintained here for backwards compatibility.
|
||||
kwargs['input'] = '' if kwargs.get('universal_newlines', False) else b''
|
||||
# NOTE(dhellmann): end copied from subprocess.py ^^
|
||||
|
||||
cmd = kwargs.get("args")
|
||||
if cmd is None:
|
||||
cmd = popenargs[0]
|
||||
LOG.info('$ {}'.format(' '.join(cmd)))
|
||||
|
||||
if 'stderr' not in kwargs:
|
||||
kwargs['stderr'] = subprocess.PIPE
|
||||
|
||||
completed = subprocess.run(*popenargs,
|
||||
stdout=subprocess.PIPE,
|
||||
timeout=timeout,
|
||||
check=True,
|
||||
**kwargs)
|
||||
|
||||
if completed.stderr:
|
||||
_multi_line_log(logging.WARNING, completed.stderr.decode('utf-8'))
|
||||
|
||||
return completed.stdout
|
||||
|
|
@ -15,10 +15,11 @@
|
|||
import logging
|
||||
import os
|
||||
import os.path
|
||||
import subprocess
|
||||
|
||||
import requests
|
||||
|
||||
from openstack_releases import processutils
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -33,7 +34,7 @@ def get_sdist_name(workdir, repo):
|
|||
# Use tox to set up a virtualenv so we can install the
|
||||
# dependencies for the package. This only seems to be
|
||||
# necessary for pbr, but...
|
||||
subprocess.check_output(
|
||||
processutils.check_output(
|
||||
['tox', '-e', 'venv', '--notest'],
|
||||
cwd=dest,
|
||||
)
|
||||
|
|
@ -44,10 +45,10 @@ def get_sdist_name(workdir, repo):
|
|||
# Run it once and discard the result to ensure any setup_requires
|
||||
# dependencies are installed.
|
||||
cmd = [python, 'setup.py', '--name']
|
||||
subprocess.check_output(cmd, cwd=dest)
|
||||
processutils.check_output(cmd, cwd=dest)
|
||||
# Run it again to get a clean version of the name.
|
||||
print('Running: %s in %s' % (' '.join(cmd), dest))
|
||||
out = subprocess.check_output(cmd, cwd=dest).decode('utf-8')
|
||||
out = processutils.check_output(cmd, cwd=dest).decode('utf-8')
|
||||
print('Results: %s' % (out,))
|
||||
name = out.splitlines()[-1].strip()
|
||||
return name
|
||||
|
|
|
|||
|
|
@ -17,11 +17,11 @@
|
|||
|
||||
import logging
|
||||
import os.path
|
||||
import subprocess
|
||||
|
||||
import pkg_resources
|
||||
|
||||
from openstack_releases import gitutils
|
||||
from openstack_releases import processutils
|
||||
from openstack_releases import pythonutils
|
||||
from openstack_releases import versionutils
|
||||
|
||||
|
|
@ -89,7 +89,7 @@ def get_min_specifier(specifier_set):
|
|||
def get_requirements_at_ref(workdir, repo, ref):
|
||||
"Check out the repo at the ref and load the list of requirements."
|
||||
dest = gitutils.clone_repo(workdir, repo, ref=ref)
|
||||
subprocess.check_call(['python', 'setup.py', 'sdist'], cwd=dest)
|
||||
processutils.check_call(['python', 'setup.py', 'sdist'], cwd=dest)
|
||||
sdist_name = pythonutils.get_sdist_name(workdir, repo)
|
||||
requirements_filename = os.path.join(
|
||||
dest, sdist_name + '.egg-info', 'requires.txt',
|
||||
|
|
|
|||
Loading…
Reference in New Issue