Some tweaks to get closer to Python 3 compat
Convert to use idioms that work for both python 3 and python 2.6+ and ensure that a suitable version of dependencies is included for python 3 compatibility. Update python-jenkins to 0.3.3 as the earliest version that supports python 3 without any known regressions. Add an extra parser check for missing 'command' due to changes in how argparse works under python 3. Where contents should be retained, to access the first element of a dict in both python 2 and 3, 'next(iter(dict.items()))' is used as the standard idiom to replace 'dict.items()[0]' as 'items()' returns an iterator in python 3 which cannot be indexed. Using 'next(iter(..))' allows for both lists and iterators to be passed in without unnecessary conversion of iterators to lists which would be true of 'list(dict.items())[0]'. Alternatively, where further access to the data is not required, 'dict.popitem()' is used. Change-Id: If4b35e2ceee8239379700e22eb79a3eaa04d6f0f
This commit is contained in:
parent
c99cbccb8e
commit
1d7647fa85
|
@ -17,6 +17,7 @@
|
|||
|
||||
import errno
|
||||
import os
|
||||
import operator
|
||||
import sys
|
||||
import hashlib
|
||||
import yaml
|
||||
|
@ -31,8 +32,9 @@ import logging
|
|||
import copy
|
||||
import itertools
|
||||
import fnmatch
|
||||
import six
|
||||
from jenkins_jobs.errors import JenkinsJobsException
|
||||
import local_yaml
|
||||
import jenkins_jobs.local_yaml as local_yaml
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
MAGIC_MANAGE_STRING = "<!-- Managed by Jenkins Job Builder -->"
|
||||
|
@ -82,7 +84,7 @@ def deep_format(obj, paramdict):
|
|||
# limitations on the values in paramdict - the post-format result must
|
||||
# still be valid YAML (so substituting-in a string containing quotes, for
|
||||
# example, is problematic).
|
||||
if isinstance(obj, basestring):
|
||||
if hasattr(obj, 'format'):
|
||||
try:
|
||||
result = re.match('^{obj:(?P<key>\w+)}$', obj)
|
||||
if result is not None:
|
||||
|
@ -142,7 +144,7 @@ class YamlParser(object):
|
|||
" not a {cls}".format(fname=getattr(fp, 'name', fp),
|
||||
cls=type(data)))
|
||||
for item in data:
|
||||
cls, dfn = item.items()[0]
|
||||
cls, dfn = next(iter(item.items()))
|
||||
group = self.data.get(cls, {})
|
||||
if len(item.items()) > 1:
|
||||
n = None
|
||||
|
@ -209,7 +211,7 @@ class YamlParser(object):
|
|||
for jobspec in project.get('jobs', []):
|
||||
if isinstance(jobspec, dict):
|
||||
# Singleton dict containing dict of job-specific params
|
||||
jobname, jobparams = jobspec.items()[0]
|
||||
jobname, jobparams = jobspec.popitem()
|
||||
if not isinstance(jobparams, dict):
|
||||
jobparams = {}
|
||||
else:
|
||||
|
@ -225,7 +227,7 @@ class YamlParser(object):
|
|||
for group_jobspec in group['jobs']:
|
||||
if isinstance(group_jobspec, dict):
|
||||
group_jobname, group_jobparams = \
|
||||
group_jobspec.items()[0]
|
||||
group_jobspec.popitem()
|
||||
if not isinstance(group_jobparams, dict):
|
||||
group_jobparams = {}
|
||||
else:
|
||||
|
@ -275,7 +277,7 @@ class YamlParser(object):
|
|||
expanded_values = {}
|
||||
for (k, v) in values:
|
||||
if isinstance(v, dict):
|
||||
inner_key = v.iterkeys().next()
|
||||
inner_key = next(iter(v))
|
||||
expanded_values[k] = inner_key
|
||||
expanded_values.update(v[inner_key])
|
||||
else:
|
||||
|
@ -295,6 +297,8 @@ class YamlParser(object):
|
|||
# us guarantee a group of parameters will not be added a
|
||||
# second time.
|
||||
uniq = json.dumps(expanded, sort_keys=True)
|
||||
if six.PY3:
|
||||
uniq = uniq.encode('utf-8')
|
||||
checksum = hashlib.md5(uniq).hexdigest()
|
||||
|
||||
# Lookup the checksum
|
||||
|
@ -364,7 +368,7 @@ class ModuleRegistry(object):
|
|||
Mod = entrypoint.load()
|
||||
mod = Mod(self)
|
||||
self.modules.append(mod)
|
||||
self.modules.sort(lambda a, b: cmp(a.sequence, b.sequence))
|
||||
self.modules.sort(key=operator.attrgetter('sequence'))
|
||||
if mod.component_type is not None:
|
||||
self.modules_by_component_type[mod.component_type] = mod
|
||||
|
||||
|
@ -408,7 +412,7 @@ class ModuleRegistry(object):
|
|||
|
||||
if isinstance(component, dict):
|
||||
# The component is a singleton dictionary of name: dict(args)
|
||||
name, component_data = component.items()[0]
|
||||
name, component_data = next(iter(component.items()))
|
||||
if template_data:
|
||||
# Template data contains values that should be interpolated
|
||||
# into the component definition
|
||||
|
@ -610,7 +614,7 @@ class Builder(object):
|
|||
self.load_files(input_fn)
|
||||
self.parser.generateXML(names)
|
||||
|
||||
self.parser.jobs.sort(lambda a, b: cmp(a.name, b.name))
|
||||
self.parser.jobs.sort(key=operator.attrgetter('name'))
|
||||
|
||||
for job in self.parser.jobs:
|
||||
if names and not matches(job.name, names):
|
||||
|
|
|
@ -14,12 +14,11 @@
|
|||
# under the License.
|
||||
|
||||
import argparse
|
||||
import ConfigParser
|
||||
from six.moves import configparser, StringIO
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
import cStringIO
|
||||
|
||||
from jenkins_jobs.builder import Builder
|
||||
from jenkins_jobs.errors import JenkinsJobsException
|
||||
|
@ -95,6 +94,8 @@ def main(argv=None):
|
|||
|
||||
parser = create_parser()
|
||||
options = parser.parse_args(argv)
|
||||
if not options.command:
|
||||
parser.error("Must specify a 'command' to be performed")
|
||||
if (options.log_level is not None):
|
||||
options.log_level = getattr(logging, options.log_level.upper(),
|
||||
logger.getEffectiveLevel())
|
||||
|
@ -115,9 +116,9 @@ def setup_config_settings(options):
|
|||
'jenkins_jobs.ini')
|
||||
if os.path.isfile(localconf):
|
||||
conf = localconf
|
||||
config = ConfigParser.ConfigParser()
|
||||
config = configparser.ConfigParser()
|
||||
## Load default config always
|
||||
config.readfp(cStringIO.StringIO(DEFAULT_CONF))
|
||||
config.readfp(StringIO(DEFAULT_CONF))
|
||||
if os.path.isfile(conf):
|
||||
logger.debug("Reading config from {0}".format(conf))
|
||||
conffp = open(conf, 'r')
|
||||
|
@ -152,11 +153,11 @@ def execute(options, config):
|
|||
# https://bugs.launchpad.net/openstack-ci/+bug/1259631
|
||||
try:
|
||||
user = config.get('jenkins', 'user')
|
||||
except (TypeError, ConfigParser.NoOptionError):
|
||||
except (TypeError, configparser.NoOptionError):
|
||||
user = None
|
||||
try:
|
||||
password = config.get('jenkins', 'password')
|
||||
except (TypeError, ConfigParser.NoOptionError):
|
||||
except (TypeError, configparser.NoOptionError):
|
||||
password = None
|
||||
|
||||
builder = Builder(config.get('jenkins', 'url'),
|
||||
|
|
|
@ -163,7 +163,7 @@ class LocalLoader(OrderedConstructor, yaml.Loader):
|
|||
self.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
|
||||
type(self).construct_yaml_map)
|
||||
|
||||
if isinstance(self.stream, file):
|
||||
if hasattr(self.stream, 'name'):
|
||||
self.search_path.add(os.path.normpath(
|
||||
os.path.dirname(self.stream.name)))
|
||||
self.search_path.add(os.path.normpath(os.path.curdir))
|
||||
|
|
|
@ -227,7 +227,7 @@ def ant(parser, xml_parent, data):
|
|||
if type(data) is str:
|
||||
# Support for short form: -ant: "target"
|
||||
data = {'targets': data}
|
||||
for setting, value in sorted(data.iteritems()):
|
||||
for setting, value in sorted(data.items()):
|
||||
if setting == 'targets':
|
||||
targets = XML.SubElement(ant, 'targets')
|
||||
targets.text = value
|
||||
|
|
|
@ -46,7 +46,7 @@ import xml.etree.ElementTree as XML
|
|||
import jenkins_jobs.modules.base
|
||||
import jenkins_jobs.errors
|
||||
import logging
|
||||
import ConfigParser
|
||||
from six.moves import configparser
|
||||
import sys
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
@ -73,8 +73,8 @@ class HipChat(jenkins_jobs.modules.base.Base):
|
|||
if self.authToken == '':
|
||||
raise jenkins_jobs.errors.JenkinsJobsException(
|
||||
"Hipchat authtoken must not be a blank string")
|
||||
except (ConfigParser.NoSectionError,
|
||||
jenkins_jobs.errors.JenkinsJobsException), e:
|
||||
except (configparser.NoSectionError,
|
||||
jenkins_jobs.errors.JenkinsJobsException) as e:
|
||||
logger.fatal("The configuration file needs a hipchat section" +
|
||||
" containing authtoken:\n{0}".format(e))
|
||||
sys.exit(1)
|
||||
|
|
|
@ -439,7 +439,7 @@ def cloverphp(parser, xml_parent, data):
|
|||
|
||||
metrics = data.get('metric-targets', [])
|
||||
# list of dicts to dict
|
||||
metrics = dict(kv for m in metrics for kv in m.iteritems())
|
||||
metrics = dict(kv for m in metrics for kv in m.items())
|
||||
|
||||
# Populate defaults whenever nothing has been filled by user.
|
||||
for default in default_metrics.keys():
|
||||
|
@ -889,7 +889,7 @@ def xunit(parser, xml_parent, data):
|
|||
supported_types = []
|
||||
|
||||
for configured_type in data['types']:
|
||||
type_name = configured_type.keys()[0]
|
||||
type_name = next(iter(configured_type.keys()))
|
||||
if type_name not in implemented_types:
|
||||
logger.warn("Requested xUnit type '%s' is not yet supported",
|
||||
type_name)
|
||||
|
@ -900,7 +900,7 @@ def xunit(parser, xml_parent, data):
|
|||
# Generate XML for each of the supported framework types
|
||||
xmltypes = XML.SubElement(xunit, 'types')
|
||||
for supported_type in supported_types:
|
||||
framework_name = supported_type.keys()[0]
|
||||
framework_name = next(iter(supported_type.keys()))
|
||||
xmlframework = XML.SubElement(xmltypes,
|
||||
types_to_plugin_types[framework_name])
|
||||
|
||||
|
@ -924,9 +924,10 @@ def xunit(parser, xml_parent, data):
|
|||
"Unrecognized threshold, should be 'failed' or 'skipped'")
|
||||
continue
|
||||
elname = "org.jenkinsci.plugins.xunit.threshold.%sThreshold" \
|
||||
% t.keys()[0].title()
|
||||
% next(iter(t.keys())).title()
|
||||
el = XML.SubElement(xmlthresholds, elname)
|
||||
for threshold_name, threshold_value in t.values()[0].items():
|
||||
for threshold_name, threshold_value in \
|
||||
next(iter(t.values())).items():
|
||||
# Normalize and craft the element name for this threshold
|
||||
elname = "%sThreshold" % threshold_name.lower().replace(
|
||||
'new', 'New')
|
||||
|
@ -3509,7 +3510,8 @@ def ruby_metrics(parser, xml_parent, data):
|
|||
XML.SubElement(el, 'metric').text = 'TOTAL_COVERAGE'
|
||||
else:
|
||||
XML.SubElement(el, 'metric').text = 'CODE_COVERAGE'
|
||||
for threshold_name, threshold_value in t.values()[0].items():
|
||||
for threshold_name, threshold_value in \
|
||||
next(iter(t.values())).items():
|
||||
elname = threshold_name.lower()
|
||||
XML.SubElement(el, elname).text = str(threshold_value)
|
||||
else:
|
||||
|
|
|
@ -161,9 +161,9 @@ remoteName/\*')
|
|||
data['remotes'] = [{data.get('name', 'origin'): data.copy()}]
|
||||
for remoteData in data['remotes']:
|
||||
huser = XML.SubElement(user, 'hudson.plugins.git.UserRemoteConfig')
|
||||
remoteName = remoteData.keys()[0]
|
||||
remoteName = next(iter(remoteData.keys()))
|
||||
XML.SubElement(huser, 'name').text = remoteName
|
||||
remoteParams = remoteData.values()[0]
|
||||
remoteParams = next(iter(remoteData.values()))
|
||||
if 'refspec' in remoteParams:
|
||||
refspec = remoteParams['refspec']
|
||||
else:
|
||||
|
@ -368,7 +368,7 @@ def store(parser, xml_parent, data):
|
|||
pundles = XML.SubElement(scm, 'pundles')
|
||||
for pundle_spec in pundle_specs:
|
||||
pundle = XML.SubElement(pundles, '{0}.PundleSpec'.format(namespace))
|
||||
pundle_type = pundle_spec.keys()[0]
|
||||
pundle_type = next(iter(pundle_spec))
|
||||
pundle_name = pundle_spec[pundle_type]
|
||||
if pundle_type not in valid_pundle_types:
|
||||
raise JenkinsJobsException(
|
||||
|
@ -507,9 +507,9 @@ def tfs(parser, xml_parent, data):
|
|||
server.
|
||||
:arg str login: The user name that is registered on the server. The user
|
||||
name must contain the name and the domain name. Entered as
|
||||
domain\\\user or user\@domain (optional).
|
||||
domain\\\\user or user\@domain (optional).
|
||||
**NOTE**: You must enter in at least two slashes for the
|
||||
domain\\\user format in JJB YAML. It will be rendered normally.
|
||||
domain\\\\user format in JJB YAML. It will be rendered normally.
|
||||
:arg str use-update: If true, Hudson will not delete the workspace at end
|
||||
of each build. This causes the artifacts from the previous build to
|
||||
remain when a new build starts. (default true)
|
||||
|
|
|
@ -91,7 +91,7 @@ def build_gerrit_triggers(xml_parent, data):
|
|||
'hudsontrigger.events'
|
||||
|
||||
trigger_on_events = XML.SubElement(xml_parent, 'triggerOnEvents')
|
||||
for config_key, tag_name in available_simple_triggers.iteritems():
|
||||
for config_key, tag_name in available_simple_triggers.items():
|
||||
if data.get(config_key, False):
|
||||
XML.SubElement(trigger_on_events,
|
||||
'%s.%s' % (tag_namespace, tag_name))
|
||||
|
@ -453,7 +453,7 @@ def pollurl(parser, xml_parent, data):
|
|||
str(bool(check_content)).lower()
|
||||
content_types = XML.SubElement(entry, 'contentTypes')
|
||||
for entry in check_content:
|
||||
type_name = entry.keys()[0]
|
||||
type_name = next(iter(entry.keys()))
|
||||
if type_name not in valid_content_types:
|
||||
raise JenkinsJobsException('check-content must be one of : %s'
|
||||
% ', '.join(valid_content_types.
|
||||
|
|
|
@ -35,6 +35,8 @@ The above URL is the default.
|
|||
http://ci.openstack.org/zuul/launchers.html#zuul-parameters
|
||||
"""
|
||||
|
||||
import itertools
|
||||
|
||||
|
||||
def zuul():
|
||||
"""yaml: zuul
|
||||
|
@ -152,7 +154,7 @@ class Zuul(jenkins_jobs.modules.base.Base):
|
|||
|
||||
def handle_data(self, parser):
|
||||
changed = False
|
||||
jobs = (parser.data.get('job', {}).values() +
|
||||
jobs = itertools.chain(parser.data.get('job', {}).values(),
|
||||
parser.data.get('job-template', {}).values())
|
||||
for job in jobs:
|
||||
triggers = job.get('triggers')
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
argparse
|
||||
six>=1.5.2
|
||||
PyYAML
|
||||
python-jenkins
|
||||
python-jenkins>=0.3.3
|
||||
pbr>=0.8.2,<1.0
|
||||
|
|
|
@ -26,7 +26,7 @@ import json
|
|||
import operator
|
||||
import testtools
|
||||
import xml.etree.ElementTree as XML
|
||||
from ConfigParser import ConfigParser
|
||||
from six.moves import configparser
|
||||
import jenkins_jobs.local_yaml as yaml
|
||||
from jenkins_jobs.builder import XmlJob, YamlParser, ModuleRegistry
|
||||
from jenkins_jobs.modules import (project_flow,
|
||||
|
@ -86,7 +86,7 @@ class BaseTestCase(object):
|
|||
|
||||
def _read_yaml_content(self):
|
||||
yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
|
||||
with file(yaml_filepath, 'r') as yaml_file:
|
||||
with open(yaml_filepath, 'r') as yaml_file:
|
||||
yaml_content = yaml.load(yaml_file)
|
||||
return yaml_content
|
||||
|
||||
|
@ -118,8 +118,7 @@ class BaseTestCase(object):
|
|||
pub.gen_xml(parser, xml_project, yaml_content)
|
||||
|
||||
# Prettify generated XML
|
||||
pretty_xml = unicode(XmlJob(xml_project, 'fixturejob').output(),
|
||||
'utf-8')
|
||||
pretty_xml = XmlJob(xml_project, 'fixturejob').output().decode('utf-8')
|
||||
|
||||
self.assertThat(
|
||||
pretty_xml,
|
||||
|
@ -137,7 +136,7 @@ class SingleJobTestCase(BaseTestCase):
|
|||
yaml_filepath = os.path.join(self.fixtures_path, self.in_filename)
|
||||
|
||||
if self.conf_filename:
|
||||
config = ConfigParser()
|
||||
config = configparser.ConfigParser()
|
||||
conf_filepath = os.path.join(self.fixtures_path,
|
||||
self.conf_filename)
|
||||
config.readfp(open(conf_filepath))
|
||||
|
@ -152,8 +151,8 @@ class SingleJobTestCase(BaseTestCase):
|
|||
parser.jobs.sort(key=operator.attrgetter('name'))
|
||||
|
||||
# Prettify generated XML
|
||||
pretty_xml = unicode("\n".join(job.output() for job in parser.jobs),
|
||||
'utf-8')
|
||||
pretty_xml = u"\n".join(job.output().decode('utf-8')
|
||||
for job in parser.jobs)
|
||||
|
||||
self.assertThat(
|
||||
pretty_xml,
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
import os
|
||||
import ConfigParser
|
||||
import cStringIO
|
||||
from six.moves import configparser, StringIO
|
||||
import io
|
||||
import codecs
|
||||
import mock
|
||||
import testtools
|
||||
|
@ -22,15 +22,15 @@ class CmdTests(testtools.TestCase):
|
|||
User passes no args, should fail with SystemExit
|
||||
"""
|
||||
with mock.patch('sys.stderr'):
|
||||
self.assertRaises(SystemExit, self.parser.parse_args, [])
|
||||
self.assertRaises(SystemExit, cmd.main, [])
|
||||
|
||||
def test_non_existing_config_dir(self):
|
||||
"""
|
||||
Run test mode and pass a non-existing configuration directory
|
||||
"""
|
||||
args = self.parser.parse_args(['test', 'foo'])
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.readfp(cStringIO.StringIO(cmd.DEFAULT_CONF))
|
||||
config = configparser.ConfigParser()
|
||||
config.readfp(StringIO(cmd.DEFAULT_CONF))
|
||||
self.assertRaises(IOError, cmd.execute, args, config)
|
||||
|
||||
def test_non_existing_config_file(self):
|
||||
|
@ -38,8 +38,8 @@ class CmdTests(testtools.TestCase):
|
|||
Run test mode and pass a non-existing configuration file
|
||||
"""
|
||||
args = self.parser.parse_args(['test', 'non-existing.yaml'])
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.readfp(cStringIO.StringIO(cmd.DEFAULT_CONF))
|
||||
config = configparser.ConfigParser()
|
||||
config.readfp(StringIO(cmd.DEFAULT_CONF))
|
||||
self.assertRaises(IOError, cmd.execute, args, config)
|
||||
|
||||
def test_non_existing_job(self):
|
||||
|
@ -52,8 +52,8 @@ class CmdTests(testtools.TestCase):
|
|||
'cmd-001.yaml'),
|
||||
'invalid'])
|
||||
args.output_dir = mock.MagicMock()
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.readfp(cStringIO.StringIO(cmd.DEFAULT_CONF))
|
||||
config = configparser.ConfigParser()
|
||||
config.readfp(StringIO(cmd.DEFAULT_CONF))
|
||||
cmd.execute(args, config) # probably better to fail here
|
||||
|
||||
def test_valid_job(self):
|
||||
|
@ -65,8 +65,8 @@ class CmdTests(testtools.TestCase):
|
|||
'cmd-001.yaml'),
|
||||
'foo-job'])
|
||||
args.output_dir = mock.MagicMock()
|
||||
config = ConfigParser.ConfigParser()
|
||||
config.readfp(cStringIO.StringIO(cmd.DEFAULT_CONF))
|
||||
config = configparser.ConfigParser()
|
||||
config.readfp(StringIO(cmd.DEFAULT_CONF))
|
||||
cmd.execute(args, config) # probably better to fail here
|
||||
|
||||
def test_console_output(self):
|
||||
|
@ -74,15 +74,14 @@ class CmdTests(testtools.TestCase):
|
|||
Run test mode and verify that resulting XML gets sent to the console.
|
||||
"""
|
||||
|
||||
console_out = cStringIO.StringIO()
|
||||
console_out = io.BytesIO()
|
||||
with mock.patch('sys.stdout', console_out):
|
||||
cmd.main(['test', os.path.join(self.fixtures_path,
|
||||
'cmd-001.yaml')])
|
||||
xml_content = u"%s" % codecs.open(os.path.join(self.fixtures_path,
|
||||
xml_content = codecs.open(os.path.join(self.fixtures_path,
|
||||
'cmd-001.xml'),
|
||||
'r',
|
||||
'utf-8').read()
|
||||
self.assertEqual(console_out.getvalue(), xml_content)
|
||||
'r', 'utf-8').read()
|
||||
self.assertEqual(console_out.getvalue().decode('utf-8'), xml_content)
|
||||
|
||||
def test_config_with_test(self):
|
||||
"""
|
||||
|
|
Loading…
Reference in New Issue