diff --git a/.zuul.yaml b/.zuul.yaml index e5755d68..cdde30a4 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1,6 +1,8 @@ --- - project: templates: + - openstack-python36-jobs + - openstack-python37-jobs - publish-tox-docs-infra check: jobs: diff --git a/elastic_recheck/bot.py b/elastic_recheck/bot.py index ed66beae..16b34568 100755 --- a/elastic_recheck/bot.py +++ b/elastic_recheck/bot.py @@ -250,13 +250,13 @@ class ChannelConfig(object): data['#' + key] = data.pop(key) self.channels = data.keys() self.events = {} - for channel, val in self.data.iteritems(): + for channel, val in self.data.items(): for event in val['events']: event_set = self.events.get(event, set()) event_set.add(channel) self.events[event] = event_set self.projects = {} - for channel, val in self.data.iteritems(): + for channel, val in self.data.items(): for project in val['projects']: project_set = self.projects.get(project, set()) project_set.add(channel) diff --git a/elastic_recheck/cmd/check_success.py b/elastic_recheck/cmd/check_success.py index a47a4a1a..73182bdb 100755 --- a/elastic_recheck/cmd/check_success.py +++ b/elastic_recheck/cmd/check_success.py @@ -98,12 +98,12 @@ def classifying_rate(fails, data): print("Classification percentage: %2.2f%%" % ((float(count) / float(total)) * 100.0)) sort = sorted( - bad_jobs.iteritems(), + bad_jobs.items(), key=operator.itemgetter(1), reverse=True) print("Job fails with most unclassified errors") for s in sort: - print " %3s : %s" % (s[1], s[0]) + print(" %3s : %s" % (s[1], s[0])) def _status_count(results): @@ -183,10 +183,10 @@ def collect_metrics(classifier, fails): def print_metrics(data, with_lp=False): - print "Elastic recheck known issues" - print + print("Elastic recheck known issues") + print() - sorted_data = sorted(data.iteritems(), + sorted_data = sorted(data.items(), key=lambda x: -x[1]['fails']) for d in sorted_data: bug = d[0] @@ -195,13 +195,13 @@ def print_metrics(data, with_lp=False): % (bug, data['query'].rstrip())) if with_lp: get_launchpad_bug(d[0]) - print "Hits" + print("Hits") for s in data['hits']: - print " %s: %s" % (s, data['hits'][s]) - print "Percentage of Gate Queue Job failures triggered by this bug" + print(" %s: %s" % (s, data['hits'][s])) + print("Percentage of Gate Queue Job failures triggered by this bug") for s in data['percentages']: - print " %s: %2.2f%%" % (s, data['percentages'][s]) - print + print(" %s: %2.2f%%" % (s, data['percentages'][s])) + print() def get_launchpad_bug(bug): @@ -209,11 +209,11 @@ def get_launchpad_bug(bug): 'production', LPCACHEDIR) lp_bug = lp.bugs[bug] - print "Title: %s" % lp_bug.title + print("Title: %s" % lp_bug.title) targets = map(lambda x: (x.bug_target_name, x.status), lp_bug.bug_tasks) - print "Project: Status" + print("Project: Status") for target, status in targets: - print " %s: %s" % (target, status) + print(" %s: %s" % (target, status)) def main(): diff --git a/elastic_recheck/cmd/query.py b/elastic_recheck/cmd/query.py index 9d8e5a65..f212ef4f 100755 --- a/elastic_recheck/cmd/query.py +++ b/elastic_recheck/cmd/query.py @@ -42,7 +42,7 @@ IGNORED_ATTRIBUTES = [ def analyze_attributes(attributes): analysis = {} - for attribute, values in attributes.iteritems(): + for attribute, values in attributes.items(): if attribute[0] == '@' or attribute == 'message': # skip meta attributes and raw messages continue @@ -50,7 +50,7 @@ def analyze_attributes(attributes): analysis[attribute] = [] total_hits = sum(values.values()) - for value_hash, hits in values.iteritems(): + for value_hash, hits in values.items(): value = json.loads(value_hash) analysis[attribute].append((100.0 * hits / total_hits, value)) @@ -78,13 +78,13 @@ def query(query_file_name, config=None, days=DEFAULT_NUMBER_OF_DAYS, attributes = {} for hit in r.hits['hits']: - for key, value in hit['_source'].iteritems(): + for key, value in hit['_source'].items(): value_hash = json.dumps(value) attributes.setdefault(key, {}).setdefault(value_hash, 0) attributes[key][value_hash] += 1 analysis = analyze_attributes(attributes) - for attribute, results in sorted(analysis.iteritems()): + for attribute, results in sorted(analysis.items()): if not verbose and attribute in IGNORED_ATTRIBUTES: # skip less-than-useful attributes to reduce noise in the report continue @@ -92,7 +92,7 @@ def query(query_file_name, config=None, days=DEFAULT_NUMBER_OF_DAYS, print(attribute) for percentage, value in itertools.islice(results, quantity): if isinstance(value, list): - value = ' '.join(unicode(x) for x in value) + value = ' '.join(str(x) for x in value) print(' %d%% %s' % (percentage, value)) diff --git a/elastic_recheck/cmd/uncategorized_fails.py b/elastic_recheck/cmd/uncategorized_fails.py index 40adbb82..07d0d671 100755 --- a/elastic_recheck/cmd/uncategorized_fails.py +++ b/elastic_recheck/cmd/uncategorized_fails.py @@ -176,7 +176,7 @@ def classifying_rate(fails, data, engine, classifier, ls_url): classification rate. For every failure in the gate queue did we find a match for it. """ - found_fails = {k: False for (k, v) in fails.iteritems()} + found_fails = {k: False for (k, v) in fails.items()} for bugnum in data: bug = data[bugnum] @@ -237,7 +237,7 @@ def classifying_rate(fails, data, engine, classifier, ls_url): (float(bad_jobs[job]) / float(total_job_failures[job])) * 100.0) sort = sorted( - bad_jobs.iteritems(), + bad_jobs.items(), key=operator.itemgetter(1), reverse=True) diff --git a/elastic_recheck/config.py b/elastic_recheck/config.py index baff1fec..edb8c014 100644 --- a/elastic_recheck/config.py +++ b/elastic_recheck/config.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -import ConfigParser +from six.moves import configparser import os import re @@ -26,6 +26,7 @@ JOBS_RE = 'dsvm' CI_USERNAME = 'jenkins' GERRIT_QUERY_FILE = 'queries' +GERRIT_HOST = 'review.openstack.org' PID_FN = '/var/run/elastic-recheck/elastic-recheck.pid' @@ -101,7 +102,7 @@ class Config(object): if config_obj: config = config_obj else: - config = ConfigParser.ConfigParser( + config = configparser.ConfigParser( {'es_url': ES_URL, 'ls_url': LS_URL, 'db_uri': DB_URI, @@ -110,7 +111,7 @@ class Config(object): 'jobs_re': JOBS_RE, 'pidfile': PID_FN, 'index_format': DEFAULT_INDEX_FORMAT, - 'query_file': GERRIT_QUERY_FILE, + 'query_file': GERRIT_QUERY_FILE } ) config.read(config_file) @@ -129,8 +130,15 @@ class Config(object): if config.has_section('gerrit'): self.gerrit_user = config.get('gerrit', 'user') self.gerrit_query_file = config.get('gerrit', 'query_file') - self.gerrit_host = config.get('gerrit', 'host', - 'review.openstack.org') + # workaround for python api change https://docs.python.org/3/library/configparser.html#fallback-values + try: + self.gerrit_host = config.get('gerrit', + 'host', + fallback=GERRIT_HOST) + except TypeError: + self.gerrit_host = config.get('gerrit', + 'host', + GERRIT_HOST) self.gerrit_host_key = config.get('gerrit', 'key') if config.has_section('ircbot'): diff --git a/elastic_recheck/elasticRecheck.py b/elastic_recheck/elasticRecheck.py index e6ec1a7a..f691d6fa 100644 --- a/elastic_recheck/elasticRecheck.py +++ b/elastic_recheck/elasticRecheck.py @@ -90,7 +90,7 @@ class FailJob(object): self.url = url # The last set of characters of the URL are the first 7 digits # of the build_uuid. - self.build_short_uuid = filter(None, url.split('/'))[-1] + self.build_short_uuid = list(filter(None, url.split('/')))[-1] def __repr__(self): return self.name diff --git a/elastic_recheck/tests/functional/test_gerrit_comment.py b/elastic_recheck/tests/functional/test_gerrit_comment.py index a8aedd40..5c651469 100644 --- a/elastic_recheck/tests/functional/test_gerrit_comment.py +++ b/elastic_recheck/tests/functional/test_gerrit_comment.py @@ -12,8 +12,8 @@ # License for the specific language governing permissions and limitations # under the License. -import ConfigParser import gerritlib +from six.moves import configparser import testtools from elastic_recheck import elasticRecheck @@ -23,7 +23,7 @@ class TestGerritComment(testtools.TestCase): def setUp(self): super(TestGerritComment, self).setUp() - config = ConfigParser.ConfigParser({'server_password': None}) + config = configparser.ConfigParser({'server_password': None}) config.read('elasticRecheck.conf') self.user = config.get('gerrit', 'user') key = config.get('gerrit', 'key') diff --git a/elastic_recheck/tests/unit/cmd/test_uncategorized_fails.py b/elastic_recheck/tests/unit/cmd/test_uncategorized_fails.py index f4a7c0fd..b6526d72 100644 --- a/elastic_recheck/tests/unit/cmd/test_uncategorized_fails.py +++ b/elastic_recheck/tests/unit/cmd/test_uncategorized_fails.py @@ -58,4 +58,4 @@ class TestUncategorizedFails(testtools.TestCase): self.assertThat(all_fails['integrated_gate'], testtools.matchers.HasLength(1)) self.assertIn('gate-tempest-dsvm-full', - all_fails['integrated_gate'].keys()[0]) + list(all_fails['integrated_gate'].keys())[0]) diff --git a/elastic_recheck/tests/unit/test_bot.py b/elastic_recheck/tests/unit/test_bot.py index 50e8fc64..4a410f8c 100644 --- a/elastic_recheck/tests/unit/test_bot.py +++ b/elastic_recheck/tests/unit/test_bot.py @@ -12,7 +12,7 @@ # License for the specific language governing permissions and limitations # under the License. -import ConfigParser +from six.moves import configparser import unittest import yaml @@ -49,7 +49,9 @@ def _set_fake_config(fake_config): class TestBot(unittest.TestCase): def setUp(self): super(TestBot, self).setUp() - self.fake_config = ConfigParser.ConfigParser({'server_password': None}) + self.fake_config = configparser.ConfigParser( + {'server_password': None}, + allow_no_value=True) _set_fake_config(self.fake_config) config = er_conf.Config(config_obj=self.fake_config) self.channel_config = bot.ChannelConfig(yaml.load( @@ -93,7 +95,9 @@ class TestBotWithTestTools(tests.TestCase): self.useFixture(fixtures.MonkeyPatch( 'gerritlib.gerrit.Gerrit', fg.Gerrit)) - self.fake_config = ConfigParser.ConfigParser({'server_password': None}) + self.fake_config = configparser.ConfigParser( + {'server_password': None}, + allow_no_value=True) _set_fake_config(self.fake_config) config = er_conf.Config(config_obj=self.fake_config) self.channel_config = bot.ChannelConfig(yaml.load( diff --git a/elastic_recheck/tests/unit/test_query.py b/elastic_recheck/tests/unit/test_query.py index 3a8eb543..48c9c63c 100644 --- a/elastic_recheck/tests/unit/test_query.py +++ b/elastic_recheck/tests/unit/test_query.py @@ -11,7 +11,7 @@ # under the License. import json -import StringIO +from six import StringIO import sys import mock @@ -25,7 +25,7 @@ class TestQueryCmd(unit.UnitTestCase): def setUp(self): super(TestQueryCmd, self).setUp() self._stdout = sys.stdout - sys.stdout = StringIO.StringIO() + sys.stdout = StringIO() def tearDown(self): super(TestQueryCmd, self).tearDown() diff --git a/elastic_recheck/tests/unit/test_results.py b/elastic_recheck/tests/unit/test_results.py index 5c271ea3..fec94117 100644 --- a/elastic_recheck/tests/unit/test_results.py +++ b/elastic_recheck/tests/unit/test_results.py @@ -59,7 +59,7 @@ class TestBasicParsing(tests.TestCase): facets = results.FacetSet() facets.detect_facets(result_set, ["build_status"]) - self.assertEqual(facets.keys(), ['FAILURE']) + self.assertEqual(list(facets.keys()), ['FAILURE']) data = load_sample(1226337) result_set = results.ResultSet(data) @@ -87,18 +87,15 @@ class TestBasicParsing(tests.TestCase): facets.detect_facets(result_set, ["timestamp", "build_status", "build_uuid"]) self.assertEqual(len(facets.keys()), 14) - print facets[1382104800000].keys() - self.assertEqual(facets[1382104800000].keys(), ["FAILURE"]) + print(facets[1382104800000].keys()) + self.assertEqual(list(facets[1382104800000].keys()), ["FAILURE"]) self.assertEqual(len(facets[1382104800000]["FAILURE"]), 2) - self.assertEqual(facets[1382101200000].keys(), ["FAILURE"]) + self.assertEqual(list(facets[1382101200000].keys()), ["FAILURE"]) # NOTE(mriedem): We can't mock built-ins so we have to override utcnow(). class MockDatetimeToday(datetime.datetime): - def __init__(self, *args): - super(MockDatetimeToday, self).__init__(*args) - @classmethod def utcnow(cls): # One hour and one second into today. @@ -108,9 +105,6 @@ class MockDatetimeToday(datetime.datetime): class MockDatetimeYesterday(datetime.datetime): - def __init__(self, *args): - super(MockDatetimeYesterday, self).__init__(*args) - @classmethod def utcnow(cls): # 59 minutes and 59 seconds into today. diff --git a/requirements.txt b/requirements.txt index e3a3351f..b2fe51b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,8 +9,9 @@ irc>=15.0.1 pyyaml lockfile Babel>=0.9.6 -httplib2<0.12.0 # required by launchpadlib, see https://bugs.launchpad.net/lazr.restfulclient/+bug/1803402 -launchpadlib +lazr.restfulclient>=0.14.2 # LGPL +httplib2>=0.12.0 # MIT License +launchpadlib>=1.10.6 # LGPL Jinja2 requests subunit2sql>=0.9.0 diff --git a/setup.cfg b/setup.cfg index e9b0c649..0fcaa68c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,8 @@ classifier = Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 Programming Language :: Python :: 3 - Programming Language :: Python :: 3.3 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 [files] packages = diff --git a/tools/unaccounted_rechecks.py b/tools/unaccounted_rechecks.py index 5115061d..03e973b2 100755 --- a/tools/unaccounted_rechecks.py +++ b/tools/unaccounted_rechecks.py @@ -101,12 +101,12 @@ def cross_ref_with_er(changes, dirname): def summarize_changes(changes): no_er = {} - print "Summary" - print "%4.4s - Total Rechecks" % (len(changes)) - print "%4.4s - Total w/Bug" % ( - len([c for c in changes if c['bug'] != 'no bug'])) - print "%4.4s - Total w/Bug and new recheck" % ( - len([c for c in changes if (c['bug'] != 'no bug' and not c['er'])])) + print("Summary") + print("%4.4s - Total Rechecks" % (len(changes))) + print("%4.4s - Total w/Bug" % ( + len([c for c in changes if c['bug'] != 'no bug']))) + print("%4.4s - Total w/Bug and new recheck" % ( + len([c for c in changes if (c['bug'] != 'no bug' and not c['er'])]))) for c in changes: bug = c['bug'] @@ -115,12 +115,12 @@ def summarize_changes(changes): no_er[bug] = {'count': 0, 'reviews': []} no_er[bug]['count'] += 1 no_er[bug]['reviews'].append(c['review']) - print - print "New bugs" - for k, v in no_er.iteritems(): - print "Bug %s found %d times" % (k, v['count']) + print() + print("New bugs") + for k, v in no_er.items(): + print("Bug %s found %d times" % (k, v['count'])) for rev in v['reviews']: - print " - %s" % rev + print(" - %s" % rev) def main(): diff --git a/tox.ini b/tox.ini index 197eb9c2..cd148849 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 1.6 -envlist = py27,pep8,queries,docs +envlist = pep8,py37,py36,py27,queries,docs skipsdist = True [testenv] diff --git a/web_server.py b/web_server.py index 2ed5bd73..6036131a 100755 --- a/web_server.py +++ b/web_server.py @@ -40,7 +40,7 @@ class ERHandler(BaseHTTPServer.BaseHTTPRequestHandler): # if the file exists locally, we'll serve it up directly fname = "web/share" + self.path if os.path.isfile(fname): - print "found local file %s" % (fname) + print("found local file %s" % (fname)) self.send_response(200, "Success") self.end_headers() with open(fname) as f: @@ -73,7 +73,7 @@ class ERHandler(BaseHTTPServer.BaseHTTPRequestHandler): return # Fall through for paths we don't understand - print "Unknown path requested: %s" % self.path + print("Unknown path requested: %s" % self.path) def parse_opts(): @@ -90,9 +90,9 @@ def main(): server_address = ('', opts.port) httpd = BaseHTTPServer.HTTPServer(server_address, ERHandler) - print "Test Server is running at http://localhost:%s" % opts.port - print "Ctrl-C to exit" - print + print("Test Server is running at http://localhost:%s" % opts.port) + print("Ctrl-C to exit") + print() while True: httpd.handle_request() @@ -101,5 +101,5 @@ if __name__ == '__main__': try: main() except KeyboardInterrupt: - print "\n" - print "Thanks for testing! Please come again." + print("\n") + print("Thanks for testing! Please come again.")