From de255d9b8bfd8426206344e701eab002f2149765 Mon Sep 17 00:00:00 2001 From: David Ostrovsky Date: Sun, 14 Jan 2018 11:09:43 +0100 Subject: [PATCH 1/5] Bazel: Make build tool chain python 3 compatible This change establishes Python 3 compatibility for the major part of Gerrit build tool chain. Only two scripts remain non Python 3 compatible: * Documentation/replace_macros.py * tools/bzl/license-map.py Those scripts explicitly invoked with python2 version. Test Plan: a. Python 2.7 * Switch to system where /usr/bin/python points to Python 2.7 * bazel build release * bazel test //... * tools/eclipse/project.py b. Python 3.6 * Switch to system where /usr/bin/python points to Python 3.6 * bazel build release * bazel test //... * tools/eclipse/project.py Pre-requisites for the test plan: In case bazel action and repository caching is activated on the SUT, the caches would need to be wiped out, to make sure that the complete build tool chain was tested. On my system I had to run these commands: * bazel clean --expunge_async * rm -rf ~/.gerritcodereview/buck-cache/downloaded-artifacts/* * rm -rf ~/.gerritcodereview/bazel-cache/cas/* * rm -rf ~/.gerritcodereview/bazel-cache/repository/* Bug: Issue 8151 Change-Id: Iece59d0c5149b77a02754b3fed4ce84d5d8085ee (cherry picked from commit 0c9e13c11a9a0facd1538d6e5b45b02a90db5ab3) --- Documentation/replace_macros.py | 2 +- tools/bzl/license.bzl | 2 +- tools/eclipse/project.py | 2 +- tools/js/bowerutil.py | 4 ++-- tools/js/download_bower.py | 2 +- tools/js/npm_pack.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Documentation/replace_macros.py b/Documentation/replace_macros.py index baf08e7d58..2996a98aea 100755 --- a/Documentation/replace_macros.py +++ b/Documentation/replace_macros.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python2 # coding=utf-8 # Copyright (C) 2013 The Android Open Source Project # diff --git a/tools/bzl/license.bzl b/tools/bzl/license.bzl index 38dfbe5e28..357817303a 100644 --- a/tools/bzl/license.bzl +++ b/tools/bzl/license.bzl @@ -25,7 +25,7 @@ def license_map(name, targets = [], opts = [], **kwargs): # post process the XML into our favorite format. native.genrule( name = "gen_license_txt_" + name, - cmd = "python $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)), + cmd = "python2 $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)), outs = [ name + ".txt" ], tools = tools, **kwargs diff --git a/tools/eclipse/project.py b/tools/eclipse/project.py index 97c2b12b3f..86bcf5569f 100755 --- a/tools/eclipse/project.py +++ b/tools/eclipse/project.py @@ -262,7 +262,7 @@ def gen_factorypath(ext): doc.writexml(fd, addindent='\t', newl='\n', encoding='UTF-8') try: - ext_location = retrieve_ext_location() + ext_location = retrieve_ext_location().decode("utf-8") gen_project(args.project_name) gen_classpath(ext_location) gen_factorypath(ext_location) diff --git a/tools/js/bowerutil.py b/tools/js/bowerutil.py index 8e8e835a46..c2e11cdde2 100644 --- a/tools/js/bowerutil.py +++ b/tools/js/bowerutil.py @@ -40,7 +40,7 @@ def hash_bower_component(hash_obj, path): if f == '.bower.json': continue p = os.path.join(root, f) - hash_obj.update(p[len(path)+1:]) - hash_obj.update(open(p).read()) + hash_obj.update(p[len(path)+1:].encode("utf-8")) + hash_obj.update(open(p, "rb").read()) return hash_obj diff --git a/tools/js/download_bower.py b/tools/js/download_bower.py index 80cb56e402..3db39d5bfa 100755 --- a/tools/js/download_bower.py +++ b/tools/js/download_bower.py @@ -68,7 +68,7 @@ def ignore_deps(info): deps = info.get('dependencies') if deps: with open(os.path.join('.bowerrc'), 'w') as f: - json.dump({'ignoredDependencies': deps.keys()}, f) + json.dump({'ignoredDependencies': list(deps.keys())}, f) def cache_entry(name, package, version, sha1): diff --git a/tools/js/npm_pack.py b/tools/js/npm_pack.py index 9eb6e34365..52dc512056 100755 --- a/tools/js/npm_pack.py +++ b/tools/js/npm_pack.py @@ -36,7 +36,7 @@ def is_bundled(tar): def bundle_dependencies(): with open('package.json') as f: package = json.load(f) - package['bundledDependencies'] = package['dependencies'].keys() + package['bundledDependencies'] = list(package['dependencies'].keys()) with open('package.json', 'w') as f: json.dump(package, f) From 03b8b2caada96eff5d35347a9199c5e7e25a50b5 Mon Sep 17 00:00:00 2001 From: Paladox none Date: Fri, 2 Mar 2018 15:48:43 +0000 Subject: [PATCH 2/5] license and doc: Add support for python3 With homebrew's recent change of making python3 the default over python2, lets support python3. Bug: Issue 8474 Change-Id: I8dba441c8d717ae06d156c9201a02926884fd5f0 --- Documentation/replace_macros.py | 12 ++++++++---- tools/bzl/license-map.py | 9 +++++++-- tools/bzl/license.bzl | 2 +- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/Documentation/replace_macros.py b/Documentation/replace_macros.py index 2996a98aea..c76d133936 100755 --- a/Documentation/replace_macros.py +++ b/Documentation/replace_macros.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python # coding=utf-8 # Copyright (C) 2013 The Android Open Source Project # @@ -229,12 +229,16 @@ opts.add_option('--no-searchbox', action="store_false", dest='searchbox', options, _ = opts.parse_args() try: - out_file = open(options.out, 'w') - src_file = open(options.src, 'r') + try: + out_file = open(options.out, 'w', errors='ignore') + src_file = open(options.src, 'r', errors='ignore') + except TypeError: + out_file = open(options.out, 'w') + src_file = open(options.src, 'r') last_line = '' ignore_next_line = False last_title = '' - for line in src_file.xreadlines(): + for line in src_file: if PAT_GERRIT.match(last_line): # Case of "GERRIT\n------" at the footer out_file.write(GERRIT_UPLINK) diff --git a/tools/bzl/license-map.py b/tools/bzl/license-map.py index 1c8db72ef5..74a84cc3e4 100644 --- a/tools/bzl/license-map.py +++ b/tools/bzl/license-map.py @@ -113,8 +113,13 @@ for n in sorted(graph.keys()): print() print("[[%s_license]]" % safename) print("----") - with open(n[2:].replace(":", "/")) as fd: - copyfileobj(fd, stdout) + filename = n[2:].replace(":", "/") + try: + with open(filename, errors='ignore') as fd: + copyfileobj(fd, stdout) + except TypeError: + with open(filename) as fd: + copyfileobj(fd, stdout) print() print("----") print() diff --git a/tools/bzl/license.bzl b/tools/bzl/license.bzl index 357817303a..38dfbe5e28 100644 --- a/tools/bzl/license.bzl +++ b/tools/bzl/license.bzl @@ -25,7 +25,7 @@ def license_map(name, targets = [], opts = [], **kwargs): # post process the XML into our favorite format. native.genrule( name = "gen_license_txt_" + name, - cmd = "python2 $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)), + cmd = "python $(location //tools/bzl:license-map.py) %s %s > $@" % (" ".join(opts), " ".join(xmls)), outs = [ name + ".txt" ], tools = tools, **kwargs From ffb6e8ca6e0b0fb38d8b36037ce6d064730297d6 Mon Sep 17 00:00:00 2001 From: David Pursehouse Date: Thu, 8 Mar 2018 08:43:02 +0900 Subject: [PATCH 3/5] project.py: decode byte output from check_result When running project.py under Python 3, the output is a byte sequence, and the bazel location gets written to the .bazel_path file as: bazel=b'/usr/local/bin/bazel' This causes launching the project in Eclipse to fail. To prevent this, decode the byte sequence back to a string. Change-Id: Ia0e1c7f1e4f934e245dc85d83a01089034679024 --- tools/eclipse/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/eclipse/project.py b/tools/eclipse/project.py index 86bcf5569f..833474a1b5 100755 --- a/tools/eclipse/project.py +++ b/tools/eclipse/project.py @@ -57,7 +57,7 @@ def retrieve_ext_location(): return check_output(['bazel', 'info', 'output_base']).strip() def gen_bazel_path(): - bazel = check_output(['which', 'bazel']).strip() + bazel = check_output(['which', 'bazel']).strip().decode('UTF-8') with open(path.join(ROOT, ".bazel_path"), 'w') as fd: fd.write("bazel=%s\n" % bazel) fd.write("PATH=%s\n" % environ["PATH"]) From 945ea9fc4108083e39c7e20c620200197e99b899 Mon Sep 17 00:00:00 2001 From: David Pursehouse Date: Thu, 8 Mar 2018 10:43:02 +0900 Subject: [PATCH 4/5] merge_jars.py: Fix for python 3 compatibility * ZipFile.read returns byte and not string * dict.iteritems() was removed in Python 3 Change-Id: Id3c0edf7f554150c948cbfdba0b1b24a5d3ad51e --- tools/merge_jars.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/merge_jars.py b/tools/merge_jars.py index 46016c0d21..89e83ca4f3 100755 --- a/tools/merge_jars.py +++ b/tools/merge_jars.py @@ -39,12 +39,12 @@ try: continue elif n.startswith(SERVICES): # Concatenate all provider configuration files. - services[n] += inzip.read(n) + services[n] += inzip.read(n).decode("UTF-8") continue outzip.writestr(info, inzip.read(n)) seen.add(n) - for n, v in services.iteritems(): + for n, v in list(services.items()): outzip.writestr(n, v) except Exception as err: exit('Failed to merge jars: %s' % err) From 9276890d38e9d2760654c6522bc1f09b9e5d6d9e Mon Sep 17 00:00:00 2001 From: David Pursehouse Date: Thu, 8 Mar 2018 20:52:37 +0900 Subject: [PATCH 5/5] Document that the build works with Python 2 or 3 Change-Id: If824dc80e49a54e3255ebe12fba1ec2981a4d255 --- Documentation/dev-bazel.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/dev-bazel.txt b/Documentation/dev-bazel.txt index a613103b2c..45a6bfe75b 100644 --- a/Documentation/dev-bazel.txt +++ b/Documentation/dev-bazel.txt @@ -3,7 +3,7 @@ [[installation]] == Installation -You need to use Python 2, Java 8, and Node.js for building gerrit. +You need to use Python (2 or 3), Java 8, and Node.js for building gerrit. You can install Bazel from the bazel.io: https://www.bazel.io/versions/master/docs/install.html