diff --git a/openstack_election/cmds/change_owners.py b/openstack_election/cmds/change_owners.py new file mode 100644 index 00000000..1f9e285c --- /dev/null +++ b/openstack_election/cmds/change_owners.py @@ -0,0 +1,125 @@ +# Copyright (c) 2016 OpenStack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an "AS +# IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language +# governing permissions and limitations under the License. + +# Description: When run using OpenStack's Gerrit server, this builds +# YAML representations of aggregate change owner details and change +# counts for each governance project-team, as well as a combined set +# for all teams. + +# Rationale: The OpenStack Technical Committee and Project Team Lead +# elections need electorate rolls taken from "Active Technical +# Contributors" to any repos under official project-teams over a +# particular timeframe. Similarly, the OpenStack Foundation gives +# summit registration discount codes to contributors meeting similar +# criteria. The Gerrit REST API provides access to all the data +# necessary to identify these individuals. + +# Use: The results end up in files named for each +# official governance project-team (or "all") ending with a .yaml +# extension. At the time of writing, it takes approximately 30 +# minutes to run on a well-connected machine with 70-80ms round-trip +# latency to review.openstack.org. + +# An example for generating the March 2016 technical election rolls: +# +# $ virtualenv venv +# [...] +# $ ./venv/bin/pip install /path/to/openstack/election +# [...] +# $ owners -a 2015-03-04 \ +# -b 2016-03-04 -o owners -r march-2016-elections +# MISSING: ansible-build-image +# MERGING DUPLICATE ACCOUNT: 8074 into 2467 +# [...blah, blah, blah...wait for completion...] +# +# TODO(fungi): Add a pass which will correctly generate the +# stable_branch_maintenance.* files. In the meantime, to properly +# generate the SBM PTL electorate, run a second time with a +# different -o of sbm, adding the -n and -s options, and then copy +# the full electorate over like: +# +# $ owners -a 2015-03-04 \ +# -b 2016-03-04 -o sbm -r march-2016-elections \ +# -n -s 'branch:^stable/.*' +# [...wait for completion again...] +# $ cp sbm/_electorate.txt owners/stable_branch_maintenance.txt +# $ cp sbm/_all_owners.yaml owners/stable_branch_maintenance.yaml +# +# Once complete, make a compressed tarball of the owners directory +# and send it attached to a PGP/MIME signed message to the appointed +# election officials. The various *.txt files are lists of the +# preferred addresses of all valid voters for the various PTL +# elections (whose team names correspond to the file names), +# suitable for passing directly to CIVS. The similarly named *.yaml +# files are detailed structured data about the same sets of voters, +# for use in validating the address lists. The _electorate.txt file +# is the equivalent address list for the TC election voters, and its +# corresponding structured data is in _all_owners.yaml. + +# You can also do interesting analysis on _all_owners.yaml, for +# example: +# +# $ ./venv/bin/python +# >>> import yaml +# >>> +# >>> o = yaml.load(open('owners/_all_owners.yaml')) +# >>> for c in range(5): +# ... print('Owners of at least %s changes: %s' % ( +# ... c+1, +# ... len({k: v for k, v in o.iteritems() if v['count'] > c}))) +# ... +# Owners of at least 1 changes: 3239 +# Owners of at least 2 changes: 2352 +# Owners of at least 3 changes: 1924 +# Owners of at least 4 changes: 1682 +# Owners of at least 5 changes: 1504 + + +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import argparse +import sys + +from openstack_election import owners + + +def usage(argv=sys.argv): + """Parse command line argument""" + parser = argparse.ArgumentParser( + description="When run using OpenStack's Gerrit server, this builds " + "YAML representations of aggregate change owner details and change " + "counts for each governance project-team, as well as a combined set " + "for all teams. Before and after dates/times should be supplied in " + "formats Gerrit accepts: https://review.openstack.org/Documentation/" + "user-search.html#search-operators") + parser.add_argument("-a", "--after", help="Start date for matching merges") + parser.add_argument("-b", "--before", help="End date for matching merges") + parser.add_argument("-c", "--config", help="Path to script configuration") + parser.add_argument("-i", "--ignore", help="Account Id numbers to skip", + action='append') + parser.add_argument("-n", "--no-extra-atcs", help='Omit "extra ATCs"', + dest='no_extra_atcs', action='store_true') + parser.add_argument("-o", "--outdir", help="Create an output directory") + parser.add_argument("-r", "--ref", help="Specify a Governance refname") + parser.add_argument("-s", "--sieve", help="Add Gerrit query parameters") + + return parser.parse_args(argv[1:]) + + +def main(): + options = usage() + + return owners.main(options) diff --git a/openstack_election/cmds/generate_rolls.py b/openstack_election/cmds/generate_rolls.py index 95c7c005..8c7b34e2 100755 --- a/openstack_election/cmds/generate_rolls.py +++ b/openstack_election/cmds/generate_rolls.py @@ -26,6 +26,24 @@ from openstack_election import owners from openstack_election import utils +def change_owners_options_proxy(after, before, ref, outdir='./', sieve=None, + no_extra_atcs=False): + options = argparse.Namespace() + + options.config = None + options.ignore = [] + + options.after = after + options.before = before + options.outdir = outdir + options.ref = ref + + options.no_extra_atcs = no_extra_atcs + options.sieve = sieve + + return options + + def main(): start = utils.conf['timeframe']['start'] end = utils.conf['timeframe']['end'] @@ -61,16 +79,20 @@ def main(): os.chdir(os.path.dirname(args.rolls_dir)) print("Starting roll generation @%s" % time.ctime()) - owners.main(["owners.py", "-a", args.after, "-b", args.before, - "-o", args.tag, "-r", args.tag]) + options = change_owners_options_proxy(args.after, args.before, + args.tag, args.tag) + owners.main(options) print("Finished roll generation @%s" % time.ctime()) if args.with_stable: tmp_dir = tempfile.mkdtemp(prefix='election.') print("Starting (Stable) roll generation @%s" % time.ctime()) - owners.main(["owners.py", "-a", args.after, "-b", args.before, - "-o", tmp_dir, "-r", args.tag, "-n", - "-s", "branch:^stable/.*"]) + # owners.main() potentially mutates options so create a fresh one + options = change_owners_options_proxy(args.after, args.before, + args.tag, tmp_dir, + no_extra_atcs=True, + sieve="branch:^stable/.*") + owners.main(options) print("Finished (Stable) roll generation @%s" % time.ctime()) shutil.copy("%s/_electorate.txt" % tmp_dir, "./%s/stable_branch_maintenance.txt" % args.tag) diff --git a/openstack_election/owners.py b/openstack_election/owners.py index bd7ea79b..1fbccd9d 100644 --- a/openstack_election/owners.py +++ b/openstack_election/owners.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - # Copyright (c) 2016 OpenStack Foundation # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -19,77 +17,8 @@ # counts for each governance project-team, as well as a combined set # for all teams. -# Rationale: The OpenStack Technical Committee and Project Team Lead -# elections need electorate rolls taken from "Active Technical -# Contributors" to any repos under official project-teams over a -# particular timeframe. Similarly, the OpenStack Foundation gives -# summit registration discount codes to contributors meeting similar -# criteria. The Gerrit REST API provides access to all the data -# necessary to identify these individuals. - -# Use: The results end up in files named for each -# official governance project-team (or "all") ending with a .yaml -# extension. At the time of writing, it takes approximately 30 -# minutes to run on a well-connected machine with 70-80ms round-trip -# latency to review.openstack.org. - -# An example for generating the March 2016 technical election rolls: -# -# $ virtualenv venv -# [...] -# $ ./venv/bin/pip install pyyaml requests -# [...] -# $ ./venv/bin/python tools/owners.py -a 2015-03-04 \ -# -b 2016-03-04 -o owners -r march-2016-elections -# MISSING: ansible-build-image -# MERGING DUPLICATE ACCOUNT: 8074 into 2467 -# [...blah, blah, blah...wait for completion...] -# -# TODO(fungi): Add a pass which will correctly generate the -# stable_branch_maintenance.* files. In the meantime, to properly -# generate the SBM PTL electorate, run a second time with a -# different -o of sbm, adding the -n and -s options, and then copy -# the full electorate over like: -# -# $ ./venv/bin/python tools/owners.py -a 2015-03-04 \ -# -b 2016-03-04 -o sbm -r march-2016-elections \ -# -n -s 'branch:^stable/.*' -# [...wait for completion again...] -# $ cp sbm/_electorate.txt owners/stable_branch_maintenance.txt -# $ cp sbm/_all_owners.yaml owners/stable_branch_maintenance.yaml -# -# Once complete, make a compressed tarball of the owners directory -# and send it attached to a PGP/MIME signed message to the appointed -# election officials. The various *.txt files are lists of the -# preferred addresses of all valid voters for the various PTL -# elections (whose team names correspond to the file names), -# suitable for passing directly to CIVS. The similarly named *.yaml -# files are detailed structured data about the same sets of voters, -# for use in validating the address lists. The _electorate.txt file -# is the equivalent address list for the TC election voters, and its -# corresponding structured data is in _all_owners.yaml. - -# You can also do interesting analysis on _all_owners.yaml, for -# example: -# -# $ ./venv/bin/python -# >>> import yaml -# >>> -# >>> o = yaml.load(open('owners/_all_owners.yaml')) -# >>> for c in range(5): -# ... print('Owners of at least %s changes: %s' % ( -# ... c+1, -# ... len({k: v for k, v in o.iteritems() if v['count'] > c}))) -# ... -# Owners of at least 1 changes: 3239 -# Owners of at least 2 changes: 2352 -# Owners of at least 3 changes: 1924 -# Owners of at least 4 changes: 1682 -# Owners of at least 5 changes: 1504 - - from __future__ import print_function -import argparse + import csv import datetime import json @@ -229,36 +158,12 @@ def lookup_member(email): return decode_json(raw) -def usage(argv): - """Parse command line argument""" - parser = argparse.ArgumentParser( - description="When run using OpenStack's Gerrit server, this builds " - "YAML representations of aggregate change owner details and change " - "counts for each governance project-team, as well as a combined set " - "for all teams. Before and after dates/times should be supplied in " - "formats Gerrit accepts: https://review.openstack.org/Documentation/" - "user-search.html#search-operators") - parser.add_argument("-a", "--after", help="Start date for matching merges") - parser.add_argument("-b", "--before", help="End date for matching merges") - parser.add_argument("-c", "--config", help="Path to script configuration") - parser.add_argument("-i", "--ignore", help="Account Id numbers to skip", - action='append') - parser.add_argument("-n", "--no-extra-atcs", help='Omit "extra ATCs"', - dest='no_extra_atcs', action='store_true') - parser.add_argument("-o", "--outdir", help="Create an output directory") - parser.add_argument("-r", "--ref", help="Specify a Governance refname") - parser.add_argument("-s", "--sieve", help="Add Gerrit query parameters") - return parser.parse_args(argv[1:]) - - -def main(argv=sys.argv): +def main(options): """The giant pile of spaghetti which does everything else""" # Record the start time for use later start = datetime.datetime.utcnow() - options = usage(argv) - # If we're supplied a configuration file, use it if options.config: config = yaml.safe_load(open(options.config)) @@ -719,7 +624,3 @@ def main(argv=sys.argv): fd = open(os.path.join(outdir, '%s.txt' % normalized_project), 'w') fd.writelines(electorate) fd.close() - - -if __name__ == "__main__": - main() diff --git a/setup.cfg b/setup.cfg index 30f0e5d5..26f9b074 100644 --- a/setup.cfg +++ b/setup.cfg @@ -29,6 +29,7 @@ console_scripts = search-rolls = openstack_election.cmds.search_rolls:main create-directories = openstack_election.cmds.create_directories:main generate-rolls = openstack_election.cmds.generate_rolls:main + owners = openstack_election.cmds.change_owners:main [build_sphinx] all_files = 1