236 lines
7.7 KiB
Python
Executable File
236 lines
7.7 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
# Copyright (C) 2013-2014 OpenStack Foundation
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
# implied.
|
|
#
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
# Soren Hansen wrote the original version of this script.
|
|
# James Blair hacked it up to include email addresses from gerrit.
|
|
# Jeremy Stanley overhauled it for gerrit 2.8 and our governance repo.
|
|
|
|
import csv
|
|
import datetime
|
|
import json
|
|
import optparse
|
|
import os
|
|
import os.path
|
|
import re
|
|
|
|
import paramiko
|
|
import requests
|
|
import yaml
|
|
|
|
MAILTO_RE = re.compile('mailto:(.*)')
|
|
USERNAME_RE = re.compile('username:(.*)')
|
|
EXTRA_ATC_RE = re.compile('^[^#][^:]*: ([^\(]*) \(([^@]*@[^\)]*)\) \[[^\[]*\]')
|
|
PROJECTS_URL = ('https://git.openstack.org/cgit/openstack/governance/plain'
|
|
'/reference/projects.yaml')
|
|
EXTRA_ATCS_URL = ('https://git.openstack.org/cgit/openstack/governance/plain'
|
|
'/reference/extra-atcs')
|
|
|
|
|
|
class Account(object):
|
|
def __init__(self, num):
|
|
self.num = num
|
|
self.full_name = ''
|
|
self.emails = []
|
|
self.username = None
|
|
|
|
|
|
def get_account(accounts, num):
|
|
a = accounts.get(num)
|
|
if not a:
|
|
a = Account(num)
|
|
accounts[num] = a
|
|
return a
|
|
|
|
|
|
def repo_stats(repo, output, begin, end, keyfile, user):
|
|
accounts = {}
|
|
|
|
for row in open('accounts.tab'):
|
|
if not row.startswith('registered_on'):
|
|
row = row.split('\t')
|
|
num = int(row[13])
|
|
name = row[1]
|
|
email = row[2]
|
|
a = get_account(accounts, num)
|
|
a.full_name = name
|
|
if email and email != 'NULL':
|
|
a.emails.append(email)
|
|
|
|
for row in open('emails.tab'):
|
|
if not row.startswith('account_id'):
|
|
num, email, pw, external = row.split('\t')
|
|
num = int(num)
|
|
a = get_account(accounts, num)
|
|
if email and email != 'NULL' and email not in a.emails:
|
|
a.emails.append(email)
|
|
m = MAILTO_RE.match(external)
|
|
if m:
|
|
if m.group(1) not in a.emails:
|
|
a.emails.append(m.group(1))
|
|
m = USERNAME_RE.match(external)
|
|
if m:
|
|
if a.username:
|
|
print a.num
|
|
print a.username
|
|
raise Exception("Already a username")
|
|
a.username = m.group(1)
|
|
|
|
username_accounts = {}
|
|
for a in accounts.values():
|
|
username_accounts[a.username] = a
|
|
|
|
atcs = []
|
|
|
|
QUERY = "project:%s status:merged" % repo
|
|
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
client.load_system_host_keys()
|
|
client.connect(
|
|
'review.opendev.org', port=29418,
|
|
key_filename=os.path.expanduser(keyfile), username=user)
|
|
stdin, stdout, stderr = client.exec_command(
|
|
'gerrit query %s --all-approvals --format JSON' % QUERY)
|
|
|
|
done = False
|
|
last_sortkey = ''
|
|
begin_time = datetime.datetime(
|
|
int(begin[0:4]), int(begin[4:6]), int(begin[6:8]),
|
|
int(begin[8:10]), int(begin[10:12]), int(begin[12:14]))
|
|
end_time = datetime.datetime(
|
|
int(end[0:4]), int(end[4:6]), int(end[6:8]),
|
|
int(end[8:10]), int(end[10:12]), int(end[12:14]))
|
|
|
|
count = 0
|
|
earliest = datetime.datetime.now()
|
|
while not done:
|
|
for l in stdout:
|
|
data = json.loads(l)
|
|
if 'rowCount' in data:
|
|
if data['rowCount'] < 500:
|
|
done = True
|
|
continue
|
|
count += 1
|
|
last_sortkey = data['sortKey']
|
|
if 'owner' not in data:
|
|
continue
|
|
if 'username' not in data['owner']:
|
|
continue
|
|
account = username_accounts[data['owner']['username']]
|
|
approved = False
|
|
for ps in data['patchSets']:
|
|
if 'approvals' not in ps:
|
|
continue
|
|
for aprv in ps['approvals']:
|
|
if aprv['type'] != 'SUBM':
|
|
continue
|
|
ts = datetime.datetime.fromtimestamp(aprv['grantedOn'])
|
|
if ts < begin_time or ts > end_time:
|
|
continue
|
|
approved = True
|
|
if ts < earliest:
|
|
earliest = ts
|
|
if approved and account not in atcs:
|
|
atcs.append(account)
|
|
if not done:
|
|
stdin, stdout, stderr = client.exec_command(
|
|
'gerrit query %s resume_sortkey:%s --all-approvals'
|
|
' --format JSON' % (QUERY, last_sortkey))
|
|
|
|
print 'repo: %s' % repo
|
|
print 'examined %s changes' % count
|
|
print 'earliest timestamp: %s' % earliest
|
|
writer = csv.writer(open(output, 'w'))
|
|
for a in atcs:
|
|
writer.writerow([a.username, a.full_name] + a.emails)
|
|
print
|
|
|
|
|
|
def get_repos(url, keyfile, user):
|
|
client = paramiko.SSHClient()
|
|
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
|
|
client.load_system_host_keys()
|
|
client.connect(
|
|
'review.opendev.org', port=29418,
|
|
key_filename=os.path.expanduser(keyfile), username=user)
|
|
stdin, stdout, stderr = client.exec_command('gerrit ls-projects')
|
|
gerrit_repos = [x.strip() for x in stdout]
|
|
|
|
projects_yaml = yaml.load(requests.get(url).text)
|
|
repos = []
|
|
for team in projects_yaml:
|
|
for deliver in projects_yaml[team]['deliverables']:
|
|
for repo in projects_yaml[team]['deliverables'][deliver]['repos']:
|
|
for r in gerrit_repos:
|
|
if re.search('/%s$' % repo.split('/')[1], r):
|
|
repos.append(r)
|
|
break
|
|
return repos
|
|
|
|
|
|
def get_extra_atcs(url):
|
|
extra_atcs = []
|
|
for line in requests.get(url).text.split('\n'):
|
|
if line and not line.startswith('#'):
|
|
extra_atcs.append(line)
|
|
return extra_atcs
|
|
|
|
|
|
def main():
|
|
now = ''.join(
|
|
'%02d' % x for x in datetime.datetime.utcnow().utctimetuple()[:6])
|
|
|
|
optparser = optparse.OptionParser()
|
|
optparser.add_option(
|
|
'-b', '--begin', help='begin date/time (e.g. 20131017000000)')
|
|
optparser.add_option(
|
|
'-e', '--end', default=now, help='end date/time (default is now)')
|
|
optparser.add_option(
|
|
'-k', '--keyfile', default='~/.ssh/id_rsa',
|
|
help='SSH key (default is ~/.ssh/id_rsa)')
|
|
optparser.add_option(
|
|
'-r', '--ref', default='',
|
|
help='governance git ref (e.g. sept-2014-elections')
|
|
optparser.add_option(
|
|
'-u', '--user', default=os.environ['USER'],
|
|
help='SSH username (default is $USER)')
|
|
options, args = optparser.parse_args()
|
|
|
|
if options.ref:
|
|
projects_url = '%s?id=%s' % (PROJECTS_URL, options.ref)
|
|
extra_atcs_url = '%s?id=%s' % (EXTRA_ATCS_URL, options.ref)
|
|
else:
|
|
projects_url = PROJECTS_URL
|
|
extra_atcs_url = EXTRA_ATCS_URL
|
|
|
|
for repo in get_repos(projects_url, options.keyfile, options.user):
|
|
output = 'out/%s.csv' % repo.split('/')[-1]
|
|
repo_stats(repo, output, options.begin, options.end,
|
|
options.keyfile, options.user)
|
|
|
|
writer = csv.writer(open('out/extra-atcs.csv', 'w'))
|
|
for atc in get_extra_atcs(extra_atcs_url):
|
|
try:
|
|
writer.writerow([''] + list(EXTRA_ATC_RE.match(atc).groups()))
|
|
except AttributeError:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|