Nova CI reporting.
This commit is contained in:
Michael Still
parent
c8ad51b188
commit
0123b34bb5
65
analyse.py
65
analyse.py
|
|
@ -11,7 +11,9 @@ CI_SYSTEM = ['Jenkins',
|
|||
'Hyper-V CI',
|
||||
'VMware Mine Sweeper',
|
||||
'Docker CI',
|
||||
'NEC OpenStack CI']
|
||||
'NEC OpenStack CI',
|
||||
'XenServer CI',
|
||||
'IBM PowerKVM Testing']
|
||||
|
||||
|
||||
def read_remote_lines(url):
|
||||
|
|
@ -55,15 +57,22 @@ if __name__ == '__main__':
|
|||
continue
|
||||
if j['change']['project'] != 'openstack/nova':
|
||||
continue
|
||||
if j['change']['branch'] != 'master':
|
||||
continue
|
||||
|
||||
if j['type'] == 'patchset-created':
|
||||
number = j['change']['number']
|
||||
patchset = j['patchSet']['number']
|
||||
timestamp = j['patchSet']['createdOn']
|
||||
patchsets['%s,%s' % (number, patchset)] = \
|
||||
{'__created__': timestamp}
|
||||
patchsets.setdefault(number, {})
|
||||
patchsets[number][patchset] = {'__created__': timestamp}
|
||||
|
||||
elif j['type'] == 'comment-added':
|
||||
if j['comment'].startswith('Starting check jobs'):
|
||||
continue
|
||||
if j['comment'].startswith('Starting gate jobs'):
|
||||
continue
|
||||
|
||||
if not 'approvals' in j:
|
||||
j['approvals'] = [{'type': 'CRVW', 'value': 0}]
|
||||
|
||||
|
|
@ -75,20 +84,52 @@ if __name__ == '__main__':
|
|||
|
||||
number = j['change']['number']
|
||||
patchset = j['patchSet']['number']
|
||||
timestamp = j['patchSet']['createdOn']
|
||||
patchsets.setdefault(number, {})
|
||||
patchsets[number].setdefault(patchset, {})
|
||||
|
||||
verified = []
|
||||
if author in patchsets[number].get(patchset, {}):
|
||||
verified = patchsets[number][patchset][author]
|
||||
for approval in j['approvals']:
|
||||
verified.append('%s:%s' % (approval['type'],
|
||||
approval.get('value')))
|
||||
|
||||
key = '%s,%s' % (number, patchset)
|
||||
patchsets.setdefault(key, {})
|
||||
patchsets[key][author] = (timestamp, verified)
|
||||
if approval.get('value') in ['1', '2']:
|
||||
sentiment = 'Positive'
|
||||
elif approval.get('value') in ['-1', '-2']:
|
||||
sentiment = 'Negative'
|
||||
elif (author == 'Hyper-V CI'
|
||||
and j['comment'].startswith('Build succeeded.')
|
||||
and j['comment'].find(
|
||||
'Test run failed in') != -1):
|
||||
sentiment = 'Negative, buried in comment'
|
||||
elif (author == 'XenServer CI'
|
||||
and j['comment'].startswith('Passed using')):
|
||||
sentiment = 'Positive comment'
|
||||
elif (author == 'XenServer CI'
|
||||
and j['comment'].startswith('Failed using')):
|
||||
sentiment = 'Negative comment'
|
||||
elif j['comment'].startswith('Build succeeded.'):
|
||||
sentiment = 'Positive comment'
|
||||
elif j['comment'].startswith('Build successful.'):
|
||||
sentiment = 'Positive comment'
|
||||
elif j['comment'].startswith('Build failed.'):
|
||||
sentiment = 'Negative comment'
|
||||
else:
|
||||
sentiment = 'Unknown'
|
||||
|
||||
verified.append(('%s:%s' % (approval['type'],
|
||||
approval.get('value')),
|
||||
j['comment'].split('\n')[0],
|
||||
sentiment))
|
||||
patchsets[number][patchset][author] = verified
|
||||
|
||||
elif j['type'] in ['change-abandoned',
|
||||
'change-merged',
|
||||
'change-restored',
|
||||
'change-merged']:
|
||||
# These special cases might cause a CI system to stop
|
||||
# running its tests
|
||||
number = j['change']['number']
|
||||
patchsets.setdefault(number, {})
|
||||
patchsets[number]['__exemption__'] = j['type']
|
||||
|
||||
elif j['type'] in ['change-restored',
|
||||
'ref-updated']:
|
||||
pass
|
||||
|
||||
|
|
|
|||
16799
patchsets.json
16799
patchsets.json
File diff suppressed because it is too large
Load Diff
200
report.py
200
report.py
|
|
@ -3,90 +3,154 @@
|
|||
import datetime
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
CI_SYSTEM = [
|
||||
'Jenkins',
|
||||
'Docker CI',
|
||||
'Hyper-V CI',
|
||||
'IBM PowerKVM Testing',
|
||||
'NEC OpenStack CI',
|
||||
'VMware Mine Sweeper',
|
||||
'XenServer CI',
|
||||
'turbo-hipster',
|
||||
]
|
||||
|
||||
SENTIMENTS = [
|
||||
'Positive',
|
||||
'Negative',
|
||||
'Positive comment',
|
||||
'Negative comment',
|
||||
'Negative, buried in comment',
|
||||
'Unknown'
|
||||
]
|
||||
|
||||
|
||||
def patch_list_as_html(l):
|
||||
out = []
|
||||
for p in sorted(l):
|
||||
number, patch = p.split(',')
|
||||
out.append('<a href="http://review.openstack.org/#/c/%s/%s">%s,%s</a>'
|
||||
% (number, patch, number, patch))
|
||||
return ', '.join(out)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
with open('patchsets.json') as f:
|
||||
patchsets = json.loads(f.read())
|
||||
|
||||
# Summarize
|
||||
timeslots = {}
|
||||
for patchset in patchsets:
|
||||
if not '__created__' in patchsets[patchset]:
|
||||
# This is more complicated than it looks because we need to handle
|
||||
# patchsets which are uploaded so rapidly that older patchsets aren't
|
||||
# finished testing.
|
||||
total_patches = 0
|
||||
total_votes = {}
|
||||
missed_votes = {}
|
||||
sentiments = {}
|
||||
passed_votes = {}
|
||||
failed_votes = {}
|
||||
unparsed_votes = {}
|
||||
|
||||
for number in patchsets:
|
||||
if patchsets[number].get('__exemption__'):
|
||||
continue
|
||||
created = patchsets[patchset]['__created__']
|
||||
|
||||
patches = sorted(patchsets[number].keys())
|
||||
valid_patches = []
|
||||
|
||||
created_dt = datetime.datetime.fromtimestamp(created)
|
||||
timeslot = datetime.datetime(created_dt.year,
|
||||
created_dt.month,
|
||||
created_dt.day,
|
||||
created_dt.hour).strftime('%Y%m%d %H%M')
|
||||
|
||||
timeslots.setdefault(timeslot, {})
|
||||
timeslots[timeslot].setdefault('__total__', 0)
|
||||
timeslots[timeslot]['__total__'] += 1
|
||||
|
||||
for author in patchsets[patchset]:
|
||||
if author == '__created__':
|
||||
# Determine how long a patch was valid for. If it wasn't valid for
|
||||
# at least three hours, disgard.
|
||||
for patch in patches:
|
||||
if not '__created__' in patchsets[number][patch]:
|
||||
continue
|
||||
|
||||
author_vote = json.dumps((author, patchsets[patchset][author][1]))
|
||||
timeslots[timeslot].setdefault(author_vote, 0)
|
||||
timeslots[timeslot][author_vote] += 1
|
||||
uploaded = datetime.datetime.fromtimestamp(
|
||||
patchsets[number][patch]['__created__'])
|
||||
obsoleted = datetime.datetime.fromtimestamp(
|
||||
patchsets[number].get(str(int(patch) + 1), {}).get(
|
||||
'__created__', time.time()))
|
||||
valid_for = obsoleted - uploaded
|
||||
|
||||
#print '%s,%s,%s,%s' %(patchset,
|
||||
# author,
|
||||
# patchsets[patchset][author][0] - created,
|
||||
# patchsets[patchset][author][1])
|
||||
|
||||
# Report
|
||||
for timeslot in sorted(timeslots.keys()):
|
||||
authors = {}
|
||||
for author_vote in timeslots[timeslot]:
|
||||
if author_vote == '__total__':
|
||||
if valid_for < datetime.timedelta(hours=3):
|
||||
continue
|
||||
|
||||
try:
|
||||
author, vote = json.loads(author_vote)
|
||||
count = timeslots[timeslot][author_vote]
|
||||
valid_patches.append(patch)
|
||||
|
||||
authors.setdefault(author, {})
|
||||
authors[author].setdefault('+', 0)
|
||||
authors[author].setdefault('-', 0)
|
||||
authors[author].setdefault('0', 0)
|
||||
authors[author].setdefault('?', 0)
|
||||
total_patches += len(valid_patches)
|
||||
|
||||
clean_votes = []
|
||||
for single in vote:
|
||||
if not single.endswith(':0'):
|
||||
clean_votes.append(single)
|
||||
vote = clean_votes
|
||||
for patch in valid_patches:
|
||||
for author in patchsets[number][patch]:
|
||||
if author == '__created__':
|
||||
continue
|
||||
|
||||
if len(vote) > 1:
|
||||
print '*** Multiple vote %s ***' % vote
|
||||
v = '?'
|
||||
elif len(vote) == 0:
|
||||
v = '0'
|
||||
else:
|
||||
vote = vote[0]
|
||||
votetype, votevalue = vote.split(':')
|
||||
if votevalue in ['1', '2']:
|
||||
v = '+'
|
||||
elif votevalue in ['-1', '-2']:
|
||||
v = '-'
|
||||
total_votes.setdefault(author, 0)
|
||||
total_votes[author] += 1
|
||||
|
||||
for vote, msg, sentiment in patchsets[number][patch][author]:
|
||||
if sentiment.startswith('Positive'):
|
||||
passed_votes.setdefault(author, 0)
|
||||
passed_votes[author] += 1
|
||||
elif sentiment.startswith('Negative'):
|
||||
failed_votes.setdefault(author, 0)
|
||||
failed_votes[author] += 1
|
||||
else:
|
||||
v = '0'
|
||||
authors[author][v] += count
|
||||
unparsed_votes.setdefault(author, 0)
|
||||
unparsed_votes[author] += 1
|
||||
|
||||
sentiments.setdefault(author, {})
|
||||
sentiments[author].setdefault(sentiment, [])
|
||||
sentiments[author][sentiment].append(
|
||||
'%s,%s' % (number, patch))
|
||||
|
||||
except Exception, e:
|
||||
print '*** Could not decode %s (%s) ***' % (author_vote, e)
|
||||
for author in CI_SYSTEM:
|
||||
if not author in patchsets[number][patch]:
|
||||
missed_votes.setdefault(author, [])
|
||||
missed_votes[author].append('%s,%s' % (number, patch))
|
||||
|
||||
sys.stdout.write('%s ' % timeslot)
|
||||
for author in authors:
|
||||
sys.stdout.write('%s(' % author)
|
||||
votes = []
|
||||
for vote in ['-', '0', '+', '?']:
|
||||
votes.append('%s' % authors[author][vote])
|
||||
sys.stdout.write(','.join(votes))
|
||||
sys.stdout.write(') ')
|
||||
sys.stdout.write('\n')
|
||||
print '<b>Valid patches in report period: %d</b><ul>' % total_patches
|
||||
for author in CI_SYSTEM:
|
||||
if not author in total_votes:
|
||||
print ('<li><font color=blue>No votes recorded for '
|
||||
'<b>%s</b></font></li>'
|
||||
% author)
|
||||
continue
|
||||
|
||||
percentage = (total_votes[author] * 100.0 / total_patches)
|
||||
|
||||
if percentage < 95.0:
|
||||
print '<font color=red>'
|
||||
|
||||
passed = passed_votes.get(author, 0)
|
||||
failed = failed_votes.get(author, 0)
|
||||
unparsed = unparsed_votes.get(author, 0)
|
||||
total = passed + failed + unparsed
|
||||
pass_percentage = passed * 100.0 / total
|
||||
fail_percentage = failed * 100.0 / total
|
||||
unparsed_percentage = unparsed * 100.0 / total
|
||||
print ('<li><b>%s</b> voted on %d patchsets (%.02f%%), '
|
||||
'passing %d (%.02f%%), failing %s (%.02f%%) and '
|
||||
'unparsed %d (%.02f%%)'
|
||||
% (author, total_votes[author], percentage, passed,
|
||||
pass_percentage, failed, fail_percentage, unparsed,
|
||||
unparsed_percentage))
|
||||
|
||||
if percentage < 95.0:
|
||||
print '</font>'
|
||||
|
||||
print '</li><ul>'
|
||||
print ('<li>Missed %d: %s</li>'
|
||||
% (len(missed_votes.get(author, [])),
|
||||
patch_list_as_html(missed_votes.get(author, []))))
|
||||
print '<li>Sentiment:</li><ul>'
|
||||
for sentiment in SENTIMENTS:
|
||||
count = len(sentiments.get(author, {}).get(sentiment, []))
|
||||
if count > 0:
|
||||
print '<li>%s: %d' % (sentiment, count )
|
||||
if sentiment != 'Positive':
|
||||
print ('(%s)'
|
||||
% patch_list_as_html(sentiments[author][sentiment]))
|
||||
print '</li>'
|
||||
|
||||
print '</ul></ul>'
|
||||
|
||||
print '</ul>'
|
||||
|
|
|
|||
184
skipped.json
184
skipped.json
|
|
@ -1,184 +0,0 @@
|
|||
{
|
||||
"Ailing Zhang": 1,
|
||||
"Dan Prince": 11,
|
||||
"Baodong (Robert) Li": 2,
|
||||
"Andrew Laski": 16,
|
||||
"Ryan Hsu": 8,
|
||||
"Daniel Kuffner": 2,
|
||||
"Maithem": 3,
|
||||
"Shane Wang": 4,
|
||||
"Dirk Mueller": 1,
|
||||
"Jason Dillaman": 1,
|
||||
"Subbu": 4,
|
||||
"Cedric Brandily": 1,
|
||||
"Facundo Farias": 2,
|
||||
"Sridevi Koushik": 1,
|
||||
"Monty Taylor": 2,
|
||||
"jan grant": 2,
|
||||
"Sabari Murugesan": 1,
|
||||
"Eric Harney": 1,
|
||||
"John Warren": 1,
|
||||
"Xiang Hui": 1,
|
||||
"Yuiko Takada": 3,
|
||||
"Christopher Yeoh": 35,
|
||||
"Sean Dague": 8,
|
||||
"Alan Kavanagh": 1,
|
||||
"Eric Brown": 3,
|
||||
"Noorul Islam K M": 1,
|
||||
"Pavel Kirpichyov": 1,
|
||||
"Ryan Moore": 1,
|
||||
"Josh Durgin": 1,
|
||||
"Andrea Rosa": 6,
|
||||
"Ionut Artarisi": 1,
|
||||
"Alvaro Lopez Garcia": 3,
|
||||
"Sidharth Surana": 8,
|
||||
"Inbar Shapira": 2,
|
||||
"lawrancejing": 1,
|
||||
"Marcos Ferm\u00edn Lobo": 3,
|
||||
"Sean M. Collins": 2,
|
||||
"Guillaume Thouvenin": 7,
|
||||
"Steve Kowalik": 1,
|
||||
"jichenjc": 36,
|
||||
"Juan Manuel Oll\u00e9": 1,
|
||||
"Ken'ichi Ohmichi": 53,
|
||||
"Leandro Ignacio Costantino": 7,
|
||||
"Michael Still": 35,
|
||||
"Mikhail Durnosvistov": 1,
|
||||
"Chris Krelle": 3,
|
||||
"Shlomi Sasson": 2,
|
||||
"timello": 1,
|
||||
"Kravchenko Pavel": 1,
|
||||
"wingwj": 2,
|
||||
"Hirofumi Ichihara": 1,
|
||||
"Aditi Raveesh": 3,
|
||||
"Roman Vyalov": 1,
|
||||
"\u00c9douard Thuleau": 2,
|
||||
"Vui Lam": 9,
|
||||
"Solly Ross": 2,
|
||||
"Qiu Yu": 14,
|
||||
"Xavier Queralt": 4,
|
||||
"Jaesang Lee": 1,
|
||||
"Nikola Dipanov": 12,
|
||||
"Bob Ball": 7,
|
||||
"Jay Lau": 65,
|
||||
"sahid": 35,
|
||||
"XiaoLiang Hu": 1,
|
||||
"Telles Mota Vidal N\u00f3brega": 4,
|
||||
"Geza Gemes": 2,
|
||||
"Rick Harris": 3,
|
||||
"Matt Dietz": 16,
|
||||
"Paul Murray": 6,
|
||||
"mark mcclain": 1,
|
||||
"Clark Boylan": 1,
|
||||
"Alessandro Pilotti": 14,
|
||||
"Liyi Meng": 1,
|
||||
"Lee Yarwood": 1,
|
||||
"xing-yang": 1,
|
||||
"John Haan": 3,
|
||||
"Zhi Yan Liu": 2,
|
||||
"Boris Pavlovic": 1,
|
||||
"Christopher Lefelhocz": 1,
|
||||
"Khanh-Toan TRAN": 3,
|
||||
"Elastic Recheck": 61,
|
||||
"Kaitlin Farr": 3,
|
||||
"xu-haiwei": 19,
|
||||
"Alexander Gorodnev": 4,
|
||||
"yasunori jitsukawa": 2,
|
||||
"Mathew Odden": 2,
|
||||
"Tracy Jones": 3,
|
||||
"Alexey Ovchinnikov": 6,
|
||||
"dave-mcnally": 4,
|
||||
"Shuangtai Tian": 24,
|
||||
"Xinyuan Huang": 1,
|
||||
"Ghe Rivero": 2,
|
||||
"Sreeram Yerrapragada": 2,
|
||||
"lifeless": 5,
|
||||
"Russell Bryant": 50,
|
||||
"Vish Ishaya": 1,
|
||||
"Aaron Rosen": 9,
|
||||
"John Garbutt": 11,
|
||||
"Sylvain Bauza": 1,
|
||||
"David Xie": 1,
|
||||
"Doug Hellmann": 1,
|
||||
"Roman Bogorodskiy": 2,
|
||||
"S\u00e9bastien Han": 2,
|
||||
"Debo~ Dutta": 1,
|
||||
"Sumanth Nagadavalli": 1,
|
||||
"Joshua Hesketh": 32,
|
||||
"liusheng": 4,
|
||||
"Daniel Berrange": 80,
|
||||
"Alex Xu": 8,
|
||||
"Vincent Untz": 1,
|
||||
"Wangpan": 12,
|
||||
"Sabari": 27,
|
||||
"Sergey Vilgelm": 1,
|
||||
"Phil Day": 7,
|
||||
"David Ripton": 29,
|
||||
"Melanie Witt": 1,
|
||||
"Mark McLoughlin": 26,
|
||||
"Eiichi Aikawa": 13,
|
||||
"Edward Hope-Morley": 4,
|
||||
"Chris Behrens": 15,
|
||||
"Alexis Lee": 1,
|
||||
"Kiyohiro Adachi": 1,
|
||||
"Arnaud Legendre": 5,
|
||||
"Yathiraj Udupi": 2,
|
||||
"Matt Riedemann": 53,
|
||||
"Matt Fischer": 1,
|
||||
"Pedro Marques": 1,
|
||||
"Radoslav Gerganov": 19,
|
||||
"Trivial Rebase": 35,
|
||||
"Gast\u00f3n Severina": 1,
|
||||
"Devananda van der Veen": 5,
|
||||
"Lin Tan": 8,
|
||||
"ChangBo Guo": 1,
|
||||
"Chen Xiao": 2,
|
||||
"Michael H Wilson": 2,
|
||||
"Lucas Alvares Gomes": 2,
|
||||
"Chris Buccella": 4,
|
||||
"Li Yingjun": 5,
|
||||
"David Jia": 1,
|
||||
"Venkatesh Sampath": 1,
|
||||
"Matthew Gilliard": 27,
|
||||
"LaunchpadSync": 74,
|
||||
"Ilya Pekelny": 1,
|
||||
"Rafael Folco": 2,
|
||||
"Hans Lindgren": 5,
|
||||
"Petrut Lucian": 2,
|
||||
"Ben Nemec": 3,
|
||||
"Joe Gordon": 37,
|
||||
"Dong Liu": 1,
|
||||
"Robert Tingirica": 1,
|
||||
"garyk": 147,
|
||||
"Alex Glikson": 1,
|
||||
"Brian Elliott": 7,
|
||||
"ijw-ubuntu": 2,
|
||||
"justinsb": 4,
|
||||
"haruka tanizawa": 2,
|
||||
"Roman Podoliaka": 1,
|
||||
"Michael Davies": 12,
|
||||
"p-draigbrady": 9,
|
||||
"Yaguang Tang": 5,
|
||||
"Matthew Booth": 8,
|
||||
"Dmitry Shulyak": 1,
|
||||
"lizheming": 2,
|
||||
"Victor Sergeyev": 6,
|
||||
"Haomeng,Wang": 1,
|
||||
"Dan Smith": 54,
|
||||
"gongysh": 2,
|
||||
"Tiantian Gao": 3,
|
||||
"Richard Jones": 1,
|
||||
"Kevin L. Mitchell": 30,
|
||||
"Vladik Romanovsky": 10,
|
||||
"Sam Morrison": 1,
|
||||
"Shawn Hartsock": 8,
|
||||
"Dazhao Yu": 2,
|
||||
"Aneesh Puliyedath Udumbath": 1,
|
||||
"Matthew Oliver": 1,
|
||||
"Lianhao Lu": 6,
|
||||
"Angus Salkeld": 2,
|
||||
"huangtianhua": 3,
|
||||
"Rushi Agrawal": 3,
|
||||
"Alan Pevec": 5,
|
||||
"Sandy Walsh": 1
|
||||
}
|
||||
Loading…
Reference in New Issue