Automatically update the Third-Party CI users group

This gets rid of the stupid users.py file and having people needing to
manually add their CI system to the repo. It will now update every 5
min (and on startup) via the ls-members Gerrit cli tool.

Change-Id: I2a588cacc50fc7867f880ddeaf7d49cd3ae9ed0e
This commit is contained in:
Patrick East 2015-06-15 13:49:12 -07:00
parent 780e54568d
commit 2c3257b3d6
3 changed files with 57 additions and 690 deletions

View File

@ -167,6 +167,17 @@ class Gerrit(object):
chunk = _query_chunk("%s %s" % (query, sortkey))
return alldata
def lsMembers(self, group):
cmd = "gerrit ls-members '%s'" % group
out, err = self._ssh(cmd)
if not out:
return False
lines = out.split('\n')
if not lines:
return False
return lines
def _open(self):
client = paramiko.SSHClient()
client.load_system_host_keys()

View File

@ -8,30 +8,20 @@ import config
import db_helper
from infra import gerrit
import logger
import users
USERS_UPDATE_INTERVAL = 60 * 5 # Seconds
class GerritCIListener():
def __init__(self):
self.ci_user_names = []
self.ci_users = {}
self.cfg = config.Config()
self.db = db_helper.DBHelper(self.cfg).get()
logger.init(self.cfg)
self.log = logger.get('scoreboard-gerrit-listener')
def get_thirdparty_users(self):
# TODO: figure out how to do the authentication..
# thirdparty_group = '95d633d37a5d6b06df758e57b1370705ec071a57'
# url = 'http://review.openstack.org/groups/%s/members' % thirdparty_group
# members = eval(urllib.urlopen(url).read())
members = users.third_party_group
for account in members:
username = account[u'username']
self.ci_user_names.append(username)
self.g = None
def is_ci_user(self, username):
# TODO: query this from gerrit. Maybe save a copy in the db?
return (username in self.ci_user_names) or (username == u'jenkins')
return username in self.ci_users
def determine_result(self, event):
approvals = event.get(u'approvals', None)
@ -97,27 +87,60 @@ class GerritCIListener():
self.log.info('Inserting %s' % review_num_patchset)
self.db.test_results.insert(patchset)
def periodic_query_users(self):
self.log.info('Updating Third-Party CI user list')
# Query for the most up to date list of users
raw_members = self.g.lsMembers('Third-Party CI')
# Cut off the first line, we don't care about the column headers
del raw_members[0]
current_members = {}
# Format the results for easier consumption
for line in raw_members:
if line:
id, user_name, full_name, email = line.split('\t')
current_members[user_name] = {
'gerrit_id': id,
'full_name': full_name,
'email': email,
}
# Add a special one for jenkins, its not part of the group but
# we want to count it too.
current_members['jenkins'] = {
'gerrit_id': -1,
'full_name': 'jenkins',
'email': ''
}
self.ci_users = current_members
self.log.debug('Updated ci group members: ' + str(self.ci_users))
# Schedule to re-run again
threading.Timer(USERS_UPDATE_INTERVAL, self.periodic_query_users).start()
def run(self):
# TODO: Maybe split this into its own process? Its kind of annoying that
# when modifying the UI portion of the project it stops gathering data..
hostname = self.cfg.gerrit_hostname()
username = self.cfg.gerrit_user()
port = self.cfg.gerrit_port()
keyfile = self.cfg.gerrit_key()
g = gerrit.Gerrit(hostname, username, port=port, keyfile=keyfile)
g.startWatching()
self.g = gerrit.Gerrit(hostname, username, port=port, keyfile=keyfile)
self.g.startWatching()
self.get_thirdparty_users()
self.periodic_query_users()
while True:
event = g.getEvent()
event = self.g.getEvent()
try:
self.handle_gerrit_event(event)
except:
self.log.exception('Failed to handle gerrit event: ')
g.eventDone()
self.log.exception('Failed to handle gerrit event: ' + str(event))
self.g.eventDone()
if __name__ == '__main__':
listener = GerritCIListener()

View File

@ -1,667 +0,0 @@
# TODO: Seriously... get rid of this thing...
# email addresses have been removed
third_party_group =[
{
u"_account_id": 12040,
u"name": u"A10 Networks CI",
u"username": u"a10networks-ci",
u"avatars": []
},
{
u"_account_id": 9845,
u"name": u"Arista CI",
u"username": u"arista-test",
u"avatars": []
},
{
u"_account_id": 9787,
u"name": u"Big Switch CI",
u"username": u"bsn",
u"avatars": []
},
{
u"_account_id": 10624,
u"name": u"Brocade ADX CI",
u"username": u"pattabi-ayyasami-ci",
u"avatars": []
},
{
u"_account_id": 9753,
u"name": u"Brocade BNA CI",
u"username": u"brocade_jenkins",
u"avatars": []
},
{
u"_account_id": 13396,
u"name": u"Brocade Fibre CI",
u"username": u"brocade-fibre-ci",
u"avatars": []
},
{
u"_account_id": 13051,
u"name": u"Brocade LBaaS CI",
u"username": u"brocade-lbaas-ci",
u"avatars": []
},
{
u"_account_id": 10294,
u"name": u"Brocade VDX CI",
u"username": u"bci",
u"avatars": []
},
{
u"_account_id": 10692,
u"name": u"Brocade Vyatta CI",
u"username": u"brocade-oss-service",
u"avatars": []
},
{
u"_account_id": 14214,
u"name": u"Cisco APIC CI",
u"username": u"cisco_apic_ci",
u"avatars": []
},
{
u"_account_id": 10192,
u"name": u"Cisco CI",
u"username": u"cisco_neutron_ci",
u"avatars": []
},
{
u"_account_id": 14212,
u"name": u"Cisco Tail-f CI",
u"username": u"cisco_tailf_ci",
u"avatars": []
},
{
u"_account_id": 14208,
u"name": u"Cisco ml2 CI",
u"username": u"cisco_ml2_ci",
u"avatars": []
},
{
u"_account_id": 9897,
u"name": u"Citrix NetScaler CI",
u"username": u"NetScalerAts",
u"avatars": []
},
{
u"_account_id": 10385,
u"name": u"Citrix XenServer CI",
u"username": u"citrix_xenserver_ci",
u"avatars": []
},
{
u"_account_id": 14259,
u"name": u"CloudByte CI",
u"username": u"CloudbyteCI",
u"avatars": []
},
{
u"_account_id": 14206,
u"name": u"CloudFounders OpenvStorage CI",
u"username": u"cloudfoundersci",
u"avatars": []
},
{
u"_account_id": 10193,
u"name": u"Compass CI",
u"username": u"compass_ci",
u"avatars": []
},
{
u"_account_id": 13049,
u"name": u"Coraid CI",
u"username": u"coraid-ci",
u"avatars": []
},
{
u"_account_id": 9578,
u"name": u"DB Datasets CI",
u"username": u"turbo-hipster",
u"avatars": []
},
{
u"_account_id": 12032,
u"name": u"Datera CI",
u"username": u"datera-ci",
u"avatars": []
},
{
u"_account_id": 13141,
u"name": u"Dell AFC CI",
u"username": u"dell-afc-ci",
u"avatars": []
},
{
u"_account_id": 15249,
u"name": u"Dell Storage CI",
u"username": u"dell-storage-ci",
u"avatars": []
},
{
u"_account_id": 12779,
u"name": u"Dell StorageCenter CI",
u"username": u"dell-storagecenter-ci",
u"avatars": []
},
{
u"_account_id": 5171,
u"name": u"Deprecated Citrix CI",
u"username": u"citrixjenkins",
u"avatars": []
},
{
u"_account_id": 10240,
u"name": u"Deprecated Designate CI",
u"username": u"designate-jenkins",
u"avatars": []
},
{
u"_account_id": 13143,
u"name": u"Deprecated Nexenta ISCSI NFS CI",
u"username": u"nexenta-iscsi-nfs-ci",
u"avatars": []
},
{
u"_account_id": 11016,
u"name": u"Deprecated RAX Heat CI",
u"username": u"raxheatci",
u"avatars": []
},
{
u"_account_id": 7092,
u"name": u"Deprecated Trove CI",
u"username": u"reddwarf",
u"avatars": []
},
{
u"_account_id": 10183,
u"name": u"Docker CI",
u"username": u"docker-ci",
u"avatars": []
},
{
u"_account_id": 12016,
u"name": u"EMC VMAX CI",
u"username": u"emc-vmax-ci",
u"avatars": []
},
{
u"_account_id": 12017,
u"name": u"EMC VNX CI",
u"username": u"emc-vnx-ci",
u"avatars": []
},
{
u"_account_id": 12018,
u"name": u"EMC ViPR CI",
u"username": u"emc-vipr-ci",
u"avatars": []
},
{
u"_account_id": 12033,
u"name": u"EMC XIO CI",
u"username": u"emc-xio-ci",
u"avatars": []
},
{
u"_account_id": 9885,
u"name": u"Embrane CI",
u"username": u"eci",
u"avatars": []
},
{
u"_account_id": 10387,
u"name": u"Freescale CI",
u"username": u"freescale-ci",
u"avatars": []
},
{
u"_account_id": 10808,
u"name": u"Fuel Bot",
u"username": u"fuel-watcher",
u"avatars": []
},
{
u"_account_id": 8971,
u"name": u"Fuel CI",
u"username": u"fuel-ci",
u"avatars": []
},
{
u"_account_id": 12489,
u"name": u"Fusion-io CI",
u"username": u"fusionio-ci",
u"avatars": []
},
{
u"_account_id": 12368,
u"name": u"GlobalLogic CI",
u"username": u"globallogic-ci",
u"avatars": []
},
{
u"_account_id": 12778,
u"name": u"HDS HNAS CI",
u"username": u"hds-hnas-ci",
u"avatars": []
},
{
u"_account_id": 16269,
u"name": u"Hitachi HBSD CI",
u"username": u"hitachi-hbsd-ci",
u"avatars": []
},
{
u"_account_id": 16660,
u"name": u"Hitachi HBSD2 CI",
u"username": u"hitachi-hbsd2-ci",
u"avatars": []
},
{
u"_account_id": 11811,
u"name": u"HP Storage CI",
u"username": u"hp-cinder-ci",
u"avatars": []
},
{
u"_account_id": 10503,
u"name": u"Huawei ML2 CI",
u"username": u"huawei-ci",
u"avatars": []
},
{
u"_account_id": 13628,
u"name": u"Huawei Volume CI",
u"username": u"huawei-volume-ci",
u"avatars": []
},
{
u"_account_id": 12251,
u"name": u"IB IPAM CI",
u"username": u"ib-ipam-ci",
u"avatars": []
},
{
u"_account_id": 9751,
u"name": u"IBM DB2 CI",
u"username": u"ibmdb2",
u"avatars": []
},
{
u"_account_id": 12499,
u"name": u"IBM DS8000 CI",
u"username": u"ibm-ds8k-ci",
u"avatars": []
},
{
u"_account_id": 12540,
u"name": u"IBM DS8000 CI",
u"username": u"ed3ed3",
u"avatars": []
},
{
u"_account_id": 12370,
u"name": u"IBM FlashSystem CI",
u"username": u"ibm-flashsystem-ci",
u"avatars": []
},
{
u"_account_id": 12491,
u"name": u"IBM GPFS CI",
u"username": u"ibm-gpfs-ci",
u"avatars": []
},
{
u"_account_id": 12492,
u"name": u"IBM NAS CI",
u"username": u"ibm-ibmnas-ci",
u"avatars": []
},
{
u"_account_id": 10118,
u"name": u"IBM PowerKVM CI",
u"username": u"powerkvm",
u"avatars": []
},
{
u"_account_id": 9752,
u"name": u"IBM PowerVC CI",
u"username": u"ibmpwrvc",
u"avatars": []
},
{
u"_account_id": 9846,
u"name": u"IBM SDN-VE CI",
u"username": u"ibmsdnve",
u"avatars": []
},
{
u"_account_id": 12202,
u"name": u"IBM Storwize CI",
u"username": u"ibm-storwize-ci",
u"avatars": []
},
{
u"_account_id": 12493,
u"name": u"IBM XIV CI",
u"username": u"ibm-xiv-ci",
u"avatars": []
},
{
u"_account_id": 10623,
u"name": u"IBM ZVM CI",
u"username": u"ibm-zvm-ci",
u"avatars": []
},
{
u"_account_id": 12081,
u"name": u"IBM xCAT CI",
u"username": u"ibm-xcat-ci",
u"avatars": []
},
{
u"_account_id": 14571,
u"name": u"Intel Networking CI",
u"username": u"intel-networking-ci",
u"avatars": []
},
{
u"_account_id": 11103,
u"name": u"Intel PCI CI",
u"username": u"intelotccloud",
u"avatars": []
},
{
u"_account_id": 10307,
u"name": u"Jay Pipes Testing CI",
u"username": u"jaypipes-testing",
u"avatars": []
},
{
u"_account_id": 10867,
u"name": u"LVS CI",
u"username": u"lvstest",
u"avatars": []
},
{
u"_account_id": 10712,
u"name": u"MagnetoDB CI",
u"username": u"jenkins-magnetodb",
u"avatars": []
},
{
u"_account_id": 9732,
u"name": u"Mellanox CI",
u"username": u"mellanox",
u"avatars": []
},
{
u"_account_id": 10121,
u"name": u"Metaplugin CI",
u"username": u"metaplugintest",
u"avatars": []
},
{
u"_account_id": 5170,
u"name": u"Microsoft Hyper-V CI",
u"username": u"hyper-v-ci",
u"avatars": []
},
{
u"_account_id": 13394,
u"name": u"Microsoft iSCSI CI",
u"username": u"microsoft-iscsi-ci",
u"avatars": []
},
{
u"_account_id": 9925,
u"name": u"Midokura CI",
u"username": u"midokura",
u"avatars": []
},
{
u"_account_id": 7821,
u"name": u"Murano CI",
u"username": u"murano-ci",
u"avatars": []
},
{
u"_account_id": 10116,
u"name": u"NEC CI",
u"username": u"nec-openstack-ci",
u"avatars": []
},
{
u"_account_id": 10621,
u"name": u"NetApp CI",
u"username": u"netapp-ci",
u"avatars": []
},
{
u"_account_id": 15764,
u"name": u"NetApp FC CI",
u"username": u"netapp-fc-ci",
u"avatars": []
},
{
u"_account_id": 13395,
u"name": u"NetApp NAS CI",
u"username": u"netapp-nas-ci",
u"avatars": []
},
{
u"_account_id": 15239,
u"name": u"Nexenta CI check",
u"username": u"hodos",
u"avatars": []
},
{
u"_account_id": 13627,
u"name": u"Nexenta iSCSI CI",
u"username": u"nexenta-iscsi-ci",
u"avatars": []
},
{
u"_account_id": 7246,
u"name": u"Nicira Bot",
u"username": u"nicirabot",
u"avatars": []
},
{
u"_account_id": 11611,
u"name": u"Nimble Storage CI",
u"username": u"nimbleci",
u"avatars": []
},
{
u"_account_id": 10184,
u"name": u"Nuage CI",
u"username": u"nuage-ci",
u"avatars": []
},
{
u"_account_id": 10153,
u"name": u"One Convergence CI",
u"username": u"oneconvergence",
u"avatars": []
},
{
u"_account_id": 12250,
u"name": u"Open Attic CI",
u"username": u"open-attic-ci",
u"avatars": []
},
{
u"_account_id": 9682,
u"name": u"OpenContrail CI",
u"username": u"contrail",
u"avatars": []
},
{
u"_account_id": 10386,
u"name": u"OpenDaylight CI",
u"username": u"odl-jenkins",
u"avatars": []
},
{
u"_account_id": 13144,
u"name": u"Oracle ZFSSA CI",
u"username": u"oracle-zfssa-ci",
u"avatars": []
},
{
u"_account_id": 10117,
u"name": u"PLUMgrid CI",
u"username": u"plumgrid-ci",
u"avatars": []
},
{
u"_account_id": 12780,
u"name": u"ProphetStor CI",
u"username": u"prophetstor-ci",
u"avatars": []
},
{
u"_account_id": 7360,
u"name": u"Puppet CI",
u"username": u"puppet-openstack-ci-user",
u"avatars": []
},
{
u"_account_id": 9133,
u"name": u"Puppet Ceph CI",
u"username": u"puppetceph",
u"avatars": []
},
{
u"_account_id": 12369,
u"name": u"Pure Storage CI",
u"username": u"purestorage-cinder-ci",
u"avatars": []
},
{
u"_account_id": 9828,
u"name": u"Radware CI",
u"username": u"radware3rdpartytesting",
u"avatars": []
},
{
u"_account_id": 9009,
u"name": u"Red Hat CI",
u"username": u"redhatci",
u"avatars": []
},
{
u"_account_id": 9681,
u"name": u"Ryu CI",
u"username": u"neutronryu",
u"avatars": []
},
{
u"_account_id": 12494,
u"name": u"SUSE Cloud CI",
u"username": u"suse-cloud-ci",
u"avatars": []
},
{
u"_account_id": 12034,
u"name": u"Sacharya CI",
u"username": u"sacharya-ci",
u"avatars": []
},
{
u"_account_id": 7213,
u"name": u"Sahara Hadoop Cluster CI",
u"username": u"savanna-ci",
u"avatars": []
},
{
u"_account_id": 12249,
u"name": u"Scality CI",
u"username": u"scality-ci",
u"avatars": []
},
{
u"_account_id": 2166,
u"name": u"SmokeStack CI",
u"username": u"smokestack",
u"avatars": []
},
{
u"_account_id": 12019,
u"name": u"Snabb NFV CI",
u"username": u"snabb-nfv-ci",
u"avatars": []
},
{
u"_account_id": 10622,
u"name": u"SolidFire CI",
u"username": u"sfci",
u"avatars": []
},
{
u"_account_id": 13052,
u"name": u"SwiftStack Cluster CI",
u"username": u"swiftstack-cluster-ci",
u"avatars": []
},
{
u"_account_id": 11258,
u"name": u"THSTACK CI",
u"username": u"thstack-ci",
u"avatars": []
},
{
u"_account_id": 9695,
u"name": u"Tail-f NCS CI",
u"username": u"tailfncs",
u"avatars": []
},
{
u"_account_id": 13050,
u"name": u"VMWare Congress CI",
u"username": u"vmware-congress-ci",
u"avatars": []
},
{
u"_account_id": 9008,
u"name": u"VMware NSX CI",
u"username": u"vmwareminesweeper",
u"avatars": []
},
{
u"_account_id": 7584,
u"name": u"Vanilla Bot",
u"username": u"vanillabot",
u"avatars": []
},
{
u"_account_id": 9884,
u"name": u"Wherenow.org CI",
u"username": u"wherenowjenkins",
u"avatars": []
},
{
u"_account_id": 13142,
u"name": u"X-IO ISE CI",
u"username": u"x-io-ise-ci",
u"avatars": []
},
{
u"_account_id": 10119,
u"name": u"vArmour CI",
u"username": u"varmourci",
u"avatars": []
},
{
u"_account_id": 15286,
u"name": u"XenProject CI",
u"username": u"XenProject-CI",
u"avatars": []
}
]