From 2c3257b3d6f628e78fbcb0bcba31cd1fadf91240 Mon Sep 17 00:00:00 2001 From: Patrick East Date: Mon, 15 Jun 2015 13:49:12 -0700 Subject: [PATCH] Automatically update the Third-Party CI users group This gets rid of the stupid users.py file and having people needing to manually add their CI system to the repo. It will now update every 5 min (and on startup) via the ls-members Gerrit cli tool. Change-Id: I2a588cacc50fc7867f880ddeaf7d49cd3ae9ed0e --- monitoring/scoreboard/infra/gerrit.py | 11 + .../scoreboard/scoreboard_gerrit_listener.py | 69 +- monitoring/scoreboard/users.py | 667 ------------------ 3 files changed, 57 insertions(+), 690 deletions(-) delete mode 100644 monitoring/scoreboard/users.py diff --git a/monitoring/scoreboard/infra/gerrit.py b/monitoring/scoreboard/infra/gerrit.py index 1e7888b..612102c 100644 --- a/monitoring/scoreboard/infra/gerrit.py +++ b/monitoring/scoreboard/infra/gerrit.py @@ -167,6 +167,17 @@ class Gerrit(object): chunk = _query_chunk("%s %s" % (query, sortkey)) return alldata + def lsMembers(self, group): + cmd = "gerrit ls-members '%s'" % group + out, err = self._ssh(cmd) + if not out: + return False + lines = out.split('\n') + if not lines: + return False + + return lines + def _open(self): client = paramiko.SSHClient() client.load_system_host_keys() diff --git a/monitoring/scoreboard/scoreboard_gerrit_listener.py b/monitoring/scoreboard/scoreboard_gerrit_listener.py index 14dd8e0..0a4120f 100755 --- a/monitoring/scoreboard/scoreboard_gerrit_listener.py +++ b/monitoring/scoreboard/scoreboard_gerrit_listener.py @@ -8,30 +8,20 @@ import config import db_helper from infra import gerrit import logger -import users +USERS_UPDATE_INTERVAL = 60 * 5 # Seconds class GerritCIListener(): def __init__(self): - self.ci_user_names = [] + self.ci_users = {} self.cfg = config.Config() self.db = db_helper.DBHelper(self.cfg).get() logger.init(self.cfg) self.log = logger.get('scoreboard-gerrit-listener') - - def get_thirdparty_users(self): - # TODO: figure out how to do the authentication.. - # thirdparty_group = '95d633d37a5d6b06df758e57b1370705ec071a57' - # url = 'http://review.openstack.org/groups/%s/members' % thirdparty_group - # members = eval(urllib.urlopen(url).read()) - members = users.third_party_group - for account in members: - username = account[u'username'] - self.ci_user_names.append(username) + self.g = None def is_ci_user(self, username): - # TODO: query this from gerrit. Maybe save a copy in the db? - return (username in self.ci_user_names) or (username == u'jenkins') + return username in self.ci_users def determine_result(self, event): approvals = event.get(u'approvals', None) @@ -97,27 +87,60 @@ class GerritCIListener(): self.log.info('Inserting %s' % review_num_patchset) self.db.test_results.insert(patchset) + def periodic_query_users(self): + self.log.info('Updating Third-Party CI user list') + + # Query for the most up to date list of users + raw_members = self.g.lsMembers('Third-Party CI') + + # Cut off the first line, we don't care about the column headers + del raw_members[0] + + current_members = {} + + # Format the results for easier consumption + for line in raw_members: + if line: + id, user_name, full_name, email = line.split('\t') + current_members[user_name] = { + 'gerrit_id': id, + 'full_name': full_name, + 'email': email, + } + + # Add a special one for jenkins, its not part of the group but + # we want to count it too. + current_members['jenkins'] = { + 'gerrit_id': -1, + 'full_name': 'jenkins', + 'email': '' + } + + self.ci_users = current_members + + self.log.debug('Updated ci group members: ' + str(self.ci_users)) + + # Schedule to re-run again + threading.Timer(USERS_UPDATE_INTERVAL, self.periodic_query_users).start() + def run(self): - # TODO: Maybe split this into its own process? Its kind of annoying that - # when modifying the UI portion of the project it stops gathering data.. hostname = self.cfg.gerrit_hostname() username = self.cfg.gerrit_user() port = self.cfg.gerrit_port() keyfile = self.cfg.gerrit_key() - g = gerrit.Gerrit(hostname, username, port=port, keyfile=keyfile) - g.startWatching() + self.g = gerrit.Gerrit(hostname, username, port=port, keyfile=keyfile) + self.g.startWatching() - self.get_thirdparty_users() + self.periodic_query_users() while True: - event = g.getEvent() + event = self.g.getEvent() try: self.handle_gerrit_event(event) except: - self.log.exception('Failed to handle gerrit event: ') - g.eventDone() - + self.log.exception('Failed to handle gerrit event: ' + str(event)) + self.g.eventDone() if __name__ == '__main__': listener = GerritCIListener() diff --git a/monitoring/scoreboard/users.py b/monitoring/scoreboard/users.py deleted file mode 100644 index 40127eb..0000000 --- a/monitoring/scoreboard/users.py +++ /dev/null @@ -1,667 +0,0 @@ - -# TODO: Seriously... get rid of this thing... - -# email addresses have been removed - -third_party_group =[ - { - u"_account_id": 12040, - u"name": u"A10 Networks CI", - u"username": u"a10networks-ci", - u"avatars": [] - }, - { - u"_account_id": 9845, - u"name": u"Arista CI", - u"username": u"arista-test", - u"avatars": [] - }, - { - u"_account_id": 9787, - u"name": u"Big Switch CI", - u"username": u"bsn", - u"avatars": [] - }, - { - u"_account_id": 10624, - u"name": u"Brocade ADX CI", - u"username": u"pattabi-ayyasami-ci", - u"avatars": [] - }, - { - u"_account_id": 9753, - u"name": u"Brocade BNA CI", - u"username": u"brocade_jenkins", - u"avatars": [] - }, - { - u"_account_id": 13396, - u"name": u"Brocade Fibre CI", - u"username": u"brocade-fibre-ci", - u"avatars": [] - }, - { - u"_account_id": 13051, - u"name": u"Brocade LBaaS CI", - u"username": u"brocade-lbaas-ci", - u"avatars": [] - }, - { - u"_account_id": 10294, - u"name": u"Brocade VDX CI", - u"username": u"bci", - u"avatars": [] - }, - { - u"_account_id": 10692, - u"name": u"Brocade Vyatta CI", - u"username": u"brocade-oss-service", - u"avatars": [] - }, - { - u"_account_id": 14214, - u"name": u"Cisco APIC CI", - u"username": u"cisco_apic_ci", - u"avatars": [] - }, - { - u"_account_id": 10192, - u"name": u"Cisco CI", - u"username": u"cisco_neutron_ci", - u"avatars": [] - }, - { - u"_account_id": 14212, - u"name": u"Cisco Tail-f CI", - u"username": u"cisco_tailf_ci", - u"avatars": [] - }, - { - u"_account_id": 14208, - u"name": u"Cisco ml2 CI", - u"username": u"cisco_ml2_ci", - u"avatars": [] - }, - { - u"_account_id": 9897, - u"name": u"Citrix NetScaler CI", - u"username": u"NetScalerAts", - u"avatars": [] - }, - { - u"_account_id": 10385, - u"name": u"Citrix XenServer CI", - u"username": u"citrix_xenserver_ci", - u"avatars": [] - }, - { - u"_account_id": 14259, - u"name": u"CloudByte CI", - u"username": u"CloudbyteCI", - u"avatars": [] - }, - { - u"_account_id": 14206, - u"name": u"CloudFounders OpenvStorage CI", - u"username": u"cloudfoundersci", - u"avatars": [] - }, - { - u"_account_id": 10193, - u"name": u"Compass CI", - u"username": u"compass_ci", - u"avatars": [] - }, - { - u"_account_id": 13049, - u"name": u"Coraid CI", - u"username": u"coraid-ci", - u"avatars": [] - }, - { - u"_account_id": 9578, - u"name": u"DB Datasets CI", - u"username": u"turbo-hipster", - u"avatars": [] - }, - { - u"_account_id": 12032, - u"name": u"Datera CI", - u"username": u"datera-ci", - u"avatars": [] - }, - { - u"_account_id": 13141, - u"name": u"Dell AFC CI", - u"username": u"dell-afc-ci", - u"avatars": [] - }, - { - u"_account_id": 15249, - u"name": u"Dell Storage CI", - u"username": u"dell-storage-ci", - u"avatars": [] - }, - { - u"_account_id": 12779, - u"name": u"Dell StorageCenter CI", - u"username": u"dell-storagecenter-ci", - u"avatars": [] - }, - { - u"_account_id": 5171, - u"name": u"Deprecated Citrix CI", - u"username": u"citrixjenkins", - u"avatars": [] - }, - { - u"_account_id": 10240, - u"name": u"Deprecated Designate CI", - u"username": u"designate-jenkins", - u"avatars": [] - }, - { - u"_account_id": 13143, - u"name": u"Deprecated Nexenta ISCSI NFS CI", - u"username": u"nexenta-iscsi-nfs-ci", - u"avatars": [] - }, - { - u"_account_id": 11016, - u"name": u"Deprecated RAX Heat CI", - u"username": u"raxheatci", - u"avatars": [] - }, - { - u"_account_id": 7092, - u"name": u"Deprecated Trove CI", - u"username": u"reddwarf", - u"avatars": [] - }, - { - u"_account_id": 10183, - u"name": u"Docker CI", - u"username": u"docker-ci", - u"avatars": [] - }, - { - u"_account_id": 12016, - u"name": u"EMC VMAX CI", - u"username": u"emc-vmax-ci", - u"avatars": [] - }, - { - u"_account_id": 12017, - u"name": u"EMC VNX CI", - u"username": u"emc-vnx-ci", - u"avatars": [] - }, - { - u"_account_id": 12018, - u"name": u"EMC ViPR CI", - u"username": u"emc-vipr-ci", - u"avatars": [] - }, - { - u"_account_id": 12033, - u"name": u"EMC XIO CI", - u"username": u"emc-xio-ci", - u"avatars": [] - }, - { - u"_account_id": 9885, - u"name": u"Embrane CI", - u"username": u"eci", - u"avatars": [] - }, - { - u"_account_id": 10387, - u"name": u"Freescale CI", - u"username": u"freescale-ci", - u"avatars": [] - }, - { - u"_account_id": 10808, - u"name": u"Fuel Bot", - u"username": u"fuel-watcher", - u"avatars": [] - }, - { - u"_account_id": 8971, - u"name": u"Fuel CI", - u"username": u"fuel-ci", - u"avatars": [] - }, - { - u"_account_id": 12489, - u"name": u"Fusion-io CI", - u"username": u"fusionio-ci", - u"avatars": [] - }, - { - u"_account_id": 12368, - u"name": u"GlobalLogic CI", - u"username": u"globallogic-ci", - u"avatars": [] - }, - { - u"_account_id": 12778, - u"name": u"HDS HNAS CI", - u"username": u"hds-hnas-ci", - u"avatars": [] - }, - { - u"_account_id": 16269, - u"name": u"Hitachi HBSD CI", - u"username": u"hitachi-hbsd-ci", - u"avatars": [] - }, - { - u"_account_id": 16660, - u"name": u"Hitachi HBSD2 CI", - u"username": u"hitachi-hbsd2-ci", - u"avatars": [] - }, - { - u"_account_id": 11811, - u"name": u"HP Storage CI", - u"username": u"hp-cinder-ci", - u"avatars": [] - }, - { - u"_account_id": 10503, - u"name": u"Huawei ML2 CI", - u"username": u"huawei-ci", - u"avatars": [] - }, - { - u"_account_id": 13628, - u"name": u"Huawei Volume CI", - u"username": u"huawei-volume-ci", - u"avatars": [] - }, - { - u"_account_id": 12251, - u"name": u"IB IPAM CI", - u"username": u"ib-ipam-ci", - u"avatars": [] - }, - { - u"_account_id": 9751, - u"name": u"IBM DB2 CI", - u"username": u"ibmdb2", - u"avatars": [] - }, - { - u"_account_id": 12499, - u"name": u"IBM DS8000 CI", - u"username": u"ibm-ds8k-ci", - u"avatars": [] - }, - { - u"_account_id": 12540, - u"name": u"IBM DS8000 CI", - u"username": u"ed3ed3", - u"avatars": [] - }, - { - u"_account_id": 12370, - u"name": u"IBM FlashSystem CI", - u"username": u"ibm-flashsystem-ci", - u"avatars": [] - }, - { - u"_account_id": 12491, - u"name": u"IBM GPFS CI", - u"username": u"ibm-gpfs-ci", - u"avatars": [] - }, - { - u"_account_id": 12492, - u"name": u"IBM NAS CI", - u"username": u"ibm-ibmnas-ci", - u"avatars": [] - }, - { - u"_account_id": 10118, - u"name": u"IBM PowerKVM CI", - u"username": u"powerkvm", - u"avatars": [] - }, - { - u"_account_id": 9752, - u"name": u"IBM PowerVC CI", - u"username": u"ibmpwrvc", - u"avatars": [] - }, - { - u"_account_id": 9846, - u"name": u"IBM SDN-VE CI", - u"username": u"ibmsdnve", - u"avatars": [] - }, - { - u"_account_id": 12202, - u"name": u"IBM Storwize CI", - u"username": u"ibm-storwize-ci", - u"avatars": [] - }, - { - u"_account_id": 12493, - u"name": u"IBM XIV CI", - u"username": u"ibm-xiv-ci", - u"avatars": [] - }, - { - u"_account_id": 10623, - u"name": u"IBM ZVM CI", - u"username": u"ibm-zvm-ci", - u"avatars": [] - }, - { - u"_account_id": 12081, - u"name": u"IBM xCAT CI", - u"username": u"ibm-xcat-ci", - u"avatars": [] - }, - { - u"_account_id": 14571, - u"name": u"Intel Networking CI", - u"username": u"intel-networking-ci", - u"avatars": [] - }, - { - u"_account_id": 11103, - u"name": u"Intel PCI CI", - u"username": u"intelotccloud", - u"avatars": [] - }, - { - u"_account_id": 10307, - u"name": u"Jay Pipes Testing CI", - u"username": u"jaypipes-testing", - u"avatars": [] - }, - { - u"_account_id": 10867, - u"name": u"LVS CI", - u"username": u"lvstest", - u"avatars": [] - }, - { - u"_account_id": 10712, - u"name": u"MagnetoDB CI", - u"username": u"jenkins-magnetodb", - u"avatars": [] - }, - { - u"_account_id": 9732, - u"name": u"Mellanox CI", - u"username": u"mellanox", - u"avatars": [] - }, - { - u"_account_id": 10121, - u"name": u"Metaplugin CI", - u"username": u"metaplugintest", - u"avatars": [] - }, - { - u"_account_id": 5170, - u"name": u"Microsoft Hyper-V CI", - u"username": u"hyper-v-ci", - u"avatars": [] - }, - { - u"_account_id": 13394, - u"name": u"Microsoft iSCSI CI", - u"username": u"microsoft-iscsi-ci", - u"avatars": [] - }, - { - u"_account_id": 9925, - u"name": u"Midokura CI", - u"username": u"midokura", - u"avatars": [] - }, - { - u"_account_id": 7821, - u"name": u"Murano CI", - u"username": u"murano-ci", - u"avatars": [] - }, - { - u"_account_id": 10116, - u"name": u"NEC CI", - u"username": u"nec-openstack-ci", - u"avatars": [] - }, - { - u"_account_id": 10621, - u"name": u"NetApp CI", - u"username": u"netapp-ci", - u"avatars": [] - }, - { - u"_account_id": 15764, - u"name": u"NetApp FC CI", - u"username": u"netapp-fc-ci", - u"avatars": [] - }, - { - u"_account_id": 13395, - u"name": u"NetApp NAS CI", - u"username": u"netapp-nas-ci", - u"avatars": [] - }, - { - u"_account_id": 15239, - u"name": u"Nexenta CI check", - u"username": u"hodos", - u"avatars": [] - }, - { - u"_account_id": 13627, - u"name": u"Nexenta iSCSI CI", - u"username": u"nexenta-iscsi-ci", - u"avatars": [] - }, - { - u"_account_id": 7246, - u"name": u"Nicira Bot", - u"username": u"nicirabot", - u"avatars": [] - }, - { - u"_account_id": 11611, - u"name": u"Nimble Storage CI", - u"username": u"nimbleci", - u"avatars": [] - }, - { - u"_account_id": 10184, - u"name": u"Nuage CI", - u"username": u"nuage-ci", - u"avatars": [] - }, - { - u"_account_id": 10153, - u"name": u"One Convergence CI", - u"username": u"oneconvergence", - u"avatars": [] - }, - { - u"_account_id": 12250, - u"name": u"Open Attic CI", - u"username": u"open-attic-ci", - u"avatars": [] - }, - { - u"_account_id": 9682, - u"name": u"OpenContrail CI", - u"username": u"contrail", - u"avatars": [] - }, - { - u"_account_id": 10386, - u"name": u"OpenDaylight CI", - u"username": u"odl-jenkins", - u"avatars": [] - }, - { - u"_account_id": 13144, - u"name": u"Oracle ZFSSA CI", - u"username": u"oracle-zfssa-ci", - u"avatars": [] - }, - { - u"_account_id": 10117, - u"name": u"PLUMgrid CI", - u"username": u"plumgrid-ci", - u"avatars": [] - }, - { - u"_account_id": 12780, - u"name": u"ProphetStor CI", - u"username": u"prophetstor-ci", - u"avatars": [] - }, - { - u"_account_id": 7360, - u"name": u"Puppet CI", - u"username": u"puppet-openstack-ci-user", - u"avatars": [] - }, - { - u"_account_id": 9133, - u"name": u"Puppet Ceph CI", - u"username": u"puppetceph", - u"avatars": [] - }, - { - u"_account_id": 12369, - u"name": u"Pure Storage CI", - u"username": u"purestorage-cinder-ci", - u"avatars": [] - }, - { - u"_account_id": 9828, - u"name": u"Radware CI", - u"username": u"radware3rdpartytesting", - u"avatars": [] - }, - { - u"_account_id": 9009, - u"name": u"Red Hat CI", - u"username": u"redhatci", - u"avatars": [] - }, - { - u"_account_id": 9681, - u"name": u"Ryu CI", - u"username": u"neutronryu", - u"avatars": [] - }, - { - u"_account_id": 12494, - u"name": u"SUSE Cloud CI", - u"username": u"suse-cloud-ci", - u"avatars": [] - }, - { - u"_account_id": 12034, - u"name": u"Sacharya CI", - u"username": u"sacharya-ci", - u"avatars": [] - }, - { - u"_account_id": 7213, - u"name": u"Sahara Hadoop Cluster CI", - u"username": u"savanna-ci", - u"avatars": [] - }, - { - u"_account_id": 12249, - u"name": u"Scality CI", - u"username": u"scality-ci", - u"avatars": [] - }, - { - u"_account_id": 2166, - u"name": u"SmokeStack CI", - u"username": u"smokestack", - u"avatars": [] - }, - { - u"_account_id": 12019, - u"name": u"Snabb NFV CI", - u"username": u"snabb-nfv-ci", - u"avatars": [] - }, - { - u"_account_id": 10622, - u"name": u"SolidFire CI", - u"username": u"sfci", - u"avatars": [] - }, - { - u"_account_id": 13052, - u"name": u"SwiftStack Cluster CI", - u"username": u"swiftstack-cluster-ci", - u"avatars": [] - }, - { - u"_account_id": 11258, - u"name": u"THSTACK CI", - u"username": u"thstack-ci", - u"avatars": [] - }, - { - u"_account_id": 9695, - u"name": u"Tail-f NCS CI", - u"username": u"tailfncs", - u"avatars": [] - }, - { - u"_account_id": 13050, - u"name": u"VMWare Congress CI", - u"username": u"vmware-congress-ci", - u"avatars": [] - }, - { - u"_account_id": 9008, - u"name": u"VMware NSX CI", - u"username": u"vmwareminesweeper", - u"avatars": [] - }, - { - u"_account_id": 7584, - u"name": u"Vanilla Bot", - u"username": u"vanillabot", - u"avatars": [] - }, - { - u"_account_id": 9884, - u"name": u"Wherenow.org CI", - u"username": u"wherenowjenkins", - u"avatars": [] - }, - { - u"_account_id": 13142, - u"name": u"X-IO ISE CI", - u"username": u"x-io-ise-ci", - u"avatars": [] - }, - { - u"_account_id": 10119, - u"name": u"vArmour CI", - u"username": u"varmourci", - u"avatars": [] - }, - { - u"_account_id": 15286, - u"name": u"XenProject CI", - u"username": u"XenProject-CI", - u"avatars": [] - } -]