Move general purpose functions out of owners
Change-Id: I17c16d7464647f8d81cdb7cf45dba8e63be5a005
This commit is contained in:
Tony Breeds
parent
af28fc385c
commit
7a802b65ad
|
|
@ -21,13 +21,12 @@ from __future__ import print_function
|
|||
|
||||
import csv
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
from openstack_election import utils
|
||||
|
||||
try:
|
||||
from string import maketrans
|
||||
except ImportError: # Python3
|
||||
|
|
@ -91,73 +90,6 @@ def date_merged(change, after=None, before=None):
|
|||
return date
|
||||
|
||||
|
||||
def requester(url, params={}, headers={}):
|
||||
"""A requests wrapper to consistently retry HTTPS queries"""
|
||||
|
||||
# Try up to 3 times
|
||||
retry = requests.Session()
|
||||
retry.mount("https://", requests.adapters.HTTPAdapter(max_retries=3))
|
||||
return retry.get(url=url, params=params, headers=headers)
|
||||
|
||||
|
||||
def decode_json(raw):
|
||||
"""Trap JSON decoding failures and provide more detailed errors"""
|
||||
|
||||
# Gerrit's REST API prepends a JSON-breaker to avoid XSS vulnerabilities
|
||||
if raw.text.startswith(")]}'"):
|
||||
trimmed = raw.text[4:]
|
||||
else:
|
||||
trimmed = raw.text
|
||||
|
||||
# Try to decode and bail with much detail if it fails
|
||||
try:
|
||||
decoded = json.loads(trimmed)
|
||||
except Exception:
|
||||
print('\nrequest returned %s error to query:\n\n %s\n'
|
||||
'\nwith detail:\n\n %s\n' % (raw, raw.url, trimmed),
|
||||
file=sys.stderr)
|
||||
raise
|
||||
return decoded
|
||||
|
||||
|
||||
def query_gerrit(method, params={}):
|
||||
"""Query the Gerrit REST API"""
|
||||
|
||||
# The base URL to Gerrit REST API
|
||||
GERRIT_API_URL = 'https://review.openstack.org/'
|
||||
|
||||
raw = requester(GERRIT_API_URL + method, params=params,
|
||||
headers={'Accept': 'application/json'})
|
||||
return decode_json(raw)
|
||||
|
||||
|
||||
def get_from_cgit(project, obj, params={}):
|
||||
"""Retrieve a file from the cgit interface"""
|
||||
|
||||
url = 'http://git.openstack.org/cgit/' + project + '/plain/' + obj
|
||||
raw = requester(url, params=params,
|
||||
headers={'Accept': 'application/json'})
|
||||
return yaml.safe_load(raw.text)
|
||||
|
||||
|
||||
def lookup_member(email):
|
||||
"""A requests wrapper to querying the OSF member directory API"""
|
||||
|
||||
# The OpenStack foundation member directory lookup API endpoint
|
||||
MEMBER_LOOKUP_URL = 'https://openstackid-resources.openstack.org/'
|
||||
|
||||
# URL pattern for querying foundation members by E-mail address
|
||||
raw = requester(MEMBER_LOOKUP_URL + '/api/public/v1/members',
|
||||
params={'filter[]': [
|
||||
'group_slug==foundation-members',
|
||||
'email==' + email,
|
||||
]},
|
||||
headers={'Accept': 'application/json'},
|
||||
)
|
||||
|
||||
return decode_json(raw)
|
||||
|
||||
|
||||
def main(options):
|
||||
"""The giant pile of spaghetti which does everything else"""
|
||||
|
||||
|
|
@ -241,17 +173,17 @@ def main(options):
|
|||
# TODO(fungi): make this a configurable option so that you can
|
||||
# for example supply a custom project list for running elections
|
||||
# in unofficial teams
|
||||
gov_projects = get_from_cgit('openstack/governance',
|
||||
'reference/projects.yaml',
|
||||
{'h': ref})
|
||||
gov_projects = utils.get_from_cgit('openstack/governance',
|
||||
'reference/projects.yaml',
|
||||
{'h': ref})
|
||||
|
||||
# The set of retired or removed "legacy" projects from governance
|
||||
# are merged into the main dict if their retired-on date falls
|
||||
# later than the after parameter for the qualifying time period
|
||||
# TODO(fungi): make this a configurable option
|
||||
old_projects = get_from_cgit('openstack/governance',
|
||||
'reference/legacy.yaml',
|
||||
{'h': ref})
|
||||
old_projects = utils.get_from_cgit('openstack/governance',
|
||||
'reference/legacy.yaml',
|
||||
{'h': ref})
|
||||
for project in old_projects:
|
||||
for deliverable in old_projects[project]['deliverables']:
|
||||
retired = old_projects[project]['deliverables'][deliverable].get(
|
||||
|
|
@ -277,7 +209,7 @@ def main(options):
|
|||
# in governance during transitions and also to filter out repos
|
||||
# listed in governance which don't actually exist
|
||||
ger_repos = dict(
|
||||
[(x.split('/')[-1], x) for x in query_gerrit('projects/')])
|
||||
[(x.split('/')[-1], x) for x in utils.query_gerrit('projects/')])
|
||||
|
||||
# This will be populated with change owners mapped to the
|
||||
# project-teams maintaining their respective Git repositories
|
||||
|
|
@ -321,7 +253,7 @@ def main(options):
|
|||
offset = 0
|
||||
changes = []
|
||||
while offset >= 0:
|
||||
changes += query_gerrit('changes/', params={
|
||||
changes += utils.query_gerrit('changes/', params={
|
||||
'q': 'project:%s %s' % (ger_repos[repo], match),
|
||||
'n': '100',
|
||||
'start': offset,
|
||||
|
|
@ -367,7 +299,7 @@ def main(options):
|
|||
if new:
|
||||
# Get the set of all E-mail addresses
|
||||
# Gerrit knows for this owner's account
|
||||
emails = query_gerrit(
|
||||
emails = utils.query_gerrit(
|
||||
'accounts/%s/emails'
|
||||
% change['owner']['_account_id'])
|
||||
|
||||
|
|
@ -552,7 +484,7 @@ def main(options):
|
|||
'addresses found for account %s' % owner, file=sys.stderr)
|
||||
continue
|
||||
for email in [owners[owner]['preferred']] + owners[owner]['extra']:
|
||||
member = lookup_member(email)
|
||||
member = utils.lookup_member(email)
|
||||
if member['data']:
|
||||
owners[owner]['member'] = member['data'][0]['id']
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import os
|
|||
import pickle
|
||||
import pytz
|
||||
import requests
|
||||
import sys
|
||||
import time
|
||||
import yaml
|
||||
|
||||
|
|
@ -27,7 +28,6 @@ from six.moves.urllib.parse import quote_plus
|
|||
from six.moves.urllib.request import urlopen
|
||||
|
||||
from openstack_election import config
|
||||
from openstack_election import owners
|
||||
|
||||
|
||||
# Library constants
|
||||
|
|
@ -52,6 +52,64 @@ def requester(url, params={}, headers={}):
|
|||
return retry.get(url=url, params=params, headers=headers)
|
||||
|
||||
|
||||
def decode_json(raw):
|
||||
"""Trap JSON decoding failures and provide more detailed errors"""
|
||||
|
||||
# Gerrit's REST API prepends a JSON-breaker to avoid XSS vulnerabilities
|
||||
if raw.text.startswith(")]}'"):
|
||||
trimmed = raw.text[4:]
|
||||
else:
|
||||
trimmed = raw.text
|
||||
|
||||
# Try to decode and bail with much detail if it fails
|
||||
try:
|
||||
decoded = json.loads(trimmed)
|
||||
except Exception:
|
||||
print('\nrequest returned %s error to query:\n\n %s\n'
|
||||
'\nwith detail:\n\n %s\n' % (raw, raw.url, trimmed),
|
||||
file=sys.stderr)
|
||||
raise
|
||||
return decoded
|
||||
|
||||
|
||||
def query_gerrit(method, params={}):
|
||||
"""Query the Gerrit REST API"""
|
||||
|
||||
# The base URL to Gerrit REST API
|
||||
GERRIT_API_URL = 'https://review.openstack.org/'
|
||||
|
||||
raw = requester(GERRIT_API_URL + method, params=params,
|
||||
headers={'Accept': 'application/json'})
|
||||
return decode_json(raw)
|
||||
|
||||
|
||||
def get_from_cgit(project, obj, params={}):
|
||||
"""Retrieve a file from the cgit interface"""
|
||||
|
||||
url = 'http://git.openstack.org/cgit/' + project + '/plain/' + obj
|
||||
raw = requester(url, params=params,
|
||||
headers={'Accept': 'application/json'})
|
||||
return yaml.safe_load(raw.text)
|
||||
|
||||
|
||||
def lookup_member(email):
|
||||
"""A requests wrapper to querying the OSF member directory API"""
|
||||
|
||||
# The OpenStack foundation member directory lookup API endpoint
|
||||
MEMBER_LOOKUP_URL = 'https://openstackid-resources.openstack.org/'
|
||||
|
||||
# URL pattern for querying foundation members by E-mail address
|
||||
raw = requester(MEMBER_LOOKUP_URL + '/api/public/v1/members',
|
||||
params={'filter[]': [
|
||||
'group_slug==foundation-members',
|
||||
'email==' + email,
|
||||
]},
|
||||
headers={'Accept': 'application/json'},
|
||||
)
|
||||
|
||||
return decode_json(raw)
|
||||
|
||||
|
||||
def load_exceptions():
|
||||
global exceptions
|
||||
exceptions = {}
|
||||
|
|
@ -73,6 +131,7 @@ def gerrit_datetime(dt):
|
|||
return dt.strftime('%Y-%m-%d %H:%M:%S %z')
|
||||
|
||||
|
||||
# TODO(tonyb): this is now basically a duplicate of query_gerrit()
|
||||
def gerrit_query(url, params=None):
|
||||
r = requester(url, params=params)
|
||||
if r.status_code == 200:
|
||||
|
|
@ -226,7 +285,7 @@ def build_candidates_list(election=conf['release']):
|
|||
for candidate_file in file_list:
|
||||
filepath = os.path.join(project_prefix, candidate_file)
|
||||
email = get_email(filepath)
|
||||
member = owners.lookup_member(email)
|
||||
member = lookup_member(email)
|
||||
candidates_list.append(
|
||||
{
|
||||
'url': ('%s/%s/plain/%s' %
|
||||
|
|
|
|||
Loading…
Reference in New Issue