[zanata] Fixes zanata_users with REST API

Zanata >=4 which is now on production utilizes REST API [1].
This commit fixes zanata_users.py to make compatible with
Zanata REST API and syncs translation_team.yaml up to date.

[1] http://zanata.org/zanata-platform/rest-api-docs/resource_LocalesResource.html

Change-Id: Ia99db310cff236797ef403ba5534d853ed63e3e3
This commit is contained in:
Ian Y. Choi 2018-07-24 22:50:58 +09:00
parent 994b3ed97f
commit 876d6cb939
2 changed files with 84 additions and 69 deletions

View File

@ -20,14 +20,11 @@ Th:
- phithak
- sawangpong
ar:
coordinators:
- ianychoi
coordinators: []
language: Arabic
reviewers:
- ianychoi
reviewers: []
translators:
- hosam
- ianychoi
- synax
- welabbar
- yasir
@ -87,6 +84,7 @@ de:
- senk
- vcomas3
translators:
- alexanderloew2
- aperri
- atalanttore
- cduch
@ -183,6 +181,7 @@ es:
- hybridpollo
- iranzo
- jfrancoa
- jmont
- jorpilo
- josemhb
- mariantb
@ -207,6 +206,7 @@ es-MX:
- sergioerazo
translators:
- heleno_jimenez
- jmont
- scuellarv
- sergioerazo
- toxickore
@ -267,6 +267,7 @@ fr:
- corin
- croe
- didier
- emacchi
- eouzans
- fdotfr
- flowalrave
@ -280,6 +281,7 @@ fr:
- jftalta
- kodokuu
- liced
- linkid
- lnic
- makowals
- mmarin
@ -310,15 +312,18 @@ hi:
- chandankumar
hu:
coordinators:
- dabosshun
- gergelys
- marcus
- szilvia
language: Hungarian
reviewers:
- dabosshun
- gergelys
- marcus
- szilvia
translators:
- dabosshun
- gergelys
- marcus
- szilvia
@ -376,6 +381,7 @@ ja:
- hidekazuna
- k_makabe007
- ken_ishii
- kuroshun
- macjack
- makita
- mariojv
@ -443,6 +449,7 @@ ko-KR:
- ianychoi
- jaewook_oh
- janghe11
- jc01rho
- jitaejang
- johjuhyun
- jongwooh
@ -552,6 +559,7 @@ pt-BR:
- diegodalmolin
- dritec
- dsantos
- ebrandi
- ericbaum
- fcpimenta
- franciosi
@ -567,11 +575,13 @@ pt-BR:
- lveneziano
- marcelodieder
- marcio
- marcocristofolini
- mariaandrada
- marquesc
- melotto
- mrbits
- nokostya
- northward
- raildomascena
- raquellira
- rbraga
@ -671,6 +681,7 @@ tr-TR:
- oguzy
- tuna_verid
- yasemindemiral
- ziya_karakaya
vi-VN:
coordinators:
- kiennt26
@ -679,10 +690,12 @@ vi-VN:
- kiennt26
translators:
- kiennt26
- nguyentrihai
zh-CN:
coordinators:
- daisyycguo
- vuuv
- zzxwill
language: Chinese (China)
reviewers:
- actionchen
@ -692,6 +705,7 @@ zh-CN:
- dliu
- fifieldt
- jimmyli
- johnwoo_lee
- lindayang
- liuchaoxin
- liujunpeng
@ -701,6 +715,7 @@ zh-CN:
- vuuv
- yanheven
- zhaochao
- zzxwill
translators:
- 979zhang
- actionchen
@ -718,7 +733,9 @@ zh-CN:
- coffee
- cuixiaoyu
- daisyycguo
- dayou
- dliu
- domine
- e010
- fanzhang
- fifieldt
@ -737,7 +754,10 @@ zh-CN:
- irina
- javeme
- jelanliu
- jenner
- jiangyikun
- jimmyli
- johnwoo_lee
- lapalm
- lawrence
- leiyahsuai
@ -780,10 +800,14 @@ zh-CN:
- wangfang
- woshishui419
- wuyongpeng
- xhkjoe
- xiangbinbin
- xiangqiao
- xuyang
- xywang
- yagehu
- yaguang
- yanghongyu
- yanghy
- yanheven
- yihleong
@ -798,10 +822,13 @@ zh-CN:
- zheng_minxian
- zhengjie7800
- zhonghuali
- zhongjun2
- zhongtian
- zhouchangxun
- zhsj
- zhubx007
- zhugaoxiao
- zzxwill
zh-TW:
coordinators:
- ccheng
@ -826,6 +853,7 @@ zh-TW:
- bokaitseng
- ccheng
- danny
- dayou
- derekjhyang
- e010
- elliswu

View File

@ -15,14 +15,16 @@
import argparse
from collections import OrderedDict
import json
import logging
import random
import bs4
import requests
import yaml
base_url = "https://translate.openstack.org/%s"
yaml_comment = """\
ZANATA_URI = 'https://translate.openstack.org/rest/%s'
LOG = logging.getLogger('zanata_users')
YAML_COMMENT = """\
# This file is generated by tools/zanata/zanata_users.py.
# For more information, see
# https://docs.openstack.org/i18n/latest/tools.html#sync-the-translator-list-with-zanata
@ -39,51 +41,47 @@ class ZanataUtility(object):
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'
]
def read_uri(self, uri):
headers = {
'User-Agent': random.choice(ZanataUtility.user_agents)
}
req = requests.get(url=uri, headers=headers)
return req.text
def read_uri(self, uri, headers):
try:
headers['User-Agent'] = random.choice(ZanataUtility.user_agents)
req = requests.get(uri, headers=headers)
return req.text
except Exception as e:
LOG.error('Error "%(error)s" while reading uri %(uri)s',
{'error': e, 'uri': uri})
raise
def iter_language_members(self, uri):
data = self.read_uri(base_url % uri)
soup = bs4.BeautifulSoup(data, 'html.parser')
users = soup.find('ul', {'class': 'list--stats'}) \
.findAll('li', {'class': 'l--pad-all-quarter'})
def read_json_from_uri(self, uri):
data = self.read_uri(uri, {'Accept': 'application/json'})
try:
return json.loads(data)
except Exception as e:
LOG.error('Error "%(error)s" parsing json from uri %(uri)s',
{'error': e, 'uri': uri})
raise
for user in users:
user_id = user.find('a').text.strip()
def get_locale_members(self, language_id):
uri = ZANATA_URI % ('locales/locale/%s/members' % language_id)
LOG.debug("Reading the list of locale members in %s from %s"
% (language_id, uri))
locale_members_data = self.read_json_from_uri(uri)
return locale_members_data
roles_tag = user.find('ul', {'class': 'list--horizontal'}) \
.find('li')
roles = roles_tag.text.strip().split(', ')
for role_name in roles:
yield role_name, user_id
def get_languages(self):
data = self.read_uri(base_url % 'language/list')
soup = bs4.BeautifulSoup(data, 'html.parser')
def get_locales(self):
# The default value of sizePerPage is 10, which needs to be increased
# http://zanata.org/zanata-platform/rest-api-docs/resource_LocalesResource.html
uri = ZANATA_URI % 'locales?sizePerPage=255'
LOG.debug("Reading the list of locales from %s" % uri)
locales_data = self.read_json_from_uri(uri)
languages = {}
ul = soup.find('ul', {'class': 'list--stats'}).findAll('li')
for li in ul:
lang_tag = li.find('h3', {'class': 'list__title'}).text
language = lang_tag.split('\n')[1].lstrip()
for locale in locales_data['results']:
span_txt = li.find('span', {'class': 'list__item__meta'}).text
language_meta = span_txt.split(' ')[0]
member_url = li.find('a')['href']
span_txt = li.find('span', {'class': 'txt--understated'}).text
total_user = span_txt.lstrip().rstrip()
if total_user == '0':
# No need to store language information with no members
if locale['memberCount'] == 0:
continue
languages[language_meta] = {
'language': language,
'member_url': member_url,
languages[locale['id']] = {
'language': locale['localeDetails']['displayName'],
'coordinators': [],
'reviewers': [],
'translators': [],
@ -93,40 +91,29 @@ class ZanataUtility(object):
def save_to_yaml(data, output_file):
with open(output_file, 'w') as out:
out.write(yaml_comment)
out.write(YAML_COMMENT)
for (k, v) in data.items():
yaml.safe_dump({k: v}, out, allow_unicode=True, indent=4,
encoding='utf-8', default_flow_style=False)
def convert_role_name(role):
roles = {
'Translator': 'translators',
'Reviewer': 'reviewers',
'Coordinator': 'coordinators'
}
return roles.get(role)
def collect_zanata_language_and_members():
zanata = ZanataUtility()
print("Retreiving language list")
languages = zanata.get_languages()
LOG.info("Retrieving language list")
languages = zanata.get_locales()
for language in languages.keys():
print("Getting member list from language %s" % language)
member_url = languages[language].pop('member_url')
for role, user_id in zanata.iter_language_members(member_url):
role = convert_role_name(role)
if not role:
print('[Warn] Unknown role : %s' % role)
continue
LOG.info("Getting member list from language %s" % language)
for member in zanata.get_locale_members(language):
if member['isTranslator']:
languages[language]['translators'].append(member['username'])
if member['isReviewer']:
languages[language]['reviewers'].append(member['username'])
if member['isCoordinator']:
languages[language]['coordinators'].append(member['username'])
languages[language][role].append(user_id)
if role == 'coordinators':
languages[language]['translators'].append(user_id)
languages[language]['reviewers'].append(user_id)
# Sort each member list alphabetically
for role in ['translators', 'reviewers', 'coordinators']:
languages[language][role].sort()