[zanata] Fixes zanata_users with REST API
Zanata >=4 which is now on production utilizes REST API [1]. This commit fixes zanata_users.py to make compatible with Zanata REST API and syncs translation_team.yaml up to date. [1] http://zanata.org/zanata-platform/rest-api-docs/resource_LocalesResource.html Change-Id: Ia99db310cff236797ef403ba5534d853ed63e3e3
This commit is contained in:
parent
994b3ed97f
commit
876d6cb939
|
@ -20,14 +20,11 @@ Th:
|
|||
- phithak
|
||||
- sawangpong
|
||||
ar:
|
||||
coordinators:
|
||||
- ianychoi
|
||||
coordinators: []
|
||||
language: Arabic
|
||||
reviewers:
|
||||
- ianychoi
|
||||
reviewers: []
|
||||
translators:
|
||||
- hosam
|
||||
- ianychoi
|
||||
- synax
|
||||
- welabbar
|
||||
- yasir
|
||||
|
@ -87,6 +84,7 @@ de:
|
|||
- senk
|
||||
- vcomas3
|
||||
translators:
|
||||
- alexanderloew2
|
||||
- aperri
|
||||
- atalanttore
|
||||
- cduch
|
||||
|
@ -183,6 +181,7 @@ es:
|
|||
- hybridpollo
|
||||
- iranzo
|
||||
- jfrancoa
|
||||
- jmont
|
||||
- jorpilo
|
||||
- josemhb
|
||||
- mariantb
|
||||
|
@ -207,6 +206,7 @@ es-MX:
|
|||
- sergioerazo
|
||||
translators:
|
||||
- heleno_jimenez
|
||||
- jmont
|
||||
- scuellarv
|
||||
- sergioerazo
|
||||
- toxickore
|
||||
|
@ -267,6 +267,7 @@ fr:
|
|||
- corin
|
||||
- croe
|
||||
- didier
|
||||
- emacchi
|
||||
- eouzans
|
||||
- fdotfr
|
||||
- flowalrave
|
||||
|
@ -280,6 +281,7 @@ fr:
|
|||
- jftalta
|
||||
- kodokuu
|
||||
- liced
|
||||
- linkid
|
||||
- lnic
|
||||
- makowals
|
||||
- mmarin
|
||||
|
@ -310,15 +312,18 @@ hi:
|
|||
- chandankumar
|
||||
hu:
|
||||
coordinators:
|
||||
- dabosshun
|
||||
- gergelys
|
||||
- marcus
|
||||
- szilvia
|
||||
language: Hungarian
|
||||
reviewers:
|
||||
- dabosshun
|
||||
- gergelys
|
||||
- marcus
|
||||
- szilvia
|
||||
translators:
|
||||
- dabosshun
|
||||
- gergelys
|
||||
- marcus
|
||||
- szilvia
|
||||
|
@ -376,6 +381,7 @@ ja:
|
|||
- hidekazuna
|
||||
- k_makabe007
|
||||
- ken_ishii
|
||||
- kuroshun
|
||||
- macjack
|
||||
- makita
|
||||
- mariojv
|
||||
|
@ -443,6 +449,7 @@ ko-KR:
|
|||
- ianychoi
|
||||
- jaewook_oh
|
||||
- janghe11
|
||||
- jc01rho
|
||||
- jitaejang
|
||||
- johjuhyun
|
||||
- jongwooh
|
||||
|
@ -552,6 +559,7 @@ pt-BR:
|
|||
- diegodalmolin
|
||||
- dritec
|
||||
- dsantos
|
||||
- ebrandi
|
||||
- ericbaum
|
||||
- fcpimenta
|
||||
- franciosi
|
||||
|
@ -567,11 +575,13 @@ pt-BR:
|
|||
- lveneziano
|
||||
- marcelodieder
|
||||
- marcio
|
||||
- marcocristofolini
|
||||
- mariaandrada
|
||||
- marquesc
|
||||
- melotto
|
||||
- mrbits
|
||||
- nokostya
|
||||
- northward
|
||||
- raildomascena
|
||||
- raquellira
|
||||
- rbraga
|
||||
|
@ -671,6 +681,7 @@ tr-TR:
|
|||
- oguzy
|
||||
- tuna_verid
|
||||
- yasemindemiral
|
||||
- ziya_karakaya
|
||||
vi-VN:
|
||||
coordinators:
|
||||
- kiennt26
|
||||
|
@ -679,10 +690,12 @@ vi-VN:
|
|||
- kiennt26
|
||||
translators:
|
||||
- kiennt26
|
||||
- nguyentrihai
|
||||
zh-CN:
|
||||
coordinators:
|
||||
- daisyycguo
|
||||
- vuuv
|
||||
- zzxwill
|
||||
language: Chinese (China)
|
||||
reviewers:
|
||||
- actionchen
|
||||
|
@ -692,6 +705,7 @@ zh-CN:
|
|||
- dliu
|
||||
- fifieldt
|
||||
- jimmyli
|
||||
- johnwoo_lee
|
||||
- lindayang
|
||||
- liuchaoxin
|
||||
- liujunpeng
|
||||
|
@ -701,6 +715,7 @@ zh-CN:
|
|||
- vuuv
|
||||
- yanheven
|
||||
- zhaochao
|
||||
- zzxwill
|
||||
translators:
|
||||
- 979zhang
|
||||
- actionchen
|
||||
|
@ -718,7 +733,9 @@ zh-CN:
|
|||
- coffee
|
||||
- cuixiaoyu
|
||||
- daisyycguo
|
||||
- dayou
|
||||
- dliu
|
||||
- domine
|
||||
- e010
|
||||
- fanzhang
|
||||
- fifieldt
|
||||
|
@ -737,7 +754,10 @@ zh-CN:
|
|||
- irina
|
||||
- javeme
|
||||
- jelanliu
|
||||
- jenner
|
||||
- jiangyikun
|
||||
- jimmyli
|
||||
- johnwoo_lee
|
||||
- lapalm
|
||||
- lawrence
|
||||
- leiyahsuai
|
||||
|
@ -780,10 +800,14 @@ zh-CN:
|
|||
- wangfang
|
||||
- woshishui419
|
||||
- wuyongpeng
|
||||
- xhkjoe
|
||||
- xiangbinbin
|
||||
- xiangqiao
|
||||
- xuyang
|
||||
- xywang
|
||||
- yagehu
|
||||
- yaguang
|
||||
- yanghongyu
|
||||
- yanghy
|
||||
- yanheven
|
||||
- yihleong
|
||||
|
@ -798,10 +822,13 @@ zh-CN:
|
|||
- zheng_minxian
|
||||
- zhengjie7800
|
||||
- zhonghuali
|
||||
- zhongjun2
|
||||
- zhongtian
|
||||
- zhouchangxun
|
||||
- zhsj
|
||||
- zhubx007
|
||||
- zhugaoxiao
|
||||
- zzxwill
|
||||
zh-TW:
|
||||
coordinators:
|
||||
- ccheng
|
||||
|
@ -826,6 +853,7 @@ zh-TW:
|
|||
- bokaitseng
|
||||
- ccheng
|
||||
- danny
|
||||
- dayou
|
||||
- derekjhyang
|
||||
- e010
|
||||
- elliswu
|
||||
|
|
|
@ -15,14 +15,16 @@
|
|||
|
||||
import argparse
|
||||
from collections import OrderedDict
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
|
||||
import bs4
|
||||
import requests
|
||||
import yaml
|
||||
|
||||
base_url = "https://translate.openstack.org/%s"
|
||||
yaml_comment = """\
|
||||
ZANATA_URI = 'https://translate.openstack.org/rest/%s'
|
||||
LOG = logging.getLogger('zanata_users')
|
||||
YAML_COMMENT = """\
|
||||
# This file is generated by tools/zanata/zanata_users.py.
|
||||
# For more information, see
|
||||
# https://docs.openstack.org/i18n/latest/tools.html#sync-the-translator-list-with-zanata
|
||||
|
@ -39,51 +41,47 @@ class ZanataUtility(object):
|
|||
'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko'
|
||||
]
|
||||
|
||||
def read_uri(self, uri):
|
||||
headers = {
|
||||
'User-Agent': random.choice(ZanataUtility.user_agents)
|
||||
}
|
||||
req = requests.get(url=uri, headers=headers)
|
||||
return req.text
|
||||
def read_uri(self, uri, headers):
|
||||
try:
|
||||
headers['User-Agent'] = random.choice(ZanataUtility.user_agents)
|
||||
req = requests.get(uri, headers=headers)
|
||||
return req.text
|
||||
except Exception as e:
|
||||
LOG.error('Error "%(error)s" while reading uri %(uri)s',
|
||||
{'error': e, 'uri': uri})
|
||||
raise
|
||||
|
||||
def iter_language_members(self, uri):
|
||||
data = self.read_uri(base_url % uri)
|
||||
soup = bs4.BeautifulSoup(data, 'html.parser')
|
||||
users = soup.find('ul', {'class': 'list--stats'}) \
|
||||
.findAll('li', {'class': 'l--pad-all-quarter'})
|
||||
def read_json_from_uri(self, uri):
|
||||
data = self.read_uri(uri, {'Accept': 'application/json'})
|
||||
try:
|
||||
return json.loads(data)
|
||||
except Exception as e:
|
||||
LOG.error('Error "%(error)s" parsing json from uri %(uri)s',
|
||||
{'error': e, 'uri': uri})
|
||||
raise
|
||||
|
||||
for user in users:
|
||||
user_id = user.find('a').text.strip()
|
||||
def get_locale_members(self, language_id):
|
||||
uri = ZANATA_URI % ('locales/locale/%s/members' % language_id)
|
||||
LOG.debug("Reading the list of locale members in %s from %s"
|
||||
% (language_id, uri))
|
||||
locale_members_data = self.read_json_from_uri(uri)
|
||||
return locale_members_data
|
||||
|
||||
roles_tag = user.find('ul', {'class': 'list--horizontal'}) \
|
||||
.find('li')
|
||||
roles = roles_tag.text.strip().split(', ')
|
||||
|
||||
for role_name in roles:
|
||||
yield role_name, user_id
|
||||
|
||||
def get_languages(self):
|
||||
data = self.read_uri(base_url % 'language/list')
|
||||
soup = bs4.BeautifulSoup(data, 'html.parser')
|
||||
def get_locales(self):
|
||||
# The default value of sizePerPage is 10, which needs to be increased
|
||||
# http://zanata.org/zanata-platform/rest-api-docs/resource_LocalesResource.html
|
||||
uri = ZANATA_URI % 'locales?sizePerPage=255'
|
||||
LOG.debug("Reading the list of locales from %s" % uri)
|
||||
locales_data = self.read_json_from_uri(uri)
|
||||
languages = {}
|
||||
ul = soup.find('ul', {'class': 'list--stats'}).findAll('li')
|
||||
for li in ul:
|
||||
lang_tag = li.find('h3', {'class': 'list__title'}).text
|
||||
language = lang_tag.split('\n')[1].lstrip()
|
||||
for locale in locales_data['results']:
|
||||
|
||||
span_txt = li.find('span', {'class': 'list__item__meta'}).text
|
||||
language_meta = span_txt.split(' ')[0]
|
||||
|
||||
member_url = li.find('a')['href']
|
||||
|
||||
span_txt = li.find('span', {'class': 'txt--understated'}).text
|
||||
total_user = span_txt.lstrip().rstrip()
|
||||
if total_user == '0':
|
||||
# No need to store language information with no members
|
||||
if locale['memberCount'] == 0:
|
||||
continue
|
||||
|
||||
languages[language_meta] = {
|
||||
'language': language,
|
||||
'member_url': member_url,
|
||||
languages[locale['id']] = {
|
||||
'language': locale['localeDetails']['displayName'],
|
||||
'coordinators': [],
|
||||
'reviewers': [],
|
||||
'translators': [],
|
||||
|
@ -93,40 +91,29 @@ class ZanataUtility(object):
|
|||
|
||||
def save_to_yaml(data, output_file):
|
||||
with open(output_file, 'w') as out:
|
||||
out.write(yaml_comment)
|
||||
out.write(YAML_COMMENT)
|
||||
for (k, v) in data.items():
|
||||
yaml.safe_dump({k: v}, out, allow_unicode=True, indent=4,
|
||||
encoding='utf-8', default_flow_style=False)
|
||||
|
||||
|
||||
def convert_role_name(role):
|
||||
roles = {
|
||||
'Translator': 'translators',
|
||||
'Reviewer': 'reviewers',
|
||||
'Coordinator': 'coordinators'
|
||||
}
|
||||
return roles.get(role)
|
||||
|
||||
|
||||
def collect_zanata_language_and_members():
|
||||
zanata = ZanataUtility()
|
||||
|
||||
print("Retreiving language list")
|
||||
languages = zanata.get_languages()
|
||||
LOG.info("Retrieving language list")
|
||||
languages = zanata.get_locales()
|
||||
|
||||
for language in languages.keys():
|
||||
print("Getting member list from language %s" % language)
|
||||
member_url = languages[language].pop('member_url')
|
||||
for role, user_id in zanata.iter_language_members(member_url):
|
||||
role = convert_role_name(role)
|
||||
if not role:
|
||||
print('[Warn] Unknown role : %s' % role)
|
||||
continue
|
||||
LOG.info("Getting member list from language %s" % language)
|
||||
|
||||
for member in zanata.get_locale_members(language):
|
||||
if member['isTranslator']:
|
||||
languages[language]['translators'].append(member['username'])
|
||||
if member['isReviewer']:
|
||||
languages[language]['reviewers'].append(member['username'])
|
||||
if member['isCoordinator']:
|
||||
languages[language]['coordinators'].append(member['username'])
|
||||
|
||||
languages[language][role].append(user_id)
|
||||
if role == 'coordinators':
|
||||
languages[language]['translators'].append(user_id)
|
||||
languages[language]['reviewers'].append(user_id)
|
||||
# Sort each member list alphabetically
|
||||
for role in ['translators', 'reviewers', 'coordinators']:
|
||||
languages[language][role].sort()
|
||||
|
|
Loading…
Reference in New Issue