From 876d6cb939081925027b0ae0c8cf605db2910046 Mon Sep 17 00:00:00 2001 From: "Ian Y. Choi" Date: Tue, 24 Jul 2018 22:50:58 +0900 Subject: [PATCH] [zanata] Fixes zanata_users with REST API Zanata >=4 which is now on production utilizes REST API [1]. This commit fixes zanata_users.py to make compatible with Zanata REST API and syncs translation_team.yaml up to date. [1] http://zanata.org/zanata-platform/rest-api-docs/resource_LocalesResource.html Change-Id: Ia99db310cff236797ef403ba5534d853ed63e3e3 --- tools/zanata/translation_team.yaml | 38 ++++++++-- tools/zanata/zanata_users.py | 115 +++++++++++++---------------- 2 files changed, 84 insertions(+), 69 deletions(-) diff --git a/tools/zanata/translation_team.yaml b/tools/zanata/translation_team.yaml index 8f65160..72ca380 100644 --- a/tools/zanata/translation_team.yaml +++ b/tools/zanata/translation_team.yaml @@ -20,14 +20,11 @@ Th: - phithak - sawangpong ar: - coordinators: - - ianychoi + coordinators: [] language: Arabic - reviewers: - - ianychoi + reviewers: [] translators: - hosam - - ianychoi - synax - welabbar - yasir @@ -87,6 +84,7 @@ de: - senk - vcomas3 translators: + - alexanderloew2 - aperri - atalanttore - cduch @@ -183,6 +181,7 @@ es: - hybridpollo - iranzo - jfrancoa + - jmont - jorpilo - josemhb - mariantb @@ -207,6 +206,7 @@ es-MX: - sergioerazo translators: - heleno_jimenez + - jmont - scuellarv - sergioerazo - toxickore @@ -267,6 +267,7 @@ fr: - corin - croe - didier + - emacchi - eouzans - fdotfr - flowalrave @@ -280,6 +281,7 @@ fr: - jftalta - kodokuu - liced + - linkid - lnic - makowals - mmarin @@ -310,15 +312,18 @@ hi: - chandankumar hu: coordinators: + - dabosshun - gergelys - marcus - szilvia language: Hungarian reviewers: + - dabosshun - gergelys - marcus - szilvia translators: + - dabosshun - gergelys - marcus - szilvia @@ -376,6 +381,7 @@ ja: - hidekazuna - k_makabe007 - ken_ishii + - kuroshun - macjack - makita - mariojv @@ -443,6 +449,7 @@ ko-KR: - ianychoi - jaewook_oh - janghe11 + - jc01rho - jitaejang - johjuhyun - jongwooh @@ -552,6 +559,7 @@ pt-BR: - diegodalmolin - dritec - dsantos + - ebrandi - ericbaum - fcpimenta - franciosi @@ -567,11 +575,13 @@ pt-BR: - lveneziano - marcelodieder - marcio + - marcocristofolini - mariaandrada - marquesc - melotto - mrbits - nokostya + - northward - raildomascena - raquellira - rbraga @@ -671,6 +681,7 @@ tr-TR: - oguzy - tuna_verid - yasemindemiral + - ziya_karakaya vi-VN: coordinators: - kiennt26 @@ -679,10 +690,12 @@ vi-VN: - kiennt26 translators: - kiennt26 + - nguyentrihai zh-CN: coordinators: - daisyycguo - vuuv + - zzxwill language: Chinese (China) reviewers: - actionchen @@ -692,6 +705,7 @@ zh-CN: - dliu - fifieldt - jimmyli + - johnwoo_lee - lindayang - liuchaoxin - liujunpeng @@ -701,6 +715,7 @@ zh-CN: - vuuv - yanheven - zhaochao + - zzxwill translators: - 979zhang - actionchen @@ -718,7 +733,9 @@ zh-CN: - coffee - cuixiaoyu - daisyycguo + - dayou - dliu + - domine - e010 - fanzhang - fifieldt @@ -737,7 +754,10 @@ zh-CN: - irina - javeme - jelanliu + - jenner + - jiangyikun - jimmyli + - johnwoo_lee - lapalm - lawrence - leiyahsuai @@ -780,10 +800,14 @@ zh-CN: - wangfang - woshishui419 - wuyongpeng + - xhkjoe - xiangbinbin - xiangqiao - xuyang + - xywang + - yagehu - yaguang + - yanghongyu - yanghy - yanheven - yihleong @@ -798,10 +822,13 @@ zh-CN: - zheng_minxian - zhengjie7800 - zhonghuali + - zhongjun2 - zhongtian - zhouchangxun - zhsj + - zhubx007 - zhugaoxiao + - zzxwill zh-TW: coordinators: - ccheng @@ -826,6 +853,7 @@ zh-TW: - bokaitseng - ccheng - danny + - dayou - derekjhyang - e010 - elliswu diff --git a/tools/zanata/zanata_users.py b/tools/zanata/zanata_users.py index 7bce44c..4fc3c10 100755 --- a/tools/zanata/zanata_users.py +++ b/tools/zanata/zanata_users.py @@ -15,14 +15,16 @@ import argparse from collections import OrderedDict +import json +import logging import random -import bs4 import requests import yaml -base_url = "https://translate.openstack.org/%s" -yaml_comment = """\ +ZANATA_URI = 'https://translate.openstack.org/rest/%s' +LOG = logging.getLogger('zanata_users') +YAML_COMMENT = """\ # This file is generated by tools/zanata/zanata_users.py. # For more information, see # https://docs.openstack.org/i18n/latest/tools.html#sync-the-translator-list-with-zanata @@ -39,51 +41,47 @@ class ZanataUtility(object): 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko' ] - def read_uri(self, uri): - headers = { - 'User-Agent': random.choice(ZanataUtility.user_agents) - } - req = requests.get(url=uri, headers=headers) - return req.text + def read_uri(self, uri, headers): + try: + headers['User-Agent'] = random.choice(ZanataUtility.user_agents) + req = requests.get(uri, headers=headers) + return req.text + except Exception as e: + LOG.error('Error "%(error)s" while reading uri %(uri)s', + {'error': e, 'uri': uri}) + raise - def iter_language_members(self, uri): - data = self.read_uri(base_url % uri) - soup = bs4.BeautifulSoup(data, 'html.parser') - users = soup.find('ul', {'class': 'list--stats'}) \ - .findAll('li', {'class': 'l--pad-all-quarter'}) + def read_json_from_uri(self, uri): + data = self.read_uri(uri, {'Accept': 'application/json'}) + try: + return json.loads(data) + except Exception as e: + LOG.error('Error "%(error)s" parsing json from uri %(uri)s', + {'error': e, 'uri': uri}) + raise - for user in users: - user_id = user.find('a').text.strip() + def get_locale_members(self, language_id): + uri = ZANATA_URI % ('locales/locale/%s/members' % language_id) + LOG.debug("Reading the list of locale members in %s from %s" + % (language_id, uri)) + locale_members_data = self.read_json_from_uri(uri) + return locale_members_data - roles_tag = user.find('ul', {'class': 'list--horizontal'}) \ - .find('li') - roles = roles_tag.text.strip().split(', ') - - for role_name in roles: - yield role_name, user_id - - def get_languages(self): - data = self.read_uri(base_url % 'language/list') - soup = bs4.BeautifulSoup(data, 'html.parser') + def get_locales(self): + # The default value of sizePerPage is 10, which needs to be increased + # http://zanata.org/zanata-platform/rest-api-docs/resource_LocalesResource.html + uri = ZANATA_URI % 'locales?sizePerPage=255' + LOG.debug("Reading the list of locales from %s" % uri) + locales_data = self.read_json_from_uri(uri) languages = {} - ul = soup.find('ul', {'class': 'list--stats'}).findAll('li') - for li in ul: - lang_tag = li.find('h3', {'class': 'list__title'}).text - language = lang_tag.split('\n')[1].lstrip() + for locale in locales_data['results']: - span_txt = li.find('span', {'class': 'list__item__meta'}).text - language_meta = span_txt.split(' ')[0] - - member_url = li.find('a')['href'] - - span_txt = li.find('span', {'class': 'txt--understated'}).text - total_user = span_txt.lstrip().rstrip() - if total_user == '0': + # No need to store language information with no members + if locale['memberCount'] == 0: continue - languages[language_meta] = { - 'language': language, - 'member_url': member_url, + languages[locale['id']] = { + 'language': locale['localeDetails']['displayName'], 'coordinators': [], 'reviewers': [], 'translators': [], @@ -93,40 +91,29 @@ class ZanataUtility(object): def save_to_yaml(data, output_file): with open(output_file, 'w') as out: - out.write(yaml_comment) + out.write(YAML_COMMENT) for (k, v) in data.items(): yaml.safe_dump({k: v}, out, allow_unicode=True, indent=4, encoding='utf-8', default_flow_style=False) -def convert_role_name(role): - roles = { - 'Translator': 'translators', - 'Reviewer': 'reviewers', - 'Coordinator': 'coordinators' - } - return roles.get(role) - - def collect_zanata_language_and_members(): zanata = ZanataUtility() - print("Retreiving language list") - languages = zanata.get_languages() + LOG.info("Retrieving language list") + languages = zanata.get_locales() for language in languages.keys(): - print("Getting member list from language %s" % language) - member_url = languages[language].pop('member_url') - for role, user_id in zanata.iter_language_members(member_url): - role = convert_role_name(role) - if not role: - print('[Warn] Unknown role : %s' % role) - continue + LOG.info("Getting member list from language %s" % language) + + for member in zanata.get_locale_members(language): + if member['isTranslator']: + languages[language]['translators'].append(member['username']) + if member['isReviewer']: + languages[language]['reviewers'].append(member['username']) + if member['isCoordinator']: + languages[language]['coordinators'].append(member['username']) - languages[language][role].append(user_id) - if role == 'coordinators': - languages[language]['translators'].append(user_id) - languages[language]['reviewers'].append(user_id) # Sort each member list alphabetically for role in ['translators', 'reviewers', 'coordinators']: languages[language][role].sort()