Add translation jobs

Add upstream_translation_update and propose_translation_update jobs.
They are close enough to put them both in the same commit.

Adds a legacy role to prep the upper-constraints file for tox.
Eventually this should work like the tox role, as an env variable passed
along for tox to read.

Adds a role to prep zanata client on the node in order to use the client
to interact with the Zanata server.

Change-Id: If736642860c5b0d1523215f90c64e858387d4254
Depends-On: I014d559c330803213105ce1bf83a41a3c157df9c
Signed-off-by: Jesse Keating <omgjlk@us.ibm.com>

jenkins/scripts/common_translation_update.sh

@@ -57,7 +57,7 @@ function get_modulename {
     local project=$1
     local target=$2
 
-    /usr/local/jenkins/slave_scripts/get-modulename.py \
+    $SCRIPTSDIR/get-modulename.py \
         -p $project -t $target
 }
 
@@ -134,7 +134,7 @@ function setup_project {
 
     local exclude='.tox/**'
 
-    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
+    $SCRIPTSDIR/create-zanata-xml.py \
         -p $project -v $version --srcdir . --txdir . \
         -r '**/*.pot' '{path}/{locale_with_underscore}/LC_MESSAGES/{filename}.po' \
         -e "$exclude" -f zanata.xml
@@ -214,7 +214,7 @@ function setup_manuals {
         ZANATA_RULES="$ZANATA_RULES -r ./releasenotes/source/locale/releasenotes.pot releasenotes/source/locale/{locale_with_underscore}/LC_MESSAGES/releasenotes.po"
     fi
 
-    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
+    $SCRIPTSDIR/create-zanata-xml.py \
         -p $project -v $version --srcdir . --txdir . \
         $ZANATA_RULES -e "$EXCLUDE" \
         -f zanata.xml
@@ -228,7 +228,7 @@ function setup_training_guides {
     # Update the .pot file
     tox -e generatepot-training
 
-    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
+    $SCRIPTSDIR/create-zanata-xml.py \
         -p $project -v $version \
         --srcdir doc/upstream-training/source/locale \
         --txdir doc/upstream-training/source/locale \
@@ -243,7 +243,7 @@ function setup_i18n {
     # Update the .pot file
     tox -e generatepot
 
-    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
+    $SCRIPTSDIR/create-zanata-xml.py \
         -p $project -v $version \
         --srcdir doc/source/locale \
         --txdir doc/source/locale \
@@ -265,7 +265,7 @@ function setup_reactjs_project {
     # Transform them into .pot files
     npm run json2pot
 
-    /usr/local/jenkins/slave_scripts/create-zanata-xml.py \
+    $SCRIPTSDIR/create-zanata-xml.py \
         -p $project -v $version --srcdir . --txdir . \
         -r '**/*.pot' '{path}/{locale}.po' \
         -e "$exclude" -f zanata.xml

jenkins/scripts/propose_translation_update.sh

@@ -20,7 +20,8 @@ JOBNAME=$3
 # allow /'s in version names.
 ZANATA_VERSION=${BRANCH//\//-}
 
-source /usr/local/jenkins/slave_scripts/common_translation_update.sh
+SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $SCRIPTSDIR/common_translation_update.sh
 
 init_branch $BRANCH
 

jenkins/scripts/upstream_translation_update.sh

@@ -19,14 +19,15 @@ JOBNAME=$2
 # allow /'s in version names.
 ZANATA_VERSION=${ZUUL_REFNAME//\//-}
 
-source /usr/local/jenkins/slave_scripts/common_translation_update.sh
+SCRIPTSDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+source $SCRIPTSDIR/common_translation_update.sh
 
 init_branch $ZUUL_REFNAME
 
 # List of all modules to copy POT files from
 ALL_MODULES=""
 
-if ! /usr/local/jenkins/slave_scripts/query-zanata-project-version.py \
+if ! $SCRIPTSDIR/query-zanata-project-version.py \
     -p $PROJECT -v $ZANATA_VERSION; then
     # Exit successfully so that lack of a version doesn't cause the jenkins
     # jobs to fail. This is necessary because not all branches of a project

playbooks/translation/propose-translation-update.yaml

@@ -0,0 +1,11 @@
+---
+- hosts: all
+  roles:
+    - prep-zanata
+    - legacy-zuul-git-prep-upper-constraints
+
+  tasks:
+    - name: Run propose_translation_update.sh script
+      command: "{{ ansible_user_dir }}/scripts/upstream_translation_update.sh {{ zuul.project.short_name }} {{ zuul.branch }} {{ zuul.job }}"
+      args:
+        chdir: "src/{{ zuul.project.canonical_name }}"

playbooks/translation/upstream-translation-update.yaml

@@ -0,0 +1,15 @@
+---
+- hosts: all
+  roles:
+    - prep-zanata
+    - legacy-zuul-git-prep-upper-constraints
+
+  tasks:
+    - name: Run upstream_translation_update.sh script
+      command: "{{ ansible_user_dir }}/scripts/upstream_translation_update.sh {{ zuul.project.short_name }} {{ zuul.job }}"
+      args:
+        chdir: "src/{{ zuul.project.canonical_name }}"
+
+- hosts: all
+  roles:
+    - fetch-translation-output

roles/legacy-zuul-git-prep-upper-constraints/README.rst

@@ -0,0 +1,4 @@
+Copy openstack/requirements/upper-constraints.txt file to project dir
+
+.. note:: This role is deprecated and only exists for transition purposes.
+          It should not be used for new jobs.

roles/legacy-zuul-git-prep-upper-constraints/tasks/main.yaml

@@ -0,0 +1,6 @@
+- name: Copy the upper-constraints.txt file to the project dir
+  copy:
+    # copy happens entirely on the remote node
+    remote_src: yes
+    src: "{{ ansible_user_dir }}/src/git.openstack.org/openstack/requirements/upper-constraints.txt"
+    dest: "{{ zuul.project.src_dir }}/upper-constraints.txt"

roles/prep-zanata/README.rst

@@ -0,0 +1,29 @@
+Prepare zanata client use
+
+**Role Variables**
+
+.. zuul:rolevar:: zanata_api_credentials
+
+  Complex argument which contains the ssh key information. It is
+  expected that this argument comes from a `Secret`
+
+    .. zuul:rolevar:: server_id
+
+        This is the ID of the zanata server to use
+
+    .. zuul:rolevar:: url
+
+        The url to the zanata server
+
+    .. zuul:rolevar:: username
+
+        The username to use with the zanata server
+
+    .. zuul:rolevar:: key
+
+        The key to login with
+
+.. zuul:rolevar:: zanata_client_version
+   :default: 3.8.1
+
+  The version of zanata client to install

roles/prep-zanata/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+zanata_client_version: 3.8.1

roles/prep-zanata/tasks/main.yaml

@@ -0,0 +1,57 @@
+---
+# This is all bad and I feel bad, but it's translated from
+# https://github.com/openstack-infra/puppet-zanata/blob/master/manifests/client.pp
+
+- name: install necessary packages
+  apt:
+    name: default-jre-headless
+
+- name: ensure zanata install dir
+  file:
+    path: /opt/zanata
+    owner: "{{ ansible_ssh_user }}"
+  become: true
+
+- name: extract zanata client tarball
+  unarchive:
+    src: "https://search.maven.org/remotecontent?filepath=org/zanata/zanata-cli/{{ zanata_client_version }}/zanata-cli-{{ zanata_client_version }}-dist.tar.gz"
+    remote_src: yes
+    dest: "/opt/zanata/"
+    creates: "/opt/zanata/zanata-cli-{{ version }}/bin/zanata-cli"
+
+- name: ensure zanata-cli perms
+  file:
+    path: "/opt/zanata/zanata-cli-{{ version }}/bin/zanata-cli"
+    mode: 0755
+
+- name: link zanata-cli
+  file:
+    path: /usr/local/bin/zanata-cli
+    src: "/opt/zanata/zanata-cli-{{ version }}/bin/zanata-cli"
+    state: link
+
+# This is a preview module in Ansible 2.3. It may not work.
+- name: import cert to java keystore
+  java_cert:
+    cert_url: "{{ zanata_api_credentials.server_name }}:443"
+    keystore_path: /etc/ssl/certs/java/cacerts
+    keystore_pass: changeit
+    keystore_create: true
+  become: true
+  # Use sudo to ensure root ownership
+
+- name: set permissions for cacert
+  file:
+    path: /etc/ssl/certs/java/cacerts
+    mode: 0644
+  become: true
+
+- name: ensure zanata config dir
+  file:
+    path: ~/.config
+    state: directory
+
+- name: write out zanata config
+  template:
+    src: zanata.ini
+    dest: ~/.config/zanata.ini

roles/prep-zanata/templates/zanata.ini

@@ -0,0 +1,4 @@
+[servers]
+{{ zanata_api_credentials.server_id }}.url={{ zanata_api_credentials.url }}
+{{ zanata_api_credentials.server_id }}.username={{ zanata_api_credentials.username }}
+{{ zanata_api_credentials.server_id }}.key={{ zanata_api_credentials.key }}

zuul.d/jobs.yaml

@@ -450,3 +450,26 @@
     files:
       - zuul/layout.yaml
       - ^jenkins/jobs/.*
+
+- job:
+    name: upstream-translation-update
+    parent: propose-updates
+    run: playbooks/translation/upstream-translation-update
+    post-run: playbooks/publish/openstack-artifacts.yaml
+    vars:
+      fileserver_leading_path: translation-source
+    required-projects:
+      - openstack/requirements
+    secrets:
+      - secret: site_tarballs
+        name: fileserver
+      - zanata_api_credentials
+
+- job:
+    name: propose-translation-update
+    parent: propose-updates
+    run: playbooks/translation/propose-translation-update
+    required-projects:
+      - openstack/requirements
+    secrets:
+      - zanata_api_credentials

zuul.d/secrets.yaml

@@ -481,3 +481,21 @@
           Z4ZLhXqOWRyH0Ibm91/We6njx/c/5wEreOCMqFs4hZcPbKXH772LZ1k5MH2kUpr+4jxxz
           HPQUNxVHExtCUgJxsmWWRdQ1rEna8/HsNPCbk2Kk4S2bpFZ/NZqaSwmgo0JJSV9SYbNO/
           +Ci2z269z5YGw0fapcO3/+X0lRHqjvZ6Wiz0zXdRlkTJYttw+uyK30bH/cmLn0=
+
+- secret:
+    name: zanata_api_credentials
+    data:
+      server_id: translate_openstack_org
+      url: https://translate.openstack.org/
+      username: infra
+      key: !encrypted/pkcs1-oaep
+        - o3C44R9+6Dd/WTKGL7TY+R3YOT8pjNhK6+Y0KmNppgLj8OtN8K1yNz8VggTOvQE1fhX9U
+          VXtSF6V2HAeRw4OexAMsdiXsDZwxcPSOd+h4sF3OMUEMRNxn8ePeHYmuCDw1GR2TcrZU0
+          iB/23AfQ9ea5igvYKuFr6c0rBxICAtdOqQNCxUESumMJLDLFslsmN11BtH9DFmVJ/iUm1
+          bnDvKd9lLg3dF4iLXdPz30347D/P0994c761HWB3XCiHnJlNdhaH41JbZn+z+03/XMXlu
+          InAzQGbiGLvcZSKnIylZFXFnj7R2CmDrPdZLUo8UYftZUl2yuhOeF3hLT0oQC9gArpS6K
+          9mrNdFCQkp5lw2nGE2T9seYQX2l7F+nfDpCD08DDMjdIG91mZLh4FwfJgLjCqYucAQab/
+          RQX5qtXyu4PJXWDgycfiovBjr7bYJolDar9GHeKEfcWnsfU/W0wCPjS4nvEae0xJQ+7vb
+          Acp4f7EGs1Sn7b/c2er9a6w6Gl80+ZVGWi0ojozsiZTbXgI4MVTJmXsKfydSBff9/H3jl
+          MAfjUMs0405CLtPZZG8k+KyG5kDAbOHCrK6ALaLqwVtt9YAdxEdkASJsMuQSk6BbdDJCI
+          fxtaJjXVTHQDYdouqbIWGMNypwKmxFub5lWainhbOKdvhl3CZDuuUq33k+jCkQ=