diff --git a/jenkins/jobs/include-build-releasenotes.sh b/jenkins/jobs/include-build-releasenotes.sh new file mode 100755 index 0000000000..6f784f3f08 --- /dev/null +++ b/jenkins/jobs/include-build-releasenotes.sh @@ -0,0 +1,117 @@ +#!/bin/bash -xe + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +DOCNAME=releasenotes +DIRECTORY=releasenotes + +script_path=/usr/local/jenkins/slave_scripts + +# This file always exists in OpenStack CI jobs, check for it so that +# it can be used manually as well. +if [ -e "$(pwd)/upper-constraints.txt" ]; then + export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt +fi + +if [ ! -e ${DIRECTORY}/source/locale/ ]; then + echo "No translations found, only building normal release notes" + $script_path/run-tox.sh releasenotes + exit 0 +fi + +# TODO(amotoki): releasenote conf.py files in most projects do not +# define 'locale_dirs' setting which is required for i18n. +# Remove this once repositories are changed. +if ! grep -q -E '^locale_dirs *=' $DIRECTORY/source/conf.py; then + echo "locale_dirs = ['locale/']" >> $DIRECTORY/source/conf.py +fi + +REFERENCES=`mktemp` +trap "rm -f -- '$REFERENCES'" EXIT + +# Extract translations +tox -e venv -- sphinx-build -b gettext \ + -d ${DIRECTORY}/build/doctrees.gettext \ + ${DIRECTORY}/source/ \ + ${DIRECTORY}/source/locale/ + +# Add links for translations to index file +cat <> ${REFERENCES} + +Translated Release Notes +======================== + +EOF + +# Check all language translation resources +for locale in `find ${DIRECTORY}/source/locale/ -maxdepth 1 -type d` ; do + # Skip if it is not a valid language translation resource. + if [ ! -e ${locale}/LC_MESSAGES/${DOCNAME}.po ]; then + continue + fi + language=$(basename $locale) + + echo "Building $language translation" + + # Prepare all translation resources + for pot in ${DIRECTORY}/source/locale/*.pot ; do + # Get filename + resname=$(basename ${pot} .pot) + + # Merge all translation resources. Note this is done the same + # way as done in common_translation_update.sh where we merge + # all strings together in a single file. + msgmerge --silent -o \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po \ + ${pot} + # Compile all translation resources + msgfmt -o \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.mo \ + ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${resname}.po + done + + # Build translated document + tox -e venv -- sphinx-build -b html -D language=${language} \ + -d "${DIRECTORY}/build/doctrees.${language}" \ + ${DIRECTORY}/source/ ${DIRECTORY}/build/html/${language} + + # Reference translated document from index file + echo "* \`${language} <${language}/index.html>\`__" >> ${REFERENCES} + + # Remove newly created files + git clean -f -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.po + git clean -f -x -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/*.mo + # revert changes to po file + git reset -q ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po + git checkout -- ${DIRECTORY}/source/locale/${language}/LC_MESSAGES/${DOCNAME}.po +done + +# Now append our references to the index file. We cannot do this +# earlier since the sphinx commands will read this file. +cat ${REFERENCES} >> ${DIRECTORY}/source/index.rst + +# Remove newly created pot files +rm -f ${DIRECTORY}/source/locale/*.pot + +# Now build releasenotes with reference to translations +$script_path/run-tox.sh releasenotes + +# Revert any changes to the index file. +git checkout -- ${DIRECTORY}/source/index.rst + +# TODO(amotoki): Revert conf.py now as we might have changed this to +# enable internationalization. +# Remove this again later. +git checkout -- ${DIRECTORY}/source/conf.py diff --git a/jenkins/jobs/openstack-publish-jobs.yaml b/jenkins/jobs/openstack-publish-jobs.yaml index 653de3352a..714de74c25 100644 --- a/jenkins/jobs/openstack-publish-jobs.yaml +++ b/jenkins/jobs/openstack-publish-jobs.yaml @@ -119,6 +119,22 @@ - upload-releasenotes-draft - console-log +- job-template: + name: 'gate-{name}-i18n-releasenotes-nv' + node: ubuntu-xenial + + builders: + - print-template-name: + template-name: "{template-name}" + - zuul-git-prep-upper-constraints: + - install-distro-packages + - revoke-sudo + - shell: + !include-raw-escape: include-build-releasenotes.sh + + publishers: + - upload-releasenotes-draft + - console-log - job-group: name: openstack-publish-jobs diff --git a/jenkins/jobs/projects.yaml b/jenkins/jobs/projects.yaml index c1ad015a81..b741b907a0 100644 --- a/jenkins/jobs/projects.yaml +++ b/jenkins/jobs/projects.yaml @@ -5414,6 +5414,7 @@ node: - ubuntu-trusty - ubuntu-xenial + - gate-{name}-i18n-releasenotes-nv - project: name: horizon-cisco-ui @@ -8620,6 +8621,7 @@ - gate-{name}-tox-{envlist}-{node}: envlist: checksyntax node: ubuntu-xenial + - gate-{name}-i18n-releasenotes-nv - project: name: openstack-resource-agents-specs diff --git a/tools/jenkins-projects-checks.py b/tools/jenkins-projects-checks.py index 6c009bbfb0..fd198cb37a 100755 --- a/tools/jenkins-projects-checks.py +++ b/tools/jenkins-projects-checks.py @@ -14,12 +14,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import glob import sys -import yaml - import voluptuous as v +# The files uses YAML extensions like !include, therefore use the +# jenkins-job-builder yaml parser for loading. +from jenkins_jobs import local_yaml + + BUILDER = v.Schema({ v.Required('name'): v.All(str), v.Required('builders'): v.All(list), @@ -63,6 +67,7 @@ PUBLISHER = v.Schema({ 'description': v.All(str) }) + def normalize(s): "Normalize string for comparison." return s.lower().replace("_", "-") @@ -114,7 +119,7 @@ def validate_jobs(): print("=====================") for job_file in glob.glob('jenkins/jobs/*.yaml'): - jobs = yaml.load(open(job_file)) + jobs = local_yaml.load(io.open(job_file, 'r', encoding='utf-8')) for item in jobs: if 'builder' in item: schema = BUILDER diff --git a/tox.ini b/tox.ini index 410038eae6..1e008c4663 100644 --- a/tox.ini +++ b/tox.ini @@ -68,6 +68,7 @@ commands = deps = PyYAML voluptuous + jenkins-job-builder commands = {toxinidir}/tools/jenkins-projects-checks.py diff --git a/zuul/layout.yaml b/zuul/layout.yaml index c1f6ddd2df..c73f14c7e6 100755 --- a/zuul/layout.yaml +++ b/zuul/layout.yaml @@ -3319,6 +3319,14 @@ jobs: - '^tox.ini' - '^.*requirements.txt$' + - name: ^gate-.*-i18n-releasenotes-nv$ + success-pattern: http://docs-draft.openstack.org/{build.parameters[LOG_PATH]}/releasenotes/build/html/ + branch: ^(?!stable/(?:juno|kilo)).*$ + files: + - '^releasenotes/.*' + - '^tox.ini' + - '^.*requirements.txt$' + # Some projects do not have releasenotes until the Mitaka cycle - name: ^gate-(keystoneauth|keystonemiddleware|networking-midonet|openstack-manuals|os-client-config|packstack|os-brick|puppet-.*)-releasenotes$ branch: ^(?!stable/(?:juno|kilo|liberty)).*$ @@ -10020,6 +10028,7 @@ projects: - gate-horizon-dsvm-integration-current-ubuntu-xenial - gate-horizon-dsvm-integration-deprecated-ubuntu-trusty - gate-horizon-dsvm-integration-deprecated-ubuntu-xenial + - gate-horizon-i18n-releasenotes-nv gate: - gate-horizon-tox-py27dj18-ubuntu-trusty - gate-horizon-tox-py27dj18-ubuntu-xenial @@ -12740,6 +12749,7 @@ projects: - name: translation-jobs-mitaka - name: release-notes-jobs check: + - gate-openstack-manuals-i18n-releasenotes-nv - gate-openstack-manuals-tox-checksyntax-ubuntu-xenial gate: - gate-openstack-manuals-tox-checksyntax-ubuntu-xenial