diff --git a/translation-cleanup.sh b/translation-cleanup.sh index 75532a4..a0e4ee3 100755 --- a/translation-cleanup.sh +++ b/translation-cleanup.sh @@ -24,55 +24,31 @@ # License for the specific language governing permissions and limitations # under the License. -if [[ $# -lt 2 ]]; then - echo "Usage: $0 series projectname" +if [[ $# -lt 3 ]]; then + echo "Usage: $0 project-config/jenkins/scripts/path series projectname" echo - echo "Example: $0 kilo keystone" + echo "Example: $0 project-config/jenkins/scripts/ stable/liberty keystone" exit 2 fi -SERIES=$1 -PROJECT=$2 -BRANCH="stable/$SERIES" +PATH_CREATE_ZANATA=$1 +BRANCH=$2 +PROJECT=$3 +QUIET="--quiet" -# Initial transifex setup -function setup_translation { - # Track in HAS_CONFIG whether we run "tx init" since calling it - # will add the file .tx/config - and "tx set" might update it. If - # "tx set" updates .tx/config, we need to handle the file if it - # existed before. - HAS_CONFIG=1 - - # Initialize the transifex client, if there's no .tx directory - if [ ! -d .tx ] ; then - tx init --host=https://www.transifex.com - HAS_CONFIG=0 - fi -} - -# Setup a project for transifex +# Setup a project for Zanata function setup_project { local project=$1 + local version=${2:-master} - # Transifex project name does not include "." - tx_project=${project/\./} - tx set --auto-local -r ${tx_project}.${tx_project}-translations \ - "${project}/locale//LC_MESSAGES/${project}.po" \ - --source-lang en \ - --source-file ${project}/locale/${project}.pot -t PO \ - --execute + $PATH_CREATE_ZANATA/create-zanata-xml.py -p $project \ + -v $version --srcdir ${project}/locale --txdir ${project}/locale \ + -f zanata.xml --no-verify } - # Propose patch using COMMIT_MSG function send_patch { - # Revert any changes done to .tx/config - if [ $HAS_CONFIG -eq 1 ]; then - git reset -q .tx/config - git checkout -- .tx/config - fi - # Don't send a review if nothing has changed. if [ `git diff --cached |wc -l` -gt 0 ]; then # Commit and review @@ -95,27 +71,6 @@ function setup_loglevel_vars { LKEYWORD['critical']='_LC' } -# Setup transifex configuration for log level message translation. -# Needs variables setup via setup_loglevel_vars. -function setup_loglevel_project { - project=$1 - - # Transifex project name does not include "." - tx_project=${project/\./} - - for level in $LEVELS ; do - # Bootstrapping: Create file if it does not exist yet, - # otherwise "tx set" will fail. - if [ ! -e ${project}/locale/${project}-log-${level}.pot ]; then - touch ${project}/locale/${project}-log-${level}.pot - fi - tx set --auto-local -r ${tx_project}.${tx_project}-log-${level}-translations \ - "${project}/locale//LC_MESSAGES/${project}-log-${level}.po" \ - --source-lang en \ - --source-file ${project}/locale/${project}-log-${level}.pot -t PO \ - --execute - done -} # Run extract_messages for user visible messages and log messages. # Needs variables setup via setup_loglevel_vars. @@ -123,9 +78,9 @@ function extract_messages_log { project=$1 # Update the .pot files - python setup.py extract_messages + python setup.py $QUIET extract_messages --keyword "_C:1c,2 _P:1,2" for level in $LEVELS ; do - python setup.py extract_messages --no-default-keywords \ + python setup.py $QUIET extract_messages --no-default-keywords \ --keyword ${LKEYWORD[$level]} \ --output-file ${project}/locale/${project}-log-${level}.pot done @@ -145,7 +100,7 @@ function filter_commits { # Don't send files where the only things which have changed are # the creation date, the version number, the revision date, # comment lines, or diff file information. - for f in `git diff --cached --name-only`; do + for f in $(git diff --cached --name-only --diff-filter=AM); do # It's ok if the grep fails set +e changed=$(git diff --cached "$f" \ @@ -164,49 +119,75 @@ function filter_commits { function cleanup_po_files { local project=$1 - for i in `find $project/locale -name *.po `; do - # Output goes to stderr, so redirect to stdout to catch it. - trans=`msgfmt --statistics -o /dev/null $i 2>&1` - check="^0 translated messages" - if [[ $trans =~ $check ]] ; then - # Nothing is translated, remove the file. - git rm -f $i + # Note that the po files do not contain untranslated strings, we need + # the pot file to figure out the total number of strings. + for s in `find $project/ -name *.pot`; do + bs=`basename $s` + trans=`msgfmt --statistics -o /dev/null $s 2>&1` + if [[ $trans =~ " untranslated message" ]] ; then + total_no=`echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/'` else - if [[ $trans =~ " translated message" ]] ; then - trans_no=`echo $trans|sed -e 's/ translated message.*$//'` - else - trans_no=0 - fi - if [[ $trans =~ " untranslated message" ]] ; then - untrans_no=`echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/'` - else - untrans_no=0 - fi - let total=$trans_no+$untrans_no - let ratio=100*$trans_no/$total - # Since we only download files that are at least - # translated to 66 per cent, let's delete those that have - # signficantly less translations. - # For the release we delete files that are less than 66 - # per cent translated. - if [[ "$ratio" -lt "66" ]] ; then - git rm -f $i - fi + total_no=0 fi + echo "$bs $total_no" + if [ $total_no -eq 0 ] ; then + continue + fi + po=${bs/.pot/.po} + for i in `find $project/ -name $po `; do + # Output goes to stderr, so redirect to stdout to catch it. + trans=`msgfmt --statistics -o /dev/null $i 2>&1` + check="^0 translated messages" + if [[ $trans =~ $check ]] ; then + # Nothing is translated, remove the file. + echo "Nothing translated" + git rm -f $i + else + if [[ $trans =~ " translated message" ]] ; then + trans_no=`echo $trans|sed -e 's/ translated message.*$//'` + else + trans_no=0 + fi + if [[ $trans =~ " untranslated message" ]] ; then + untrans_no=`echo $trans|sed -e 's/^.* \([0-9]*\) untranslated message.*/\1/'` + else + untrans_no=0 + fi + let ratio=100*$trans_no/$total_no + # Since we only download files that are at least + # translated to 75 per cent, let's delete those that have + # signficantly less translations. + # For now we delete files that suddenly are less than 66 + # per cent translated. + if [[ "$ratio" -lt "66" ]] ; then + git rm -f $i + echo "Removed $i with $ratio %" + fi + fi + done done } -if [ $# -ne 1 ] ; then - echo "Script needs to called with single argument: Name of current project" - exit 1 -fi +# Reduce size of po files. This reduces the amount of content imported +# and makes for fewer imports. +# This does not touch the pot files. This way we can reconstruct the po files +# using "msgmerge POTFILE POFILE -o COMPLETEPOFILE". +function compress_po_files { + local directory=$1 + + for i in $(find $directory -name *.po) ; do + msgattrib --translated --no-location --sort-output "$i" \ + --output="${i}.tmp" + mv "${i}.tmp" "$i" + done +} # Note: Branch need to be exists, you can create it with: # git checkout --track origin/$BRANCH git checkout $BRANCH git pull -git checkout -b translations +git checkout -B translations # For partial translations and choosing 75% , see # https://bugs.launchpad.net/horizon/+bug/1317794. The number 66 % @@ -215,40 +196,30 @@ git checkout -b translations set +e read -d '' COMMIT_MSG <