From 4bf96b7cab784ba65c1417b96c27764adc3e4a22 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Mon, 11 Aug 2014 14:59:18 +0530 Subject: [PATCH] Compile user manual translations --- .gitignore | 1 + setup/parallel_build.py | 16 +++++++++++++-- setup/translations.py | 45 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 8e4c5adcd4..e35e87c506 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ resources/images.qrc manual/.build/ manual/cli/ manual/template_ref.rst +manual/locale build dist docs diff --git a/setup/parallel_build.py b/setup/parallel_build.py index df8e9e84b8..9f3fe26800 100644 --- a/setup/parallel_build.py +++ b/setup/parallel_build.py @@ -8,10 +8,11 @@ __copyright__ = '2014, Kovid Goyal ' import subprocess from multiprocessing.dummy import Pool +from functools import partial from setup.build_environment import cpu_count -def run_worker(job): +def run_worker(job, decorate=True): cmd, human_text = job human_text = human_text or b' '.join(cmd) try: @@ -19,7 +20,8 @@ def run_worker(job): except Exception as err: return False, human_text, unicode(err) stdout, stderr = p.communicate() - stdout = human_text + b'\n' + (stdout or b'') + if decorate: + stdout = human_text + b'\n' + (stdout or b'') ok = p.returncode == 0 return ok, stdout, (stderr or b'') @@ -37,3 +39,13 @@ def parallel_build(jobs, log, verbose=True): return False return True +def parallel_check_output(jobs, log): + p = Pool(cpu_count) + for ok, stdout, stderr in p.imap( + partial(run_worker, decorate=False), ((j, '') for j in jobs)): + if not ok: + log(stdout) + if stderr: + log(stderr) + raise SystemExit(1) + yield stdout diff --git a/setup/translations.py b/setup/translations.py index 0b1eda5306..ea9f9d9b84 100644 --- a/setup/translations.py +++ b/setup/translations.py @@ -6,11 +6,12 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, tempfile, shutil, subprocess, glob, re, time, textwrap, cPickle, shlex +import os, tempfile, shutil, subprocess, glob, re, time, textwrap, cPickle, shlex, json from locale import normalize as normalize_locale from functools import partial from setup import Command, __appname__, __version__, require_git_master +from setup.parallel_build import parallel_check_output def qt_sources(): qtdir = '/usr/src/qt5' @@ -238,6 +239,7 @@ class Translations(POT): # {{{ self.write_stats() self.freeze_locales() + self.compile_user_manual_translations() def check_iso639(self, path): from calibre.utils.localization import langnames_to_langcodes @@ -297,12 +299,53 @@ class Translations(POT): # {{{ import cPickle cPickle.dump(stats, open(dest, 'wb'), -1) + def compile_user_manual_translations(self): + self.info('Compiling user manual translations...') + srcbase = self.j(self.d(self.SRC), 'translations', 'manual') + destbase = self.j(self.d(self.SRC), 'manual', 'locale') + complete = {} + for x in os.listdir(srcbase): + q = self.j(srcbase, x) + if not os.path.isdir(q): + continue + dest = self.j(destbase, x, 'LC_MESSAGES') + if os.path.exists(dest): + shutil.rmtree(dest) + os.makedirs(dest) + jobs = [] + for po in os.listdir(q): + if not po.endswith('.po'): + continue + jobs.append([ + 'msgfmt', '--statistics', '-o', self.j( + dest, po.rpartition('.')[0] + '.mo'), self.j(q, po)]) + stats = tuple(parallel_check_output(jobs, self.info)) + translated = untranslated = 0 + for line in stats: + m = re.search('(\d+).+(\d+)', line) + if m is None: + translated += int(re.search('\d+', line).group()) + else: + translated += int(m.group(1)) + untranslated += int(m.group(2)) + stats = {'translated':translated, 'untranslated':untranslated} + with open(self.j(self.d(dest), 'stats.json'), 'wb') as f: + json.dump(stats, f) + total = translated + untranslated + if total and (translated / float(total)) > 0.75: + complete[x] = stats + with open(self.j(destbase, 'completed.json'), 'wb') as f: + json.dump(complete, f) + def clean(self): if os.path.exists(self.stats): os.remove(self.stats) zf = self.DEST + '.zip' if os.path.exists(zf): os.remove(zf) + destbase = self.j(self.d(self.SRC), 'manual', 'locale') + if os.path.exists(destbase): + shutil.rmtree(destbase) # }}}