Compile user manual translations

.gitignore

@@ -9,6 +9,7 @@ resources/images.qrc
 manual/.build/
 manual/cli/
 manual/template_ref.rst
+manual/locale
 build
 dist
 docs

setup/parallel_build.py

@@ -8,10 +8,11 @@ __copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import subprocess
 from multiprocessing.dummy import Pool
+from functools import partial
 
 from setup.build_environment import cpu_count
 
-def run_worker(job):
+def run_worker(job, decorate=True):
     cmd, human_text = job
     human_text = human_text or b' '.join(cmd)
     try:
@@ -19,7 +20,8 @@ def run_worker(job):
     except Exception as err:
         return False, human_text, unicode(err)
     stdout, stderr = p.communicate()
-    stdout = human_text + b'\n' + (stdout or b'')
+    if decorate:
+        stdout = human_text + b'\n' + (stdout or b'')
     ok = p.returncode == 0
     return ok, stdout, (stderr or b'')
 
@@ -37,3 +39,13 @@ def parallel_build(jobs, log, verbose=True):
             return False
     return True
 
+def parallel_check_output(jobs, log):
+    p = Pool(cpu_count)
+    for ok, stdout, stderr in p.imap(
+            partial(run_worker, decorate=False), ((j, '') for j in jobs)):
+        if not ok:
+            log(stdout)
+            if stderr:
+                log(stderr)
+            raise SystemExit(1)
+        yield stdout

setup/translations.py

@@ -6,11 +6,12 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, tempfile, shutil, subprocess, glob, re, time, textwrap, cPickle, shlex
+import os, tempfile, shutil, subprocess, glob, re, time, textwrap, cPickle, shlex, json
 from locale import normalize as normalize_locale
 from functools import partial
 
 from setup import Command, __appname__, __version__, require_git_master
+from setup.parallel_build import parallel_check_output
 
 def qt_sources():
     qtdir = '/usr/src/qt5'
@@ -238,6 +239,7 @@ class Translations(POT):  # {{{
 
         self.write_stats()
         self.freeze_locales()
+        self.compile_user_manual_translations()
 
     def check_iso639(self, path):
         from calibre.utils.localization import langnames_to_langcodes
@@ -297,12 +299,53 @@ class Translations(POT):  # {{{
         import cPickle
         cPickle.dump(stats, open(dest, 'wb'), -1)
 
+    def compile_user_manual_translations(self):
+        self.info('Compiling user manual translations...')
+        srcbase = self.j(self.d(self.SRC), 'translations', 'manual')
+        destbase = self.j(self.d(self.SRC), 'manual', 'locale')
+        complete = {}
+        for x in os.listdir(srcbase):
+            q = self.j(srcbase, x)
+            if not os.path.isdir(q):
+                continue
+            dest = self.j(destbase, x, 'LC_MESSAGES')
+            if os.path.exists(dest):
+                shutil.rmtree(dest)
+            os.makedirs(dest)
+            jobs = []
+            for po in os.listdir(q):
+                if not po.endswith('.po'):
+                    continue
+                jobs.append([
+                    'msgfmt', '--statistics', '-o', self.j(
+                        dest, po.rpartition('.')[0] + '.mo'), self.j(q, po)])
+            stats = tuple(parallel_check_output(jobs, self.info))
+            translated = untranslated = 0
+            for line in stats:
+                m = re.search('(\d+).+(\d+)', line)
+                if m is None:
+                    translated += int(re.search('\d+', line).group())
+                else:
+                    translated += int(m.group(1))
+                    untranslated += int(m.group(2))
+            stats = {'translated':translated, 'untranslated':untranslated}
+            with open(self.j(self.d(dest), 'stats.json'), 'wb') as f:
+                json.dump(stats, f)
+            total = translated + untranslated
+            if total and (translated / float(total)) > 0.75:
+                complete[x] = stats
+        with open(self.j(destbase, 'completed.json'), 'wb') as f:
+            json.dump(complete, f)
+
     def clean(self):
         if os.path.exists(self.stats):
             os.remove(self.stats)
         zf = self.DEST + '.zip'
         if os.path.exists(zf):
             os.remove(zf)
+        destbase = self.j(self.d(self.SRC), 'manual', 'locale')
+        if os.path.exists(destbase):
+            shutil.rmtree(destbase)
 
 # }}}