User Manual: Build localized versions

manual/Makefile

@@ -3,7 +3,7 @@
 
 # You can set these variables from the command line.
 SPHINXOPTS   =
-SPHINXBUILD  = sphinx-build
+SPHINXBUILD  = sphinx-build2
 PAPER        =
 
 ALLSPHINXOPTS = -d .build/doctrees -D latex_paper_size=$(PAPER) \

manual/build.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+# vim:fileencoding=utf-8
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__ = 'GPL v3'
+__copyright__ = '2014, Kovid Goyal <kovid at kovidgoyal.net>'
+
+from calibre.utils.icu import upper  # ensure encoding is set to utf-8
+del upper
+import sys, os, subprocess, shutil
+
+j, d, a = os.path.join, os.path.dirname, os.path.abspath
+
+def build_manual(language, base):
+
+    def sphinx_build(builder='html', bdir='html', t=None):
+        destdir = j(base, bdir)
+        os.makedirs(destdir)
+        ans = ['sphinx-build2', '-D', ('language=' + language), '-q', '-b', builder]
+        if builder == 'html':
+            ans += ['-w', j(destdir, 'sphinx-build-warnings.txt')]
+        if t:
+            ans += ['-t', t]
+        ans += ['-d', j(base, 'doctrees'), '.', destdir]
+        print(' '.join(ans))
+        subprocess.check_call(ans)
+        return destdir
+
+    onlinedir = sphinx_build(t='online')
+    epubdir = sphinx_build('myepub', 'epub')
+    latexdir = sphinx_build('mylatex', 'latex')
+    pwd = os.getcwdu()
+    os.chdir(latexdir)
+    def run_cmd(cmd):
+        p = subprocess.Popen(cmd, stdout=open(os.devnull, 'wb'), stdin=subprocess.PIPE)
+        p.stdin.close()
+        return p.wait()
+    try:
+        for i in xrange(3):
+            run_cmd(['pdflatex', '-interaction=nonstopmode', 'calibre.tex'])
+        run_cmd(['makeindex', '-s', 'python.ist', 'calibre.idx'])
+        for i in xrange(2):
+            run_cmd(['pdflatex', '-interaction=nonstopmode', 'calibre.tex'])
+        if not os.path.exists('calibre.pdf'):
+            print('Failed to build pdf file, see calibre.log in the latex directory', file=sys.stderr)
+            raise SystemExit(1)
+    finally:
+        os.chdir(pwd)
+    epub_dest = j(onlinedir, 'calibre.epub')
+    pdf_dest = j(onlinedir, 'calibre.pdf')
+    shutil.copyfile(j(epubdir, 'calibre.epub'), epub_dest)
+    shutil.copyfile(j(latexdir, 'calibre.pdf'), pdf_dest)
+    from calibre.ebooks.oeb.polish.container import epub_to_azw3
+    epub_to_azw3(epub_dest)
+
+if __name__ == '__main__':
+    language, base, src, __appname__, __version__ = sys.argv[1:]
+    os.chdir(src)
+    os.environ['__appname__'] = __appname__
+    os.environ['__version__'] = __version__
+    build_manual(language, base)

manual/conf.py

@@ -66,6 +66,7 @@ today_fmt = '%B %d, %Y'
 # List of documents that shouldn't be included in the build.
 unused_docs = ['global', 'cli/global']
 
+locale_dirs = ['locale/']
 
 # If true, '()' will be appended to :func: etc. cross-reference text.
 #add_function_parentheses = True

manual/custom.py

@@ -4,10 +4,8 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import sys, os, re, textwrap
-
+import init_calibre
-sys.path.insert(0, os.path.abspath('../src'))
+del init_calibre
-sys.extensions_location = '../src/calibre/plugins'
-sys.resources_location  = '../resources'
 
 from sphinx.util.console import bold
 

setup/parallel_build.py

@@ -21,7 +21,7 @@ def run_worker(job, decorate=True):
         return False, human_text, unicode(err)
     stdout, stderr = p.communicate()
     if decorate:
-        stdout = human_text + b'\n' + (stdout or b'')
+        stdout = bytes(human_text) + b'\n' + (stdout or b'')
     ok = p.returncode == 0
     return ok, stdout, (stderr or b'')
 

setup/publish.py

@@ -6,10 +6,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, shutil, subprocess, glob
+import os, shutil, subprocess, glob, tempfile, json, time, filecmp
 
 from setup import Command, __appname__, __version__, require_clean_git, require_git_master
-
+from setup.parallel_build import parallel_build
 
 class Stage1(Command):
 
@@ -82,59 +82,53 @@ class Manual(Command):
     description='''Build the User Manual '''
 
     def run(self, opts):
-        cwd = os.path.abspath(os.getcwd())
+        tdir = self.j(tempfile.gettempdir(), 'user-manual-build')
-        os.chdir(os.path.join(self.SRC, '..', 'manual'))
+        if os.path.exists(tdir):
-        try:
+            shutil.rmtree(tdir)
+        os.mkdir(tdir)
+        st = time.time()
         for d in ('.build', 'cli'):
             if os.path.exists(d):
                 shutil.rmtree(d)
             os.makedirs(d)
-            if not os.path.exists('.build'+os.sep+'html'):
+        jobs = []
-                os.makedirs('.build'+os.sep+'html')
+        mandir = self.j(self.d(self.SRC), 'manual')
-            os.environ['__appname__'] = __appname__
+        for language in (['en'] + list(json.load(open(self.j(self.d(self.SRC), 'manual', 'locale', 'completed.json'), 'rb')))):
-            os.environ['__version__'] = __version__
+            jobs.append((['calibre-debug', self.j(self.d(self.SRC), 'manual', 'build.py'),
-            subprocess.check_call(['sphinx-build2', '-b', 'html', '-t', 'online',
+                          language, self.j(tdir, language), mandir, __appname__, __version__],
-                                   '-d', '.build/doctrees', '.', '.build/html'])
+                         '\n\n**************** Building translations for: %s'%language))
-            with self:
+        self.info('Building translations for %d languages' % len(jobs))
-                # This is needed as without it the ToC is doubled in EPUB and
+        if not parallel_build(jobs, self.info):
-                # the both the ToC and the content is doubled in PDF
+            raise SystemExit(1)
-                subprocess.check_call(['sphinx-build', '-b', 'myepub', '-d',
+        os.chdir(self.j(tdir, 'en', 'html'))
-                                   '.build/doctrees', '.', '.build/epub'])
+        for x in os.listdir(tdir):
-                subprocess.check_call(['sphinx-build', '-b', 'mylatex', '-d',
+            if x != 'en':
-                                   '.build/doctrees', '.', '.build/latex'])
+                shutil.copytree(self.j(tdir, x, 'html'), x)
-            pwd = os.getcwdu()
+                self.replace_with_symlinks(x)
-            os.chdir('.build/latex')
+            else:
-            subprocess.check_call(['make', 'all-pdf'], stdout=open(os.devnull,
+                os.symlink('..', 'en')
-                'wb'))
+        self.info('Built manual for %d languages in %s minutes' % (len(jobs), int((time.time() - st)/60.)))
-            os.chdir(pwd)
+
-            epub_dest = self.j('.build', 'html', 'calibre.epub')
+    def replace_with_symlinks(self, lang_dir):
-            pdf_dest = self.j('.build', 'html', 'calibre.pdf')
+        ' Replace all identical files with symlinks to save disk space/upload bandwidth '
-            shutil.copyfile(self.j('.build', 'epub', 'calibre.epub'), epub_dest)
+        from calibre import walk
-            shutil.copyfile(self.j('.build', 'latex', 'calibre.pdf'), pdf_dest)
+        base = self.a(lang_dir)
-            subprocess.check_call(['ebook-convert', epub_dest,
+        for f in walk(base):
-                epub_dest.rpartition('.')[0] + '.azw3',
+            r = os.path.relpath(f, base)
-                '--page-breaks-before=/', '--disable-font-rescaling',
+            orig = self.j(self.d(base), r)
-                '--chapter=/'])
+            try:
-        finally:
+                sz = os.stat(orig).st_size
-            os.chdir(cwd)
+            except EnvironmentError:
+                continue
+            if sz == os.stat(f).st_size and filecmp._do_cmp(f, orig):
+                os.remove(f)
+                os.symlink(os.path.relpath(orig, self.d(f)), f)
 
     def clean(self):
         path = os.path.join(self.SRC, 'calibre', 'manual', '.build')
         if os.path.exists(path):
             shutil.rmtree(path)
 
-    def __enter__(self):
-        with open('index.rst', 'r+b') as f:
-            raw = self.orig_index = f.read()
-            f.seek(0)
-            f.truncate()
-            pos = raw.index(b'.. REMOVE_IN_PDF')
-            f.write(raw[:pos])
-
-    def __exit__(self, *args):
-        with open('index.rst', 'wb') as f:
-            f.write(self.orig_index)
-
 class TagRelease(Command):
 
     description = 'Tag a new release in git'

setup/upload.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 
 import os, subprocess, hashlib, shutil, glob, stat, sys, time
 from subprocess import check_call
-from tempfile import NamedTemporaryFile, mkdtemp
+from tempfile import NamedTemporaryFile, mkdtemp, gettempdir
 from zipfile import ZipFile
 
 if __name__ == '__main__':
@@ -129,8 +129,7 @@ def run_remote_upload(args):
 class UploadInstallers(Command):  # {{{
 
     def add_options(self, parser):
-        parser.add_option('--replace', default=False, action='store_true', help=
+        parser.add_option('--replace', default=False, action='store_true', help='Replace existing installers')
-                'Replace existing installers')
 
     def run(self, opts):
         all_possible = set(installers())
@@ -207,7 +206,7 @@ class UploadInstallers(Command):  # {{{
 
 class UploadUserManual(Command):  # {{{
     description = 'Build and upload the User Manual'
-    sub_commands = ['manual']
+    # sub_commands = ['manual']
 
     def build_plugin_example(self, path):
         from calibre import CurrentDir
@@ -232,9 +231,10 @@ class UploadUserManual(Command):  # {{{
         for x in glob.glob(self.j(path, '*')):
             self.build_plugin_example(x)
 
+        srcdir = self.j(gettempdir(), 'user-manual-build', 'en', 'html') + '/'
         for host in ('download', 'files'):
-            check_call(' '.join(['rsync', '-z', '-r', '--progress',
+            check_call(' '.join(['rsync', '-zrl', '--progress',
-                'manual/.build/html/', '%s:/srv/manual/' % host]), shell=True)
+                srcdir, '%s:/srv/manual/' % host]), shell=True)
 # }}}
 
 class UploadDemo(Command):  # {{{
@@ -332,4 +332,3 @@ def test_google_uploader():
 
 if __name__ == '__main__':
     test_google_uploader()
-