diff --git a/manual/Makefile b/manual/Makefile index a21de12bed..303436d807 100644 --- a/manual/Makefile +++ b/manual/Makefile @@ -3,7 +3,7 @@ # You can set these variables from the command line. SPHINXOPTS = -SPHINXBUILD = sphinx-build +SPHINXBUILD = sphinx-build2 PAPER = ALLSPHINXOPTS = -d .build/doctrees -D latex_paper_size=$(PAPER) \ diff --git a/manual/build.py b/manual/build.py new file mode 100644 index 0000000000..8787273818 --- /dev/null +++ b/manual/build.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# vim:fileencoding=utf-8 +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2014, Kovid Goyal ' + +from calibre.utils.icu import upper # ensure encoding is set to utf-8 +del upper +import sys, os, subprocess, shutil + +j, d, a = os.path.join, os.path.dirname, os.path.abspath + +def build_manual(language, base): + + def sphinx_build(builder='html', bdir='html', t=None): + destdir = j(base, bdir) + os.makedirs(destdir) + ans = ['sphinx-build2', '-D', ('language=' + language), '-q', '-b', builder] + if builder == 'html': + ans += ['-w', j(destdir, 'sphinx-build-warnings.txt')] + if t: + ans += ['-t', t] + ans += ['-d', j(base, 'doctrees'), '.', destdir] + print(' '.join(ans)) + subprocess.check_call(ans) + return destdir + + onlinedir = sphinx_build(t='online') + epubdir = sphinx_build('myepub', 'epub') + latexdir = sphinx_build('mylatex', 'latex') + pwd = os.getcwdu() + os.chdir(latexdir) + def run_cmd(cmd): + p = subprocess.Popen(cmd, stdout=open(os.devnull, 'wb'), stdin=subprocess.PIPE) + p.stdin.close() + return p.wait() + try: + for i in xrange(3): + run_cmd(['pdflatex', '-interaction=nonstopmode', 'calibre.tex']) + run_cmd(['makeindex', '-s', 'python.ist', 'calibre.idx']) + for i in xrange(2): + run_cmd(['pdflatex', '-interaction=nonstopmode', 'calibre.tex']) + if not os.path.exists('calibre.pdf'): + print('Failed to build pdf file, see calibre.log in the latex directory', file=sys.stderr) + raise SystemExit(1) + finally: + os.chdir(pwd) + epub_dest = j(onlinedir, 'calibre.epub') + pdf_dest = j(onlinedir, 'calibre.pdf') + shutil.copyfile(j(epubdir, 'calibre.epub'), epub_dest) + shutil.copyfile(j(latexdir, 'calibre.pdf'), pdf_dest) + from calibre.ebooks.oeb.polish.container import epub_to_azw3 + epub_to_azw3(epub_dest) + +if __name__ == '__main__': + language, base, src, __appname__, __version__ = sys.argv[1:] + os.chdir(src) + os.environ['__appname__'] = __appname__ + os.environ['__version__'] = __version__ + build_manual(language, base) diff --git a/manual/conf.py b/manual/conf.py index ea853e7f1f..5123ecb5f3 100644 --- a/manual/conf.py +++ b/manual/conf.py @@ -66,6 +66,7 @@ today_fmt = '%B %d, %Y' # List of documents that shouldn't be included in the build. unused_docs = ['global', 'cli/global'] +locale_dirs = ['locale/'] # If true, '()' will be appended to :func: etc. cross-reference text. #add_function_parentheses = True diff --git a/manual/custom.py b/manual/custom.py index a24466427e..ed0b841536 100644 --- a/manual/custom.py +++ b/manual/custom.py @@ -4,10 +4,8 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal ' import sys, os, re, textwrap - -sys.path.insert(0, os.path.abspath('../src')) -sys.extensions_location = '../src/calibre/plugins' -sys.resources_location = '../resources' +import init_calibre +del init_calibre from sphinx.util.console import bold diff --git a/setup/parallel_build.py b/setup/parallel_build.py index 9f3fe26800..0016b964eb 100644 --- a/setup/parallel_build.py +++ b/setup/parallel_build.py @@ -21,7 +21,7 @@ def run_worker(job, decorate=True): return False, human_text, unicode(err) stdout, stderr = p.communicate() if decorate: - stdout = human_text + b'\n' + (stdout or b'') + stdout = bytes(human_text) + b'\n' + (stdout or b'') ok = p.returncode == 0 return ok, stdout, (stderr or b'') diff --git a/setup/publish.py b/setup/publish.py index 891c0f209a..a68c733e1b 100644 --- a/setup/publish.py +++ b/setup/publish.py @@ -6,10 +6,10 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, shutil, subprocess, glob +import os, shutil, subprocess, glob, tempfile, json, time, filecmp from setup import Command, __appname__, __version__, require_clean_git, require_git_master - +from setup.parallel_build import parallel_build class Stage1(Command): @@ -82,59 +82,53 @@ class Manual(Command): description='''Build the User Manual ''' def run(self, opts): - cwd = os.path.abspath(os.getcwd()) - os.chdir(os.path.join(self.SRC, '..', 'manual')) - try: - for d in ('.build', 'cli'): - if os.path.exists(d): - shutil.rmtree(d) - os.makedirs(d) - if not os.path.exists('.build'+os.sep+'html'): - os.makedirs('.build'+os.sep+'html') - os.environ['__appname__'] = __appname__ - os.environ['__version__'] = __version__ - subprocess.check_call(['sphinx-build2', '-b', 'html', '-t', 'online', - '-d', '.build/doctrees', '.', '.build/html']) - with self: - # This is needed as without it the ToC is doubled in EPUB and - # the both the ToC and the content is doubled in PDF - subprocess.check_call(['sphinx-build', '-b', 'myepub', '-d', - '.build/doctrees', '.', '.build/epub']) - subprocess.check_call(['sphinx-build', '-b', 'mylatex', '-d', - '.build/doctrees', '.', '.build/latex']) - pwd = os.getcwdu() - os.chdir('.build/latex') - subprocess.check_call(['make', 'all-pdf'], stdout=open(os.devnull, - 'wb')) - os.chdir(pwd) - epub_dest = self.j('.build', 'html', 'calibre.epub') - pdf_dest = self.j('.build', 'html', 'calibre.pdf') - shutil.copyfile(self.j('.build', 'epub', 'calibre.epub'), epub_dest) - shutil.copyfile(self.j('.build', 'latex', 'calibre.pdf'), pdf_dest) - subprocess.check_call(['ebook-convert', epub_dest, - epub_dest.rpartition('.')[0] + '.azw3', - '--page-breaks-before=/', '--disable-font-rescaling', - '--chapter=/']) - finally: - os.chdir(cwd) + tdir = self.j(tempfile.gettempdir(), 'user-manual-build') + if os.path.exists(tdir): + shutil.rmtree(tdir) + os.mkdir(tdir) + st = time.time() + for d in ('.build', 'cli'): + if os.path.exists(d): + shutil.rmtree(d) + os.makedirs(d) + jobs = [] + mandir = self.j(self.d(self.SRC), 'manual') + for language in (['en'] + list(json.load(open(self.j(self.d(self.SRC), 'manual', 'locale', 'completed.json'), 'rb')))): + jobs.append((['calibre-debug', self.j(self.d(self.SRC), 'manual', 'build.py'), + language, self.j(tdir, language), mandir, __appname__, __version__], + '\n\n**************** Building translations for: %s'%language)) + self.info('Building translations for %d languages' % len(jobs)) + if not parallel_build(jobs, self.info): + raise SystemExit(1) + os.chdir(self.j(tdir, 'en', 'html')) + for x in os.listdir(tdir): + if x != 'en': + shutil.copytree(self.j(tdir, x, 'html'), x) + self.replace_with_symlinks(x) + else: + os.symlink('..', 'en') + self.info('Built manual for %d languages in %s minutes' % (len(jobs), int((time.time() - st)/60.))) + + def replace_with_symlinks(self, lang_dir): + ' Replace all identical files with symlinks to save disk space/upload bandwidth ' + from calibre import walk + base = self.a(lang_dir) + for f in walk(base): + r = os.path.relpath(f, base) + orig = self.j(self.d(base), r) + try: + sz = os.stat(orig).st_size + except EnvironmentError: + continue + if sz == os.stat(f).st_size and filecmp._do_cmp(f, orig): + os.remove(f) + os.symlink(os.path.relpath(orig, self.d(f)), f) def clean(self): path = os.path.join(self.SRC, 'calibre', 'manual', '.build') if os.path.exists(path): shutil.rmtree(path) - def __enter__(self): - with open('index.rst', 'r+b') as f: - raw = self.orig_index = f.read() - f.seek(0) - f.truncate() - pos = raw.index(b'.. REMOVE_IN_PDF') - f.write(raw[:pos]) - - def __exit__(self, *args): - with open('index.rst', 'wb') as f: - f.write(self.orig_index) - class TagRelease(Command): description = 'Tag a new release in git' diff --git a/setup/upload.py b/setup/upload.py index 6fc2cba8aa..f910bb1ae1 100644 --- a/setup/upload.py +++ b/setup/upload.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' import os, subprocess, hashlib, shutil, glob, stat, sys, time from subprocess import check_call -from tempfile import NamedTemporaryFile, mkdtemp +from tempfile import NamedTemporaryFile, mkdtemp, gettempdir from zipfile import ZipFile if __name__ == '__main__': @@ -129,8 +129,7 @@ def run_remote_upload(args): class UploadInstallers(Command): # {{{ def add_options(self, parser): - parser.add_option('--replace', default=False, action='store_true', help= - 'Replace existing installers') + parser.add_option('--replace', default=False, action='store_true', help='Replace existing installers') def run(self, opts): all_possible = set(installers()) @@ -207,7 +206,7 @@ class UploadInstallers(Command): # {{{ class UploadUserManual(Command): # {{{ description = 'Build and upload the User Manual' - sub_commands = ['manual'] + # sub_commands = ['manual'] def build_plugin_example(self, path): from calibre import CurrentDir @@ -232,9 +231,10 @@ class UploadUserManual(Command): # {{{ for x in glob.glob(self.j(path, '*')): self.build_plugin_example(x) + srcdir = self.j(gettempdir(), 'user-manual-build', 'en', 'html') + '/' for host in ('download', 'files'): - check_call(' '.join(['rsync', '-z', '-r', '--progress', - 'manual/.build/html/', '%s:/srv/manual/' % host]), shell=True) + check_call(' '.join(['rsync', '-zrl', '--progress', + srcdir, '%s:/srv/manual/' % host]), shell=True) # }}} class UploadDemo(Command): # {{{ @@ -332,4 +332,3 @@ def test_google_uploader(): if __name__ == '__main__': test_google_uploader() -