From e1226bbf17b32d640bd5cfca5ce10398a9763af3 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 13 Jan 2016 12:04:09 +0530 Subject: [PATCH] E-book viewer: Run input plugins in a worker process Prevents bugs in input plugins from causing memory leaks when opening multiple books in the same viewer process --- .../ebooks/conversion/plugins/epub_input.py | 1 - src/calibre/ebooks/oeb/iterator/book.py | 83 ++++++++++--------- src/calibre/gui2/viewer/main.py | 11 ++- 3 files changed, 52 insertions(+), 43 deletions(-) diff --git a/src/calibre/ebooks/conversion/plugins/epub_input.py b/src/calibre/ebooks/conversion/plugins/epub_input.py index 9d29fbbbf2..8aa1bcf4ae 100644 --- a/src/calibre/ebooks/conversion/plugins/epub_input.py +++ b/src/calibre/ebooks/conversion/plugins/epub_input.py @@ -227,7 +227,6 @@ class EPUBInput(InputFormatPlugin): self.removed_cover = self.rationalize_cover(opf, log) - self.optimize_opf_parsing = opf for x in opf.itermanifest(): if x.get('media-type', '') == 'application/x-dtbook+xml': raise ValueError( diff --git a/src/calibre/ebooks/oeb/iterator/book.py b/src/calibre/ebooks/oeb/iterator/book.py index 83652ba0d5..437d1aaa93 100644 --- a/src/calibre/ebooks/oeb/iterator/book.py +++ b/src/calibre/ebooks/oeb/iterator/book.py @@ -45,6 +45,46 @@ def write_oebbook(oeb, path): if f.endswith('.opf'): return f +def extract_book(pathtoebook, tdir, log=None, view_kepub=False, processed=False, only_input_plugin=False): + from calibre.ebooks.conversion.plumber import Plumber, create_oebbook + from calibre.utils.logging import default_log + log = log or default_log + plumber = Plumber(pathtoebook, tdir, log, view_kepub=view_kepub) + plumber.setup_options() + if pathtoebook.lower().endswith('.opf'): + plumber.opts.dont_package = True + if hasattr(plumber.opts, 'no_process'): + plumber.opts.no_process = True + + plumber.input_plugin.for_viewer = True + with plumber.input_plugin, open(plumber.input, 'rb') as inf: + pathtoopf = plumber.input_plugin(inf, + plumber.opts, plumber.input_fmt, log, {}, tdir) + + if not only_input_plugin: + # Run the HTML preprocess/parsing from the conversion pipeline as + # well + if (processed or plumber.input_fmt.lower() in {'pdb', 'pdf', 'rb'} and + not hasattr(pathtoopf, 'manifest')): + if hasattr(pathtoopf, 'manifest'): + pathtoopf = write_oebbook(pathtoopf, tdir) + pathtoopf = create_oebbook(log, pathtoopf, plumber.opts) + + if hasattr(pathtoopf, 'manifest'): + pathtoopf = write_oebbook(pathtoopf, tdir) + + book_format = os.path.splitext(pathtoebook)[1][1:].upper() + if getattr(plumber.input_plugin, 'is_kf8', False): + fs = ':joint' if getattr(plumber.input_plugin, 'mobi_is_joint', False) else '' + book_format = 'KF8' + fs + return book_format, pathtoopf, plumber.input_fmt + +def run_extract_book(*args, **kwargs): + from calibre.utils.ipc.simple_worker import fork_job + ans = fork_job('calibre.ebooks.oeb.iterator.book', 'extract_book', args=args, kwargs=kwargs, timeout=3000, no_output=True) + return ans['result'] + + class EbookIterator(BookmarksMixin): CHARACTERS_PER_PAGE = 1000 @@ -93,45 +133,12 @@ class EbookIterator(BookmarksMixin): ''' Convert an ebook file into an exploded OEB book suitable for display in viewers/preprocessing etc. ''' - from calibre.ebooks.conversion.plumber import Plumber, create_oebbook - self.delete_on_exit = [] self._tdir = TemporaryDirectory('_ebook_iter') self.base = self._tdir.__enter__() - plumber = Plumber(self.pathtoebook, self.base, self.log, view_kepub=view_kepub) - plumber.setup_options() - if self.pathtoebook.lower().endswith('.opf'): - plumber.opts.dont_package = True - if hasattr(plumber.opts, 'no_process'): - plumber.opts.no_process = True - - plumber.input_plugin.for_viewer = True - with plumber.input_plugin, open(plumber.input, 'rb') as inf: - self.pathtoopf = plumber.input_plugin(inf, - plumber.opts, plumber.input_fmt, self.log, - {}, self.base) - - if not only_input_plugin: - # Run the HTML preprocess/parsing from the conversion pipeline as - # well - if (processed or plumber.input_fmt.lower() in {'pdb', 'pdf', 'rb'} and - not hasattr(self.pathtoopf, 'manifest')): - if hasattr(self.pathtoopf, 'manifest'): - self.pathtoopf = write_oebbook(self.pathtoopf, self.base) - self.pathtoopf = create_oebbook(self.log, self.pathtoopf, - plumber.opts) - - if hasattr(self.pathtoopf, 'manifest'): - self.pathtoopf = write_oebbook(self.pathtoopf, self.base) - - self.book_format = os.path.splitext(self.pathtoebook)[1][1:].upper() - if getattr(plumber.input_plugin, 'is_kf8', False): - fs = ':joint' if getattr(plumber.input_plugin, 'mobi_is_joint', False) else '' - self.book_format = 'KF8' + fs - - self.opf = getattr(plumber.input_plugin, 'optimize_opf_parsing', None) - if self.opf is None: - self.opf = OPF(self.pathtoopf, os.path.dirname(self.pathtoopf)) + self.book_format, self.pathtoopf, input_fmt = run_extract_book( + self.pathtoebook, self.base, only_input_plugin=only_input_plugin, view_kepub=view_kepub, processed=processed) + self.opf = OPF(self.pathtoopf, os.path.dirname(self.pathtoopf)) self.language = self.opf.language if self.language: self.language = self.language.lower() @@ -140,7 +147,7 @@ class EbookIterator(BookmarksMixin): self.spine = [] Spiny = partial(SpineItem, read_anchor_map=read_anchor_map, read_links=read_links, run_char_count=run_char_count, from_epub=self.book_format == 'EPUB') - is_comic = plumber.input_fmt.lower() in {'cbc', 'cbz', 'cbr', 'cb7'} + is_comic = input_fmt.lower() in {'cbc', 'cbz', 'cbr', 'cb7'} for i in ordered: spath = i.path mt = None @@ -220,5 +227,3 @@ class EbookIterator(BookmarksMixin): os.remove(x) except: pass - - diff --git a/src/calibre/gui2/viewer/main.py b/src/calibre/gui2/viewer/main.py index 92b31fa614..e780ede872 100644 --- a/src/calibre/gui2/viewer/main.py +++ b/src/calibre/gui2/viewer/main.py @@ -17,7 +17,6 @@ from calibre.gui2 import ( Application, ORG_NAME, APP_UID, choose_files, info_dialog, error_dialog, open_url, setup_gui_option_parser) from calibre.ebooks.oeb.iterator.book import EbookIterator -from calibre.ebooks import DRMError from calibre.constants import islinux, filesystem_encoding from calibre.utils.config import Config, StringConfig, JSONConfig from calibre.customize.ui import available_input_formats @@ -33,6 +32,7 @@ dprefs.defaults['word_lookups'] = {} class Worker(Thread): def run(self): + from calibre.utils.ipc.simple_worker import WorkerError try: Thread.run(self) self.exception = self.traceback = None @@ -41,6 +41,9 @@ class Worker(Thread): 'This ebook is corrupted and cannot be opened. If you ' 'downloaded it from somewhere, try downloading it again.') self.traceback = '' + except WorkerError as err: + self.exception = Exception(_('Failed to read book, {0} click "Show Details" for more information').format(self.path_to_ebook)) + self.traceback = err.orig_tb except Exception as err: self.exception = err self.traceback = traceback.format_exc() @@ -863,19 +866,21 @@ class EbookViewer(MainWindow): self.history.clear() self.open_progress_indicator(_('Loading ebook...')) worker = Worker(target=partial(self.iterator.__enter__, view_kepub=True)) + worker.path_to_ebook = pathtoebook worker.start() while worker.isAlive(): worker.join(0.1) QApplication.processEvents() if worker.exception is not None: - if isinstance(worker.exception, DRMError): + tb = worker.traceback + if tb.strip().splitlines()[-1].startswith('DRMError:'): from calibre.gui2.dialogs.drm_error import DRMErrorMessage DRMErrorMessage(self).exec_() else: r = getattr(worker.exception, 'reason', worker.exception) error_dialog(self, _('Could not open ebook'), as_unicode(r) or _('Unknown error'), - det_msg=worker.traceback, show=True) + det_msg=tb, show=True) self.close_progress_indicator() else: self.metadata.show_opf(self.iterator.opf,