From 606f3a6dd1ec2ab73ab300ae3880af606e6eac33 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Wed, 27 Aug 2008 11:57:48 -0700 Subject: [PATCH] Fix #951 ([comic2lrf] on windows multi-threading isn't working) --- installer/windows/freeze.py | 4 +- src/calibre/debug.py | 4 +- src/calibre/ebooks/epub/__init__.py | 20 ++++++ src/calibre/ebooks/epub/from_html.py | 6 +- src/calibre/ebooks/lrf/comic/convert_from.py | 73 ++++++++++---------- src/calibre/parallel.py | 28 ++++++-- src/calibre/utils/threadpool.py | 1 - src/calibre/utils/zipfile.py | 4 +- 8 files changed, 91 insertions(+), 49 deletions(-) diff --git a/installer/windows/freeze.py b/installer/windows/freeze.py index 8a4a3ccec0..2d8932a440 100644 --- a/installer/windows/freeze.py +++ b/installer/windows/freeze.py @@ -161,12 +161,12 @@ def main(args=sys.argv): 'win32process', 'win32api', 'msvcrt', 'win32event', 'calibre.ebooks.lrf.any.*', 'calibre.ebooks.lrf.feeds.*', - 'lxml', 'lxml._elementpath', 'genshi', + 'genshi', 'path', 'pydoc', 'IPython.Extensions.*', 'calibre.web.feeds.recipes.*', 'PyQt4.QtWebKit', 'PyQt4.QtNetwork', ], - 'packages' : ['PIL'], + 'packages' : ['PIL', 'lxml'], 'excludes' : ["Tkconstants", "Tkinter", "tcl", "_imagingtk", "ImageTk", "FixTk" ], diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 12bc4f8063..9b794a3b6d 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -25,9 +25,7 @@ Run an embedded python interpreter. def update_zipfile(zipfile, mod, path): if 'win32' in sys.platform: - print 'WARNING: On Windows Vista you must run this from a console that has been started in Administrator mode.' - print 'Press Enter to continue if this is an Administrator console or Ctrl-C to Cancel' - raw_input() + print 'WARNING: On Windows Vista using this option may cause windows to put library.zip into the Virtual Store (typically located in c:\Users\username\AppData\Local\VirtualStore). If it does this you must delete it from there after you\'re done debugging).' pat = re.compile(mod.replace('.', '/')+r'\.py[co]*') name = mod.replace('.', '/') + os.path.splitext(path)[-1] update(zipfile, [pat], [path], [name]) diff --git a/src/calibre/ebooks/epub/__init__.py b/src/calibre/ebooks/epub/__init__.py index 45d5d44296..8fa259694a 100644 --- a/src/calibre/ebooks/epub/__init__.py +++ b/src/calibre/ebooks/epub/__init__.py @@ -8,8 +8,28 @@ Conversion to EPUB. ''' import sys from calibre.utils.config import Config, StringConfig +from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED from calibre.ebooks.html import config as common_config +def initialize_container(path_to_container, opf_name='metadata.opf'): + ''' + Create an empty EPUB document, with a default skeleton. + ''' + CONTAINER='''\ + + + + + + + '''%opf_name + zf = ZipFile(path_to_container, 'w') + zf.writestr('mimetype', 'application/epub+zip', compression=ZIP_DEFLATED) + zf.writestr('META-INF/', '', 0700) + zf.writestr('META-INF/container.xml', CONTAINER) + return zf + + def config(defaults=None): desc = _('Options to control the conversion to EPUB') if defaults is None: diff --git a/src/calibre/ebooks/epub/from_html.py b/src/calibre/ebooks/epub/from_html.py index 91351423ba..1c21bf4c2b 100644 --- a/src/calibre/ebooks/epub/from_html.py +++ b/src/calibre/ebooks/epub/from_html.py @@ -36,7 +36,7 @@ class HTMLProcessor(Parser): for text in get_text(self.body if self.body is not None else self.root): length, parent = len(re.sub(r'\s+', '', text)), text.getparent() #TODO: Use cssutils on self.raw_css to figure out the font size - # of this piece text and update statistics accordingly + # of this piece of text and update statistics accordingly def split(self): ''' Split into individual flows to accommodate Adobe's incompetence ''' @@ -74,13 +74,17 @@ def convert(htmlfile, opts, notification=None): mi = merge_metadata(htmlfile, opf, opts) opts.chapter = XPath(opts.chapter, namespaces={'re':'http://exslt.org/regular-expressions'}) + resource_map = parse_content(filelist, opts) + resources = [os.path.join(opts.output, 'content', f) for f in resource_map.values()] + if opf.cover and os.access(opf.cover, os.R_OK): shutil.copyfile(opf.cover, os.path.join(opts.output, 'content', 'resources', '_cover_'+os.path.splitext(opf.cover))) cpath = os.path.join(opts.output, 'content', 'resources', '_cover_'+os.path.splitext(opf.cover)) shutil.copyfile(opf.cover, cpath) resources.append(cpath) + mi.cover = cpath def main(args=sys.argv): parser = option_parser() diff --git a/src/calibre/ebooks/lrf/comic/convert_from.py b/src/calibre/ebooks/lrf/comic/convert_from.py index 70daaa5705..bb0229e7fa 100755 --- a/src/calibre/ebooks/lrf/comic/convert_from.py +++ b/src/calibre/ebooks/lrf/comic/convert_from.py @@ -7,14 +7,13 @@ __docformat__ = 'restructuredtext en' Based on ideas from comiclrf created by FangornUK. ''' -import os, sys, traceback, shutil +import os, sys, traceback, shutil, threading from uuid import uuid4 -from calibre import extract, detect_ncpus, terminal_controller, \ - __appname__, __version__ +from calibre import extract, terminal_controller, __appname__, __version__ from calibre.utils.config import Config, StringConfig from calibre.ptempfile import PersistentTemporaryDirectory -from calibre.utils.threadpool import ThreadPool, WorkRequest +from calibre.parallel import Server, ParallelJob from calibre.utils.terminfo import ProgressBar from calibre.ebooks.lrf.pylrs.pylrs import Book, BookSetting, ImageStream, ImageBlock try: @@ -84,12 +83,13 @@ class PageProcessor(list): ''' def __init__(self, path_to_page, dest, opts, num): - self.path_to_page = path_to_page - self.opts = opts - self.num = num - self.dest = dest - self.rotate = False list.__init__(self) + self.path_to_page = path_to_page + self.opts = opts + self.num = num + self.dest = dest + self.rotate = False + def __call__(self): try: @@ -100,7 +100,6 @@ class PageProcessor(list): raise IOError('Failed to read image from: %'%self.path_to_page) width = MagickGetImageWidth(img) height = MagickGetImageHeight(img) - if self.num == 0: # First image so create a thumbnail from it thumb = CloneMagickWand(img) if thumb < 0: @@ -122,7 +121,6 @@ class PageProcessor(list): MagickCropImage(split1, (width/2)-1, height, 0, 0) MagickCropImage(split2, (width/2)-1, height, width/2, 0 ) self.pages = [split2, split1] if self.opts.right2left else [split1, split2] - self.process_pages() except Exception, err: print 'Failed to process page: %s'%os.path.basename(self.path_to_page) @@ -138,23 +136,20 @@ class PageProcessor(list): PixelSetColor(pw, 'white') MagickSetImageBorderColor(wand, pw) - if self.rotate: MagickRotateImage(wand, pw, -90) # 25 percent fuzzy trim? MagickTrimImage(wand, 25*65535/100) MagickSetImagePage(wand, 0,0,0,0) #Clear page after trim, like a "+repage" - # Do the Photoshop "Auto Levels" equivalent if not self.opts.dont_normalize: MagickNormalizeImage(wand) - sizex = MagickGetImageWidth(wand) sizey = MagickGetImageHeight(wand) SCRWIDTH, SCRHEIGHT = PROFILES[self.opts.profile] - + print 77777, threading.currentThread() if self.opts.keep_aspect_ratio: # Preserve the aspect ratio by adding border aspect = float(sizex) / float(sizey) @@ -168,7 +163,6 @@ class PageProcessor(list): newsizey = int(newsizex / aspect) deltax = 0 deltay = (SCRHEIGHT - newsizey) / 2 - MagickResizeImage(wand, newsizex, newsizey, CatromFilter, 1.0) MagickSetImageBorderColor(wand, pw) MagickBorderImage(wand, pw, deltax, deltay) @@ -193,6 +187,12 @@ class PageProcessor(list): DestroyPixelWand(pw) wand = DestroyMagickWand(wand) +def process_page(path, dest, opts, num): + pp = PageProcessor(path, dest, opts, num) + with ImageMagick(): + pp() + return list(pp) + class Progress(object): def __init__(self, total, update): @@ -200,10 +200,11 @@ class Progress(object): self.update = update self.done = 0 - def __call__(self, req, res): + def __call__(self, job): self.done += 1 - self.update(float(self.done)/self.total, - _('Rendered %s')%os.path.basename(req.callable.path_to_page)) + msg = _('Rendered %s') if job.result else _('Failed %s') + msg = msg%os.path.basename(job.args[0]) + self.update(float(self.done)/self.total, msg) def process_pages(pages, opts, update): ''' @@ -211,23 +212,25 @@ def process_pages(pages, opts, update): ''' if not _imagemagick_loaded: raise RuntimeError('Failed to load ImageMagick') - with ImageMagick(): - tdir = PersistentTemporaryDirectory('_comic2lrf_pp') - processed_pages = [PageProcessor(path, tdir, opts, i) for i, path in enumerate(pages)] - tp = ThreadPool(detect_ncpus()) - update(0, '') - notify = Progress(len(pages), update) - for pp in processed_pages: - tp.putRequest(WorkRequest(pp, callback=notify)) - tp.wait() - ans, failures = [], [] + + tdir = PersistentTemporaryDirectory('_comic2lrf_pp') + notify = Progress(len(pages), update) + server = Server() + jobs = [] + for i, path in enumerate(pages): + jobs.append(ParallelJob('render_page', notify, args=[path, tdir, opts, i])) + server.add_job(jobs[-1]) + server.wait() + server.killall() + server.close() + ans, failures = [], [] - for pp in processed_pages: - if len(pp) == 0: - failures.append(os.path.basename(pp.path_to_page)) - else: - ans += pp - return ans, failures, tdir + for job in jobs: + if not job.result: + failures.append(os.path.basename(job.args[0])) + else: + ans += job.result + return ans, failures, tdir def config(defaults=None): desc = _('Options to control the conversion of comics (CBR, CBZ) files into ebooks') diff --git a/src/calibre/parallel.py b/src/calibre/parallel.py index 0a028a16bd..fba3b73e59 100644 --- a/src/calibre/parallel.py +++ b/src/calibre/parallel.py @@ -36,19 +36,22 @@ DEBUG = False #: A mapping from job names to functions that perform the jobs PARALLEL_FUNCS = { - 'any2lrf' : + 'any2lrf' : ('calibre.ebooks.lrf.any.convert_from', 'main', dict(gui_mode=True), None), - 'lrfviewer' : + 'lrfviewer' : ('calibre.gui2.lrf_renderer.main', 'main', {}, None), - 'feeds2lrf' : + 'feeds2lrf' : ('calibre.ebooks.lrf.feeds.convert_from', 'main', {}, 'notification'), - 'render_table' : + 'render_table' : ('calibre.ebooks.lrf.html.table_as_image', 'do_render', {}, None), + + 'render_page' : + ('calibre.ebooks.lrf.comic.convert_from', 'process_page', {}, None), - 'comic2lrf' : + 'comic2lrf' : ('calibre.ebooks.lrf.comic.convert_from', 'do_convert', {}, 'notification'), } @@ -657,6 +660,21 @@ class Server(Thread): if job.job_manager is not None: job.job_manager.add_job(job) + def poll(self): + ''' + Return True if the server has either working or queued jobs + ''' + with self.job_lock: + with self.working_lock: + return len(self.jobs) + len(self.working) > 0 + + def wait(self, sleep=1): + ''' + Wait until job queue is empty + ''' + while self.poll(): + time.sleep(sleep) + def run(self): while True: job = None diff --git a/src/calibre/utils/threadpool.py b/src/calibre/utils/threadpool.py index 19b1a7b038..e2a5583baf 100644 --- a/src/calibre/utils/threadpool.py +++ b/src/calibre/utils/threadpool.py @@ -46,7 +46,6 @@ __date__ = "$Date: 2006/06/23 12:32:25 $" __license__ = 'Python license' # standard library modules -import sys import threading import Queue diff --git a/src/calibre/utils/zipfile.py b/src/calibre/utils/zipfile.py index 3deba3a612..b135bde601 100644 --- a/src/calibre/utils/zipfile.py +++ b/src/calibre/utils/zipfile.py @@ -1136,7 +1136,7 @@ class ZipFile: self.filelist.append(zinfo) self.NameToInfo[zinfo.filename] = zinfo - def writestr(self, zinfo_or_arcname, bytes, permissions=0600): + def writestr(self, zinfo_or_arcname, bytes, permissions=0600, compression=ZIP_DEFLATED): """Write a file into the archive. The contents is the string 'bytes'. 'zinfo_or_arcname' is either a ZipInfo instance or the name of the file in the archive.""" @@ -1145,7 +1145,7 @@ class ZipFile: zinfo_or_arcname = zinfo_or_arcname.encode('utf-8') zinfo = ZipInfo(filename=zinfo_or_arcname, date_time=time.localtime(time.time())[:6]) - zinfo.compress_type = self.compression + zinfo.compress_type = compression zinfo.external_attr = permissions << 16 else: zinfo = zinfo_or_arcname