Fix #951 ([comic2lrf] on windows multi-threading isn't working)

installer/windows/freeze.py

@@ -161,12 +161,12 @@ def main(args=sys.argv):
                                              'win32process', 'win32api', 'msvcrt',
                                              'win32event', 'calibre.ebooks.lrf.any.*',
                                              'calibre.ebooks.lrf.feeds.*',
-                                             'lxml', 'lxml._elementpath', 'genshi',
+                                             'genshi', 
                                              'path', 'pydoc', 'IPython.Extensions.*',
                                              'calibre.web.feeds.recipes.*',
                                              'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
                                              ],
-                                  'packages'  : ['PIL'],
+                                  'packages'  : ['PIL', 'lxml'],
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",
                                                  "_imagingtk", "ImageTk", "FixTk"
                                                 ],

src/calibre/debug.py

@@ -25,9 +25,7 @@ Run an embedded python interpreter.
 
 def update_zipfile(zipfile, mod, path):
     if 'win32' in sys.platform:
-        print 'WARNING: On Windows Vista you must run this from a console that has been started in Administrator mode.'
+        print 'WARNING: On Windows Vista using this option may cause windows to put library.zip into the Virtual Store (typically located in c:\Users\username\AppData\Local\VirtualStore). If it does this you must delete it from there after you\'re done debugging).' 
-        print 'Press Enter to continue if this is an Administrator console or Ctrl-C to Cancel'
-        raw_input()
     pat = re.compile(mod.replace('.', '/')+r'\.py[co]*')
     name = mod.replace('.', '/') + os.path.splitext(path)[-1]
     update(zipfile, [pat], [path], [name])

src/calibre/ebooks/epub/__init__.py

@@ -8,8 +8,28 @@ Conversion to EPUB.
 '''
 import sys
 from calibre.utils.config import Config, StringConfig
+from calibre.utils.zipfile import ZipFile, ZIP_DEFLATED
 from calibre.ebooks.html import config as common_config
 
+def initialize_container(path_to_container, opf_name='metadata.opf'):
+    '''
+    Create an empty EPUB document, with a default skeleton.
+    '''
+    CONTAINER='''\
+<?xml version="1.0"?>
+<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container">
+   <rootfiles>
+      <rootfile full-path="%s" media-type="application/oebps-package+xml"/>
+   </rootfiles>
+</container>
+    '''%opf_name
+    zf = ZipFile(path_to_container, 'w')
+    zf.writestr('mimetype', 'application/epub+zip', compression=ZIP_DEFLATED)
+    zf.writestr('META-INF/', '', 0700)
+    zf.writestr('META-INF/container.xml', CONTAINER)
+    return zf
+    
+
 def config(defaults=None):
     desc = _('Options to control the conversion to EPUB')
     if defaults is None:

src/calibre/ebooks/epub/from_html.py

@@ -36,7 +36,7 @@ class HTMLProcessor(Parser):
         for text in get_text(self.body if self.body is not None else self.root):
             length, parent = len(re.sub(r'\s+', '', text)), text.getparent()
             #TODO: Use cssutils on self.raw_css to figure out the font size 
-            # of this piece text and update statistics accordingly        
+            # of this piece of text and update statistics accordingly        
     
     def split(self):
         ''' Split into individual flows to accommodate Adobe's incompetence '''
@@ -74,13 +74,17 @@ def convert(htmlfile, opts, notification=None):
     mi = merge_metadata(htmlfile, opf, opts)
     opts.chapter = XPath(opts.chapter, 
                     namespaces={'re':'http://exslt.org/regular-expressions'})
+    
     resource_map = parse_content(filelist, opts)
+    
     resources = [os.path.join(opts.output, 'content', f) for f in resource_map.values()]
+    
     if opf.cover and os.access(opf.cover, os.R_OK):
         shutil.copyfile(opf.cover, os.path.join(opts.output, 'content', 'resources', '_cover_'+os.path.splitext(opf.cover)))
         cpath = os.path.join(opts.output, 'content', 'resources', '_cover_'+os.path.splitext(opf.cover))
         shutil.copyfile(opf.cover, cpath)
         resources.append(cpath)
+        mi.cover = cpath
             
 def main(args=sys.argv):
     parser = option_parser()

src/calibre/ebooks/lrf/comic/convert_from.py

@@ -7,14 +7,13 @@ __docformat__ = 'restructuredtext en'
 Based on ideas from comiclrf created by FangornUK.
 '''
 
-import os, sys, traceback, shutil
+import os, sys, traceback, shutil, threading
 from uuid import uuid4
 
-from calibre import extract, detect_ncpus, terminal_controller, \
+from calibre import extract, terminal_controller, __appname__, __version__
-                    __appname__, __version__
 from calibre.utils.config import Config, StringConfig
 from calibre.ptempfile import PersistentTemporaryDirectory
-from calibre.utils.threadpool import ThreadPool, WorkRequest
+from calibre.parallel import Server, ParallelJob
 from calibre.utils.terminfo import ProgressBar
 from calibre.ebooks.lrf.pylrs.pylrs import Book, BookSetting, ImageStream, ImageBlock
 try:
@@ -84,12 +83,13 @@ class PageProcessor(list):
     '''
     
     def __init__(self, path_to_page, dest, opts, num):
-        self.path_to_page = path_to_page
-        self.opts = opts
-        self.num = num
-        self.dest = dest
-        self.rotate = False
         list.__init__(self)
+        self.path_to_page = path_to_page
+        self.opts         = opts
+        self.num          = num
+        self.dest         = dest
+        self.rotate       = False
+        
         
     def __call__(self):
         try:
@@ -100,7 +100,6 @@ class PageProcessor(list):
                 raise IOError('Failed to read image from: %'%self.path_to_page)
             width  = MagickGetImageWidth(img)
             height = MagickGetImageHeight(img)
-            
             if self.num == 0: # First image so create a thumbnail from it
                 thumb = CloneMagickWand(img)
                 if thumb < 0:
@@ -122,7 +121,6 @@ class PageProcessor(list):
                     MagickCropImage(split1, (width/2)-1, height, 0, 0)
                     MagickCropImage(split2, (width/2)-1, height, width/2, 0 )
                     self.pages = [split2, split1] if self.opts.right2left else [split1, split2]
-                    
             self.process_pages()
         except Exception, err:
             print 'Failed to process page: %s'%os.path.basename(self.path_to_page)
@@ -138,23 +136,20 @@ class PageProcessor(list):
             PixelSetColor(pw, 'white')
             
             MagickSetImageBorderColor(wand, pw)
-            
             if self.rotate:
                 MagickRotateImage(wand, pw, -90)
                 
             # 25 percent fuzzy trim?
             MagickTrimImage(wand, 25*65535/100)
             MagickSetImagePage(wand, 0,0,0,0)   #Clear page after trim, like a "+repage"
-            
             # Do the Photoshop "Auto Levels" equivalent
             if not self.opts.dont_normalize:
                 MagickNormalizeImage(wand)
-        
             sizex = MagickGetImageWidth(wand)
             sizey = MagickGetImageHeight(wand)
             
             SCRWIDTH, SCRHEIGHT = PROFILES[self.opts.profile]
-            
+            print 77777, threading.currentThread()
             if self.opts.keep_aspect_ratio: 
                 # Preserve the aspect ratio by adding border
                 aspect = float(sizex) / float(sizey)
@@ -168,7 +163,6 @@ class PageProcessor(list):
                     newsizey = int(newsizex / aspect)
                     deltax = 0
                     deltay = (SCRHEIGHT - newsizey) / 2
-        
                 MagickResizeImage(wand, newsizex, newsizey, CatromFilter, 1.0)
                 MagickSetImageBorderColor(wand, pw)
                 MagickBorderImage(wand, pw, deltax, deltay)
@@ -193,6 +187,12 @@ class PageProcessor(list):
             DestroyPixelWand(pw)
             wand = DestroyMagickWand(wand)
             
+def process_page(path, dest, opts, num):
+    pp = PageProcessor(path, dest, opts, num)
+    with ImageMagick():
+        pp()
+    return list(pp)
+            
 class Progress(object):
     
     def __init__(self, total, update):
@@ -200,10 +200,11 @@ class Progress(object):
         self.update = update
         self.done   = 0
         
-    def __call__(self, req, res):
+    def __call__(self, job):
         self.done += 1
-        self.update(float(self.done)/self.total, 
+        msg = _('Rendered %s') if job.result else _('Failed %s')
-                    _('Rendered %s')%os.path.basename(req.callable.path_to_page))
+        msg = msg%os.path.basename(job.args[0])
+        self.update(float(self.done)/self.total, msg)
 
 def process_pages(pages, opts, update):
     '''
@@ -211,23 +212,25 @@ def process_pages(pages, opts, update):
     '''
     if not _imagemagick_loaded:
         raise RuntimeError('Failed to load ImageMagick')
-    with ImageMagick():
+    
-        tdir = PersistentTemporaryDirectory('_comic2lrf_pp')
+    tdir = PersistentTemporaryDirectory('_comic2lrf_pp')
-        processed_pages = [PageProcessor(path, tdir, opts, i) for i, path in enumerate(pages)]
+    notify = Progress(len(pages), update)
-        tp = ThreadPool(detect_ncpus())
+    server = Server()
-        update(0, '')
+    jobs = []
-        notify = Progress(len(pages), update)
+    for i, path in enumerate(pages):
-        for pp in processed_pages:
+        jobs.append(ParallelJob('render_page', notify, args=[path, tdir, opts, i]))
-            tp.putRequest(WorkRequest(pp, callback=notify))
+        server.add_job(jobs[-1])
-            tp.wait()
+    server.wait()
-        ans, failures = [], []
+    server.killall()
+    server.close()
+    ans, failures = [], []
         
-        for pp in processed_pages:
+    for job in jobs:
-            if len(pp) == 0:
+        if not job.result:
-                failures.append(os.path.basename(pp.path_to_page))
+            failures.append(os.path.basename(job.args[0]))
-            else:
+        else:
-                ans += pp
+            ans += job.result
-        return ans, failures, tdir
+    return ans, failures, tdir
     
 def config(defaults=None):
     desc = _('Options to control the conversion of comics (CBR, CBZ) files into ebooks')

src/calibre/parallel.py

@@ -36,19 +36,22 @@ DEBUG = False
 
 #: A mapping from job names to functions that perform the jobs
 PARALLEL_FUNCS = {
-                  'any2lrf'      :
+      'any2lrf'      :
         ('calibre.ebooks.lrf.any.convert_from', 'main', dict(gui_mode=True), None),
 
-                  'lrfviewer'    :
+      'lrfviewer'    :
         ('calibre.gui2.lrf_renderer.main', 'main', {}, None),
 
-                  'feeds2lrf'    :
+      'feeds2lrf'    :
         ('calibre.ebooks.lrf.feeds.convert_from', 'main', {}, 'notification'),
 
-                  'render_table' :
+      'render_table' :
         ('calibre.ebooks.lrf.html.table_as_image', 'do_render', {}, None),
+        
+      'render_page' :
+        ('calibre.ebooks.lrf.comic.convert_from', 'process_page', {}, None),
 
-                  'comic2lrf'    :
+      'comic2lrf'    :
         ('calibre.ebooks.lrf.comic.convert_from', 'do_convert', {}, 'notification'),
 }
 
@@ -657,6 +660,21 @@ class Server(Thread):
         if job.job_manager is not None:
             job.job_manager.add_job(job)
             
+    def poll(self):
+        '''
+        Return True if the server has either working or queued jobs
+        '''
+        with self.job_lock:
+            with self.working_lock:
+                return len(self.jobs) + len(self.working) > 0
+            
+    def wait(self, sleep=1):
+        '''
+        Wait until job queue is empty
+        '''
+        while self.poll():
+            time.sleep(sleep)
+    
     def run(self):
         while True:
             job = None

src/calibre/utils/threadpool.py

@@ -46,7 +46,6 @@ __date__ = "$Date: 2006/06/23 12:32:25 $"
 __license__ = 'Python license'
 
 # standard library modules
-import sys
 import threading
 import Queue
 

src/calibre/utils/zipfile.py

@@ -1136,7 +1136,7 @@ class ZipFile:
         self.filelist.append(zinfo)
         self.NameToInfo[zinfo.filename] = zinfo
 
-    def writestr(self, zinfo_or_arcname, bytes, permissions=0600):
+    def writestr(self, zinfo_or_arcname, bytes, permissions=0600, compression=ZIP_DEFLATED):
         """Write a file into the archive.  The contents is the string
         'bytes'.  'zinfo_or_arcname' is either a ZipInfo instance or
         the name of the file in the archive."""
@@ -1145,7 +1145,7 @@ class ZipFile:
                 zinfo_or_arcname = zinfo_or_arcname.encode('utf-8')
             zinfo = ZipInfo(filename=zinfo_or_arcname,
                             date_time=time.localtime(time.time())[:6])
-            zinfo.compress_type = self.compression
+            zinfo.compress_type = compression
             zinfo.external_attr = permissions << 16
         else:
             zinfo = zinfo_or_arcname