IGN:Various cleanups and performance improvement in comic2lrf

src/calibre/ebooks/lrf/comic/convert_from.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 Based on ideas from comiclrf created by FangornUK.
 '''
 
-import os, sys, traceback, shutil, threading
+import os, sys, shutil, traceback
 from uuid import uuid4
 
 from calibre import extract, terminal_controller, __appname__, __version__
@@ -44,7 +44,7 @@ def extract_comic(path_to_comic_file):
     '''
     Un-archive the comic file.
     '''
-    tdir = PersistentTemporaryDirectory(suffix='comic_extract')
+    tdir = PersistentTemporaryDirectory(suffix='_comic_extract')
     extract(path_to_comic_file, tdir)
     return tdir
 
@@ -78,7 +78,7 @@ def find_pages(dir, sort_on_mtime=False, verbose=False):
 
 class PageProcessor(list):
     '''
-    Contains the actual image rendering logic. See :method:`__call__` and 
+    Contains the actual image rendering logic. See :method:`render` and 
     :method:`process_pages`.
     '''
     
@@ -89,9 +89,10 @@ class PageProcessor(list):
         self.num          = num
         self.dest         = dest
         self.rotate       = False
+        self.render()
         
         
-    def __call__(self):
+    def render(self):
         img = NewMagickWand()
         if img < 0:
             raise RuntimeError('Cannot create wand.')
@@ -106,16 +107,15 @@ class PageProcessor(list):
             MagickThumbnailImage(thumb, 60, 80)
             MagickWriteImage(thumb, os.path.join(self.dest, 'thumbnail.png'))
             DestroyMagickWand(thumb)
-        
         self.pages = [img]
         if width > height:
             if self.opts.landscape:
                 self.rotate = True
             else:
                 split1, split2 = map(CloneMagickWand, (img, img))
+                DestroyMagickWand(img)
                 if split1 < 0 or split2 < 0:
                     raise RuntimeError('Cannot create wand.')
-                DestroyMagickWand(img)
                 MagickCropImage(split1, (width/2)-1, height, 0, 0)
                 MagickCropImage(split2, (width/2)-1, height, width/2, 0 )
                 self.pages = [split2, split1] if self.opts.right2left else [split1, split2]
@@ -124,105 +124,126 @@ class PageProcessor(list):
     def process_pages(self):
         for i, wand in enumerate(self.pages):
             pw = NewPixelWand()
-            if pw < 0:
+            try:
-                raise RuntimeError('Cannot create wand.')
+                if pw < 0:
-            PixelSetColor(pw, 'white')
+                    raise RuntimeError('Cannot create wand.')
-            
+                PixelSetColor(pw, 'white')
-            MagickSetImageBorderColor(wand, pw)
-            if self.rotate:
-                MagickRotateImage(wand, pw, -90)
                 
-            # 25 percent fuzzy trim?
-            MagickTrimImage(wand, 25*65535/100)
-            MagickSetImagePage(wand, 0,0,0,0)   #Clear page after trim, like a "+repage"
-            # Do the Photoshop "Auto Levels" equivalent
-            if not self.opts.dont_normalize:
-                MagickNormalizeImage(wand)
-            sizex = MagickGetImageWidth(wand)
-            sizey = MagickGetImageHeight(wand)
-            
-            SCRWIDTH, SCRHEIGHT = PROFILES[self.opts.profile]
-            print 77777, threading.currentThread()
-            if self.opts.keep_aspect_ratio:
-                # Preserve the aspect ratio by adding border
-                aspect = float(sizex) / float(sizey)
-                if aspect <= (float(SCRWIDTH) / float(SCRHEIGHT)):
-                    newsizey = SCRHEIGHT
-                    newsizex = int(newsizey * aspect)
-                    deltax = (SCRWIDTH - newsizex) / 2
-                    deltay = 0
-                else:
-                    newsizex = SCRWIDTH
-                    newsizey = int(newsizex / aspect)
-                    deltax = 0
-                    deltay = (SCRHEIGHT - newsizey) / 2
-                MagickResizeImage(wand, newsizex, newsizey, CatromFilter, 1.0)
                 MagickSetImageBorderColor(wand, pw)
-                MagickBorderImage(wand, pw, deltax, deltay)
+                if self.rotate:
-            elif self.opts.wide:
+                    MagickRotateImage(wand, pw, -90)
-                # Keep aspect and Use device height as scaled image width so landscape mode is clean
+                    
-                aspect = float(sizex) / float(sizey)
+                # 25 percent fuzzy trim?
-                screen_aspect = float(SCRWIDTH) / float(SCRHEIGHT)
+                MagickTrimImage(wand, 25*65535/100)
-                # Get dimensions of the landscape mode screen
+                MagickSetImagePage(wand, 0,0,0,0)   #Clear page after trim, like a "+repage"
-                # Add 25px back to height for the battery bar.
+                # Do the Photoshop "Auto Levels" equivalent
-                wscreenx = SCRHEIGHT + 25
+                if not self.opts.dont_normalize:
-                wscreeny = int(wscreenx / screen_aspect)
+                    MagickNormalizeImage(wand)
-                if aspect <= screen_aspect:
+                sizex = MagickGetImageWidth(wand)
-                    newsizey = wscreeny
+                sizey = MagickGetImageHeight(wand)
-                    newsizex = int(newsizey * aspect)
+                
-                    deltax = (wscreenx - newsizex) / 2
+                SCRWIDTH, SCRHEIGHT = PROFILES[self.opts.profile]
-                    deltay = 0
+                
+                if self.opts.keep_aspect_ratio:
+                    # Preserve the aspect ratio by adding border
+                    aspect = float(sizex) / float(sizey)
+                    if aspect <= (float(SCRWIDTH) / float(SCRHEIGHT)):
+                        newsizey = SCRHEIGHT
+                        newsizex = int(newsizey * aspect)
+                        deltax = (SCRWIDTH - newsizex) / 2
+                        deltay = 0
+                    else:
+                        newsizex = SCRWIDTH
+                        newsizey = int(newsizex / aspect)
+                        deltax = 0
+                        deltay = (SCRHEIGHT - newsizey) / 2
+                    MagickResizeImage(wand, newsizex, newsizey, CatromFilter, 1.0)
+                    MagickSetImageBorderColor(wand, pw)
+                    MagickBorderImage(wand, pw, deltax, deltay)
+                elif self.opts.wide:
+                    # Keep aspect and Use device height as scaled image width so landscape mode is clean
+                    aspect = float(sizex) / float(sizey)
+                    screen_aspect = float(SCRWIDTH) / float(SCRHEIGHT)
+                    # Get dimensions of the landscape mode screen
+                    # Add 25px back to height for the battery bar.
+                    wscreenx = SCRHEIGHT + 25
+                    wscreeny = int(wscreenx / screen_aspect)
+                    if aspect <= screen_aspect:
+                        newsizey = wscreeny
+                        newsizex = int(newsizey * aspect)
+                        deltax = (wscreenx - newsizex) / 2
+                        deltay = 0
+                    else:
+                        newsizex = wscreenx
+                        newsizey = int(newsizex / aspect)
+                        deltax = 0
+                        deltay = (wscreeny - newsizey) / 2
+                    MagickResizeImage(wand, newsizex, newsizey, CatromFilter, 1.0)
+                    MagickSetImageBorderColor(wand, pw)
+                    MagickBorderImage(wand, pw, deltax, deltay)
                 else:
-                    newsizex = wscreenx
+                    MagickResizeImage(wand, SCRWIDTH, SCRHEIGHT, CatromFilter, 1.0)
-                    newsizey = int(newsizex / aspect)
+                    
-                    deltax = 0
+                if not self.opts.dont_sharpen:
-                    deltay = (wscreeny - newsizey) / 2
+                    MagickSharpenImage(wand, 0.0, 1.0)
-                MagickResizeImage(wand, newsizex, newsizey, CatromFilter, 1.0)
+                    
-                MagickSetImageBorderColor(wand, pw)
+                MagickSetImageType(wand, GrayscaleType)
-                MagickBorderImage(wand, pw, deltax, deltay)
-            else:
-                MagickResizeImage(wand, SCRWIDTH, SCRHEIGHT, CatromFilter, 1.0)
                 
-            if not self.opts.dont_sharpen:
+                if self.opts.despeckle:
-                MagickSharpenImage(wand, 0.0, 1.0)
+                    MagickDespeckleImage(wand)
                 
-            MagickSetImageType(wand, GrayscaleType)
+                MagickQuantizeImage(wand, self.opts.colors, RGBColorspace, 0, 1, 0)
+                dest = '%d_%d.png'%(self.num, i)
+                dest = os.path.join(self.dest, dest)
+                MagickWriteImage(wand, dest+'8')
+                os.rename(dest+'8', dest)
+                self.append(dest)
+            finally:
+                if pw > 0:
+                    DestroyPixelWand(pw)
+                DestroyMagickWand(wand)
             
-            if self.opts.despeckle:
+def render_pages(tasks, dest, opts, notification=None):
-                MagickDespeckleImage(wand)
+    '''
-            
+    Entry point for the job server.
-            MagickQuantizeImage(wand, self.opts.colors, RGBColorspace, 0, 1, 0)
+    '''
-            dest = '%d_%d.png'%(self.num, i)
+    failures, pages = [], []
-            dest = os.path.join(self.dest, dest)
-            MagickWriteImage(wand, dest+'8')
-            os.rename(dest+'8', dest)
-            self.append(dest)
-        
-            DestroyPixelWand(pw)
-            wand = DestroyMagickWand(wand)
-            
-def process_page(path, dest, opts, num):
-    pp = PageProcessor(path, dest, opts, num)
     with ImageMagick():
-        pp()
+        for num, path in tasks:
-    return list(pp)
+            try:
-            
+                pages.extend(PageProcessor(path, dest, opts, num))
-class Progress(object):
+                msg = _('Rendered %s') 
+            except:
+                failures.append(path)
+                msg = _('Failed %s')
+                if opts.verbose:
+                    msg += '\n' + traceback.format_exc() 
+            msg = msg%path
+            if notification is not None:
+                notification(0.5, msg)
     
-    def __init__(self, total, update, verbose):
+    return pages, failures
+        
+            
+class JobManager(object):
+    '''
+    Simple job manager responsible for keeping track of overall progress.
+    '''
+    
+    def __init__(self, total, update):
         self.total  = total
         self.update = update
         self.done   = 0
-        self.verbose = verbose
+        self.add_job        = lambda j: j
+        self.output         = lambda j: j
+        self.start_work     = lambda j: j
+        self.job_done       = lambda j: j
         
-    def __call__(self, job):
+    def status_update(self, job):
         self.done += 1
-        msg = _('Rendered %s') if job.result else _('Failed %s')
+        #msg = msg%os.path.basename(job.args[0])
-        msg = msg%os.path.basename(job.args[0])
+        self.update(float(self.done)/self.total, job.msg)
-        self.update(float(self.done)/self.total, msg)
+        
-        if not job.result and self.verbose:
-            print job.traceback
-
 def process_pages(pages, opts, update):
     '''
     Render all identified comic pages.
@@ -231,11 +252,13 @@ def process_pages(pages, opts, update):
         raise RuntimeError('Failed to load ImageMagick')
     
     tdir = PersistentTemporaryDirectory('_comic2lrf_pp')
-    notify = Progress(len(pages), update, opts.verbose)
+    job_manager = JobManager(len(pages), update)
     server = Server()
     jobs = []
-    for i, path in enumerate(pages):
+    tasks = server.split(pages)
-        jobs.append(ParallelJob('render_page', notify, args=[path, tdir, opts, i]))
+    for task in tasks:
+        jobs.append(ParallelJob('render_pages', lambda s:s, job_manager=job_manager,
+                                args=[task, tdir, opts]))
         server.add_job(jobs[-1])
     server.wait()
     server.killall()
@@ -243,10 +266,9 @@ def process_pages(pages, opts, update):
     ans, failures = [], []
         
     for job in jobs:
-        if not job.result:
+        pages, failures_ = job.result
-            failures.append(os.path.basename(job.args[0]))
+        ans += pages
-        else:
+        failures += failures_
-            ans += job.result
     return ans, failures, tdir
     
 def config(defaults=None):
@@ -282,7 +304,7 @@ def config(defaults=None):
               help=_("Don't sort the files found in the comic alphabetically by name. Instead use the order they were added to the comic."))
     c.add_opt('profile', ['-p', '--profile'], default='prs500', choices=PROFILES.keys(),
               help=_('Choose a profile for the device you are generating this LRF for. The default is the SONY PRS-500 with a screen size of 584x754 pixels. Choices are %s')%PROFILES.keys())
-    c.add_opt('verbose', ['--verbose'], default=0, action='count',
+    c.add_opt('verbose', ['-v', '--verbose'], default=0, action='count',
               help=_('Be verbose, useful for debugging. Can be specified multiple times for greater verbosity.'))
     c.add_opt('no_progress_bar', ['--no-progress-bar'], default=False,
                       help=_("Don't show progress bar."))
@@ -322,7 +344,7 @@ def create_lrf(pages, profile, opts, thumbnail=None):
 def do_convert(path_to_file, opts, notification=lambda m, p: p):
     source = path_to_file
     if not opts.title:
-        opts.title = os.path.splitext(os.path.basename(source))
+        opts.title = os.path.splitext(os.path.basename(source))[0]
     if not opts.output:
         opts.output = os.path.abspath(os.path.splitext(os.path.basename(source))[0]+'.lrf')
         

src/calibre/parallel.py

@@ -28,6 +28,7 @@ import sys, os, gc, cPickle, traceback, atexit, cStringIO, time, signal, \
        subprocess, socket, collections, binascii, re, thread, tempfile
 from select import select
 from threading import RLock, Thread, Event
+from math import ceil
 
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre import iswindows, detect_ncpus, isosx
@@ -48,8 +49,8 @@ PARALLEL_FUNCS = {
       'render_table' :
         ('calibre.ebooks.lrf.html.table_as_image', 'do_render', {}, None),
         
-      'render_page' :
+      'render_pages' :
-        ('calibre.ebooks.lrf.comic.convert_from', 'process_page', {}, None),
+        ('calibre.ebooks.lrf.comic.convert_from', 'render_pages', {}, 'notification'),
 
       'comic2lrf'    :
         ('calibre.ebooks.lrf.comic.convert_from', 'do_convert', {}, 'notification'),
@@ -532,7 +533,10 @@ class Job(object):
         self.percent = percent
         self.msg     = msg
         if self.job_manager is not None:
-            self.job_manager.status_update(self)
+            try:
+                self.job_manager.status_update(self)
+            except:
+                traceback.print_exc()
         
     def status(self):
         if self.is_running:
@@ -647,6 +651,25 @@ class Server(Thread):
         self.result_lock = RLock()
         self.pool_lock = RLock()
         self.start()
+        
+    def split(self, tasks):
+        '''
+        Split a list into a list of sub lists, with the number of sub lists being
+        no more than the number of workers this server supports. Each sublist contains
+        two tuples of the form (i, x) where x is an element fro the original list
+        and i is the index of the element x in the original list.
+        '''
+        ans, count, pos = [], 0, 0
+        delta = int(ceil(len(tasks)/float(self.number_of_workers)))
+        while count < len(tasks):
+            section = []
+            for t in tasks[pos:pos+delta]:
+                section.append((count, t))
+                count += 1
+            ans.append(section)
+            pos += delta
+        return ans
+        
 
     def close(self):
         try: