Intial implementation of comic2lrf for converting CBR, CBZ files

src/calibre/__init__.py

@@ -20,7 +20,7 @@ from PyQt4.QtGui import QDesktopServices
 
 from calibre.translations.msgfmt import make
 from calibre.ebooks.chardet import detect
-from calibre.terminfo import TerminalController
+from calibre.utils.terminfo import TerminalController
 
 terminal_controller = TerminalController(sys.stdout)
 iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower()
@@ -303,10 +303,10 @@ def filename_to_utf8(name):
 def extract(path, dir):
     ext = os.path.splitext(path)[1][1:].lower()
     extractor = None
-    if ext == 'zip':
+    if ext in ['zip', 'cbz', 'epub']:
         from calibre.libunzip import extract as zipextract
         extractor = zipextract
-    elif ext == 'rar':
+    elif ext in ['cbr', 'rar']:
         from calibre.libunrar import extract as rarextract
         extractor = rarextract
     if extractor is None:

src/calibre/ebooks/lrf/comic/__init__.py

@@ -0,0 +1,16 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Convert CBR/CBZ files to LRF.
+'''
+
+import sys
+
+def main(args=sys.argv):
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/ebooks/lrf/comic/convert_from.py

@@ -0,0 +1,298 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Based on ideas from comiclrf created by FangornUK.
+'''
+
+import os, sys, traceback, shutil
+from uuid import uuid4
+
+from calibre import extract, OptionParser, detect_ncpus, terminal_controller, \
+                    __appname__, __version__
+from calibre.ptempfile import PersistentTemporaryDirectory
+from calibre.utils.threadpool import ThreadPool, WorkRequest
+from calibre.utils.terminfo import ProgressBar
+from calibre.ebooks.lrf.pylrs.pylrs import Book, BookSetting, ImageStream, ImageBlock
+from calibre.utils.PythonMagickWand import \
+    NewMagickWand, NewPixelWand, \
+    MagickSetImageBorderColor, \
+    MagickReadImage, MagickRotateImage, \
+    MagickTrimImage, \
+    MagickNormalizeImage, MagickGetImageWidth, \
+    MagickGetImageHeight, \
+    MagickResizeImage, MagickSetImageType, \
+    GrayscaleType, CatromFilter,  MagickSetImagePage, \
+    MagickBorderImage, MagickSharpenImage, \
+    MagickQuantizeImage, RGBColorspace, \
+    MagickWriteImage, DestroyPixelWand, \
+    DestroyMagickWand, CloneMagickWand, \
+    MagickThumbnailImage, MagickCropImage, initialize, finalize
+
+PROFILES = {
+            'prs500':(584, 754),            
+            }
+
+def extract_comic(path_to_comic_file):
+    tdir = PersistentTemporaryDirectory(suffix='comic_extract')
+    extract(path_to_comic_file, tdir)
+    return tdir
+
+def find_pages(dir, sort_on_mtime=False, verbose=False):
+    extensions = ['jpeg', 'jpg', 'gif', 'png']
+    pages = []
+    for datum in os.walk(dir):
+        for name in datum[-1]:
+            path = os.path.join(datum[0], name)
+            for ext in extensions:
+                if path.lower().endswith('.'+ext):
+                    pages.append(path)
+                    break
+    if sort_on_mtime:
+        comparator = lambda x, y : cmp(os.stat(x).st_mtime, os.stat(y).st_mtime)
+    else:
+        comparator = lambda x, y : cmp(os.path.basename(x), os.path.basename(y))
+
+    pages.sort(cmp=comparator)
+    if verbose:
+        print 'Found comic pages...'
+        print '\t'+'\n\t'.join([os.path.basename(p) for p in pages])
+    return pages
+
+class PageProcessor(list):
+    
+    def __init__(self, path_to_page, dest, opts, num):
+        self.path_to_page = path_to_page
+        self.opts = opts
+        self.num = num
+        self.dest = dest
+        self.rotate = False
+        list.__init__(self)
+        
+    def __call__(self):
+        try:
+            img = NewMagickWand()
+            if img < 0:
+                raise RuntimeError('Cannot create wand.')
+            if not MagickReadImage(img, self.path_to_page):
+                raise IOError('Failed to read image from: %'%self.path_to_page)
+            width  = MagickGetImageWidth(img)
+            height = MagickGetImageHeight(img)
+            
+            if self.num == 0: # First image so create a thumbnail from it
+                thumb = CloneMagickWand(img)
+                if thumb < 0:
+                    raise RuntimeError('Cannot create wand.')
+                MagickThumbnailImage(thumb, 60, 80)
+                MagickWriteImage(thumb, os.path.join(self.dest, 'thumbnail.png'))
+                DestroyMagickWand(thumb)
+            
+            self.pages = [img]
+            
+            if width > height:
+                if self.opts.landscape:
+                    self.rotate = True
+                else: 
+                    split1, split2 = map(CloneMagickWand, (img, img))
+                    if split1 < 0 or split2 < 0:
+                        raise RuntimeError('Cannot create wand.')
+                    DestroyMagickWand(img)
+                    MagickCropImage(split1, (width/2)-1, height, 0, 0)
+                    MagickCropImage(split2, (width/2)-1, height, width/2, 0 )
+                    self.pages = [split1, split2]
+                
+            self.process_pages()
+        except Exception, err:
+            print 'Failed to process page: %s'%os.path.basename(self.path_to_page)
+            print 'Error:', err
+            if self.opts.verbose:
+                traceback.print_exc()
+        
+    def process_pages(self):
+        for i, wand in enumerate(self.pages):
+            pw = NewPixelWand()
+            if pw < 0:
+                raise RuntimeError('Cannot create wand.')
+            #flag = PixelSetColor(pw, 'white')
+            
+            MagickSetImageBorderColor(wand, pw)
+            
+            if self.rotate:
+                MagickRotateImage(wand, pw, -90)
+                
+            # 25 percent fuzzy trim?
+            MagickTrimImage(wand, 25*65535/100)
+            MagickSetImagePage(wand, 0,0,0,0)   #Clear page after trim, like a "+repage"
+            
+            # Do the Photoshop "Auto Levels" equivalent
+            if self.opts.normalize:
+                MagickNormalizeImage(wand)
+        
+            sizex = MagickGetImageWidth(wand)
+            sizey = MagickGetImageHeight(wand)
+            
+            SCRWIDTH, SCRHEIGHT = PROFILES[self.opts.profile]
+            
+            if self.opts.keep_aspect_ratio: 
+                # Preserve the aspect ratio by adding border
+                aspect = float(sizex) / float(sizey)
+                if aspect <= (float(SCRWIDTH) / float(SCRHEIGHT)):
+                    newsizey = SCRHEIGHT
+                    newsizex = int(newsizey * aspect)
+                    deltax = (SCRWIDTH - newsizex) / 2
+                    deltay = 0
+                else:
+                    newsizex = SCRWIDTH
+                    newsizey = int(newsizex / aspect)
+                    deltax = 0
+                    deltay = (SCRHEIGHT - newsizey) / 2
+        
+                MagickResizeImage(wand, newsizex, newsizey, CatromFilter, 1.0)
+                MagickSetImageBorderColor(wand, pw)
+                MagickBorderImage(wand, pw, deltax, deltay)
+            else:
+                MagickResizeImage(wand, SCRWIDTH, SCRHEIGHT, CatromFilter, 1.0)
+                
+            if self.opts.sharpen:
+                MagickSharpenImage(wand, 0.0, 1.0)
+                
+            MagickSetImageType(wand, GrayscaleType)
+            MagickQuantizeImage(wand, self.opts.colors, RGBColorspace, 0, 1, 0)
+            dest = '%d_%d%s'%(self.num, i, os.path.splitext(self.path_to_page)[-1])
+            dest = os.path.join(self.dest, dest)
+            MagickWriteImage(wand, dest)
+            self.append(dest)
+        
+            DestroyPixelWand(pw)
+            wand = DestroyMagickWand(wand)
+            
+class Progress(object):
+    
+    def __init__(self, total, update):
+        self.total  = total
+        self.update = update
+        self.done   = 0
+        
+    def __call__(self, req, res):
+        self.done += 1
+        self.update(float(self.done)/self.total, 
+                    _('Rendered %s')%os.path.basename(req.callable.path_to_page))
+
+def process_pages(pages, opts, update):
+    initialize()
+    try:
+        tdir = PersistentTemporaryDirectory('_comic2lrf_pp')
+        processed_pages = [PageProcessor(path, tdir, opts, i) for i, path in enumerate(pages)]
+        tp = ThreadPool(detect_ncpus())
+        update(0, '')
+        notify = Progress(len(pages), update)
+        for pp in processed_pages:
+            tp.putRequest(WorkRequest(pp, callback=notify))
+            tp.wait()
+        ans, failures = [], []
+        
+        for pp in processed_pages:
+            if len(pp) == 0:
+                failures.append(os.path.basename(pp.path_to_page()))
+            else:
+                ans += pp
+        return ans, failures, tdir
+    finally:
+        finalize()
+
+def option_parser():
+    parser = OptionParser(_('''\
+%prog [options] comic.cb[z|r]
+
+Convert a comic in a CBZ or CBR file to an LRF ebook. 
+	'''))
+    parser.add_option('-t', '--title', help=_('Title for generated ebook. Default is to use the filename.'), default=None)
+    parser.add_option('-a', '--author', help=_('Set the author in the metadata of the generated ebook. Default is %default'), default=_('Unknown'))
+    parser.add_option('-o', '--output', help=_('Path to output LRF file. By default a file is created in the current directory.'), default=None)
+    parser.add_option('-c', '--colors', type='int', default=64,
+    				  help=_('Number of colors for Grayscale image conversion. Default: %default'))
+    parser.add_option('-n', '--disable-normalize', dest='normalize', default=True, action='store_false',
+    				  help=_('Disable normalize (improve contrast) color range for pictures. Default: False'))
+    parser.add_option('-r', '--keep-aspect-ratio', action='store_true', default=False,
+    				  help=_('Maintain picture aspect ratio. Default is to fill the screen.'))
+    parser.add_option('-s', '--disable-sharpen', default=True, action='store_false', dest='sharpen',
+    				  help=_('Disable sharpening.'))
+    parser.add_option('-l', '--landscape', default=False, action='store_true',
+    				  help=_("Don't split landscape images into two portrait images"))
+    parser.add_option('--no-sort', default=False, action='store_true',
+                      help=_("Don't sort the files found in the comic alphabetically by name. Instead use the order they were added to the comic."))
+    parser.add_option('-p', '--profile', default='prs500', dest='profile', type='choice',
+                      choices=PROFILES.keys(), help=_('Choose a profile for the device you are generating this LRF for. The default is the SONY PRS-500 with a screen size of 584x754 pixels. Choices are %s')%PROFILES.keys())
+    parser.add_option('--verbose', default=False, action='store_true', 
+                      help=_('Be verbose, useful for debugging'))
+    parser.add_option('--no-progress-bar', default=False, action='store_true',
+                      help=_("Don't show progress bar."))
+    return parser
+
+def create_lrf(pages, profile, opts, thumbnail=None):
+    width, height = PROFILES[profile]
+    ps = {}
+    ps['topmargin']      = 0
+    ps['evensidemargin'] = 0
+    ps['oddsidemargin']  = 0
+    ps['textwidth']      = width
+    ps['textheight']     = height
+    book = Book(title=opts.title, author=opts.author,
+            bookid=uuid4().hex,
+            publisher='%s %s'%(__appname__, __version__), thumbnail=thumbnail,
+            category='Comic', pagestyledefault=ps, 
+            booksetting=BookSetting(screenwidth=width, screenheight=height))
+    for page in pages:
+        imageStream = ImageStream(page)
+        _page = book.create_page()
+        _page.append(ImageBlock(refstream=imageStream, 
+                    blockwidth=width, blockheight=height, xsize=width, 
+                    ysize=height, x1=width, y1=height))
+        book.append(_page)
+        
+    book.renderLrf(open(opts.output, 'wb'))
+    
+
+
+def main(args=sys.argv, notification=None):
+    parser = option_parser()
+    opts, args = parser.parse_args(args)
+    if len(args) < 2:
+        parser.print_help()
+        print '\nYou must specify a file to convert'
+        return 1
+    
+    if not callable(notification):
+        pb = ProgressBar(terminal_controller, _('Rendering comic pages...'), 
+                         no_progress_bar=opts.no_progress_bar)
+        notification = pb.update
+    
+    source = os.path.abspath(args[1])
+    if not opts.title:
+        opts.title = os.path.splitext(os.path.basename(source))
+    if not opts.output:
+        opts.output = os.path.abspath(os.path.splitext(os.path.basename(source))[0]+'.lrf')
+        
+    tdir  = extract_comic(source)
+    pages = find_pages(tdir, sort_on_mtime=opts.no_sort, verbose=opts.verbose)
+    if not pages:
+        raise ValueError('Could not find any pages in the comic: %s'%source)
+    pages, failures, tdir2 = process_pages(pages, opts, notification)
+    if not pages:
+        raise ValueError('Could not find any valid pages in the comic: %s'%source)
+    if failures:
+        print 'Could not process the following pages (run with --verbose to see why):'
+        for f in failures:
+            print '\t', f
+    thumbnail = os.path.join(tdir2, 'thumbnail.png')
+    if not os.access(thumbnail, os.R_OK):
+        thumbnail = None
+    create_lrf(pages, opts.profile, opts, thumbnail=thumbnail)
+    shutil.rmtree(tdir)
+    shutil.rmtree(tdir2)
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/libunzip.py

@@ -3,8 +3,6 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 
-import os
-from cStringIO import StringIO
 from calibre.utils import zipfile
 
 def update(pathtozip, patterns, filepaths, names, compression=zipfile.ZIP_DEFLATED, verbose=True):
@@ -42,34 +40,4 @@ def extract(filename, dir):
     Extract archive C{filename} into directory C{dir}
     """
     zf = zipfile.ZipFile( filename )
-    namelist = zf.namelist()
-    dirlist = filter( lambda x: x.endswith( '/' ), namelist )
-    filelist = filter( lambda x: not x.endswith( '/' ), namelist )
-    # make base
-    pushd = os.getcwd()
-    if not os.path.isdir( dir ):
-        os.mkdir( dir )
-    os.chdir( dir )
-    # create directory structure
-    dirlist.sort()
-    for dirs in dirlist:
-        dirs = dirs.split( '/' )
-        prefix = ''
-        for dir in dirs:
-            dirname = os.path.join( prefix, dir )
-            if dir and not os.path.isdir( dirname ):
-                os.mkdir( dirname )
-            prefix = dirname
-    # extract files
-    for fn in filelist:
-        if os.path.dirname(fn) and not os.path.exists(os.path.dirname(fn)):
-            os.makedirs(os.path.dirname(fn))
-        out = open( fn, 'wb' )
-        buffer = StringIO( zf.read( fn ))
-        buflen = 2 ** 20
-        datum = buffer.read( buflen )
-        while datum:
-            out.write( datum )
-            datum = buffer.read( buflen )
-        out.close()
-    os.chdir( pushd )
+    zf.extractall(dir)

src/calibre/linux.py

@@ -47,6 +47,7 @@ entry_points = {
                              'mobi2oeb  = calibre.ebooks.mobi.reader:main',
                              'lrf2html  = calibre.ebooks.lrf.html.convert_to:main',
                              'lit2oeb   = calibre.ebooks.lit.reader:main',
+                             'comic2lrf = calibre.ebooks.lrf.comic.convert_from:main',
                              'calibre-debug      = calibre.debug:main',
                              'calibredb          = calibre.library.cli:main',
                              'calibre-fontconfig = calibre.utils.fontconfig:main',
@@ -166,6 +167,7 @@ def setup_completion(fatal_errors):
         from calibre.web.feeds.recipes import titles as feed_titles
         from calibre.ebooks.lrf.feeds.convert_from import option_parser as feeds2lrf
         from calibre.ebooks.metadata.epub import option_parser as epub_meta
+        from calibre.ebooks.lrf.comic.convert_from import option_parser as comicop
         
         f = open_file('/etc/bash_completion.d/libprs500')
         f.close()
@@ -198,6 +200,7 @@ def setup_completion(fatal_errors):
         f.write(opts_and_exts('pdfrelow', pdfhtmlop, ['pdf']))
         f.write(opts_and_exts('mobi2oeb', mobioeb, ['mobi', 'prc']))
         f.write(opts_and_exts('lit2oeb', lit2oeb, ['lit']))
+        f.write(opts_and_exts('comic2lrf', comicop, ['cbz', 'cbr']))
         f.write(opts_and_words('feeds2disk', feeds2disk, feed_titles))
         f.write(opts_and_words('feeds2lrf', feeds2lrf, feed_titles))
         f.write('''

src/calibre/manual/faq.rst

@@ -133,6 +133,7 @@ The graphical user interface of |app| is not starting on Windows?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 There can be several causes for this:
 
+    * **Any windows version**: Try running it as Administrator (Right click on the icon ans select "Run as Administrator")
     * **Any windows version**: Search for the files `calibre2.ini` and `calibre.ini` on your computer and delete them. Search for the file `library1.db` and rename it (this file contains all your converted books so deleting it is not a good idea. Now try again.
     * **Windows Vista**: If the folder :file:`C:\Users\Your User Name\AppData\Local\VirtualStore\Program Files\calibre` exists, delete it. Uninstall |app|. Reboot. Re-install.
     * **Any windows version**: Search your computer for a folder named :file:`_ipython`. Delete it and try again.

src/calibre/utils/fontconfig.py

@@ -22,7 +22,7 @@ match to a given font specification. The main functions in this module are:
 .. autofunction:: match
 '''
 
-import sys, os, locale, codecs, ctypes
+import sys, os, locale, codecs
 from ctypes import cdll, c_void_p, Structure, c_int, POINTER, c_ubyte, c_char, util, \
                    pointer, byref, create_string_buffer, Union, c_char_p, c_double
 

src/calibre/utils/terminfo.py

@@ -163,36 +163,50 @@ class ProgressBar:
     
     The progress bar is colored, if the terminal supports color
     output; and adjusts to the width of the terminal.
+    
+    If the terminal doesn't have the required capabilities, it uses a
+    simple progress bar.
     """
     BAR = '%3d%% ${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}\n'
     HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n'
     
-    def __init__(self, term, header):
-        self.term = term
-        if not (self.term.CLEAR_EOL and self.term.UP and self.term.BOL):
-            raise ValueError("Terminal isn't capable enough -- you "
-            "should use a simpler progress dispaly.")
-        self.width = self.term.COLS or 75
-        self.bar = term.render(self.BAR)
-        self.header = self.term.render(self.HEADER % header.center(self.width))
-        self.cleared = 1 #: true if we haven't drawn the bar yet.
+    def __init__(self, term, header, no_progress_bar = False):
+        self.term, self.no_progress_bar = term, no_progress_bar
+        self.fancy = self.term.CLEAR_EOL and self.term.UP and self.term.BOL
+        if self.fancy:
+            self.width = self.term.COLS or 75
+            self.bar = term.render(self.BAR)
+            self.header = self.term.render(self.HEADER % header.center(self.width))
+            self.cleared = 1 #: true if we haven't drawn the bar yet.
         
     def update(self, percent, message=''):
         if isinstance(message, unicode):
-            message = message.encode('utf-8', 'ignore')
-        if self.cleared:
-            sys.stdout.write(self.header)
-            self.cleared = 0
-        n = int((self.width-10)*percent)
-        msg = message.center(self.width)
-        sys.stdout.write(
-        self.term.BOL + self.term.UP + self.term.CLEAR_EOL +
-        (self.bar % (100*percent, '='*n, '-'*(self.width-10-n))) +
-        self.term.CLEAR_EOL + msg)
-        sys.stdout.flush()
+            message = message.encode('utf-8', 'replace')
+        
+        if self.no_progress_bar:
+            if message:
+                print message
+        elif self.fancy:
+            if self.cleared:
+                sys.stdout.write(self.header)
+                self.cleared = 0
+            n = int((self.width-10)*percent)
+            msg = message.center(self.width)
+            sys.stdout.write(
+            self.term.BOL + self.term.UP + self.term.CLEAR_EOL +
+            (self.bar % (100*percent, '='*n, '-'*(self.width-10-n))) +
+            self.term.CLEAR_EOL + msg)
+            sys.stdout.flush()
+        else:
+            if not message:
+                print '%d%%'%(percent*100),
+            else:
+                print '%d%%'%(percent*100), message
+            sys.stdout.flush()
+            
     
     def clear(self):
-        if not self.cleared:
+        if self.fancy and not self.cleared:
             sys.stdout.write(self.term.BOL + self.term.CLEAR_EOL +
             self.term.UP + self.term.CLEAR_EOL +
             self.term.UP + self.term.CLEAR_EOL)

src/calibre/web/feeds/main.py

@@ -60,35 +60,15 @@ If you specify this option, any argument to %prog is ignored and a default recip
     
     return p
     
-def simple_progress_bar(percent, msg):
-    if isinstance(msg, unicode):
-        msg = msg.encode('utf-8', 'ignore')
-    if not msg:
-        print '%d%%'%(percent*100),
-    else:
-        print '%d%%'%(percent*100), msg
-    sys.stdout.flush()
-    
-def no_progress_bar(percent, msg):
-    print msg
-
 class RecipeError(Exception):
     pass
 
 def run_recipe(opts, recipe_arg, parser, notification=None, handler=None):
     if notification is None:
-        from calibre.terminfo import TerminalController, ProgressBar
+        from calibre.utils.terminfo import TerminalController, ProgressBar
         term = TerminalController(sys.stdout)
-        if opts.progress_bar:
-            try:
-                pb = ProgressBar(term, _('Fetching feeds...'))
-                notification = pb.update
-            except ValueError:
-                notification = simple_progress_bar
-                print _('Fetching feeds...')
-        else:
-            notification = no_progress_bar
-        
+        pb = ProgressBar(term, _('Fetching feeds...'), no_progress_bar=opts.progress_bar)
+        notification = pb.update
     
     recipe, is_profile = None, False
     if opts.feeds is not None:

src/calibre/web/feeds/news.py

@@ -20,7 +20,7 @@ from calibre.ebooks.metadata import MetaInformation
 from calibre.web.feeds import feed_from_xml, templates, feeds_from_index
 from calibre.web.fetch.simple import option_parser as web2disk_option_parser
 from calibre.web.fetch.simple import RecursiveFetcher
-from calibre.threadpool import WorkRequest, ThreadPool, NoResultsPending
+from calibre.utils.threadpool import WorkRequest, ThreadPool, NoResultsPending
 from calibre.ebooks.lrf.web.profiles import FullContentProfile
 from calibre.ptempfile import PersistentTemporaryFile