Sync with calibre:main

installer/linux/freeze.py

@@ -19,6 +19,8 @@ EXTRAS         = ('/usr/lib/python2.5/site-packages/PIL', os.path.expanduser('~/
 SQLITE         = '/usr/lib/libsqlite3.so.0'
 DBUS           = '/usr/lib/libdbus-1.so.3'
 LIBMNG         = '/usr/lib/libmng.so.1'
+LIBZ           = '/lib/libz.so.1'
+LIBUSB         = '/lib/libusb.so'
 
 
 CALIBRESRC     = os.path.join(CALIBREPREFIX, 'src')
@@ -117,7 +119,7 @@ binaries += [('clit', CLIT, 'BINARY'), ('pdftohtml', PDFTOHTML, 'BINARY'),
              ('libunrar.so', LIBUNRAR, 'BINARY')]
 
 print 'Adding external libraries...'
-binaries += [ (os.path.basename(x), x, 'BINARY') for x in (SQLITE, DBUS, LIBMNG)]
+binaries += [ (os.path.basename(x), x, 'BINARY') for x in (SQLITE, DBUS, LIBMNG, LIBZ, LIBUSB)]
              
 
 qt = []

resources.py

@@ -23,17 +23,19 @@ def main(args=sys.argv):
         bytes = repr(open(path, 'rb').read())
         data += key + ' = ' + bytes + '\n\n'
     
-    TPATH = '/usr/share/qt4/translations'
+    translations_found = False
-    if os.path.exists(TPATH):
+    for TPATH in ('/usr/share/qt4/translations', '/usr/lib/qt4/translations'):
-         files = glob.glob(TPATH + '/qt_??.qm')
+        if os.path.exists(TPATH):
-    
+             files = glob.glob(TPATH + '/qt_??.qm')
-         for f in files:
+        
-             key = os.path.basename(f).partition('.')[0]
+             for f in files:
-             bytes = repr(open(f, 'rb').read())
+                 key = os.path.basename(f).partition('.')[0]
-             data += key + ' = ' + bytes + '\n\n'
+                 bytes = repr(open(f, 'rb').read())
-         
+                 data += key + ' = ' + bytes + '\n\n'
-    else:
+             translations_found = True
-        print 'WARNING: Could not find Qt transations in', TPATH
+             break
+    if not translations_found:
+        print 'WARNING: Could not find Qt transations'
     
     open('src'+os.sep+__appname__+os.sep+'/resources.py', 'wb').write(data) 
     return 0

src/calibre/__init__.py

@@ -26,6 +26,7 @@ terminal_controller = TerminalController(sys.stdout)
 iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower()
 isosx     = 'darwin' in sys.platform.lower()
 islinux   = not(iswindows or isosx)
+isfrozen  = hasattr(sys, 'frozen') 
 
 try:
     locale.setlocale(locale.LC_ALL, '')
@@ -76,7 +77,11 @@ if iswindows:
     if not winutil:
         raise RuntimeError('Failed to load the winutil plugin: %s'%winutilerror)
     sys.argv[1:] = winutil.argv()[1:]
-
+    win32event = __import__('win32event')
+    winerror   = __import__('winerror')
+    win32api   = __import__('win32api')
+else:
+    import fcntl
 
 _abspath = os.path.abspath
 def my_abspath(path, encoding=sys.getfilesystemencoding()):
@@ -313,15 +318,46 @@ def extract(path, dir):
         raise Exception('Unknown archive type')
     extractor(path, dir)
 
+def get_proxies():
+        proxies = {}
+        if iswindows:
+            try:
+                winreg = __import__('_winreg')
+                settings = winreg.OpenKey(winreg.HKEY_CURRENT_USER,
+                                          'Software\\Microsoft\\Windows'
+                                          '\\CurrentVersion\\Internet Settings')
+                proxy = winreg.QueryValueEx(settings, "ProxyEnable")[0]
+                if proxy:
+                    server = str(winreg.QueryValueEx(settings, 'ProxyServer')[0])
+                    if ';' in server:
+                        for p in server.split(';'):
+                            protocol, address = p.split('=')
+                            proxies[protocol] = address
+                    else:
+                        proxies['http'] = server
+                        proxies['ftp'] =  server
+                settings.Close()
+            except Exception, e:
+                print('Unable to detect proxy settings: %s' % str(e))
+            if proxies:
+                print('Using proxies: %s' % proxies)
+        else:
+            for q in ('http', 'ftp'):
+                proxy =  os.environ.get(q+'_proxy', None)
+                if not proxy: continue
+                if proxy.startswith(q+'://'):
+                    proxy = proxy[7:]
+                proxies[q] = proxy
+        return proxies
+
+
 def browser(honor_time=False):
-    http_proxy = os.environ.get('http_proxy', None)
     opener = mechanize.Browser()
     opener.set_handle_refresh(True, honor_time=honor_time)
     opener.set_handle_robots(False)
     opener.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; i686 Linux; en_US; rv:1.8.0.4) Gecko/20060508 Firefox/1.5.0.4')]
+    http_proxy = get_proxies().get('http', None)
     if http_proxy:
-        if http_proxy.startswith('http://'):
-            http_proxy = http_proxy[7:]
         opener.set_proxies({'http':http_proxy})
     return opener
 
@@ -475,6 +511,43 @@ def _clean_lock_file(file):
     except:
         pass
 
+class LockError(Exception):
+    pass
+class ExclusiveFile(object):
+    
+    def __init__(self, path, timeout=10):
+        self.path = path
+        self.timeout = timeout
+        
+    def __enter__(self):
+        self.file  = open(self.path, 'a+b')
+        self.file.seek(0)
+        timeout = self.timeout
+        if iswindows:
+            name = ('Local\\'+(__appname__+self.file.name).replace('\\', '_'))[:201]
+            while self.timeout < 0 or timeout >= 0:
+                self.mutex = win32event.CreateMutex(None, False, name)
+                if win32api.GetLastError() != winerror.ERROR_ALREADY_EXISTS: break
+                time.sleep(1)
+                timeout -= 1
+        else:
+            while self.timeout < 0 or timeout >= 0:
+                try:
+                    fcntl.lockf(self.file.fileno(), fcntl.LOCK_EX|fcntl.LOCK_NB)
+                    break
+                except IOError:
+                    time.sleep(1)
+                    timeout -= 1
+        if timeout < 0 and self.timeout >= 0:
+            self.file.close()
+            raise LockError
+        return self.file
+                
+    def __exit__(self, type, value, traceback):
+        self.file.close()
+        if iswindows:
+            win32api.CloseHandle(self.mutex)
+
 def singleinstance(name):
     '''
     Return True if no other instance of the application identified by name is running,
@@ -483,16 +556,12 @@ def singleinstance(name):
     @type name: string
     '''
     if iswindows:
-        from win32event import CreateMutex
-        from win32api import CloseHandle, GetLastError
-        from winerror import ERROR_ALREADY_EXISTS
         mutexname = 'mutexforsingleinstanceof'+__appname__+name
-        mutex =  CreateMutex(None, False, mutexname)
+        mutex =  win32event.CreateMutex(None, False, mutexname)
         if mutex:
-            atexit.register(CloseHandle, mutex)
+            atexit.register(win32api.CloseHandle, mutex)
-        return not GetLastError() == ERROR_ALREADY_EXISTS
+        return not win32api.GetLastError() == winerror.ERROR_ALREADY_EXISTS
     else:
-        import fcntl
         global _lock_file
         path = os.path.expanduser('~/.'+__appname__+'_'+name+'.lock')
         try:

src/calibre/devices/libusb.py

@@ -8,7 +8,7 @@ from ctypes import cdll, POINTER, byref, pointer, Structure, \
                    c_ubyte, c_ushort, c_int, c_char, c_void_p, c_byte, c_uint
 from errno import EBUSY, ENOMEM
 
-from calibre import iswindows, isosx, load_library
+from calibre import iswindows, isosx, load_library, isfrozen
 
 _libusb_name = 'libusb'
 PATH_MAX = 511 if iswindows else 1024 if isosx else 4096

src/calibre/devices/prs505/driver.py

@@ -126,7 +126,36 @@ class PRS505(Device):
             self._card_prefix = re.search('/dev/%s(\w*)\s+on\s+([^\(]+)\s+'%(devname,), mount).group(2) + os.sep
             
     
+    def open_windows_nowmi(self):
+        from calibre import plugins
+        winutil = plugins['winutil'][0]
+        volumes = winutil.get_mounted_volumes_for_usb_device(self.VENDOR_ID, self.PRODUCT_ID)
+        main = None
+        for device_id in volumes.keys():
+            if 'PRS-505/UC&' in device_id:
+                main = volumes[device_id]+':\\'
+        if not main:
+            DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%self.__class__.__name__)
+        self._main_prefix = main
+        card = self._card_prefix = None
+        win32api = __import__('win32api')
+        for device_id in volumes.keys():
+            if 'PRS-505/UC:' in device_id:
+                card = volumes[device_id]+':\\'
+                try:
+                    win32api.GetVolumeInformation(card)
+                    self._card_prefix = card
+                    break
+                except:
+                    continue
+            
+    
     def open_windows(self):
+        try:
+            self.open_windows_nowmi()
+            return
+        except:
+            pass
         drives = []
         wmi = __import__('wmi', globals(), locals(), [], -1) 
         c = wmi.WMI()

src/calibre/ebooks/__init__.py

@@ -17,4 +17,4 @@ class UnknownFormatError(Exception):
 
 BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'htm', 'xhtm', 
                    'html', 'xhtml', 'epub', 'pdf', 'prc', 'mobi', 'azw', 
-                   'epub', 'fb2', 'djvu', 'lrx']
+                   'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz']

src/calibre/ebooks/lrf/comic/convert_from.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env  python
+from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
@@ -10,8 +10,9 @@ Based on ideas from comiclrf created by FangornUK.
 import os, sys, traceback, shutil
 from uuid import uuid4
 
-from calibre import extract, OptionParser, detect_ncpus, terminal_controller, \
+from calibre import extract, detect_ncpus, terminal_controller, \
                     __appname__, __version__
+from calibre.utils.config import Config, StringConfig
 from calibre.ptempfile import PersistentTemporaryDirectory
 from calibre.utils.threadpool import ThreadPool, WorkRequest
 from calibre.utils.terminfo import ProgressBar
@@ -21,7 +22,7 @@ try:
             NewMagickWand, NewPixelWand, \
             MagickSetImageBorderColor, \
             MagickReadImage, MagickRotateImage, \
-            MagickTrimImage, \
+            MagickTrimImage, PixelSetColor,\
             MagickNormalizeImage, MagickGetImageWidth, \
             MagickGetImageHeight, \
             MagickResizeImage, MagickSetImageType, \
@@ -30,12 +31,13 @@ try:
             MagickQuantizeImage, RGBColorspace, \
             MagickWriteImage, DestroyPixelWand, \
             DestroyMagickWand, CloneMagickWand, \
-            MagickThumbnailImage, MagickCropImage, initialize, finalize
+            MagickThumbnailImage, MagickCropImage, ImageMagick
     _imagemagick_loaded = True
 except:
     _imagemagick_loaded = False
 
 PROFILES = {
+            # Name : (width, height) in pixels
             'prs500':(584, 754),            
             }
 
@@ -133,7 +135,7 @@ class PageProcessor(list):
             pw = NewPixelWand()
             if pw < 0:
                 raise RuntimeError('Cannot create wand.')
-            #flag = PixelSetColor(pw, 'white')
+            PixelSetColor(pw, 'white')
             
             MagickSetImageBorderColor(wand, pw)
             
@@ -145,7 +147,7 @@ class PageProcessor(list):
             MagickSetImagePage(wand, 0,0,0,0)   #Clear page after trim, like a "+repage"
             
             # Do the Photoshop "Auto Levels" equivalent
-            if self.opts.normalize:
+            if not self.opts.dont_normalize:
                 MagickNormalizeImage(wand)
         
             sizex = MagickGetImageWidth(wand)
@@ -173,7 +175,7 @@ class PageProcessor(list):
             else:
                 MagickResizeImage(wand, SCRWIDTH, SCRHEIGHT, CatromFilter, 1.0)
                 
-            if self.opts.sharpen:
+            if not self.opts.dont_sharpen:
                 MagickSharpenImage(wand, 0.0, 1.0)
                 
             MagickSetImageType(wand, GrayscaleType)
@@ -204,8 +206,7 @@ def process_pages(pages, opts, update):
     '''
     if not _imagemagick_loaded:
         raise RuntimeError('Failed to load ImageMagick')
-    initialize()
+    with ImageMagick():
-    try:
         tdir = PersistentTemporaryDirectory('_comic2lrf_pp')
         processed_pages = [PageProcessor(path, tdir, opts, i) for i, path in enumerate(pages)]
         tp = ThreadPool(detect_ncpus())
@@ -222,37 +223,47 @@ def process_pages(pages, opts, update):
             else:
                 ans += pp
         return ans, failures, tdir
-    finally:
+    
-        finalize()
+def config(defaults=None):
+    desc = _('Options to control the conversion of comics (CBR, CBZ) files into ebooks')
+    if defaults is None:
+        c = Config('comic', desc)
+    else:
+        c = StringConfig(defaults, desc)
+    c.add_opt('title', ['-t', '--title'], 
+              help=_('Title for generated ebook. Default is to use the filename.'))
+    c.add_opt('author', ['-a', '--author'], 
+              help=_('Set the author in the metadata of the generated ebook. Default is %default'), 
+              default=_('Unknown'))
+    c.add_opt('output', ['-o', '--output'], 
+              help=_('Path to output LRF file. By default a file is created in the current directory.'))
+    c.add_opt('colors', ['-c', '--colors'], type='int', default=64,
+              help=_('Number of colors for Grayscale image conversion. Default: %default'))
+    c.add_opt('dont_normalize', ['-n', '--disable-normalize'], default=False, 
+              help=_('Disable normalize (improve contrast) color range for pictures. Default: False'))
+    c.add_opt('keep_aspect_ratio', ['-r', '--keep-aspect-ratio'], default=False,
+              help=_('Maintain picture aspect ratio. Default is to fill the screen.'))
+    c.add_opt('dont_sharpen', ['-s', '--disable-sharpen'], default=False,  
+              help=_('Disable sharpening.'))
+    c.add_opt('landscape', ['-l', '--landscape'], default=False, 
+              help=_("Don't split landscape images into two portrait images"))
+    c.add_opt('no_sort', ['--no-sort'], default=False, 
+              help=_("Don't sort the files found in the comic alphabetically by name. Instead use the order they were added to the comic."))
+    c.add_opt('profile', ['-p', '--profile'], default='prs500', choices=PROFILES.keys(), 
+              help=_('Choose a profile for the device you are generating this LRF for. The default is the SONY PRS-500 with a screen size of 584x754 pixels. Choices are %s')%PROFILES.keys())
+    c.add_opt('verbose', ['--verbose'], default=0, action='count',  
+              help=_('Be verbose, useful for debugging. Can be specified multiple times for greater verbosity.'))
+    c.add_opt('no_progress_bar', ['--no-progress-bar'], default=False, 
+                      help=_("Don't show progress bar."))
+    return c
 
 def option_parser():
-    parser = OptionParser(_('''\
+    c = config()
+    return c.option_parser(usage=_('''\
 %prog [options] comic.cb[z|r]
 
 Convert a comic in a CBZ or CBR file to an LRF ebook. 
-	'''))
+'''))
-    parser.add_option('-t', '--title', help=_('Title for generated ebook. Default is to use the filename.'), default=None)
-    parser.add_option('-a', '--author', help=_('Set the author in the metadata of the generated ebook. Default is %default'), default=_('Unknown'))
-    parser.add_option('-o', '--output', help=_('Path to output LRF file. By default a file is created in the current directory.'), default=None)
-    parser.add_option('-c', '--colors', type='int', default=64,
-    				  help=_('Number of colors for Grayscale image conversion. Default: %default'))
-    parser.add_option('-n', '--disable-normalize', dest='normalize', default=True, action='store_false',
-    				  help=_('Disable normalize (improve contrast) color range for pictures. Default: False'))
-    parser.add_option('-r', '--keep-aspect-ratio', action='store_true', default=False,
-    				  help=_('Maintain picture aspect ratio. Default is to fill the screen.'))
-    parser.add_option('-s', '--disable-sharpen', default=True, action='store_false', dest='sharpen',
-    				  help=_('Disable sharpening.'))
-    parser.add_option('-l', '--landscape', default=False, action='store_true',
-    				  help=_("Don't split landscape images into two portrait images"))
-    parser.add_option('--no-sort', default=False, action='store_true',
-                      help=_("Don't sort the files found in the comic alphabetically by name. Instead use the order they were added to the comic."))
-    parser.add_option('-p', '--profile', default='prs500', dest='profile', type='choice',
-                      choices=PROFILES.keys(), help=_('Choose a profile for the device you are generating this LRF for. The default is the SONY PRS-500 with a screen size of 584x754 pixels. Choices are %s')%PROFILES.keys())
-    parser.add_option('--verbose', default=False, action='store_true', 
-                      help=_('Be verbose, useful for debugging'))
-    parser.add_option('--no-progress-bar', default=False, action='store_true',
-                      help=_("Don't show progress bar."))
-    return parser
 
 def create_lrf(pages, profile, opts, thumbnail=None):
     width, height = PROFILES[profile]
@@ -277,22 +288,8 @@ def create_lrf(pages, profile, opts, thumbnail=None):
         
     book.renderLrf(open(opts.output, 'wb'))
     
-
+def do_convert(path_to_file, opts, notification=lambda m, p: p):
-
+    source = path_to_file
-def main(args=sys.argv, notification=None):
-    parser = option_parser()
-    opts, args = parser.parse_args(args)
-    if len(args) < 2:
-        parser.print_help()
-        print '\nYou must specify a file to convert'
-        return 1
-    
-    if not callable(notification):
-        pb = ProgressBar(terminal_controller, _('Rendering comic pages...'), 
-                         no_progress_bar=opts.no_progress_bar)
-        notification = pb.update
-    
-    source = os.path.abspath(args[1])
     if not opts.title:
         opts.title = os.path.splitext(os.path.basename(source))
     if not opts.output:
@@ -315,6 +312,24 @@ def main(args=sys.argv, notification=None):
     create_lrf(pages, opts.profile, opts, thumbnail=thumbnail)
     shutil.rmtree(tdir)
     shutil.rmtree(tdir2)
+
+
+def main(args=sys.argv, notification=None):
+    parser = option_parser()
+    opts, args = parser.parse_args(args)
+    if len(args) < 2:
+        parser.print_help()
+        print '\nYou must specify a file to convert'
+        return 1
+    
+    if not callable(notification):
+        pb = ProgressBar(terminal_controller, _('Rendering comic pages...'), 
+                         no_progress_bar=opts.no_progress_bar)
+        notification = pb.update
+    
+    source = os.path.abspath(args[1])
+    do_convert(source, opts, notification)
+    print _('Output written to'), opts.output
     return 0
 
 if __name__ == '__main__':

src/calibre/ebooks/metadata/opf.xml

@@ -1,14 +1,14 @@
 <?xml version="1.0"  encoding="UTF-8"?>
 <package version="2.0" 
-         xmlns:opf="http://www.idpf.org/2007/opf" 
+         xmlns="http://www.idpf.org/2007/opf" 
          xmlns:py="http://genshi.edgewall.org/" 
          unique-identifier="${__appname__}_id"
          
 >
-    <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
+    <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
-        <dc:title py:with="attrs={'files-as':mi.title_sort}" py:attrs="attrs">${mi.title}</dc:title>
+        <dc:title py:with="attrs={'opf:files-as':mi.title_sort}" py:attrs="attrs">${mi.title}</dc:title>
-        <dc:creator opf:role="aut" py:for="i, author in enumerate(mi.authors)" py:with="attrs={'file-as':mi.author_sort if i==0 else None}" py:attrs="attrs">${author}</dc:creator>
+        <dc:creator opf:role="aut" py:for="i, author in enumerate(mi.authors)" py:with="attrs={'opf:file-as':mi.author_sort if i==0 else None}" py:attrs="attrs">${author}</dc:creator>
-        <dc:identifier scheme="${__appname__}" id="${__appname__}_id">${mi.application_id}</dc:identifier>
+        <dc:identifier opf:scheme="${__appname__}" id="${__appname__}_id">${mi.application_id}</dc:identifier>
     
         <dc:language>${mi.language if mi.language else 'Unknown'}</dc:language>
         <dc:type py:if="mi.category">${mi.category}</dc:type>

src/calibre/ebooks/mobi/reader.py

@@ -72,43 +72,51 @@ class BookHeader(object):
         self.compression_type = raw[:2]
         self.records, self.records_size = struct.unpack('>HH', raw[8:12])
         self.encryption_type, = struct.unpack('>H', raw[12:14])
-        
-        self.doctype = raw[16:20]
-        self.length, self.type, self.codepage, self.unique_id, self.version = \
-                 struct.unpack('>LLLLL', raw[20:40])
-                
         if ident == 'TEXTREAD':
             self.codepage = 1252
-        
+        if len(raw) <= 16:
-        try:
+            self.codec = 'cp1251'
-            self.codec = {
-                      1252  : 'cp1252',
-                      65001 : 'utf-8',
-                      }[self.codepage]
-        except IndexError, KeyError:
-            print '[WARNING] Unknown codepage %d. Assuming cp-1252'%self.codepage
-            self.codec = 'cp1252'
-        
-        if ident == 'TEXTREAD' or self.length < 0xE4 or 0xE8 < self.length:
             self.extra_flags = 0
+            self.language = 'ENGLISH'
+            self.sublanguage = 'NEUTRAL'
+            self.exth_flag, self.exth = 0, None
+            self.ancient = True
         else:
-            self.extra_flags, = struct.unpack('>L', raw[0xF0:0xF4])
+            self.ancient = False
-        
+            self.doctype = raw[16:20]
-        if self.compression_type == 'DH':
+            self.length, self.type, self.codepage, self.unique_id, self.version = \
-            self.huff_offset, self.huff_number = struct.unpack('>LL', raw[0x70:0x78]) 
+                     struct.unpack('>LLLLL', raw[20:40])
-        
+                    
-        langcode  = struct.unpack('!L', raw[0x5C:0x60])[0]
+            
-        langid    = langcode & 0xFF
+            try:
-        sublangid = (langcode >> 10) & 0xFF
+                self.codec = {
-        self.language = main_language.get(langid, 'ENGLISH')
+                          1252  : 'cp1252',
-        self.sublanguage = sub_language.get(sublangid, 'NEUTRAL')
+                          65001 : 'utf-8',
-        
+                          }[self.codepage]
-        self.exth_flag, = struct.unpack('>L', raw[0x80:0x84])
+            except IndexError, KeyError:
-        self.exth = None
+                print '[WARNING] Unknown codepage %d. Assuming cp-1252'%self.codepage
-        if self.exth_flag & 0x40:
+                self.codec = 'cp1252'
-            self.exth = EXTHHeader(raw[16+self.length:], self.codec)
+            
-            self.exth.mi.uid = self.unique_id
+            if ident == 'TEXTREAD' or self.length < 0xE4 or 0xE8 < self.length:
-            self.exth.mi.language = self.language
+                self.extra_flags = 0
+            else:
+                self.extra_flags, = struct.unpack('>L', raw[0xF0:0xF4])
+            
+            if self.compression_type == 'DH':
+                self.huff_offset, self.huff_number = struct.unpack('>LL', raw[0x70:0x78]) 
+            
+            langcode  = struct.unpack('!L', raw[0x5C:0x60])[0]
+            langid    = langcode & 0xFF
+            sublangid = (langcode >> 10) & 0xFF
+            self.language = main_language.get(langid, 'ENGLISH')
+            self.sublanguage = sub_language.get(sublangid, 'NEUTRAL')
+            
+            self.exth_flag, = struct.unpack('>L', raw[0x80:0x84])
+            self.exth = None
+            if self.exth_flag & 0x40:
+                self.exth = EXTHHeader(raw[16+self.length:], self.codec)
+                self.exth.mi.uid = self.unique_id
+                self.exth.mi.language = self.language
             
 
 class MobiReader(object):
@@ -145,7 +153,6 @@ class MobiReader(object):
             else:
                 end_off = self.section_headers[section_number + 1][0]
             off = self.section_headers[section_number][0]
-            
             return raw[off:end_off]
             
         for i in range(self.num_sections):
@@ -201,6 +208,8 @@ class MobiReader(object):
         
     def cleanup_html(self):
         self.processed_html = re.sub(r'<div height="0(pt|px|ex|em|%){0,1}"></div>', '', self.processed_html)
+        if self.book_header.ancient and '<html' not in self.mobi_html[:300].lower():
+            self.processed_html = '<html><p>'+self.processed_html.replace('\n\n', '<p>')+'</html>'
     
     def cleanup_soup(self, soup):
         for tag in soup.recursiveChildGenerator():
@@ -313,7 +322,8 @@ class MobiReader(object):
             self.mobi_html = ''.join(text_sections)
         else:
             raise MobiError('Unknown compression algorithm: %s'%repr(self.book_header.compression_type))
-        
+        if self.book_header.ancient and '<html' not in self.mobi_html[:300].lower():
+            self.mobi_html = self.mobi_html.replace('\r ', '\n\n ')
         return processed_records
             
     

src/calibre/gui2/device.py

@@ -25,12 +25,7 @@ class DeviceDetector(QThread):
         self.keep_going = True
         
     def run(self):
-        _wmi = None
+        scanner = DeviceScanner()
-        if iswindows:
-            import wmi, pythoncom
-            pythoncom.CoInitialize()
-            _wmi = wmi.WMI()
-        scanner = DeviceScanner(_wmi)
         while self.keep_going:
             scanner.scan()
             for device in self.devices:

src/calibre/gui2/dialogs/comicconf.py

@@ -0,0 +1,85 @@
+from __future__ import with_statement
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+''''''
+from PyQt4.QtGui import QDialog
+from calibre.gui2.dialogs.comicconf_ui import Ui_Dialog
+from calibre.gui2 import qstring_to_unicode
+from calibre.ebooks.lrf.comic.convert_from import config
+
+def set_conversion_defaults(window):
+    d = ComicConf(window)
+    d.exec_()
+    
+def get_bulk_conversion_options(window):
+    c = config(None)
+    with open(c.config_file_path, 'rb') as f:
+        d = ComicConf(window, config_defaults=f.read())
+    if d.exec_() == QDialog.Accepted:
+        return d.config.parse()
+    
+def get_conversion_options(window, defaults, title, author):
+    if defaults is None:
+        c = config(None)
+        with open(c.config_file_path, 'rb') as f:
+            defaults = f.read()
+    defaults += '\ntitle=%s\nauthor=%s'%(repr(title), repr(author))
+    d = ComicConf(window, config_defaults=defaults, generic=False)
+    if d.exec_() == QDialog.Accepted:
+        return d.config.parse(), d.config.src
+    return None, None
+    
+
+class ComicConf(QDialog, Ui_Dialog):
+    
+    def __init__(self, window, config_defaults=None, generic=True,
+                 title=_('Set defaults for conversion of comics (CBR/CBZ files)')):
+        QDialog.__init__(self, window)
+        Ui_Dialog.__init__(self)
+        self.setupUi(self)
+        self.setWindowTitle(title)
+        self.config = config(config_defaults)
+        opts = self.config.parse()
+        if generic:
+            for i in ('title', 'author'):
+                getattr(self, 'opt_'+i).setVisible(False)
+                getattr(self, i+'_label').setVisible(False)
+        else:
+            title = opts.title
+            if not title:
+                title = _('Unknown')
+            self.setWindowTitle(_('Set options for converting %s')%title)
+            author = opts.author
+            self.opt_title.setText(title)
+            self.opt_author.setText(author)
+        self.opt_colors.setValue(opts.colors)
+        self.opt_profile.addItem(opts.profile)
+        self.opt_dont_normalize.setChecked(opts.dont_normalize)
+        self.opt_keep_aspect_ratio.setChecked(opts.keep_aspect_ratio)
+        self.opt_dont_sharpen.setChecked(opts.dont_sharpen)
+        self.opt_landscape.setChecked(opts.landscape)
+        self.opt_no_sort.setChecked(opts.no_sort)
+        
+        for opt in self.config.option_set.preferences:
+            g = getattr(self, 'opt_'+opt.name, False)
+            if opt.help and g:
+                g.setToolTip(opt.help)
+                
+    def accept(self):
+        for opt in self.config.option_set.preferences:
+            g = getattr(self, 'opt_'+opt.name, False)
+            if not g or not g.isVisible(): continue
+            if hasattr(g, 'isChecked'):
+                val = bool(g.isChecked())
+            elif hasattr(g, 'value'):
+                val = g.value()
+            elif hasattr(g, 'itemText'):
+                val = qstring_to_unicode(g.itemText(g.currentIndex()))
+            elif hasattr(g, 'text'):
+                val = qstring_to_unicode(g.text())
+            else:
+                raise Exception('Bad coding')
+            self.config.set(opt.name, val)
+        return QDialog.accept(self)

src/calibre/gui2/dialogs/comicconf.ui

@@ -0,0 +1,166 @@
+<ui version="4.0" >
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog" >
+  <property name="geometry" >
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>646</width>
+    <height>468</height>
+   </rect>
+  </property>
+  <property name="windowTitle" >
+   <string>Dialog</string>
+  </property>
+  <property name="windowIcon" >
+   <iconset resource="../images.qrc" >
+    <normaloff>:/images/convert.svg</normaloff>:/images/convert.svg</iconset>
+  </property>
+  <layout class="QGridLayout" name="gridLayout" >
+   <item row="0" column="0" >
+    <widget class="QLabel" name="title_label" >
+     <property name="text" >
+      <string>&amp;Title:</string>
+     </property>
+     <property name="buddy" >
+      <cstring>opt_title</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="1" >
+    <widget class="QLineEdit" name="opt_title" />
+   </item>
+   <item row="1" column="0" >
+    <widget class="QLabel" name="author_label" >
+     <property name="text" >
+      <string>&amp;Author(s):</string>
+     </property>
+     <property name="buddy" >
+      <cstring>opt_author</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="1" >
+    <widget class="QLineEdit" name="opt_author" />
+   </item>
+   <item row="2" column="0" >
+    <widget class="QLabel" name="label_3" >
+     <property name="text" >
+      <string>&amp;Number of Colors:</string>
+     </property>
+     <property name="buddy" >
+      <cstring>opt_colors</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="1" >
+    <widget class="QSpinBox" name="opt_colors" >
+     <property name="minimum" >
+      <number>8</number>
+     </property>
+     <property name="maximum" >
+      <number>3200000</number>
+     </property>
+     <property name="singleStep" >
+      <number>8</number>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="0" >
+    <widget class="QLabel" name="label_4" >
+     <property name="text" >
+      <string>&amp;Profile:</string>
+     </property>
+     <property name="buddy" >
+      <cstring>opt_profile</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="3" column="1" >
+    <widget class="QComboBox" name="opt_profile" />
+   </item>
+   <item row="4" column="0" >
+    <widget class="QCheckBox" name="opt_dont_normalize" >
+     <property name="text" >
+      <string>Disable &amp;normalize</string>
+     </property>
+    </widget>
+   </item>
+   <item row="5" column="0" >
+    <widget class="QCheckBox" name="opt_keep_aspect_ratio" >
+     <property name="text" >
+      <string>Keep &amp;aspect ratio</string>
+     </property>
+    </widget>
+   </item>
+   <item row="6" column="0" >
+    <widget class="QCheckBox" name="opt_dont_sharpen" >
+     <property name="text" >
+      <string>Disable &amp;Sharpening</string>
+     </property>
+    </widget>
+   </item>
+   <item row="7" column="0" >
+    <widget class="QCheckBox" name="opt_landscape" >
+     <property name="text" >
+      <string>&amp;Landscape</string>
+     </property>
+    </widget>
+   </item>
+   <item row="8" column="0" >
+    <widget class="QCheckBox" name="opt_no_sort" >
+     <property name="text" >
+      <string>Dont so&amp;rt</string>
+     </property>
+    </widget>
+   </item>
+   <item row="9" column="1" >
+    <widget class="QDialogButtonBox" name="buttonBox" >
+     <property name="orientation" >
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons" >
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../images.qrc" />
+ </resources>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/main.py

@@ -37,6 +37,7 @@ from calibre.gui2.dialogs.lrf_single import LRFSingleDialog, LRFBulkDialog
 from calibre.gui2.dialogs.config import ConfigDialog
 from calibre.gui2.dialogs.search import SearchDialog
 from calibre.gui2.dialogs.user_profiles import UserProfiles
+import calibre.gui2.dialogs.comicconf as ComicConf 
 from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 from calibre.gui2.dialogs.book_info import BookInfo
 from calibre.ebooks.metadata.meta import set_metadata
@@ -174,12 +175,14 @@ class Main(MainWindow, Ui_MainWindow):
         cm.addAction(_('Convert individually'))
         cm.addAction(_('Bulk convert'))
         cm.addSeparator()
-        cm.addAction(_('Set conversion defaults'))
+        cm.addAction(_('Set defaults for conversion to LRF'))
+        cm.addAction(_('Set defaults for conversion of comics'))
         self.action_convert.setMenu(cm)
         QObject.connect(cm.actions()[0], SIGNAL('triggered(bool)'), self.convert_single)
         QObject.connect(cm.actions()[1], SIGNAL('triggered(bool)'), self.convert_bulk)
         QObject.connect(cm.actions()[3], SIGNAL('triggered(bool)'), self.set_conversion_defaults)
-        QObject.connect(self.action_convert, SIGNAL('triggered(bool)'), self.convert_single)
+        QObject.connect(cm.actions()[4], SIGNAL('triggered(bool)'), self.set_comic_conversion_defaults)
+        QObject.connect(self.action_convert, SIGNAL('triggered(bool)'), self.convert_single)        
         self.convert_menu = cm
         self.tool_bar.widgetForAction(self.action_news).setPopupMode(QToolButton.InstantPopup)
         self.tool_bar.widgetForAction(self.action_edit).setPopupMode(QToolButton.MenuButtonPopup)
@@ -776,13 +779,26 @@ class Main(MainWindow, Ui_MainWindow):
     ############################################################################
 
     ############################### Convert ####################################
-
+    
-    def convert_bulk(self, checked):
+    def get_books_for_conversion(self):
-        rows = self.library_view.selectionModel().selectedRows()
+        rows = [r.row() for r in self.library_view.selectionModel().selectedRows()]
         if not rows or len(rows) == 0:
             d = error_dialog(self, _('Cannot convert'), _('No books selected'))
             d.exec_()
-            return
+            return [], []
+        comics, others = [], []
+        db = self.library_view.model().db
+        for r in rows:
+            formats = db.formats(r)
+            if not formats: continue
+            formats = formats.lower().split(',')
+            if 'cbr' in formats or 'cbz' in formats:
+                comics.append(r)
+            else:
+                others.append(r)
+        return comics, others
+    
+    def convert_bulk_others(self, rows):
         d = LRFBulkDialog(self)
         d.exec_()
         if d.result() != QDialog.Accepted:
@@ -827,34 +843,67 @@ class Main(MainWindow, Ui_MainWindow):
             cmdline.append(pt.name)
             id = self.job_manager.run_conversion_job(self.book_converted,
                                                         'any2lrf', args=[cmdline],
-                                    job_description='Convert book %d of %d'%(i+1, len(rows)))
+                                    job_description=_('Convert book %d of %d (%s)')%(i+1, len(rows), repr(mi.title)))
-
+                    
-
+                    
-            self.conversion_jobs[id] = (d.cover_file, pt, of, d.output_format,
+            self.conversion_jobs[id] = (d.cover_file, pt, of, d.output_format, 
                                         self.library_view.model().db.id(row))
-
-
         res = []
         for row in bad_rows:
             title = self.library_view.model().db.title(row)
             res.append('<li>%s</li>'%title)
         if res:
-            msg = '<p>Could not convert %d of %d books, because no suitable source format was found.<ul>%s</ul>'%(len(res), len(rows), '\n'.join(res))
+            msg = _('<p>Could not convert %d of %d books, because no suitable source format was found.<ul>%s</ul>')%(len(res), len(rows), '\n'.join(res))
-            warning_dialog(self, 'Could not convert some books', msg).exec_()
+            warning_dialog(self, _('Could not convert some books'), msg).exec_()
-
+        
-
+    
+    def convert_bulk(self, checked):
+        comics, others = self.get_books_for_conversion()
+        if others:
+            self.convert_bulk_others(others)
+        if comics:
+            opts = ComicConf.get_bulk_conversion_options(self)
+            if opts:
+                for i, row in enumerate(comics):
+                    options = opts.copy()
+                    mi = self.library_view.model().db.get_metadata(row)
+                    if mi.title:
+                        options.title = mi.title
+                    if mi.authors:
+                        opts.author =  ','.join(mi.authors)
+                    data = None
+                    for fmt in ['cbz', 'cbr']:
+                        try:
+                            data = self.library_view.model().db.format(row, fmt.upper())
+                            break                    
+                        except:
+                            continue
+                    
+                    pt = PersistentTemporaryFile('.'+fmt.lower())
+                    pt.write(data)
+                    pt.close()
+                    of = PersistentTemporaryFile('.lrf')
+                    of.close()
+                    setattr(options, 'output', of.name)
+                    options.verbose = 1
+                    args = [pt.name, options]
+                    id = self.job_manager.run_conversion_job(self.book_converted, 
+                                                        'comic2lrf', args=args,
+                                    job_description=_('Convert comic %d of %d (%s)')%(i+1, len(comics), repr(options.title)))
+                    self.conversion_jobs[id] = (None, pt, of, 'lrf', 
+                                        self.library_view.model().db.id(row))
+                        
+            
     def set_conversion_defaults(self, checked):
         d = LRFSingleDialog(self, None, None)
         d.exec_()
-
+        
-    def convert_single(self, checked):
+    def set_comic_conversion_defaults(self, checked):
-        rows = self.library_view.selectionModel().selectedRows()
+        ComicConf.set_conversion_defaults(self)
-        if not rows or len(rows) == 0:
+    
-            d = error_dialog(self, _('Cannot convert'), _('No books selected'))
+    def convert_single_others(self, rows):
-            d.exec_()
-
         changed = False
-        for row in [r.row() for r in rows]:
+        for row in rows:
             d = LRFSingleDialog(self, self.library_view.model().db, row)
             if d.selected_format:
                 d.exec_()
@@ -870,16 +919,58 @@ class Main(MainWindow, Ui_MainWindow):
                     cmdline.append(pt.name)
                     id = self.job_manager.run_conversion_job(self.book_converted,
                                                         'any2lrf', args=[cmdline],
-                                    job_description='Convert book:'+d.title())
+                                    job_description=_('Convert book: ')+d.title())
-
+                    
-
+                    
                     self.conversion_jobs[id] = (d.cover_file, pt, of, d.output_format, d.id)
                     changed = True
         if changed:
             self.library_view.model().resort(reset=False)
             self.library_view.model().research()
-
+        
-
+    
+    def convert_single(self, checked):
+        comics, others = self.get_books_for_conversion()
+        if others:
+            self.convert_single_others(others)
+        changed = False
+        db = self.library_view.model().db
+        for row in comics:
+            mi = db.get_metadata(row)
+            title = author = _('Unknown')
+            if mi.title:
+                title = mi.title
+            if mi.authors:
+                author =  ','.join(mi.authors)
+            defaults = db.conversion_options(db.id(row), 'comic')
+            opts, defaults = ComicConf.get_conversion_options(self, defaults, title, author)
+            if defaults is not None:
+                db.set_conversion_options(db.id(row), 'comic', defaults)
+            if opts is None: continue
+            for fmt in ['cbz', 'cbr']:
+                try:
+                    data = db.format(row, fmt.upper())
+                    break                    
+                except:
+                    continue
+            pt = PersistentTemporaryFile('.'+fmt)
+            pt.write(data)
+            pt.close()
+            of = PersistentTemporaryFile('.lrf')
+            of.close()
+            opts.output = of.name
+            opts.verbose = 1
+            args = [pt.name, opts]
+            changed = True
+            id = self.job_manager.run_conversion_job(self.book_converted, 
+                         'comic2lrf', args=args,
+                         job_description=_('Convert comic: ')+opts.title)
+            self.conversion_jobs[id] = (None, pt, of, 'lrf', 
+                                        self.library_view.model().db.id(row))
+        if changed:
+            self.library_view.model().resort(reset=False)
+            self.library_view.model().research()
+                    
     def book_converted(self, id, description, result, exception, formatted_traceback, log):
         of, fmt, book_id = self.conversion_jobs.pop(id)[2:]
         if exception:

src/calibre/gui2/status.py

@@ -5,7 +5,7 @@ import re
 from PyQt4.QtGui import QStatusBar, QMovie, QLabel, QFrame, QHBoxLayout, QPixmap, \
                         QVBoxLayout, QSizePolicy, QToolButton, QIcon
 from PyQt4.QtCore import Qt, QSize, SIGNAL
-from calibre import fit_image
+from calibre import fit_image, preferred_encoding
 from calibre.gui2 import qstring_to_unicode
 
 class BookInfoDisplay(QFrame):
@@ -77,8 +77,12 @@ class BookInfoDisplay(QFrame):
         for key in data.keys():
             txt = data[key]
             #txt = '<br />\n'.join(textwrap.wrap(txt, 120))
-            rows += '<tr><td><b>%s:</b></td><td>%s</td></tr>'%(key, txt)
+            if isinstance(key, str):
-        self.book_data.setText('<table>'+rows+'</table>')
+                key = key.decode(preferred_encoding, 'replace')
+            if isinstance(txt, str):
+                txt = txt.decode(preferred_encoding, 'replace')
+            rows += u'<tr><td><b>%s:</b></td><td>%s</td></tr>'%(key, txt)
+        self.book_data.setText(u'<table>'+rows+u'</table>')
         
         self.clear_message()
         self.setVisible(True)

src/calibre/library/database.py

@@ -1092,7 +1092,7 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
             self.set_tags(id, val.split(','), append=False)
 
     def set_conversion_options(self, id, format, options):
-        data = sqlite.Binary(cPickle.dumps(options))
+        data = sqlite.Binary(cPickle.dumps(options, -1))
         oid = self.conn.execute('SELECT id FROM conversion_options WHERE book=? AND format=?', (id, format.upper())).fetchone()
         if oid:
             self.conn.execute('UPDATE conversion_options SET data=? WHERE id=?', (data, oid[0]))

src/calibre/linux.py

@@ -1,8 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import shutil
 ''' Post installation script for linux '''
-import sys, os, re
+import sys, os, re, shutil
 
 from subprocess import check_call, call
 from calibre import __version__, __appname__

src/calibre/linux_installer.py

@@ -215,8 +215,12 @@ cd $frozen_path
 '''
 
 def extract_tarball(tar, destdir):
+    print 'Extracting application files...'
     if hasattr(tar, 'read'):
-        tarfile.open(fileobj=tar, mode='r').extractall(destdir)
+        try:
+            tarfile.open(fileobj=tar, mode='r').extractall(destdir)
+        except: # tarfile.py on Fedora 9 is buggy
+            subprocess.check_call(['tar', 'xjf', tar.name, '-C', destdir])
     else:
         tarfile.open(tar, 'r').extractall(destdir)
 
@@ -239,6 +243,7 @@ def do_postinstall(destdir):
     try:
         os.chdir(destdir)
         os.environ['LD_LIBRARY_PATH'] = destdir+':'+os.environ.get('LD_LIBRARY_PATH', '')
+        os.environ['PYTHONPATH'] = destdir
         subprocess.call((os.path.join(destdir, 'calibre_postinstall'), '--save-manifest-to', t.name))
     finally:
         os.chdir(cwd)

src/calibre/parallel.py

@@ -48,6 +48,9 @@ PARALLEL_FUNCS = {
         
                   'render_table' : 
         ('calibre.ebooks.lrf.html.table_as_image', 'do_render', {}, None),
+        
+                  'comic2lrf'    :
+        ('calibre.ebooks.lrf.comic.convert_from', 'do_convert', {}, 'notification'),  
 }
 
 

src/calibre/utils/PythonMagickWand.py

@@ -104,6 +104,14 @@ def finalize():
         _magick.MagickWandTerminus()
         _initialized = False
         
+class ImageMagick(object):
+    
+    def __enter__(self):
+        initialize()
+        
+    def __exit__(self, *args):
+        finalize()
+        
         
 class MetricType(ctypes.c_int): pass
 UndefinedMetric = MetricType(0)

src/calibre/utils/config.py

@@ -0,0 +1,269 @@
+from __future__ import with_statement
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+Manage application-wide preferences.
+'''
+import os, re, cPickle
+from copy import deepcopy
+from PyQt4.QtCore import QString
+from calibre import iswindows, isosx, OptionParser, ExclusiveFile, LockError 
+from collections import defaultdict
+
+if iswindows:
+    from calibre import plugins
+    config_dir = plugins['winutil'][0].special_folder_path(plugins['winutil'][0].CSIDL_APPDATA)
+    if not os.access(config_dir, os.W_OK|os.X_OK):
+        config_dir = os.path.expanduser('~')
+    config_dir = os.path.join(config_dir, 'calibre')
+elif isosx:
+    config_dir = os.path.expanduser('~/Library/Preferences/calibre')
+else:
+    config_dir = os.path.expanduser('~/.config/calibre')
+
+if not os.path.exists(config_dir):
+    os.makedirs(config_dir)
+    
+class Option(object):
+    
+    def __init__(self, name, switches=[], help='', type=None, choices=None, 
+                 check=None, group=None, default=None, action=None, metavar=None):
+        if choices:
+            type = 'choice'
+        
+        self.name     = name
+        self.switches = switches
+        self.help     = help.replace('%default', repr(default)) if help else None
+        self.type     = type
+        self.choices  = choices
+        self.check    = check
+        self.group    = group
+        self.default  = default
+        self.action   = action
+        self.metavar  = metavar
+        
+    def __eq__(self, other):
+        return self.name == getattr(other, 'name', None)
+        
+class OptionValues(object):
+    
+    def copy(self):
+        return deepcopy(self)
+
+class OptionSet(object):
+    
+    OVERRIDE_PAT = re.compile(r'#{3,100} Override Options #{15}(.*?)#{3,100} End Override #{3,100}', 
+                              re.DOTALL|re.IGNORECASE)
+    
+    def __init__(self, description=''):
+        self.description = description
+        self.preferences = []
+        self.group_list  = []
+        self.groups      = {}
+        self.set_buffer  = {}
+    
+    def has_option(self, name_or_option_object):
+        if name_or_option_object in self.preferences:
+            return True
+        for p in self.preferences:
+            if p.name == name_or_option_object:
+                return True
+        return False
+    
+    def add_group(self, name, description=''):
+        if name in self.group_list:
+            raise ValueError('A group by the name %s already exists in this set'%name)
+        self.groups[name] = description
+        self.group_list.append(name)
+        
+    def add_opt(self, name, switches=[], help=None, type=None, choices=None, 
+                 group=None, default=None, action=None, metavar=None):
+        '''
+        Add an option to this section.
+        
+        :param name:       The name of this option. Must be a valid Python identifier.
+                           Must also be unique in this OptionSet and all its subsets. 
+        :param switches:   List of command line switches for this option 
+                           (as supplied to :module:`optparse`). If empty, this
+                           option will not be added to the command line parser.
+        :param help:       Help text.
+        :param type:       Type checking of option values. Supported types are:
+                           `None, 'choice', 'complex', 'float', 'int', 'long', 'string'`.
+        :param choices:    List of strings or `None`.
+        :param group:      Group this option belongs to. You must previously 
+                           have created this group with a call to :method:`add_group`.
+        :param default:    The default value for this option.
+        :param action:     The action to pass to optparse. Supported values are:
+                           `None, 'count'`. For choices and boolean options,
+                           action is automatically set correctly.
+        '''
+        pref = Option(name, switches=switches, help=help, type=type, choices=choices, 
+                 group=group, default=default, action=action, metavar=None)
+        if group is not None and group not in self.groups.keys():
+            raise ValueError('Group %s has not been added to this section'%group)
+        if pref in self.preferences:
+            raise ValueError('An option with the name %s already exists in this set.'%name)
+        self.preferences.append(pref)
+        
+    def option_parser(self, user_defaults=None, usage='', gui_mode=False):
+        parser = OptionParser(usage, gui_mode=gui_mode)
+        groups = defaultdict(lambda : parser)
+        for group, desc in self.groups.items():
+            groups[group] = parser.add_group(group, desc)
+        
+        for pref in self.preferences:
+            if not pref.switches:
+                continue
+            g = groups[pref.group]
+            action = pref.action
+            if action is None:
+                action = 'store'
+                if pref.default is True or pref.default is False:
+                    action = 'store_' + ('false' if pref.default else 'true')
+            args = dict(
+                        dest=pref.name,
+                        help=pref.help,
+                        metavar=pref.metavar,
+                        type=pref.type,
+                        choices=pref.choices,
+                        default=getattr(user_defaults, pref.name, pref.default),
+                        action=action,
+                        )
+            g.add_option(*pref.switches, **args)
+            
+            
+        return parser
+    
+    def get_override_section(self, src):
+        match = self.OVERRIDE_PAT.search(src)
+        if match:
+            return match.group()
+        return ''
+    
+    def parse_string(self, src):
+        options = {'cPickle':cPickle}
+        if src is not None:
+            exec src in options
+        opts = OptionValues()
+        for pref in self.preferences:
+            setattr(opts, pref.name, options.get(pref.name, pref.default))
+            
+        return opts
+    
+    def render_group(self, name, desc, opts):
+        prefs = [pref for pref in self.preferences if pref.group == name]
+        lines = ['### Begin group: %s'%(name if name else 'DEFAULT')]
+        if desc:
+            lines += map(lambda x: '# '+x for x in desc.split('\n'))
+        lines.append(' ')
+        for pref in prefs:
+            lines.append('# '+pref.name.replace('_', ' '))
+            if pref.help:
+                lines += map(lambda x: '# ' + x, pref.help.split('\n'))
+            lines.append('%s = %s'%(pref.name, 
+                            self.serialize_opt(getattr(opts, pref.name, pref.default))))
+            lines.append(' ')
+        return '\n'.join(lines)
+        
+    def serialize_opt(self, val):
+        if val is val is True or val is False or val is None or \
+           isinstance(val, (int, float, long, basestring)):
+            return repr(val)
+        if isinstance(val, QString):
+            return repr(unicode(val))
+        pickle = cPickle.dumps(val, -1)
+        return 'cPickle.loads(%s)'%repr(pickle)
+    
+    def serialize(self, opts):
+        src = '# %s\n\n'%(self.description.replace('\n', '\n# '))
+        groups = [self.render_group(name, self.groups.get(name, ''), opts) \
+                                        for name in [None] + self.group_list]
+        return src + '\n\n'.join(groups)
+    
+class Config(object):
+    
+    def __init__(self, basename, description=''):
+        self.config_file_path = os.path.join(config_dir, basename+'.py')
+        self.option_set       = OptionSet(description=description)
+        self.add_opt          = self.option_set.add_opt
+        self.add_group        = self.option_set.add_group
+        
+    def option_parser(self, usage='', gui_mode=False):
+        return self.option_set.option_parser(user_defaults=self.parse(), 
+                                             usage=usage, gui_mode=gui_mode)
+        
+    def parse(self):
+        try:
+            with ExclusiveFile(self.config_file_path) as f:
+                src = f.read()
+        except LockError:
+            raise IOError('Could not lock config file: %s'%self.config_file_path)
+        return self.option_set.parse_string(src)
+    
+    def set(self, name, val):
+        if not self.option_set.has_option(name):
+            raise ValueError('The option %s is not defined.'%name)
+        try:
+            with ExclusiveFile(self.config_file_path) as f:
+                src = f.read()
+                opts = self.option_set.parse_string(src)
+                setattr(opts, name, val)
+                footer = self.option_set.get_override_section(src)
+                src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n'
+                f.seek(0)
+                f.truncate()
+                f.write(src)
+        except LockError:
+            raise IOError('Could not lock config file: %s'%self.config_file_path)
+            
+class StringConfig(object):
+    
+    def __init__(self, src, description=''):
+        self.src = src
+        self.option_set       = OptionSet(description=description)
+        self.add_opt          = self.option_set.add_opt
+        self.option_parser    = self.option_set.option_parser
+        
+    def option_parser(self, usage='', gui_mode=False):
+        return self.option_set.option_parser(user_defaults=self.parse(), 
+                                             usage=usage, gui_mode=gui_mode)
+    
+    def parse(self):
+        return self.option_set.parse_string(self.src)
+    
+    def set(self, name, val):
+        if not self.option_set.has_option(name):
+            raise ValueError('The option %s is not defined.'%name)
+        opts = self.option_set.parse_string(self.src)
+        setattr(opts, name, val)
+        footer = self.option_set.get_override_section(self.src)
+        self.src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n'
+    
+    
+        
+    
+if __name__ == '__main__':
+    import subprocess
+    from PyQt4.Qt import QByteArray
+    c = Config('test', 'test config')
+    
+    c.add_opt('one', ['-1', '--one'], help="This is option #1")
+    c.set('one', u'345')
+    
+    c.add_opt('two', help="This is option #2")
+    c.set('two', 345)
+    
+    c.add_opt('three', help="This is option #3")
+    c.set('three', QString(u'aflatoon'))
+    
+    c.add_opt('four', help="This is option #4")
+    c.set('four', QByteArray('binary aflatoon'))
+    
+    subprocess.call(['pygmentize', os.path.expanduser('~/.config/calibre/test.py')])
+    
+    opts = c.parse()
+    for i in ('one', 'two', 'three', 'four'):
+        print i, repr(getattr(opts, i))
+        

src/calibre/utils/genshi/core.py

@@ -491,7 +491,9 @@ class Markup(unicode):
         if hasattr(text, '__html__'):
             return Markup(text.__html__())
 
-        text = unicode(text).replace('&', '&') \
+        if isinstance(text, str):
+            text = text.decode('utf-8', 'replace')
+        text = text.replace('&', '&') \
                             .replace('<', '&lt;') \
                             .replace('>', '&gt;')
         if quotes:

src/calibre/utils/genshi/output.py

@@ -573,6 +573,7 @@ class NamespaceFlattener(object):
     def __call__(self, stream):
         prefixes = dict([(v, [k]) for k, v in self.prefixes.items()])
         namespaces = {XML_NAMESPACE.uri: ['xml']}
+        default = prefixes.get('', [''])
         def _push_ns(prefix, uri):
             namespaces.setdefault(uri, []).append(prefix)
             prefixes.setdefault(prefix, []).append(uri)
@@ -596,14 +597,14 @@ class NamespaceFlattener(object):
 
                 tagname = tag.localname
                 tagns = tag.namespace
-                if tagns:
+                if tagns and tagns != default[-1]:
                     if tagns in namespaces:
                         prefix = namespaces[tagns][-1]
                         if prefix:
                             tagname = u'%s:%s' % (prefix, tagname)
                     else:
                         _push_ns_attr((u'xmlns', tagns))
-                        _push_ns('', tagns)
+                        default.push(tagns)
 
                 new_attrs = []
                 for attr, value in attrs:
@@ -626,7 +627,7 @@ class NamespaceFlattener(object):
             elif kind is END:
                 tagname = data.localname
                 tagns = data.namespace
-                if tagns:
+                if tagns and tagns != default[-1]:
                     prefix = namespaces[tagns][-1]
                     if prefix:
                         tagname = u'%s:%s' % (prefix, tagname)
@@ -634,12 +635,19 @@ class NamespaceFlattener(object):
 
             elif kind is START_NS:
                 prefix, uri = data
-                if uri not in namespaces:
+                push_attr = False
-                    prefix = prefixes.get(uri, [prefix])[-1]
+                if prefix is '' and default[-1] != uri:
+                    default.append(uri)
                     _push_ns_attr(_make_ns_attr(prefix, uri))
-                _push_ns(prefix, uri)
+                elif uri not in namespaces:
+                    prefix = namespaces.get(uri, [prefix])[-1]
+                    _push_ns_attr(_make_ns_attr(prefix, uri))
+                if prefix is not '':
+                    _push_ns(prefix, uri)
 
             elif kind is END_NS:
+                if data is '':
+                    default.pop()
                 if data in prefixes:
                     uris = prefixes.get(data)
                     uri = uris.pop()

src/calibre/web/feeds/main.py

@@ -67,7 +67,7 @@ def run_recipe(opts, recipe_arg, parser, notification=None, handler=None):
     if notification is None:
         from calibre.utils.terminfo import TerminalController, ProgressBar
         term = TerminalController(sys.stdout)
-        pb = ProgressBar(term, _('Fetching feeds...'), no_progress_bar=opts.progress_bar)
+        pb = ProgressBar(term, _('Fetching feeds...'), no_progress_bar=not opts.progress_bar)
         notification = pb.update
     
     recipe, is_profile = None, False

src/calibre/web/feeds/news.py

@@ -7,7 +7,7 @@ Defines various abstract base classes that can be subclassed to create powerful
 __docformat__ = "restructuredtext en"
 
 
-import logging, os, cStringIO, time, traceback, re, urlparse
+import logging, os, cStringIO, time, traceback, re, urlparse, sys
 from collections import defaultdict
 from functools import partial
 
@@ -533,6 +533,9 @@ class BasicNewsRecipe(object, LoggingInterface):
                             self.image_map[feed.image_url] = img
                         except:
                             pass
+            if isinstance(feed.image_url, str):
+                feed.image_url = feed.image_url.decode(sys.getfilesystemencoding(), 'strict')
+                
                 
         templ = templates.FeedTemplate()
         return templ.generate(feed, self.description_limiter).render(doctype='xhtml')