Merge oeb2mobi state.

installer/linux/freeze.py

@@ -16,6 +16,7 @@ def freeze():
     from calibre.linux import entry_points
     from calibre import walk
     from calibre.web.feeds.recipes import recipe_modules
+    from calibre.ebooks.lrf.fonts import FONT_MAP
     import calibre
     
 
@@ -37,6 +38,7 @@ def freeze():
                        '/usr/lib/libxml2.so.2',
                        '/usr/lib/libxslt.so.1',
                        '/usr/lib/libxslt.so.1',
+                       '/usr/lib/libexslt.so.0',
                        '/usr/lib/libMagickWand.so',
                        '/usr/lib/libMagickCore.so',
                        ]
@@ -72,6 +74,7 @@ def freeze():
         os.makedirs(DIST_DIR)
     
     includes = [x[0] for x in executables.values()]
+    includes += ['calibre.ebooks.lrf.fonts.prs500.'+x for x in FONT_MAP.values()]
     
     excludes = ['matplotlib', "Tkconstants", "Tkinter", "tcl", "_imagingtk", 
                 "ImageTk", "FixTk", 'wx', 'PyQt4.QtAssistant', 'PyQt4.QtOpenGL.so', 

installer/osx/freeze.py

@@ -326,7 +326,7 @@ def main():
                                        'genshi', 'calibre.web.feeds.recipes.*',
                                        'calibre.ebooks.lrf.any.*', 'calibre.ebooks.lrf.feeds.*',
                                        'keyword', 'codeop', 'pydoc', 'readline',
-                                       'BeautifulSoup'
+                                       'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*',
                                        ],
                          'packages' : ['PIL', 'Authorization', 'lxml'],
                          'excludes' : ['IPython'],

installer/windows/freeze.py

@@ -176,6 +176,7 @@ def main(args=sys.argv):
                                              'BeautifulSoup', 'pyreadline',
                                              'pydoc', 'IPython.Extensions.*',
                                              'calibre.web.feeds.recipes.*',
+                                             'calibre.ebooks.lrf.fonts.prs500.*',
                                              'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
                                              ],
                                   'packages'  : ['PIL', 'lxml', 'cherrypy'],

setup.py

@@ -380,12 +380,15 @@ if __name__ == '__main__':
     
     class build(_build):
         
-        sub_commands = \
-                        [
+        sub_commands = [
                          ('resources',    lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
                          ('translations', lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
                          ('gui',          lambda self : 'CALIBRE_BUILDBOT' not in os.environ.keys()),
-                         ] + _build.sub_commands
+                         ('build_ext',    lambda self: True),
+                         ('build_py',     lambda self: True),
+                         ('build_clib',    _build.has_c_libraries),
+                         ('build_scripts', _build.has_scripts),
+                       ]
         
     entry_points['console_scripts'].append('calibre_postinstall = calibre.linux:post_install')
     ext_modules = [

src/calibre/__init__.py

@@ -13,7 +13,8 @@ from calibre.startup import plugins, winutil, winutilerror
 from calibre.constants import iswindows, isosx, islinux, isfrozen, \
                               terminal_controller, preferred_encoding, \
                               __appname__, __version__, __author__, \
-                              win32event, win32api, winerror, fcntl
+                              win32event, win32api, winerror, fcntl, \
+                              filesystem_encoding
 import mechanize
 
 mimetypes.add_type('application/epub+zip', '.epub')
@@ -41,6 +42,28 @@ def osx_version():
             return int(m.group(1)), int(m.group(2)), int(m.group(3))
 
 
+_filename_sanitize = re.compile(r'[\xae\0\\|\?\*<":>\+\[\]/]')
+
+def sanitize_file_name(name, substitute='_', as_unicode=False):
+    '''
+    Sanitize the filename `name`. All invalid characters are replaced by `substitute`.
+    The set of invalid characters is the union of the invalid characters in Windows,
+    OS X and Linux. Also removes leading an trailing whitespace.
+    **WARNING:** This function also replaces path separators, so only pass file names
+    and not full paths to it.
+    *NOTE:* This function always returns byte strings, not unicode objects. The byte strings
+    are encoded in the filesystem encoding of the platform, or UTF-8. 
+    '''
+    if isinstance(name, unicode):
+        name = name.encode(filesystem_encoding, 'ignore')
+    one = _filename_sanitize.sub(substitute, name)
+    one = re.sub(r'\s', ' ', one).strip()
+    one = re.sub(r'^\.+$', '_', one)
+    if as_unicode:
+        one = one.decode(filesystem_encoding)
+    return one
+
+
 class CommandLineError(Exception):
     pass
 
@@ -201,13 +224,6 @@ class CurrentDir(object):
     def __exit__(self, *args):
         os.chdir(self.cwd)
 
-def sanitize_file_name(name):
-    '''
-    Remove characters that are illegal in filenames from name.
-    Also remove path separators. All illegal characters are replaced by
-    underscores.
-    '''
-    return re.sub(r'\s', ' ', re.sub(r'[\xae"\'\|\~\:\?\\\/]|^-', '_', name.strip()))
 
 def detect_ncpus():
     """Detects the number of effective CPUs in the system"""
@@ -366,6 +382,12 @@ def strftime(fmt, t=time.localtime()):
         return plugins['winutil'][0].strftime(fmt, t)
     return time.strftime(fmt, t).decode(preferred_encoding, 'replace')
 
+def my_unichr(num):
+    try:
+        return unichr(num)
+    except ValueError:
+        return u'?'
+
 def entity_to_unicode(match, exceptions=[], encoding='cp1252'):
     '''
     @param match: A match object such that '&'+match.group(1)';' is the entity.
@@ -381,7 +403,7 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252'):
     if ent.startswith(u'#x'):
         num = int(ent[2:], 16)
         if encoding is None or num > 255:
-            return unichr(num)
+            return my_unichr(num)
         return chr(num).decode(encoding)
     if ent.startswith(u'#'):
         try:
@@ -389,13 +411,13 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252'):
         except ValueError:
             return '&'+ent+';'
         if encoding is None or num > 255:
-            return unichr(num)
+            return my_unichr(num)
         try:
             return chr(num).decode(encoding)
         except UnicodeDecodeError:
-            return unichr(num)
+            return my_unichr(num)
     try:
-        return unichr(name2codepoint[ent])
+        return my_unichr(name2codepoint[ent])
     except KeyError:
         return '&'+ent+';'
 

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.121'
+__version__   = '0.4.126'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.
@@ -29,6 +29,10 @@ winerror   = __import__('winerror') if iswindows else None
 win32api   = __import__('win32api') if iswindows else None
 fcntl      = None if iswindows else __import__('fcntl')
 
+filesystem_encoding = sys.getfilesystemencoding()
+if filesystem_encoding is None: filesystem_encoding = 'utf-8'
+
+
 ################################################################################
 plugins = None
 if plugins is None:

src/calibre/customize/ui.py

@@ -105,31 +105,44 @@ def reread_metadata_plugins():
     for plugin in _initialized_plugins:
         if isinstance(plugin, MetadataReaderPlugin):
             for ft in plugin.file_types:
-                _metadata_readers[ft] = plugin
+                if not _metadata_readers.has_key(ft):
+                    _metadata_readers[ft] = []
+                _metadata_readers[ft].append(plugin)
         elif isinstance(plugin, MetadataWriterPlugin):
             for ft in plugin.file_types:
-                _metadata_writers[ft] = plugin 
+                if not _metadata_writers.has_key(ft):
+                    _metadata_writers[ft] = []
+                _metadata_writers[ft].append(plugin) 
+                
+    
                 
 def get_file_type_metadata(stream, ftype):
     mi = MetaInformation(None, None)
-    try:
-        plugin = _metadata_readers[ftype.lower().strip()]
+    ftype = ftype.lower().strip()
+    if _metadata_readers.has_key(ftype):
+        for plugin in _metadata_readers[ftype]:
             if not is_disabled(plugin):
                 with plugin:
+                    try:
                         mi = plugin.get_metadata(stream, ftype.lower().strip())
+                        break
                     except:
-        pass
+                        continue
     return mi
 
 def set_file_type_metadata(stream, mi, ftype):
-    try:
-        plugin = _metadata_writers[ftype.lower().strip()]
+    ftype = ftype.lower().strip()
+    if _metadata_writers.has_key(ftype):
+        for plugin in _metadata_writers[ftype]:
             if not is_disabled(plugin):
                 with plugin:
+                    try:
                         plugin.set_metadata(stream, mi, ftype.lower().strip())
+                        break
                     except:
                         traceback.print_exc()
     
+                
 def _run_filetype_plugins(path_to_file, ft=None, occasion='preprocess'):
     occasion = {'import':_on_import, 'preprocess':_on_preprocess, 
                 'postprocess':_on_postprocess}[occasion]
@@ -185,6 +198,20 @@ def add_plugin(path_to_zip_file):
     initialize_plugins()
     return plugin
 
+def remove_plugin(plugin_or_name):
+    name = getattr(plugin_or_name, 'name', plugin_or_name)
+    plugins = config['plugins']
+    removed = False
+    if name in plugins.keys():
+        removed = True
+        zfp = plugins[name]
+        if os.path.exists(zfp):
+            os.remove(zfp)
+        plugins.pop(name)
+    config['plugins'] = plugins
+    initialize_plugins()
+    return removed
+
 def is_disabled(plugin):
     return plugin.name in config['disabled_plugins']
 
@@ -237,6 +264,8 @@ def option_parser():
     '''))
     parser.add_option('-a', '--add-plugin', default=None, 
                       help=_('Add a plugin by specifying the path to the zip file containing it.'))
+    parser.add_option('-r', '--remove-plugin', default=None, 
+                      help=_('Remove a custom plugin by name. Has no effect on builtin plugins'))
     parser.add_option('--customize-plugin', default=None,
                       help=_('Customize plugin. Specify name of plugin and customization string separated by a comma.'))
     parser.add_option('-l', '--list-plugins', default=False, action='store_true',
@@ -267,6 +296,11 @@ def main(args=sys.argv):
     if opts.add_plugin is not None:
         plugin = add_plugin(opts.add_plugin)
         print 'Plugin added:', plugin.name, plugin.version
+    if opts.remove_plugin is not None:
+        if remove_plugin(opts.remove_plugin):
+            print 'Plugin removed'
+        else:
+            print 'No custom pluginnamed', opts.remove_plugin
     if opts.customize_plugin is not None:
         name, custom = opts.customize_plugin.split(',')
         plugin = find_plugin(name.strip())

src/calibre/devices/__init__.py

@@ -1,5 +1,6 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+
 '''
 Device drivers.
 '''
@@ -8,8 +9,9 @@ def devices():
     from calibre.devices.prs500.driver import PRS500
     from calibre.devices.prs505.driver import PRS505
     from calibre.devices.prs700.driver import PRS700
+    from calibre.devices.cybookg3.driver import CYBOOKG3
     #from calibre.devices.kindle.driver import KINDLE
-    return (PRS500, PRS505, PRS700)
+    return (PRS500, PRS505, PRS700, CYBOOKG3)
 
 import time
 

src/calibre/devices/cybookg3/books.py

@@ -0,0 +1,85 @@
+__license__   = 'GPL v3'
+__copyright__ = '2009, John Schember <john at nachtimwald.com>'
+
+'''
+'''
+import os, fnmatch, time
+
+from calibre.devices.interface import BookList as _BookList
+
+EBOOK_DIR = "eBooks"
+EBOOK_TYPES = ['mobi', 'prc', 'html', 'pdf', 'rtf', 'txt']
+
+class Book(object):
+    def __init__(self, path, title, authors):
+        self.title = title
+        self.authors = authors
+        self.size = os.path.getsize(path)
+        self.datetime = time.gmtime(os.path.getctime(path))
+        self.path = path
+        self.thumbnail = None
+        self.tags = []
+        
+    @apply
+    def title_sorter():
+        doc = '''String to sort the title. If absent, title is returned'''
+        def fget(self):
+            return re.sub('^\s*A\s+|^\s*The\s+|^\s*An\s+', '', self.title).rstrip()
+        return property(doc=doc, fget=fget)
+
+    @apply
+    def thumbnail():
+        return None
+        
+    def __str__(self):
+        """ Return a utf-8 encoded string with title author and path information """
+        return self.title.encode('utf-8') + " by " + \
+               self.authors.encode('utf-8') + " at " + self.path.encode('utf-8')
+
+
+class BookList(_BookList):
+    def __init__(self, mountpath):
+        self._mountpath = mountpath
+        _BookList.__init__(self)            
+        self.return_books(mountpath)  
+
+    def return_books(self, mountpath):
+        # Get all books in all directories under the root EBOOK_DIR directory
+        for path, dirs, files in os.walk(os.path.join(mountpath, EBOOK_DIR)):
+            # Filter out anything that isn't in the list of supported ebook types
+            for book_type in EBOOK_TYPES:
+                for filename in fnmatch.filter(files, '*.%s' % (book_type)):
+                    book_title = ''
+                    book_author = ''
+                    # Calibre uses a specific format for file names. They take the form
+                    # title_-_author_number.extention We want to see if the file name is
+                    # in this format.
+                    if fnmatch.fnmatchcase(filename, '*_-_*.*'):
+                        # Get the title and author from the file name
+                        title, sep, author = filename.rpartition('_-_')
+                        author, sep, ext = author.rpartition('_')
+                        book_title = title.replace('_', ' ')
+                        book_author = author.replace('_', ' ')
+                    # if the filename did not match just set the title to
+                    # the filename without the extension
+                    else:
+                        book_title = os.path.splitext(filename)[0].replace('_', ' ')
+
+                    self.append(Book(os.path.join(path, filename), book_title, book_author))
+            
+    def add_book(self, path, title):
+        self.append(Book(path, title, ""))
+
+    def remove_book(self, path):
+        for book in self:
+            if path.endswith(book.path):
+                self.remove(book)
+                break
+            
+    def supports_tags(self):
+        ''' Return True if the the device supports tags (collections) for this book list. '''
+        return False
+    
+    def set_tags(self, book, tags):
+        pass
+

src/calibre/devices/cybookg3/driver.py

@@ -0,0 +1,332 @@
+__license__   = 'GPL v3'
+__copyright__ = '2009, John Schember <john at nachtimwald.com>'
+
+'''
+Device driver for Bookeen's Cybook Gen 3
+'''
+import os, fnmatch, shutil, time
+from itertools import cycle
+
+from calibre.devices.interface import Device
+from calibre.devices.errors import DeviceError, FreeSpaceError
+
+from calibre.devices.cybookg3.books import BookList, EBOOK_DIR, EBOOK_TYPES
+from calibre import iswindows, islinux, isosx, __appname__
+
+class CYBOOKG3(Device):
+    # Ordered list of supported formats
+    FORMATS     = EBOOK_TYPES
+    VENDOR_ID   = 0x0bda
+    PRODUCT_ID  = 0x0703
+    BCD         = 0x110
+
+    VENDOR_NAME = 'BOOKEEN'
+    PRODUCT_NAME = 'CYBOOK_GEN3'
+    
+    MAIN_MEMORY_VOLUME_LABEL  = 'Cybook Gen 3 Main Memory'
+    STORAGE_CARD_VOLUME_LABEL = 'Cybook Gen 3 Storage Card'
+    
+    FDI_TEMPLATE = \
+'''
+  <device>
+      <match key="info.category" string="volume">
+          <match key="@info.parent:@info.parent:@info.parent:@info.parent:usb.vendor_id" int="%(vendor_id)s">
+              <match key="@info.parent:@info.parent:@info.parent:@info.parent:usb.product_id" int="%(product_id)s">
+                  <match key="@info.parent:@info.parent:@info.parent:@info.parent:usb.device_revision_bcd" int="%(bcd)s">
+                      <match key="volume.is_partition" bool="false">
+                          <merge key="volume.label" type="string">%(main_memory)s</merge>
+                          <merge key="%(app)s.mainvolume" type="string">%(deviceclass)s</merge>
+                      </match>
+                  </match>
+              </match>
+          </match>
+      </match>
+  </device>
+  <device>
+      <match key="info.category" string="volume">
+          <match key="@info.parent:@info.parent:@info.parent:@info.parent:usb.vendor_id" int="%(vendor_id)s">
+              <match key="@info.parent:@info.parent:@info.parent:@info.parent:usb.product_id" int="%(product_id)s">
+                  <match key="@info.parent:@info.parent:@info.parent:@info.parent:usb.device_revision_bcd" int="%(bcd)s">
+                      <match key="volume.is_partition" bool="true">
+                          <merge key="volume.label" type="string">%(storage_card)s</merge>
+                          <merge key="%(app)s.cardvolume" type="string">%(deviceclass)s</merge>
+                      </match>
+                  </match>
+              </match>
+          </match>
+      </match>
+  </device>
+'''
+    
+    
+    def __init__(self, key='-1', log_packets=False, report_progress=None) :
+        self._main_prefix = self._card_prefix = None
+    
+    @classmethod
+    def get_fdi(cls):
+        return cls.FDI_TEMPLATE%dict(
+                                     app=__appname__,
+                                     deviceclass=cls.__name__,
+                                     vendor_id=hex(cls.VENDOR_ID),
+                                     product_id=hex(cls.PRODUCT_ID),
+                                     bcd=hex(cls.BCD),
+                                     main_memory=cls.MAIN_MEMORY_VOLUME_LABEL,
+                                     storage_card=cls.STORAGE_CARD_VOLUME_LABEL,
+                                     )
+    
+    def set_progress_reporter(self, report_progress):
+        self.report_progress = report_progress
+    
+    def get_device_information(self, end_session=True):
+        """ 
+        Ask device for device information. See L{DeviceInfoQuery}. 
+        @return: (device name, device version, software version on device, mime type)
+        """
+        return (self.__class__.__name__, '', '', '')
+    
+    def card_prefix(self, end_session=True):
+        return self._card_prefix
+    
+    @classmethod
+    def _windows_space(cls, prefix):
+        if prefix is None:
+            return 0, 0
+        win32file = __import__('win32file', globals(), locals(), [], -1)
+        try:
+            sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \
+                win32file.GetDiskFreeSpace(prefix[:-1])
+        except Exception, err:
+            if getattr(err, 'args', [None])[0] == 21: # Disk not ready
+                time.sleep(3)
+                sectors_per_cluster, bytes_per_sector, free_clusters, total_clusters = \
+                    win32file.GetDiskFreeSpace(prefix[:-1])
+            else: raise
+        mult = sectors_per_cluster * bytes_per_sector
+        return total_clusters * mult, free_clusters * mult
+    
+    def total_space(self, end_session=True):
+        msz = csz = 0
+        print self._main_prefix
+        if not iswindows:
+            if self._main_prefix is not None:
+                stats = os.statvfs(self._main_prefix)
+                msz = stats.f_frsize * (stats.f_blocks + stats.f_bavail - stats.f_bfree)
+            if self._card_prefix is not None:
+                stats = os.statvfs(self._card_prefix)
+                csz = stats.f_frsize * (stats.f_blocks + stats.f_bavail - stats.f_bfree)
+        else:
+            msz = self._windows_space(self._main_prefix)[0]
+            csz = self._windows_space(self._card_prefix)[0]
+                
+        return (msz, 0, csz)
+    
+    def free_space(self, end_session=True):
+        msz = csz = 0
+        if not iswindows:
+            if self._main_prefix is not None:
+                stats = os.statvfs(self._main_prefix)
+                msz = stats.f_frsize * stats.f_bavail
+            if self._card_prefix is not None:
+                stats = os.statvfs(self._card_prefix)
+                csz = stats.f_frsize * stats.f_bavail
+        else:
+            msz = self._windows_space(self._main_prefix)[1]
+            csz = self._windows_space(self._card_prefix)[1]
+                
+        return (msz, 0, csz)
+    
+    def books(self, oncard=False, end_session=True):
+        if oncard and self._card_prefix is None:
+            return []
+        prefix = self._card_prefix if oncard else self._main_prefix
+        bl = BookList(prefix)
+        return bl
+    
+    def upload_books(self, files, names, on_card=False, end_session=True):
+        if on_card and not self._card_prefix:
+            raise ValueError(_('The reader has no storage card connected.'))
+            
+        if not on_card:
+            path = os.path.join(self._main_prefix, EBOOK_DIR)
+        else:
+            path = os.path.join(self._card_prefix, EBOOK_DIR)
+            
+        sizes = map(os.path.getsize, files)
+        size = sum(sizes)
+    
+        if on_card and size > self.free_space()[2] - 1024*1024: 
+            raise FreeSpaceError("There is insufficient free space "+\
+                                          "on the storage card")
+        if not on_card and size > self.free_space()[0] - 2*1024*1024: 
+            raise FreeSpaceError("There is insufficient free space " +\
+                                         "in main memory")
+
+        paths = []
+        names = iter(names)
+        
+        for infile in files:
+            filepath = os.path.join(path, names.next())
+            paths.append(filepath)
+            
+            shutil.copy2(infile, filepath)
+    
+        return zip(paths, cycle([on_card]))
+    
+    @classmethod
+    def add_books_to_metadata(cls, locations, metadata, booklists):
+        for location in locations:
+            path = location[0]
+            on_card = 1 if location[1] else 0
+            booklists[on_card].add_book(path, os.path.basename(path))
+    
+    def delete_books(self, paths, end_session=True):
+        for path in paths:
+            if os.path.exists(path):
+                # Delete the ebook
+                os.unlink(path)
+                
+                filepath, ext = os.path.splitext(path)
+                basepath, filename = os.path.split(filepath)
+                
+                # Delete the ebook auxiliary file
+                if os.path.exists(filepath + '.mbp'):
+                    os.unlink(filepath + '.mbp')
+                
+                # Delete the thumbnails file auto generated for the ebook
+                for p, d, files in os.walk(basepath):
+                    for filen in fnmatch.filter(files, filename + "*.t2b"):
+                        os.unlink(os.path.join(p, filen))
+    
+    @classmethod
+    def remove_books_from_metadata(cls, paths, booklists):
+        for path in paths:
+            for bl in booklists:
+                bl.remove_book(path)
+        
+    def sync_booklists(self, booklists, end_session=True):
+        # There is no meta data on the device to update. The device is treated
+        # as a mass storage device and does not use a meta data xml file like
+        # the Sony Readers.
+        pass
+    
+    def get_file(self, path, outfile, end_session=True): 
+        path = self.munge_path(path)
+        src = open(path, 'rb')
+        shutil.copyfileobj(src, outfile, 10*1024*1024)
+    
+    def munge_path(self, path):
+        if path.startswith('/') and not (path.startswith(self._main_prefix) or \
+            (self._card_prefix and path.startswith(self._card_prefix))):
+            path = self._main_prefix + path[1:]
+        elif path.startswith('card:'):
+            path = path.replace('card:', self._card_prefix[:-1])
+        return path
+        
+    @classmethod
+    def windows_match_device(cls, device_id):
+        device_id = device_id.upper()
+        if 'VEN_'+cls.VENDOR_NAME in device_id and \
+               'PROD_'+cls.PRODUCT_NAME in device_id:
+            return True
+                    
+        vid, pid = hex(cls.VENDOR_ID)[2:], hex(cls.PRODUCT_ID)[2:]        
+        while len(vid) < 4: vid = '0' + vid
+        while len(pid) < 4: pid = '0' + pid        
+        if 'VID_'+vid in device_id and 'PID_'+pid in device_id:
+            return True
+            
+        return False
+
+    # This only supports Windows >= 2000
+    def open_windows(self):
+        drives = []
+        wmi = __import__('wmi', globals(), locals(), [], -1) 
+        c = wmi.WMI()
+        for drive in c.Win32_DiskDrive():
+            if self.__class__.windows_match_device(str(drive.PNPDeviceID)):
+                if drive.Partitions == 0:
+                    continue
+                try:
+                    partition = drive.associators("Win32_DiskDriveToDiskPartition")[0]
+                    logical_disk = partition.associators('Win32_LogicalDiskToPartition')[0]
+                    prefix = logical_disk.DeviceID+os.sep
+                    drives.append((drive.Index, prefix))
+                except IndexError:
+                    continue
+                
+        if not drives:
+            raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%self.__class__.__name__)
+        
+        drives.sort(cmp=lambda a, b: cmp(a[0], b[0]))
+        self._main_prefix = drives[0][1]
+        if len(drives) > 1:
+            self._card_prefix = drives[1][1]            
+        
+    def open_osx(self):
+        raise NotImplementedError()
+
+    def open_linux(self):
+        import dbus
+        bus = dbus.SystemBus() 
+        hm  = dbus.Interface(bus.get_object("org.freedesktop.Hal", "/org/freedesktop/Hal/Manager"), "org.freedesktop.Hal.Manager")
+        
+        def conditional_mount(dev):
+            mmo = bus.get_object("org.freedesktop.Hal", dev)
+            label = mmo.GetPropertyString('volume.label', dbus_interface='org.freedesktop.Hal.Device')
+            is_mounted = mmo.GetPropertyString('volume.is_mounted', dbus_interface='org.freedesktop.Hal.Device')
+            mount_point = mmo.GetPropertyString('volume.mount_point', dbus_interface='org.freedesktop.Hal.Device')
+            fstype = mmo.GetPropertyString('volume.fstype', dbus_interface='org.freedesktop.Hal.Device')
+            if is_mounted:
+                return str(mount_point)
+            mmo.Mount(label, fstype, ['umask=077', 'uid='+str(os.getuid()), 'sync'], 
+                          dbus_interface='org.freedesktop.Hal.Device.Volume')
+            return os.path.normpath('/media/'+label)+'/'
+        
+        mm = hm.FindDeviceStringMatch(__appname__+'.mainvolume', self.__class__.__name__)
+        if not mm:
+            raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.')%(self.__class__.__name__,))
+        self._main_prefix = None
+        for dev in mm:
+            try:
+                self._main_prefix = conditional_mount(dev)+os.sep
+                break
+            except dbus.exceptions.DBusException:
+                continue
+                
+        if not self._main_prefix:
+            raise DeviceError('Could not open device for reading. Try a reboot.')
+            
+        self._card_prefix = None
+        cards = hm.FindDeviceStringMatch(__appname__+'.cardvolume', self.__class__.__name__)
+        
+        for dev in cards:
+            try:
+                self._card_prefix = conditional_mount(dev)+os.sep
+                break
+            except:
+                import traceback
+                print traceback
+                continue
+
+    def open(self):
+        time.sleep(5)
+        self._main_prefix = self._card_prefix = None
+        if islinux:
+            try:
+                self.open_linux()
+            except DeviceError:
+                time.sleep(3)
+                self.open_linux()
+        if iswindows:
+            try:
+                self.open_windows()
+            except DeviceError:
+                time.sleep(3)
+                self.open_windows()
+        if isosx:
+            try:
+                self.open_osx()
+            except DeviceError:
+                time.sleep(3)
+                self.open_osx()
+

src/calibre/devices/interface.py

@@ -39,6 +39,18 @@ class Device(object):
         '''Return the FDI description of this device for HAL on linux.'''
         return ''
     
+    def open(self):
+        '''
+        Perform any device specific initialization. Called after the device is
+        detected but before any other functions that communicate with the device.
+        For example: For devices that present themselves as USB Mass storage
+        devices, this method would be responsible for mounting the device or
+        if the device has been automounted, for finding out where it has been 
+        mounted. The driver for the PRS505 has a implementation of this function
+        that should serve as a good example for USB Mass storage devices.
+        '''
+        raise NotImplementedError()
+    
     def set_progress_reporter(self, report_progress):
         '''
         @param report_progress: Function that is called with a % progress 

src/calibre/ebooks/epub/from_html.py

@@ -35,7 +35,7 @@ Conversion of HTML/OPF files follows several stages:
 import os, sys, cStringIO, logging, re, functools, shutil
 
 from lxml.etree import XPath
-from lxml import html
+from lxml import html, etree
 from PyQt4.Qt import QApplication, QPixmap
 
 from calibre.ebooks.html import Processor, merge_metadata, get_filelist,\
@@ -55,13 +55,13 @@ content = functools.partial(os.path.join, u'content')
 
 def remove_bad_link(element, attribute, link, pos):
     if attribute is not None:
-        if element.tag in ['link', 'img']:
+        if element.tag in ['link']:
             element.getparent().remove(element)
         else:
             element.set(attribute, '')
             del element.attrib[attribute]
 
-def check(opf_path, pretty_print):
+def check_links(opf_path, pretty_print):
     '''
     Find and remove all invalid links in the HTML files 
     '''
@@ -123,6 +123,10 @@ class HTMLProcessor(Processor, Rationalizer):
         if opts.verbose > 2:
             self.debug_tree('nocss')
             
+        if hasattr(self.body, 'xpath'):
+            for script in list(self.body.xpath('descendant::script')):
+                script.getparent().remove(script)
+            
     def convert_image(self, img):
         rpath = img.get('src', '')
         path = os.path.join(os.path.dirname(self.save_path()), *rpath.split('/'))
@@ -280,6 +284,16 @@ def find_oeb_cover(htmlfile):
     if match:
         return match.group(1)
 
+def condense_ncx(ncx_path):
+    tree = etree.parse(ncx_path)
+    for tag in tree.getroot().iter(tag=etree.Element):
+        if tag.text:
+            tag.text = tag.text.strip()
+        if tag.tail:
+            tag.tail = tag.tail.strip()
+    compressed = etree.tostring(tree.getroot(), encoding='utf-8')
+    open(ncx_path, 'wb').write(compressed)
+
 def convert(htmlfile, opts, notification=None, create_epub=True, 
             oeb_cover=False, extract_to=None):
     htmlfile = os.path.abspath(htmlfile)
@@ -362,7 +376,8 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
             if opts.show_ncx:
                 print toc
         split(opf_path, opts, stylesheet_map)
-        check(opf_path, opts.pretty_print)
+        check_links(opf_path, opts.pretty_print)
+        
         opf = OPF(opf_path, tdir)
         opf.remove_guide()
         oeb_cover_file = None
@@ -383,6 +398,13 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
             if not raw.startswith('<?xml '):
                 raw = '<?xml version="1.0"  encoding="UTF-8"?>\n'+raw
             f.write(raw)
+        ncx_path = os.path.join(os.path.dirname(opf_path), 'toc.ncx')
+        if os.path.exists(ncx_path) and os.stat(ncx_path).st_size > opts.profile.flow_size:
+            logger.info('Condensing NCX from %d bytes...'%os.stat(ncx_path).st_size)
+            condense_ncx(ncx_path)
+            if os.stat(ncx_path).st_size > opts.profile.flow_size:
+                logger.warn('NCX still larger than allowed size at %d bytes. Menu based Table of Contents may not work on device.'%os.stat(ncx_path).st_size)
+            
         if create_epub:
             epub = initialize_container(opts.output)
             epub.add_dir(tdir)

src/calibre/ebooks/html.py

@@ -314,10 +314,22 @@ def opf_traverse(opf_reader, verbose=0, encoding=None):
             
 
 convert_entities = functools.partial(entity_to_unicode, exceptions=['quot', 'apos', 'lt', 'gt', 'amp'])
+_span_pat = re.compile('<span.*?</span>', re.DOTALL|re.IGNORECASE)
+
+def sanitize_head(match):
+    x = match.group(1)
+    x = _span_pat.sub('', x)
+    return '<head>\n'+x+'\n</head>'
+    
 class PreProcessor(object):
     PREPROCESS = [
+                  # Some idiotic HTML generators (Frontpage I'm looking at you)
+                  # Put all sorts of crap into <head>. This messes up lxml
+                  (re.compile(r'<head[^>]*>(.*?)</head>', re.IGNORECASE|re.DOTALL), 
+                   sanitize_head),
                   # Convert all entities, since lxml doesn't handle them well
                   (re.compile(r'&(\S+?);'), convert_entities),
+                  
                   ]
                      
     # Fix pdftohtml markup
@@ -446,6 +458,8 @@ class Parser(PreProcessor, LoggingInterface):
     def parse_html(self):
         ''' Create lxml ElementTree from HTML '''
         self.log_info('\tParsing '+os.sep.join(self.htmlfile.path.split(os.sep)[-3:]))
+        if self.htmlfile.is_binary:
+            raise ValueError('Not a valid HTML file: '+self.htmlfile.path)
         src = open(self.htmlfile.path, 'rb').read().decode(self.htmlfile.encoding, 'replace').strip()
         src = src.replace('\x00', '')
         src = self.preprocess(src)
@@ -776,7 +790,10 @@ class Processor(Parser):
                 size = '3'
             if size and size.strip() and size.strip()[0] in ('+', '-'):
                 size = 3 + float(size) # Hack assumes basefont=3
+            try:
                 setting = 'font-size: %d%%;'%int((float(size)/3) * 100)
+            except ValueError:
+                setting = ''
             face = font.attrib.pop('face', None)
             if face is not None:
                 setting += 'font-face:%s;'%face
@@ -809,7 +826,7 @@ class Processor(Parser):
         
         css = '\n'.join(['.%s {%s;}'%(cn, setting) for \
                          setting, cn in cache.items()])
-        sheet = self.css_parser.parseString(self.preprocess_css(css))
+        sheet = self.css_parser.parseString(self.preprocess_css(css.replace(';;}', ';}')))
         for rule in sheet:
             self.stylesheet.add(rule)
         css = ''
@@ -875,7 +892,7 @@ def option_parser():
 %prog [options] file.html|opf
 
 Follow all links in an HTML file and collect them into the specified directory.
-Also collects any references resources like images, stylesheets, scripts, etc. 
+Also collects any resources like images, stylesheets, scripts, etc. 
 If an OPF file is specified instead, the list of files in its <spine> element
 is used.
 '''))

src/calibre/ebooks/lit/reader.py

@@ -7,24 +7,20 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net> ' \
     'and Marshall T. Vandegrift <llasram@gmail.com>'
 
-import sys, struct, os
+import sys, struct, cStringIO, os
 import functools
 import re
 from urlparse import urldefrag
-from cStringIO import StringIO
-from urllib import unquote as urlunquote
 from lxml import etree
 from calibre.ebooks.lit import LitError
 from calibre.ebooks.lit.maps import OPF_MAP, HTML_MAP
 import calibre.ebooks.lit.mssha1 as mssha1
-from calibre.ebooks.oeb.base import XML_PARSER, urlnormalize
+from calibre.ebooks.oeb.base import urlnormalize
 from calibre.ebooks import DRMError
 from calibre import plugins
 lzx, lxzerror = plugins['lzx']
 msdes, msdeserror = plugins['msdes']
 
-__all__ = ["LitReader"]
-
 XML_DECL = """<?xml version="1.0" encoding="UTF-8" ?>
 """
 OPF_DECL = """<?xml version="1.0" encoding="UTF-8" ?>
@@ -112,9 +108,6 @@ def consume_sized_utf8_string(bytes, zpad=False):
         pos += 1
     return u''.join(result), bytes[pos:]
 
-def encode(string):
-    return unicode(string).encode('ascii', 'xmlcharrefreplace')
-
 class UnBinary(object):
     AMPERSAND_RE = re.compile(
         r'&(?!(?:#[0-9]+|#x[0-9a-fA-F]+|[a-zA-Z_:][a-zA-Z0-9.-_:]+);)')
@@ -125,13 +118,13 @@ class UnBinary(object):
     def __init__(self, bin, path, manifest={}, map=HTML_MAP):
         self.manifest = manifest
         self.tag_map, self.attr_map, self.tag_to_attr_map = map
-        self.is_html = map is HTML_MAP
+        self.opf = map is OPF_MAP
+        self.bin = bin
         self.dir = os.path.dirname(path)
-        buf = StringIO()
-        self.binary_to_text(bin, buf)
-        self.raw = buf.getvalue().lstrip()
+        self.buf = cStringIO.StringIO()
+        self.binary_to_text()
+        self.raw = self.buf.getvalue().lstrip().decode('utf-8')
         self.escape_reserved()
-        self._tree = None
 
     def escape_reserved(self):
         raw = self.raw
@@ -158,28 +151,18 @@ class UnBinary(object):
         return '/'.join(relpath)
     
     def __unicode__(self):
-        return self.raw.decode('utf-8')
-
-    def __str__(self):
         return self.raw
     
-    def tree():
-        def fget(self):
-            if not self._tree:
-                self._tree = etree.fromstring(self.raw, parser=XML_PARSER)
-            return self._tree
-        return property(fget=fget)
-    tree = tree()
-    
-    def binary_to_text(self, bin, buf, index=0, depth=0):
+    def binary_to_text(self, base=0, depth=0):
         tag_name = current_map = None
         dynamic_tag = errors = 0
         in_censorship = is_goingdown = False
         state = 'text'
+        index = base
         flags = 0
         
-        while index < len(bin):
-            c, index = read_utf8_char(bin, index)
+        while index < len(self.bin):
+            c, index = read_utf8_char(self.bin, index)
             oc = ord(c)
             
             if state == 'text':
@@ -192,7 +175,7 @@ class UnBinary(object):
                     c = '>>'
                 elif c == '<':
                     c = '<<'
-                buf.write(encode(c))
+                self.buf.write(c.encode('ascii', 'xmlcharrefreplace'))
             
             elif state == 'get flags':
                 if oc == 0:
@@ -205,7 +188,7 @@ class UnBinary(object):
                 state = 'text' if oc == 0 else 'get attr'
                 if flags & FLAG_OPENING:
                     tag = oc
-                    buf.write('<')
+                    self.buf.write('<')
                     if not (flags & FLAG_CLOSING):
                         is_goingdown = True
                     if tag == 0x8000:
@@ -222,7 +205,7 @@ class UnBinary(object):
                         tag_name = '?'+unichr(tag)+'?'
                         current_map = self.tag_to_attr_map[tag]
                         print 'WARNING: tag %s unknown' % unichr(tag)
-                    buf.write(encode(tag_name))
+                    self.buf.write(unicode(tag_name).encode('utf-8'))
                 elif flags & FLAG_CLOSING:
                     if depth == 0:
                         raise LitError('Extra closing tag')
@@ -234,14 +217,15 @@ class UnBinary(object):
                     if not is_goingdown:
                         tag_name = None
                         dynamic_tag = 0
-                        buf.write(' />')
+                        self.buf.write(' />')
                     else:
-                        buf.write('>')
-                        index = self.binary_to_text(bin, buf, index, depth+1)
+                        self.buf.write('>')
+                        index = self.binary_to_text(base=index, depth=depth+1)
                         is_goingdown = False
                         if not tag_name:
                             raise LitError('Tag ends before it begins.')
-                        buf.write(encode(u''.join(('</', tag_name, '>'))))
+                        self.buf.write(u''.join(
+                                ('</', tag_name, '>')).encode('utf-8'))
                         dynamic_tag = 0
                         tag_name = None
                     state = 'text'
@@ -261,7 +245,7 @@ class UnBinary(object):
                         in_censorship = True
                         state = 'get value length'
                         continue
-                    buf.write(' ' + encode(attr) + '=')
+                    self.buf.write(' ' + unicode(attr).encode('utf-8') + '=')
                     if attr in ['href', 'src']:
                         state = 'get href length'
                     else:
@@ -269,39 +253,40 @@ class UnBinary(object):
             
             elif state == 'get value length':
                 if not in_censorship:
-                    buf.write('"')
+                    self.buf.write('"')
                 count = oc - 1
                 if count == 0:
                     if not in_censorship:
-                        buf.write('"')
+                        self.buf.write('"')
                     in_censorship = False
                     state = 'get attr'
                     continue
                 state = 'get value'
                 if oc == 0xffff:
                     continue
-                if count < 0 or count > (len(bin) - index):
+                if count < 0 or count > (len(self.bin) - index):
                     raise LitError('Invalid character count %d' % count)
             
             elif state == 'get value':
                 if count == 0xfffe:
                     if not in_censorship:
-                        buf.write('%s"' % (oc - 1))
+                        self.buf.write('%s"' % (oc - 1))
                     in_censorship = False
                     state = 'get attr'
                 elif count > 0:
                     if not in_censorship:
-                        buf.write(encode(c))
+                        self.buf.write(c.encode(
+                            'ascii', 'xmlcharrefreplace'))
                     count -= 1
                 if count == 0:
                     if not in_censorship:
-                        buf.write('"')
+                        self.buf.write('"')
                     in_censorship = False
                     state = 'get attr'
             
             elif state == 'get custom length':
                 count = oc - 1
-                if count <= 0 or count > len(bin)-index:
+                if count <= 0 or count > len(self.bin)-index:
                     raise LitError('Invalid character count %d' % count)
                 dynamic_tag += 1
                 state = 'get custom'
@@ -311,26 +296,26 @@ class UnBinary(object):
                 tag_name += c
                 count -= 1
                 if count == 0:
-                    buf.write(encode(tag_name))
+                    self.buf.write(unicode(tag_name).encode('utf-8'))
                     state = 'get attr'
             
             elif state == 'get attr length':
                 count = oc - 1
-                if count <= 0 or count > (len(bin) - index):
+                if count <= 0 or count > (len(self.bin) - index):
                     raise LitError('Invalid character count %d' % count)
-                buf.write(' ')
+                self.buf.write(' ')
                 state = 'get custom attr'
             
             elif state == 'get custom attr':
-                buf.write(encode(c))
+                self.buf.write(unicode(c).encode('utf-8'))
                 count -= 1
                 if count == 0:
-                    buf.write('=')
+                    self.buf.write('=')
                     state = 'get value length'
 
             elif state == 'get href length':
                 count = oc - 1
-                if count <= 0 or count > (len(bin) - index):
+                if count <= 0 or count > (len(self.bin) - index):
                     raise LitError('Invalid character count %d' % count)
                 href = ''
                 state = 'get href'
@@ -344,11 +329,10 @@ class UnBinary(object):
                     if frag:
                         path = '#'.join((path, frag))
                     path = urlnormalize(path)
-                    buf.write(encode(u'"%s"' % path))
+                    self.buf.write((u'"%s"' % path).encode('utf-8'))
                     state = 'get attr'
         return index
     
-
 class DirectoryEntry(object):
     def __init__(self, name, section, offset, size):
         self.name = name
@@ -363,7 +347,6 @@ class DirectoryEntry(object):
     def __str__(self):
         return repr(self)
 
-
 class ManifestItem(object):
     def __init__(self, original, internal, mime_type, offset, root, state):
         self.original = original
@@ -391,87 +374,65 @@ class ManifestItem(object):
             % (self.internal, self.path, self.mime_type, self.offset,
                self.root, self.state)
 
-
 def preserve(function):
     def wrapper(self, *args, **kwargs):
-        opos = self.stream.tell()
+        opos = self._stream.tell()
         try:
             return function(self, *args, **kwargs)
         finally:
-            self.stream.seek(opos)
+            self._stream.seek(opos)
     functools.update_wrapper(wrapper, function)
     return wrapper
     
-class LitFile(object):
+class LitReader(object):
     PIECE_SIZE = 16
-
-    def __init__(self, filename_or_stream):
-        if hasattr(filename_or_stream, 'read'):
-            self.stream = filename_or_stream
-        else:
-            self.stream = open(filename_or_stream, 'rb')
-        try:
-            self.opf_path = os.path.splitext(
-                os.path.basename(self.stream.name))[0] + '.opf'
-        except AttributeError:
-            self.opf_path = 'content.opf'
-        if self.magic != 'ITOLITLS':
-            raise LitError('Not a valid LIT file')
-        if self.version != 1:
-            raise LitError('Unknown LIT version %d' % (self.version,))
-        self.read_secondary_header()
-        self.read_header_pieces()
-        self.read_section_names()
-        self.read_manifest()
-        self.read_drm()
-
-    def warn(self, msg):
-        print "WARNING: %s" % (msg,)
+    XML_PARSER = etree.XMLParser(
+        recover=True, resolve_entities=False)
 
     def magic():
         @preserve
         def fget(self):
-            self.stream.seek(0)
-            return self.stream.read(8)
+            self._stream.seek(0)
+            return self._stream.read(8)
         return property(fget=fget)
     magic = magic()
     
     def version():
         def fget(self):
-            self.stream.seek(8)
-            return u32(self.stream.read(4))
+            self._stream.seek(8)
+            return u32(self._stream.read(4))
         return property(fget=fget)
     version = version()
     
     def hdr_len():
         @preserve
         def fget(self):
-            self.stream.seek(12)
-            return int32(self.stream.read(4))
+            self._stream.seek(12)
+            return int32(self._stream.read(4))
         return property(fget=fget)
     hdr_len = hdr_len()
     
     def num_pieces():
         @preserve
         def fget(self):
-            self.stream.seek(16)
-            return int32(self.stream.read(4))
+            self._stream.seek(16)
+            return int32(self._stream.read(4))
         return property(fget=fget)
     num_pieces = num_pieces()
     
     def sec_hdr_len():
         @preserve
         def fget(self):
-            self.stream.seek(20)
-            return int32(self.stream.read(4))
+            self._stream.seek(20)
+            return int32(self._stream.read(4))
         return property(fget=fget)
     sec_hdr_len = sec_hdr_len()
     
     def guid():
         @preserve
         def fget(self):
-            self.stream.seek(24)
-            return self.stream.read(16)
+            self._stream.seek(24)
+            return self._stream.read(16)
         return property(fget=fget)
     guid = guid()
     
@@ -481,27 +442,44 @@ class LitFile(object):
             size = self.hdr_len \
                 + (self.num_pieces * self.PIECE_SIZE) \
                 + self.sec_hdr_len
-            self.stream.seek(0)
-            return self.stream.read(size)
+            self._stream.seek(0)
+            return self._stream.read(size)
         return property(fget=fget)
     header = header()
     
+    def __init__(self, filename_or_stream):
+        if hasattr(filename_or_stream, 'read'):
+            self._stream = filename_or_stream
+        else:
+            self._stream = open(filename_or_stream, 'rb')
+        if self.magic != 'ITOLITLS':
+            raise LitError('Not a valid LIT file')
+        if self.version != 1:
+            raise LitError('Unknown LIT version %d' % (self.version,))
+        self.entries = {}
+        self._read_secondary_header()
+        self._read_header_pieces()
+        self._read_section_names()
+        self._read_manifest()
+        self._read_meta()
+        self._read_drm()
+
     @preserve
     def __len__(self):
-        self.stream.seek(0, 2)
-        return self.stream.tell()
+        self._stream.seek(0, 2)
+        return self._stream.tell()
 
     @preserve
-    def read_raw(self, offset, size):
-        self.stream.seek(offset)
-        return self.stream.read(size)
+    def _read_raw(self, offset, size):
+        self._stream.seek(offset)
+        return self._stream.read(size)
 
-    def read_content(self, offset, size):
-        return self.read_raw(self.content_offset + offset, size)
+    def _read_content(self, offset, size):
+        return self._read_raw(self.content_offset + offset, size)
     
-    def read_secondary_header(self):
+    def _read_secondary_header(self):
         offset = self.hdr_len + (self.num_pieces * self.PIECE_SIZE)
-        bytes = self.read_raw(offset, self.sec_hdr_len)
+        bytes = self._read_raw(offset, self.sec_hdr_len)
         offset = int32(bytes[4:])
         while offset < len(bytes):
             blocktype = bytes[offset:offset+4]
@@ -529,21 +507,21 @@ class LitFile(object):
         if not hasattr(self, 'content_offset'):
             raise LitError('Could not figure out the content offset')
     
-    def read_header_pieces(self):
+    def _read_header_pieces(self):
         src = self.header[self.hdr_len:]
         for i in xrange(self.num_pieces):
             piece = src[i * self.PIECE_SIZE:(i + 1) * self.PIECE_SIZE]
             if u32(piece[4:]) != 0 or u32(piece[12:]) != 0:
                 raise LitError('Piece %s has 64bit value' % repr(piece))
             offset, size = u32(piece), int32(piece[8:])
-            piece = self.read_raw(offset, size)
+            piece = self._read_raw(offset, size)
             if i == 0:
                 continue # Dont need this piece
             elif i == 1:
                 if u32(piece[8:])  != self.entry_chunklen or \
                    u32(piece[12:]) != self.entry_unknown:
                     raise LitError('Secondary header does not match piece')
-                self.read_directory(piece)
+                self._read_directory(piece)
             elif i == 2:
                 if u32(piece[8:])  != self.count_chunklen or \
                    u32(piece[12:]) != self.count_unknown:
@@ -554,13 +532,12 @@ class LitFile(object):
             elif i == 4:
                 self.piece4_guid = piece
                 
-    def read_directory(self, piece):
+    def _read_directory(self, piece):
         if not piece.startswith('IFCM'):
             raise LitError('Header piece #1 is not main directory.')
         chunk_size, num_chunks = int32(piece[8:12]), int32(piece[24:28])
         if (32 + (num_chunks * chunk_size)) != len(piece):
-            raise LitError('IFCM header has incorrect length')
-        self.entries = {}
+            raise LitError('IFCM HEADER has incorrect length')
         for i in xrange(num_chunks):
             offset = 32 + (i * chunk_size)
             chunk = piece[offset:offset + chunk_size]
@@ -594,17 +571,17 @@ class LitFile(object):
                 entry = DirectoryEntry(name, section, offset, size)
                 self.entries[name] = entry
 
-    def read_section_names(self):
+    def _read_section_names(self):
         if '::DataSpace/NameList' not in self.entries:
             raise LitError('Lit file does not have a valid NameList')
         raw = self.get_file('::DataSpace/NameList')
         if len(raw) < 4:
             raise LitError('Invalid Namelist section')
         pos = 4
-        num_sections = u16(raw[2:pos])
-        self.section_names = [""] * num_sections
-        self.section_data = [None] * num_sections
-        for section in xrange(num_sections):
+        self.num_sections = u16(raw[2:pos])
+        self.section_names = [""]*self.num_sections
+        self.section_data = [None]*self.num_sections
+        for section in xrange(self.num_sections):
             size = u16(raw[pos:pos+2])
             pos += 2
             size = size*2 + 2
@@ -614,12 +591,11 @@ class LitFile(object):
                 raw[pos:pos+size].decode('utf-16-le').rstrip('\000')
             pos += size
 
-    def read_manifest(self):
+    def _read_manifest(self):
         if '/manifest' not in self.entries:
             raise LitError('Lit file does not have a valid manifest')
         raw = self.get_file('/manifest')
         self.manifest = {}
-        self.paths = {self.opf_path: None}
         while raw:
             slen, raw = ord(raw[0]), raw[1:]
             if slen == 0: break
@@ -658,9 +634,28 @@ class LitFile(object):
         for item in mlist:
             if item.path[0] == '/':
                 item.path = os.path.basename(item.path)
-            self.paths[item.path] = item
 
-    def read_drm(self):
+    def _pretty_print(self, xml):
+        f = cStringIO.StringIO(xml.encode('utf-8'))
+        doc = etree.parse(f, parser=self.XML_PARSER)
+        pretty = etree.tostring(doc, encoding='ascii', pretty_print=True)
+        return XML_DECL + unicode(pretty)
+                
+    def _read_meta(self):
+        path = 'content.opf'
+        raw = self.get_file('/meta')
+        xml = OPF_DECL
+        try:
+            xml += unicode(UnBinary(raw, path, self.manifest, OPF_MAP))
+        except LitError:
+            if 'PENGUIN group' not in raw: raise
+            print "WARNING: attempting PENGUIN malformed OPF fix"
+            raw = raw.replace(
+                'PENGUIN group', '\x00\x01\x18\x00PENGUIN group', 1)
+            xml += unicode(UnBinary(raw, path, self.manifest, OPF_MAP))
+        self.meta = xml
+
+    def _read_drm(self):
         self.drmlevel = 0
         if '/DRMStorage/Licenses/EUL' in self.entries:
             self.drmlevel = 5
@@ -671,7 +666,7 @@ class LitFile(object):
         else:
             return
         if self.drmlevel < 5:
-            msdes.deskey(self.calculate_deskey(), msdes.DE1)
+            msdes.deskey(self._calculate_deskey(), msdes.DE1)
             bookkey = msdes.des(self.get_file('/DRMStorage/DRMSealed'))
             if bookkey[0] != '\000':
                 raise LitError('Unable to decrypt title key!')
@@ -679,7 +674,7 @@ class LitFile(object):
         else:
             raise DRMError("Cannot access DRM-protected book")
 
-    def calculate_deskey(self):
+    def _calculate_deskey(self):
         hashfiles = ['/meta', '/DRMStorage/DRMSource']
         if self.drmlevel == 3:
             hashfiles.append('/DRMStorage/DRMBookplate')
@@ -703,18 +698,18 @@ class LitFile(object):
     def get_file(self, name):
         entry = self.entries[name]
         if entry.section == 0:
-            return self.read_content(entry.offset, entry.size)
+            return self._read_content(entry.offset, entry.size)
         section = self.get_section(entry.section)
         return section[entry.offset:entry.offset+entry.size]
 
     def get_section(self, section):
         data = self.section_data[section]
         if not data:
-            data = self.get_section_uncached(section)
+            data = self._get_section(section)
             self.section_data[section] = data
         return data
 
-    def get_section_uncached(self, section):
+    def _get_section(self, section):
         name = self.section_names[section]
         path = '::DataSpace/Storage/' + name
         transform = self.get_file(path + '/Transform/List')
@@ -726,29 +721,29 @@ class LitFile(object):
                 raise LitError("ControlData is too short")
             guid = msguid(transform)
             if guid == DESENCRYPT_GUID:
-                content = self.decrypt(content)
+                content = self._decrypt(content)
                 control = control[csize:]
             elif guid == LZXCOMPRESS_GUID:
                 reset_table = self.get_file(
                     '/'.join(('::DataSpace/Storage', name, 'Transform',
                               LZXCOMPRESS_GUID, 'InstanceData/ResetTable')))
-                content = self.decompress(content, control, reset_table)
+                content = self._decompress(content, control, reset_table)
                 control = control[csize:]
             else:
                 raise LitError("Unrecognized transform: %s." % repr(guid))
             transform = transform[16:]
         return content
 
-    def decrypt(self, content):
+    def _decrypt(self, content):
         length = len(content)
         extra = length & 0x7
         if extra > 0:
-            self.warn("content length not a multiple of block size")
+            self._warn("content length not a multiple of block size")
             content += "\0" * (8 - extra)
         msdes.deskey(self.bookkey, msdes.DE1)
         return msdes.des(content)
 
-    def decompress(self, content, control, reset_table):
+    def _decompress(self, content, control, reset_table):
         if len(control) < 32 or control[CONTROL_TAG:CONTROL_TAG+4] != "LZXC":
             raise LitError("Invalid ControlData tag value")
         if len(reset_table) < (RESET_INTERVAL + 8):
@@ -789,7 +784,7 @@ class LitFile(object):
                         result.append(
                             lzx.decompress(content[base:size], window_bytes))
                     except lzx.LZXError:
-                        self.warn("LZX decompression error; skipping chunk")
+                        self._warn("LZX decompression error; skipping chunk")
                     bytes_remaining -= window_bytes
                     base = size
             accum += int32(reset_table[RESET_INTERVAL:])
@@ -799,88 +794,55 @@ class LitFile(object):
             try:
                 result.append(lzx.decompress(content[base:], bytes_remaining))
             except lzx.LZXError:
-                self.warn("LZX decompression error; skipping chunk")
+                self._warn("LZX decompression error; skipping chunk")
             bytes_remaining = 0
         if bytes_remaining > 0:
             raise LitError("Failed to completely decompress section")
         return ''.join(result)
 
-
-class LitReader(object):
-    def __init__(self, filename_or_stream):
-        self._litfile = LitFile(filename_or_stream)
-    
-    def namelist(self):
-        return self._litfile.paths.keys()
-
-    def exists(self, name):
-        return urlunquote(name) in self._litfile.paths
-    
-    def read_xml(self, name):
-        entry = self._litfile.paths[urlunquote(name)] if name else None
-        if entry is None:
-            content = self._read_meta()
-        elif 'spine' in entry.state:
-            internal = '/'.join(('/data', entry.internal, 'content'))
-            raw = self._litfile.get_file(internal)
-            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
-            content = unbin.tree
-        else:
-            raise LitError('Requested non-XML content as XML')
-        return content
-    
-    def read(self, name, pretty_print=False):
-        entry = self._litfile.paths[urlunquote(name)] if name else None
-        if entry is None:
-            meta = self._read_meta()
-            content = OPF_DECL + etree.tostring(
-                meta, encoding='ascii', pretty_print=pretty_print)
-        elif 'spine' in entry.state:
-            internal = '/'.join(('/data', entry.internal, 'content'))
-            raw = self._litfile.get_file(internal)
-            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
-            content = HTML_DECL
+    def get_entry_content(self, entry, pretty_print=False):
+        if 'spine' in entry.state:
+            name = '/'.join(('/data', entry.internal, 'content'))
+            path = entry.path
+            raw = self.get_file(name)
+            decl, map = (OPF_DECL, OPF_MAP) \
+                if name == '/meta' else (HTML_DECL, HTML_MAP)
+            content = decl + unicode(UnBinary(raw, path, self.manifest, map))
             if pretty_print:
-                content += etree.tostring(unbin.tree,
-                    encoding='ascii', pretty_print=True)
+                content = self._pretty_print(content)
+            content = content.encode('utf-8')
         else:
-                content += str(unbin)
-        else:
-            internal = '/'.join(('/data', entry.internal))
-            content = self._litfile.get_file(internal)
+            name = '/'.join(('/data', entry.internal))
+            content = self.get_file(name)
         return content
                     
-    def meta():
-        def fget(self):
-            return self.read(self._litfile.opf_path)
-        return property(fget=fget)
-    meta = meta()
+    def extract_content(self, output_dir=os.getcwdu(), pretty_print=False):
+        output_dir = os.path.abspath(output_dir)
+        try:
+            opf_path = os.path.splitext(
+                os.path.basename(self._stream.name))[0] + '.opf'
+        except AttributeError:
+            opf_path = 'content.opf'
+        opf_path = os.path.join(output_dir, opf_path)
+        self._ensure_dir(opf_path)
+        with open(opf_path, 'wb') as f:
+            xml = self.meta
+            if pretty_print:
+                xml = self._pretty_print(xml)
+            f.write(xml.encode('utf-8'))
+        for entry in self.manifest.values():
+            path = os.path.join(output_dir, entry.path)
+            self._ensure_dir(path)
+            with open(path, 'wb') as f:
+                f.write(self.get_entry_content(entry, pretty_print))
 
     def _ensure_dir(self, path):
         dir = os.path.dirname(path)
         if not os.path.isdir(dir):
             os.makedirs(dir)
 
-    def extract_content(self, output_dir=os.getcwdu(), pretty_print=False):
-        for name in self.namelist():
-            path = os.path.join(output_dir, name)
-            self._ensure_dir(path)
-            with open(path, 'wb') as f:
-                f.write(self.read(name, pretty_print=pretty_print))
-    
-    def _read_meta(self):
-        path = 'content.opf'
-        raw = self._litfile.get_file('/meta')
-        try:
-            unbin = UnBinary(raw, path, self._litfile.manifest, OPF_MAP)
-        except LitError:
-            if 'PENGUIN group' not in raw: raise
-            print "WARNING: attempting PENGUIN malformed OPF fix"
-            raw = raw.replace(
-                'PENGUIN group', '\x00\x01\x18\x00PENGUIN group', 1)
-            unbin = UnBinary(raw, path, self._litfile.manifest, OPF_MAP)
-        return unbin.tree
-
+    def _warn(self, msg):
+        print "WARNING: %s" % (msg,)
 
 def option_parser():
     from calibre.utils.config import OptionParser
@@ -890,8 +852,7 @@ def option_parser():
         help=_('Output directory. Defaults to current directory.'))
     parser.add_option(
         '-p', '--pretty-print', default=False, action='store_true',
-        help=_('Legibly format extracted markup.' \
-                   ' May modify meaningful whitespace.'))
+        help=_('Legibly format extracted markup. May modify meaningful whitespace.'))
     parser.add_option(
         '--verbose', default=False, action='store_true',
         help=_('Useful for debugging.'))

src/calibre/ebooks/lrf/__init__.py

@@ -108,6 +108,8 @@ def option_parser(usage, gui_mode=False):
                       help=_('Add a header to all the pages with title and author.'))
     laf.add_option('--headerformat', default="%t by %a", dest='headerformat', type='string',
                         help=_('Set the format of the header. %a is replaced by the author and %t by the title. Default is %default'))
+    laf.add_option('--header-separation', default=0, type='int', 
+                   help=_('Add extra spacing below the header. Default is %default px.'))
     laf.add_option('--override-css', default=None, dest='_override_css', type='string',
                    help=_('Override the CSS. Can be either a path to a CSS stylesheet or a string. If it is a string it is interpreted as CSS.'))
     laf.add_option('--use-spine', default=False, dest='use_spine', action='store_true',
@@ -260,10 +262,11 @@ def Book(options, logger, font_delta=0, header=None,
         hb.append(header)
         hdr.PutObj(hb)
         ps['headheight'] = profile.header_height
+        ps['headsep']    = options.header_separation
         ps['header']     = hdr
         ps['topmargin']  = 0
         ps['textheight'] = profile.screen_height - (options.bottom_margin + ps['topmargin']) \
-                                                 - ps['headheight'] - profile.fudge
+                                                 - ps['headheight'] - ps['headsep'] - profile.fudge
         
     fontsize = int(10*profile.font_size+font_delta*20)
     baselineskip = fontsize + 20

src/calibre/ebooks/lrf/epub/convert_from.py

@@ -2,14 +2,14 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import os, sys, shutil, logging
-from tempfile import mkdtemp
 from calibre.ebooks.lrf import option_parser as lrf_option_parser
 from calibre.ebooks import ConversionError, DRMError
 from calibre.ebooks.lrf.html.convert_from import process_file as html_process_file
 from calibre.ebooks.metadata.opf import OPF
 from calibre.ebooks.metadata.epub import OCFDirReader
 from calibre.utils.zipfile import ZipFile
-from calibre import __appname__, setup_cli_handlers
+from calibre import setup_cli_handlers
+from calibre.ptempfile import PersistentTemporaryDirectory
 
 
 def option_parser():
@@ -22,17 +22,16 @@ _('''Usage: %prog [options] mybook.epub
 
 def generate_html(pathtoepub, logger):
     if not os.access(pathtoepub, os.R_OK):
-        raise ConversionError, 'Cannot read from ' + pathtoepub
-    tdir = mkdtemp(prefix=__appname__+'_')
-    os.rmdir(tdir)
+        raise ConversionError('Cannot read from ' + pathtoepub)
+    tdir = PersistentTemporaryDirectory('_epub2lrf')
+    #os.rmdir(tdir)
     try:
         ZipFile(pathtoepub).extractall(tdir)
+    except:
+        raise ConversionError, '.epub extraction failed'
     if os.path.exists(os.path.join(tdir, 'META-INF', 'encryption.xml')):
             raise DRMError(os.path.basename(pathtoepub))
-    except:
-        if os.path.exists(tdir) and os.path.isdir(tdir):
-            shutil.rmtree(tdir)
-        raise ConversionError, '.epub extraction failed'
+    
     return tdir
 
 def process_file(path, options, logger=None):

src/calibre/ebooks/lrf/fonts/__init__.py

@@ -12,24 +12,62 @@ except ImportError:
 '''
 Default fonts used in the PRS500
 '''
-from calibre.ebooks.lrf.fonts.prs500 import tt0003m_, tt0011m_, tt0419m_
+
+SYSTEM_FONT_PATH = '/usr/share/fonts/truetype/ttf-liberation/'
 
 FONT_MAP = {
-            'Swis721 BT Roman'     : tt0003m_,
-            'Dutch801 Rm BT Roman' : tt0011m_,
-            'Courier10 BT Roman'   : tt0419m_,
+            'Swis721 BT Roman'     : 'tt0003m_',
+            'Dutch801 Rm BT Roman' : 'tt0011m_',
+            'Courier10 BT Roman'   : 'tt0419m_',
             }
+
+LIBERATION_FONT_MAP = {
+            'Swis721 BT Roman'     : 'LiberationSans_Regular',
+            'Dutch801 Rm BT Roman' : 'LiberationSerif_Regular',
+            'Courier10 BT Roman'   : 'LiberationMono_Regular',
+            }
+
+SYSTEM_FONT_MAP = {}
+for key, val in LIBERATION_FONT_MAP.items():
+    SYSTEM_FONT_MAP[key] = SYSTEM_FONT_PATH + val.replace('_', '-') + '.ttf'
+
 FONT_FILE_MAP = {}
 
 def get_font_path(name):
     if FONT_FILE_MAP.has_key(name) and os.access(FONT_FILE_MAP[name].name, os.R_OK):
         return FONT_FILE_MAP[name].name
+
+    # translate font into file name
+    fname = FONT_MAP[name]
+
+    # first, check configuration in /etc/
+    etc_file = os.path.join(os.path.sep, 'etc', 'calibre', 'fonts', fname + '.ttf')
+    if os.access(etc_file, os.R_OK):
+        return etc_file
+
+    # then, try calibre shipped ones
+    try:
+        try:
+            font_mod = __import__('calibre.ebooks.lrf.fonts.prs500', {}, {}, 
+                                  [fname], -1)
+            getattr(font_mod, fname)
+        except (ImportError, AttributeError):
+            font_mod = __import__('calibre.ebooks.lrf.fonts.liberation', {}, {}, 
+                                  [LIBERATION_FONT_MAP[name]], -1)
         p = PersistentTemporaryFile('.ttf', 'font_')
-    p.write(FONT_MAP[name].font_data)
+        p.write(getattr(font_mod, fname).font_data)
         p.close()
         FONT_FILE_MAP[name] = p
         return p.name
+    except ImportError:
+        pass
     
+    # finally, try system default ones
+    if SYSTEM_FONT_MAP.has_key(name) and os.access(SYSTEM_FONT_MAP[name], os.R_OK):
+        return SYSTEM_FONT_MAP[name]
+
+    # not found
+    raise SystemError, 'font %s (in file %s) not installed' % (name, fname)
 
 def get_font(name, size, encoding='unic'):
     '''

src/calibre/ebooks/lrf/html/convert_from.py

@@ -245,7 +245,6 @@ class HTMLConverter(object, LoggingInterface):
         
         self.override_css = {}
         self.override_pcss = {}
-        self.table_render_job_server = None
          
         if self._override_css is not None:
             if os.access(self._override_css, os.R_OK):
@@ -266,7 +265,6 @@ class HTMLConverter(object, LoggingInterface):
         paths = [os.path.abspath(path) for path in paths]
         paths = [path.decode(sys.getfilesystemencoding()) if not isinstance(path, unicode) else path for path in paths]
         
-        try:
         while len(paths) > 0 and self.link_level <= self.link_levels:
             for path in paths:
                 if path in self.processed_files:
@@ -298,9 +296,6 @@ class HTMLConverter(object, LoggingInterface):
         if self.base_font_size > 0:
             self.log_info('\tRationalizing font sizes...')
             self.book.rationalize_font_sizes(self.base_font_size)
-        finally:
-            if self.table_render_job_server is not None:
-                self.table_render_job_server.killall()
         
     def is_baen(self, soup):
         return bool(soup.find('meta', attrs={'name':'Publisher', 
@@ -1732,15 +1727,11 @@ class HTMLConverter(object, LoggingInterface):
                 self.process_children(tag, tag_css, tag_pseudo_css)
             elif tagname == 'table' and not self.ignore_tables and not self.in_table:
                 if self.render_tables_as_images:
-                    if self.table_render_job_server is None:
-                        from calibre.parallel import Server
-                        self.table_render_job_server = Server(number_of_workers=1)
                     print 'Rendering table...'
                     from calibre.ebooks.lrf.html.table_as_image import render_table
                     pheight = int(self.current_page.pageStyle.attrs['textheight'])
                     pwidth  = int(self.current_page.pageStyle.attrs['textwidth'])
-                    images = render_table(self.table_render_job_server,
-                                          self.soup, tag, tag_css, 
+                    images = render_table(self.soup, tag, tag_css, 
                                           os.path.dirname(self.target_prefix), 
                                           pwidth, pheight, self.profile.dpi, 
                                           self.text_size_multiplier_for_rendered_tables)
@@ -1906,6 +1897,8 @@ def process_file(path, options, logger=None):
     fpb = re.compile(options.force_page_break, re.IGNORECASE) if options.force_page_break else \
          re.compile('$')
     cq = options.chapter_attr.split(',')
+    if len(cq) < 3:
+        raise ValueError('The --chapter-attr setting must have 2 commas.')
     options.chapter_attr = [re.compile(cq[0], re.IGNORECASE), cq[1], 
                             re.compile(cq[2], re.IGNORECASE)]
     options.force_page_break = fpb
@@ -1933,7 +1926,7 @@ def process_file(path, options, logger=None):
     oname = os.path.abspath(os.path.expanduser(oname))
     conv.writeto(oname, lrs=options.lrs)
     run_plugins_on_postprocess(oname, 'lrf')
-    logger.info('Output written to %s', oname)
+    conv.log_info('Output written to %s', oname)
     conv.cleanup()
     return oname
     
@@ -1980,17 +1973,7 @@ def try_opf(path, options, logger):
                         PILImage.open(cover)
                         options.cover = cover
                     except:
-                        for prefix in opf.possible_cover_prefixes():
-                            if options.cover:
-                                break
-                            for suffix in ['.jpg', '.jpeg', '.gif', '.png', '.bmp']:
-                                cpath = os.path.join(os.path.dirname(path), prefix+suffix)
-                                try:
-                                    PILImage.open(cpath)
-                                    options.cover = cpath
-                                    break
-                                except:
-                                    continue
+                        pass
             if not getattr(options, 'cover', None) and orig_cover is not None:
                 options.cover = orig_cover
         if getattr(opf, 'spine', False):

src/calibre/ebooks/lrf/html/table_as_image.py

@@ -6,14 +6,11 @@ __docformat__ = 'restructuredtext en'
 '''
 Render HTML tables as images.
 '''
-import os, tempfile, atexit, shutil, time
-from PyQt4.Qt import QUrl, QApplication, QSize, \
+import os, tempfile, atexit, shutil
+from PyQt4.Qt import QUrl, QApplication, QSize, QEventLoop, \
                      SIGNAL, QPainter, QImage, QObject, Qt
 from PyQt4.QtWebKit import QWebPage
 
-from calibre.parallel import ParallelJob
-
-__app = None
 
 class HTMLTableRenderer(QObject):
 
@@ -27,13 +24,15 @@ class HTMLTableRenderer(QObject):
         self.app = None
         self.width, self.height, self.dpi = width, height, dpi
         self.base_dir = base_dir
+        self.images = []
+        self.tdir = tempfile.mkdtemp(prefix='calibre_render_table')
+        self.loop = QEventLoop()
         self.page = QWebPage()
         self.connect(self.page, SIGNAL('loadFinished(bool)'), self.render_html)
         self.page.mainFrame().setTextSizeMultiplier(factor)
         self.page.mainFrame().setHtml(html, 
                                 QUrl('file:'+os.path.abspath(self.base_dir)))
-        self.images = []
-        self.tdir = tempfile.mkdtemp(prefix='calibre_render_table')
+        
         
     def render_html(self, ok):
         try:
@@ -63,7 +62,7 @@ class HTMLTableRenderer(QObject):
         finally:
             QApplication.quit()
         
-def render_table(server, soup, table, css, base_dir, width, height, dpi, factor=1.0):
+def render_table(soup, table, css, base_dir, width, height, dpi, factor=1.0):
     head = ''
     for e in soup.findAll(['link', 'style']):
         head += unicode(e)+'\n\n'
@@ -83,24 +82,13 @@ def render_table(server, soup, table, css, base_dir, width, height, dpi, factor=
     </body>
 </html>
     '''%(head, width-10, style, unicode(table))
-    job = ParallelJob('render_table',  lambda j : j, None,
-                            args=[html, base_dir, width, height, dpi, factor])
-    server.add_job(job)
-    while not job.has_run:
-        time.sleep(2)
-    
-    if job.exception is not None:
-        print 'Failed to render table'
-        print job.exception
-        print job.traceback
-    images, tdir = job.result
+    images, tdir = do_render(html, base_dir, width, height, dpi, factor)
     atexit.register(shutil.rmtree, tdir)
     return images
     
 def do_render(html, base_dir, width, height, dpi, factor):
-    app = QApplication.instance()
-    if app is None:
-        app = QApplication([])
+    if QApplication.instance() is None:
+        QApplication([])
     tr = HTMLTableRenderer(html, base_dir, width, height, dpi, factor)
-    app.exec_()
+    tr.loop.exec_()
     return tr.images, tr.tdir

src/calibre/ebooks/lrf/lrfparser.py

@@ -77,7 +77,7 @@ class LRFDocument(LRFMetaFile):
         for obj in self.image_map.values() + self.font_map.values():
             open(obj.file, 'wb').write(obj.stream)            
         
-    def to_xml(self):
+    def to_xml(self, write_files=True):
         bookinfo = u'<BookInformation>\n<Info version="1.1">\n<BookInfo>\n'
         bookinfo += u'<Title reading="%s">%s</Title>\n'%(self.metadata.title_reading, self.metadata.title)
         bookinfo += u'<Author reading="%s">%s</Author>\n'%(self.metadata.author_reading, self.metadata.author)
@@ -89,8 +89,9 @@ class LRFDocument(LRFMetaFile):
         bookinfo += u'<FreeText reading="">%s</FreeText>\n</BookInfo>\n<DocInfo>\n'%(self.metadata.free_text,)
         th = self.doc_info.thumbnail
         if th:
-            prefix = sanitize_file_name(self.metadata.title)
+            prefix = sanitize_file_name(self.metadata.title, as_unicode=True)
             bookinfo += u'<CThumbnail file="%s" />\n'%(prefix+'_thumbnail.'+self.doc_info.thumbnail_extension,)
+            if write_files:
                 open(prefix+'_thumbnail.'+self.doc_info.thumbnail_extension, 'wb').write(th)
         bookinfo += u'<Language reading="">%s</Language>\n'%(self.doc_info.language,)
         bookinfo += u'<Creator reading="">%s</Creator>\n'%(self.doc_info.creator,)
@@ -127,12 +128,16 @@ class LRFDocument(LRFMetaFile):
                 objects += unicode(obj)
         styles += '</Style>\n'
         objects += '</Objects>\n'
+        if write_files:
             self.write_files()
         return '<BBeBXylog version="1.0">\n' + bookinfo + pages + styles + objects + '</BBeBXylog>'
         
 def option_parser():
     parser = OptionParser(usage=_('%prog book.lrf\nConvert an LRF file into an LRS (XML UTF-8 encoded) file'))
     parser.add_option('--output', '-o', default=None, help=_('Output LRS file'), dest='out')
+    parser.add_option('--dont-output-resources', default=True, action='store_false', 
+                      help=_('Do not save embedded image and font files to disk'), 
+                      dest='output_resources')
     parser.add_option('--verbose', default=False, action='store_true', dest='verbose')
     return parser
     
@@ -154,7 +159,7 @@ def main(args=sys.argv, logger=None):
     d = LRFDocument(open(args[1], 'rb'))
     d.parse()
     logger.info(_('Creating XML...'))
-    o.write(d.to_xml())
+    o.write(d.to_xml(write_files=opts.output_resources))
     logger.info(_('LRS written to ')+opts.out)
     return 0
 

src/calibre/ebooks/lrf/meta.py

@@ -603,12 +603,18 @@ Show/edit the metadata in an LRF file.\n\n'''),
     parser.add_option("--get-thumbnail", action="store_true", \
                     dest="get_thumbnail", default=False, \
                     help=_("Extract thumbnail from LRF file"))
+    parser.add_option('--publisher', default=None, help=_('Set the publisher'))
+    parser.add_option('--classification', default=None, help=_('Set the book classification'))
+    parser.add_option('--creator', default=None, help=_('Set the book creator'))
+    parser.add_option('--producer', default=None, help=_('Set the book producer'))
     parser.add_option('--get-cover', action='store_true', default=False, 
                       help=_('Extract cover from LRF file. Note that the LRF format has no defined cover, so we use some heuristics to guess the cover.'))
     parser.add_option('--bookid', action='store', type='string', default=None,
                       dest='book_id', help=_('Set book ID'))
-    parser.add_option("-p", "--page", action="store", type="string", \
-                    dest="page", help=_("Don't know what this is for"))
+    # The SumPage element specifies the number of "View"s (visible pages for the BookSetting element conditions) of the content.
+    # Basically, the total pages per the page size, font size, etc. when the LRF is first created. Since this will change as the book is reflowed, it is probably not worth using.
+    #parser.add_option("-p", "--page", action="store", type="string", \
+    #                dest="page", help=_("Don't know what this is for"))
     
     return parser
 
@@ -624,6 +630,8 @@ def set_metadata(stream, mi):
         lrf.free_text = mi.comments
     if mi.author_sort:
         lrf.author_reading = mi.author_sort
+    if mi.publisher:
+        lrf.publisher = mi.publisher
         
 
 def main(args=sys.argv):
@@ -644,10 +652,16 @@ def main(args=sys.argv):
         lrf.author_reading = options.author_reading
     if options.author:
         lrf.author    = options.author
+    if options.publisher:
+        lrf.publisher = options.publisher
+    if options.classification:
+        lrf.classification = options.classification
     if options.category: 
         lrf.category = options.category
-    if options.page: 
-        lrf.page = options.page
+    if options.creator:
+        lrf.creator = options.creator
+    if options.producer:
+        lrf.producer = options.producer
     if options.thumbnail:
         path = os.path.expanduser(os.path.expandvars(options.thumbnail))
         f = open(path, "rb")

src/calibre/ebooks/metadata/html.py

@@ -10,9 +10,10 @@ Try to read metadata from an HTML file.
 import re
 
 from calibre.ebooks.metadata import MetaInformation
+from calibre.ebooks.chardet import xml_to_unicode
 
 def get_metadata(stream):
-    src = stream.read()
+    src = xml_to_unicode(stream.read())[0]
     
     # Title
     title = None

src/calibre/ebooks/metadata/opf2.py

@@ -418,6 +418,7 @@ class OPF(object):
     tags_path       = XPath('descendant::*[re:match(name(), "subject", "i")]')
     isbn_path       = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
                             '(re:match(@scheme, "isbn", "i") or re:match(@opf:scheme, "isbn", "i"))]')
+    identifier_path = XPath('descendant::*[re:match(name(), "identifier", "i")]')
     application_id_path = XPath('descendant::*[re:match(name(), "identifier", "i") and '+
                             '(re:match(@opf:scheme, "calibre|libprs500", "i") or re:match(@scheme, "calibre|libprs500", "i"))]')
     manifest_path   = XPath('descendant::*[re:match(name(), "manifest", "i")]/*[re:match(name(), "item", "i")]')
@@ -719,6 +720,27 @@ class OPF(object):
         return property(fget=fget, fset=fset)
     
     
+    def guess_cover(self):
+        '''
+        Try to guess a cover. Needed for some old/badly formed OPF files.
+        '''
+        if self.base_dir and os.path.exists(self.base_dir):
+            for item in self.identifier_path(self.metadata):
+                scheme = None
+                for key in item.attrib.keys():
+                    if key.endswith('scheme'):
+                        scheme = item.get(key)
+                        break
+                if scheme is None:
+                    continue
+                if item.text:
+                    prefix = item.text.replace('-', '')
+                    for suffix in ['.jpg', '.jpeg', '.gif', '.png', '.bmp']:
+                        cpath = os.access(os.path.join(self.base_dir, prefix+suffix), os.R_OK)
+                        if os.access(os.path.join(self.base_dir, prefix+suffix), os.R_OK):
+                            return cpath
+            
+    
     @apply
     def cover():
         
@@ -728,6 +750,10 @@ class OPF(object):
                     for item in self.guide:
                         if item.type.lower() == t:
                             return item.path
+            try:
+                return self.guess_cover()
+            except:
+                pass
                         
         def fset(self, path):
             if self.guide is not None:

src/calibre/ebooks/mobi/langcodes.py

@@ -305,8 +305,8 @@ IANA_MOBI = \
             'TW': (4, 4)},
      'zu': {None: (53, 0)}}
 
-def iana2mobi(self, icode):
-    subtags = list(code.split('-'))
+def iana2mobi(icode):
+    subtags = list(icode.split('-'))
     langdict = IANA_MOBI[None]
     while len(subtags) > 0:
         lang = subtags.pop(0).lower()
@@ -316,6 +316,8 @@ def iana2mobi(self, icode):
     mcode = langdict[None]
     while len(subtags) > 0:
         subtag = subtags.pop(0)
+        if subtag not in langdict:
+            subtag = subtag.title()
         if subtag not in langdict:
             subtag = subtag.upper()
         if subtag in langdict:

src/calibre/ebooks/mobi/mobiml.py

@@ -0,0 +1,379 @@
+'''
+Transform XHTML/OPS-ish content into Mobipocket HTML 3.2.
+'''
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.cam>'
+
+import sys
+import os
+import copy
+import re
+from lxml import etree
+from calibre.ebooks.oeb.base import namespace, barename
+from calibre.ebooks.oeb.base import XHTML, XHTML_NS
+from calibre.ebooks.oeb.stylizer import Stylizer
+from calibre.ebooks.oeb.transforms.flatcss import KeyMapper
+
+MBP_NS = 'http://mobipocket.com/ns/mbp'
+def MBP(name): return '{%s}%s' % (MBP_NS, name)
+
+MOBI_NSMAP = {None: XHTML_NS, 'mbp': MBP_NS}
+
+HEADER_TAGS = set(['h1', 'h2', 'h3', 'h4', 'h5', 'h6'])
+NESTABLE_TAGS = set(['ol', 'ul', 'li', 'table', 'tr', 'td', 'th'])
+TABLE_TAGS = set(['table', 'tr', 'td', 'th'])
+SPECIAL_TAGS = set(['hr', 'br'])
+CONTENT_TAGS = set(['img', 'hr', 'br'])
+
+PAGE_BREAKS = set(['always', 'odd', 'even'])
+
+COLLAPSE = re.compile(r'[ \t\r\n\v]+')
+
+def asfloat(value):
+    if not isinstance(value, (int, long, float)):
+        return 0.0
+    return float(value)
+
+class BlockState(object):
+    def __init__(self, body):
+        self.body = body
+        self.nested = []
+        self.para = None
+        self.inline = None
+        self.anchor = None
+        self.vpadding = 0.
+        self.vmargin = 0.
+        self.pbreak = False
+        self.istate = None
+        self.content = False
+
+class FormatState(object):
+    def __init__(self):
+        self.left = 0.
+        self.halign = 'auto'
+        self.indent = 0.
+        self.fsize = 3
+        self.ids = set()
+        self.valign = 'baseline'
+        self.italic = False
+        self.bold = False
+        self.preserve = False
+        self.family = 'serif'
+        self.href = None
+        self.list_num = 0
+        self.attrib = {}
+
+    def __eq__(self, other):
+        return self.fsize == other.fsize \
+               and self.italic == other.italic \
+               and self.bold == other.bold \
+               and self.href == other.href \
+               and self.valign == other.valign \
+               and self.preserve == other.preserve \
+               and self.family == other.family
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+
+class MobiMLizer(object):
+    def transform(self, oeb, context):
+        oeb.logger.info('Converting XHTML to Mobipocket markup...')
+        self.oeb = oeb
+        self.profile = profile = context.dest
+        self.fnums = fnums = dict((v, k) for k, v in profile.fnums.items())
+        self.fmap = KeyMapper(profile.fbase, profile.fbase, fnums.keys())
+        self.remove_html_cover()
+        self.mobimlize_spine()
+
+    def remove_html_cover(self):
+        oeb = self.oeb
+        if not oeb.metadata.cover \
+           or 'cover' not in oeb.guide:
+            return
+        href = oeb.guide['cover'].href
+        del oeb.guide['cover']
+        item = oeb.manifest.hrefs[href]
+        oeb.manifest.remove(item)
+
+    def mobimlize_spine(self):
+        for item in self.oeb.spine:
+            stylizer = Stylizer(item.data, item.href, self.oeb, self.profile)
+            body = item.data.find(XHTML('body'))
+            nroot = etree.Element(XHTML('html'), nsmap=MOBI_NSMAP)
+            nbody = etree.SubElement(nroot, XHTML('body'))
+            self.mobimlize_elem(body, stylizer, BlockState(nbody),
+                                [FormatState()])
+            item.data = nroot
+
+    def mobimlize_font(self, ptsize):
+        return self.fnums[self.fmap[ptsize]]
+
+    def mobimlize_measure(self, ptsize):
+        if isinstance(ptsize, basestring):
+            return ptsize
+        fbase = self.profile.fbase
+        if ptsize < fbase:
+            return "%dpt" % int(round(ptsize))
+        return "%dem" % int(round(ptsize / fbase))
+
+    def preize_text(self, text):
+        text = unicode(text).replace(u' ', u'\xa0')
+        text = text.replace('\r\n', '\n')
+        text = text.replace('\r', '\n')
+        lines = text.split('\n')
+        result = lines[:1]
+        for line in lines[1:]:
+            result.append(etree.Element(XHTML('br')))
+            if line:
+                result.append(line)
+        return result
+    
+    def mobimlize_content(self, tag, text, bstate, istates):
+        bstate.content = True
+        istate = istates[-1]
+        para = bstate.para
+        if tag in SPECIAL_TAGS and not text:
+            para = para if para is not None else bstate.body
+        elif para is None:
+            body = bstate.body
+            if bstate.pbreak:
+                etree.SubElement(body, MBP('pagebreak'))
+                bstate.pbreak = False
+            if istate.ids:
+                for id in istate.ids:
+                    etree.SubElement(body, XHTML('a'), attrib={'id': id})
+                istate.ids.clear()
+            bstate.istate = None
+            bstate.anchor = None
+            parent = bstate.nested[-1] if bstate.nested else bstate.body
+            indent = istate.indent
+            left = istate.left
+            if indent < 0 and abs(indent) < left:
+                left += indent
+                indent = 0
+            elif indent != 0 and abs(indent) < self.profile.fbase:
+                indent = (indent / abs(indent)) * self.profile.fbase
+            if tag in NESTABLE_TAGS:
+                para = wrapper = etree.SubElement(parent, XHTML(tag))
+                bstate.nested.append(para)
+                if tag == 'li' and len(istates) > 1:
+                    istates[-2].list_num += 1
+                    para.attrib['value'] = str(istates[-2].list_num)
+            elif left > 0 and indent >= 0:
+                para = wrapper = etree.SubElement(parent, XHTML('blockquote'))
+                para = wrapper
+                emleft = int(round(left / self.profile.fbase)) - 1
+                emleft = min((emleft, 10))
+                while emleft > 0:
+                    para = etree.SubElement(para, XHTML('blockquote'))
+                    emleft -= 1
+            else:
+                ptag = tag if tag in HEADER_TAGS else 'p'
+                para = wrapper = etree.SubElement(parent, XHTML(ptag))
+            bstate.inline = bstate.para = para
+            vspace = bstate.vpadding + bstate.vmargin
+            bstate.vpadding = bstate.vmargin = 0
+            if tag not in TABLE_TAGS:
+                wrapper.attrib['height'] = self.mobimlize_measure(vspace)
+                para.attrib['width'] = self.mobimlize_measure(indent)
+            elif tag == 'table' and vspace > 0:
+                body = bstate.body
+                vspace = int(round(vspace / self.profile.fbase))
+                index = max((0, len(body) - 1))
+                while vspace > 0:
+                    body.insert(index, etree.Element(XHTML('br')))
+                    vspace -= 1
+            if istate.halign != 'auto':
+                para.attrib['align'] = istate.halign
+        pstate = bstate.istate
+        if tag in CONTENT_TAGS:
+            bstate.inline = para
+            pstate = bstate.istate = None
+            etree.SubElement(para, XHTML(tag), attrib=istate.attrib)
+        elif tag in TABLE_TAGS:
+            para.attrib['valign'] = 'top'
+        if not text:
+            return
+        if not pstate or istate != pstate:
+            inline = para
+            valign = istate.valign
+            fsize = istate.fsize
+            href = istate.href
+            if not href:
+                bstate.anchor = None
+            elif pstate and pstate.href == href:
+                inline = bstate.anchor
+            else:
+                inline = etree.SubElement(inline, XHTML('a'), href=href)
+                bstate.anchor = inline
+            if valign == 'super':
+                inline = etree.SubElement(inline, XHTML('sup'))
+            elif valign == 'sub':
+                inline = etree.SubElement(inline, XHTML('sub'))
+            if istate.family == 'monospace':
+                inline = etree.SubElement(inline, XHTML('tt'))
+            if fsize != 3:
+                inline = etree.SubElement(inline, XHTML('font'),
+                                          size=str(fsize))
+            if istate.italic:
+                inline = etree.SubElement(inline, XHTML('i'))
+            if istate.bold:
+                inline = etree.SubElement(inline, XHTML('b'))
+            bstate.inline = inline
+        bstate.istate = istate
+        inline = bstate.inline
+        content = self.preize_text(text) if istate.preserve else [text]
+        for item in content:
+            if isinstance(item, basestring):
+                if len(inline) == 0:
+                    inline.text = (inline.text or '') + item
+                else:
+                    last = inline[-1]
+                    last.tail = (last.tail or '') + item
+            else:
+                inline.append(item)
+    
+    def mobimlize_elem(self, elem, stylizer, bstate, istates):
+        if not isinstance(elem.tag, basestring) \
+           or namespace(elem.tag) != XHTML_NS:
+            return
+        style = stylizer.style(elem)
+        if style['display'] == 'none' \
+           or style['visibility'] == 'hidden':
+            return
+        tag = barename(elem.tag)
+        istate = copy.copy(istates[-1])
+        istate.list_num = 0
+        istates.append(istate)
+        left = 0
+        display = style['display']
+        isblock = not display.startswith('inline')
+        isblock = isblock and style['float'] == 'none'
+        isblock = isblock and tag != 'br'
+        if isblock:
+            bstate.para = None
+            istate.halign = style['text-align']
+            istate.indent = style['text-indent']
+            if style['margin-left'] == 'auto' \
+               and style['margin-right'] == 'auto':
+                istate.halign = 'center'
+            margin = asfloat(style['margin-left'])
+            padding = asfloat(style['padding-left'])
+            if tag != 'body':
+                left = margin + padding
+            istate.left += left
+            vmargin = asfloat(style['margin-top'])
+            bstate.vmargin = max((bstate.vmargin, vmargin))
+            vpadding = asfloat(style['padding-top'])
+            if vpadding > 0:
+                bstate.vpadding += bstate.vmargin
+                bstate.vmargin = 0
+                bstate.vpadding += vpadding
+        elif not istate.href:
+            margin = asfloat(style['margin-left'])
+            padding = asfloat(style['padding-left'])
+            lspace = margin + padding
+            if lspace > 0:
+                spaces = int(round((lspace * 3) / style['font-size']))
+                elem.text = (u'\xa0' * spaces) + (elem.text or '')
+            margin = asfloat(style['margin-right'])
+            padding = asfloat(style['padding-right'])
+            rspace = margin + padding
+            if rspace > 0:
+                spaces = int(round((rspace * 3) / style['font-size']))
+                if len(elem) == 0:
+                    elem.text = (elem.text or '') + (u'\xa0' * spaces)
+                else:
+                    last = elem[-1]
+                    last.text = (last.text or '') + (u'\xa0' * spaces)
+        if bstate.content and style['page-break-before'] in PAGE_BREAKS:
+            bstate.pbreak = True
+        istate.fsize = self.mobimlize_font(style['font-size'])
+        istate.italic = True if style['font-style'] == 'italic' else False
+        weight = style['font-weight']
+        istate.bold = weight in ('bold', 'bolder') or asfloat(weight) > 400
+        istate.preserve = (style['white-space'] in ('pre', 'pre-wrap'))
+        if 'monospace' in style['font-family']:
+            istate.family = 'monospace'
+        elif 'sans-serif' in style['font-family']:
+            istate.family = 'sans-serif'
+        else:
+            istate.family = 'serif'
+        valign = style['vertical-align']
+        if valign in ('super', 'sup') or asfloat(valign) > 0:
+            istate.valign = 'super'
+        elif valign == 'sub'  or asfloat(valign) < 0:
+            istate.valign = 'sub'
+        else:
+            istate.valign = 'baseline'
+        if 'id' in elem.attrib:
+            istate.ids.add(elem.attrib['id'])
+        if 'name' in elem.attrib:
+            istate.ids.add(elem.attrib['name'])
+        if tag == 'a' and 'href' in elem.attrib:
+            istate.href = elem.attrib['href']
+        istate.attrib.clear()
+        if tag == 'img' and 'src' in elem.attrib:
+            istate.attrib['src'] = elem.attrib['src']
+            istate.attrib['align'] = 'baseline'
+            for prop in ('width', 'height'):
+                if style[prop] != 'auto':
+                    value = style[prop]
+                    if value == getattr(self.profile, prop):
+                        result = '100%'
+                    else:
+                        ems = int(round(value / self.profile.fbase))
+                        result = "%dem" % ems
+                    istate.attrib[prop] = result
+        elif tag == 'hr' and asfloat(style['width']) > 0:
+            prop = style['width'] / self.profile.width
+            istate.attrib['width'] = "%d%%" % int(round(prop * 100))
+        elif display == 'table':
+            tag = 'table'
+        elif display == 'table-row':
+            tag = 'tr'
+        elif display == 'table-cell':
+            tag = 'td'
+        text = None
+        if elem.text:
+            if istate.preserve:
+                text = elem.text
+            elif len(elem) > 0 and elem.text.isspace():
+                text = None
+            else:
+                text = COLLAPSE.sub(' ', elem.text)
+        if text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS:
+            self.mobimlize_content(tag, text, bstate, istates)
+        for child in elem:
+            self.mobimlize_elem(child, stylizer, bstate, istates)
+            tail = None
+            if child.tail:
+                if istate.preserve:
+                    tail = child.tail
+                elif bstate.para is None and child.tail.isspace():
+                    tail = None
+                else:
+                    tail = COLLAPSE.sub(' ', child.tail)
+            if tail:
+                self.mobimlize_content(tag, tail, bstate, istates)
+        if bstate.content and style['page-break-after'] in PAGE_BREAKS:
+            bstate.pbreak = True
+        if isblock:
+            para = bstate.para
+            if para is not None and para.text == u'\xa0':
+                para.getparent().replace(para, etree.Element(XHTML('br')))
+            bstate.para = None
+            bstate.istate = None
+            vmargin = asfloat(style['margin-bottom'])
+            bstate.vmargin = max((bstate.vmargin, vmargin))
+            vpadding = asfloat(style['padding-bottom'])
+            if vpadding > 0:
+                bstate.vpadding += bstate.vmargin
+                bstate.vmargin = 0
+                bstate.vpadding += vpadding
+        if tag in NESTABLE_TAGS and bstate.nested:
+            bstate.nested.pop()
+        istates.pop()

src/calibre/ebooks/mobi/palmdoc.py

@@ -74,12 +74,13 @@ def compress_doc(data):
         else:
             j = i
             binseq = [ch]
-            while True:
+            while j < ldata:
                 ch = data[j]
                 och = ord(ch)
                 if och < 1 or (och > 8 and och < 0x80):
                     break
                 binseq.append(ch)
+                j += 1
             out.write(pack('>B', len(binseq)))
             out.write(''.join(binseq))
             i += len(binseq) - 1

src/calibre/ebooks/mobi/reader.py

@@ -22,6 +22,7 @@ from calibre.ebooks.mobi.langcodes import main_language, sub_language
 from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.metadata.opf import OPFCreator
 from calibre.ebooks.metadata.toc import TOC
+from calibre import sanitize_file_name
 
 class EXTHHeader(object):
     
@@ -200,7 +201,8 @@ class MobiReader(object):
         guide = soup.find('guide')
         for elem in soup.findAll(['metadata', 'guide']):
             elem.extract()
-        htmlfile = os.path.join(output_dir, self.name+'.html')
+        htmlfile = os.path.join(output_dir, 
+                                sanitize_file_name(self.name)+'.html')
         try:
             for ref in guide.findAll('reference', href=True):
                 ref['href'] = os.path.basename(htmlfile)+ref['href']
@@ -232,6 +234,15 @@ class MobiReader(object):
     def cleanup_soup(self, soup):
         if self.verbose:
             print 'Replacing height, width and align attributes'
+        size_map = {
+                    'xx-small' : '0.5',
+                    'x-small'  : '1',
+                    'small'    : '2',
+                    'medium'   : '3',
+                    'large'    : '4',
+                    'x-large'  : '5',
+                    'xx-large' : '6',
+                    }
         for tag in soup.recursiveChildGenerator():
             if not isinstance(tag, Tag): continue
             styles = []
@@ -246,6 +257,8 @@ class MobiReader(object):
                 pass
             try:
                 styles.append('text-indent: %s' % tag['width'])
+                if tag['width'].startswith('-'):
+                    styles.append('margin-left: %s'%(tag['width'][1:]))
                 del tag['width']
             except KeyError:
                 pass
@@ -257,6 +270,15 @@ class MobiReader(object):
             if styles:
                 tag['style'] = '; '.join(styles)
                 
+            if tag.name.lower() == 'font':
+                sz = tag.get('size', '')
+                try:
+                    float(sz)
+                except ValueError:
+                    sz = sz.lower()
+                    if sz in size_map.keys():
+                        tag['size'] = size_map[sz]
+    
     def create_opf(self, htmlfile, guide=None):
         mi = self.book_header.exth.mi
         opf = OPFCreator(os.path.dirname(htmlfile), mi)
@@ -289,6 +311,7 @@ class MobiReader(object):
                     except:
                         text = ''
                     text = ent_pat.sub(entity_to_unicode, text)
+                    if a['href'].startswith('#'):
                         tocobj.add_item(toc.partition('#')[0], a['href'][1:], text)
             if tocobj is not None:
                 opf.set_toc(tocobj)
@@ -314,6 +337,8 @@ class MobiReader(object):
             if flags & 1:
                 num += sizeof_trailing_entry(data, size - num)
             flags >>= 1
+        if self.book_header.extra_flags & 1:
+            num += (ord(data[size - num - 1]) & 0x3) + 1
         return num
 
     def text_section(self, index):

src/calibre/ebooks/mobi/writer.py

@@ -19,14 +19,23 @@ from collections import defaultdict
 from urlparse import urldefrag
 from lxml import etree
 from PIL import Image
+from calibre.ebooks.oeb.base import XML_NS, XHTML, XHTML_NS, OEB_DOCS, \
+    OEB_RASTER_IMAGES
+from calibre.ebooks.oeb.base import xpath, barename, namespace, prefixname
+from calibre.ebooks.oeb.base import FauxLogger, OEBBook
+from calibre.ebooks.oeb.profile import Context
+from calibre.ebooks.oeb.transforms.flatcss import CSSFlattener
+from calibre.ebooks.oeb.transforms.rasterize import SVGRasterizer
+from calibre.ebooks.oeb.transforms.trimmanifest import ManifestTrimmer
 from calibre.ebooks.mobi.palmdoc import compress_doc
 from calibre.ebooks.mobi.langcodes import iana2mobi
-from calibre.ebooks.lit.oeb import XML_NS, XHTML, XHTML_NS, OEB_DOCS
-from calibre.ebooks.lit.oeb import xpath, barename, namespace, prefixname
-from calibre.ebooks.lit.oeb import FauxLogger, OEBBook
+from calibre.ebooks.mobi.mobiml import MBP_NS, MBP, MobiMLizer
 
-MBP_NS = 'http://mobipocket.com/ns/mbp'
-def MBP(name): return '{%s}%s' % (MBP_NS, name)
+# TODO:
+# - Allow override CSS (?)
+# - Generate index records
+# - Generate in-content ToC
+# - Command line options, etc.
 
 EXTH_CODES = {
     'creator': 100,
@@ -43,33 +52,54 @@ EXTH_CODES = {
     'title': 503,
     }
 
+RECORD_SIZE = 0x1000
+
 UNCOMPRESSED = 1
 PALMDOC = 2
 HUFFDIC = 17480
 
-def encode(data):
-    return data.encode('ascii', 'xmlcharrefreplace')
+MAX_IMAGE_SIZE = 63 * 1024
+MAX_THUMB_SIZE = 16 * 1024
+MAX_THUMB_DIMEN = (180, 240)
 
+def encode(data):
+    return data.encode('utf-8')
+
+# Almost like the one for MS LIT, but not quite.
+DECINT_FORWARD = 0
+DECINT_BACKWARD = 1
+def decint(value, direction):
+    bytes = []
+    while True:
+        b = value & 0x7f
+        value >>= 7
+        bytes.append(b)
+        if value == 0:
+            break
+    if direction == DECINT_FORWARD:
+        bytes[0] |= 0x80
+    elif direction == DECINT_BACKWARD:
+        bytes[-1] |= 0x80
+    return ''.join(chr(b) for b in reversed(bytes))
 
 
 class Serializer(object):
     NSRMAP = {'': None, XML_NS: 'xml', XHTML_NS: '', MBP_NS: 'mbp'}
     
     def __init__(self, oeb, images):
+        oeb.logger.info('Serializing markup content...')
         self.oeb = oeb
         self.images = images
         self.id_offsets = {}
         self.href_offsets = defaultdict(list)
+        self.breaks = []
         buffer = self.buffer = StringIO()
         buffer.write('<html>')
         self.serialize_head()
         self.serialize_body()
         buffer.write('</html>')
         self.fixup_links()
-        self.raw = buffer.getvalue()
-
-    def __str__(self):
-        return self.raw
+        self.text = buffer.getvalue()
 
     def serialize_head(self):
         buffer = self.buffer
@@ -80,8 +110,12 @@ class Serializer(object):
 
     def serialize_guide(self):
         buffer = self.buffer
+        hrefs = self.oeb.manifest.hrefs
         buffer.write('<guide>')
         for ref in self.oeb.guide.values():
+            path, frag = urldefrag(ref.href)
+            if hrefs[path].media_type not in OEB_DOCS:
+                continue
             buffer.write('<reference title="%s" type="%s" '
                          % (ref.title, ref.type))
             self.serialize_href(ref.href)
@@ -100,8 +134,7 @@ class Serializer(object):
         if item and item.spine_position is None:
             return False
         id =  item.id if item else base.id
-        frag = frag if frag else 'calibre_top'
-        href = '#'.join((id, frag))
+        href = '#'.join((id, frag)) if frag else id
         buffer.write('filepos=')
         self.href_offsets[href].append(buffer.tell())
         buffer.write('0000000000')
@@ -110,23 +143,26 @@ class Serializer(object):
     def serialize_body(self):
         buffer = self.buffer
         buffer.write('<body>')
-        for item in self.oeb.spine:
+        spine = [item for item in self.oeb.spine if item.linear]
+        spine.extend([item for item in self.oeb.spine if not item.linear])
+        for item in spine:
             self.serialize_item(item)
         buffer.write('</body>')
 
     def serialize_item(self, item):
         buffer = self.buffer
-        buffer.write('<mbp:pagebreak/>')
-        # TODO: Figure out how to make the 'crossable' stuff work for
-        # non-"linear" spine items.
-        self.id_offsets[item.id + '#calibre_top'] = buffer.tell()
+        if not item.linear:
+            self.breaks.append(buffer.tell() - 1)
+        self.id_offsets[item.id] = buffer.tell()
         for elem in item.data.find(XHTML('body')):
             self.serialize_elem(elem, item)
+        buffer.write('<mbp:pagebreak/>')
 
     def serialize_elem(self, elem, item, nsrmap=NSRMAP):
-        if namespace(elem.tag) not in nsrmap:
-            return
         buffer = self.buffer
+        if not isinstance(elem.tag, basestring) \
+           or namespace(elem.tag) not in nsrmap:
+            return
         hrefs = self.oeb.manifest.hrefs
         tag = prefixname(elem.tag, nsrmap)
         for attr in ('name', 'id'):
@@ -134,6 +170,9 @@ class Serializer(object):
                 id = '#'.join((item.id, elem.attrib[attr]))
                 self.id_offsets[id] = buffer.tell()
                 del elem.attrib[attr]
+        if tag == 'a' and not elem.attrib \
+           and not len(elem) and not elem.text:
+            return
         buffer.write('<')
         buffer.write(tag)
         if elem.attrib:
@@ -149,18 +188,29 @@ class Serializer(object):
                     index = self.images[val]
                     buffer.write('recindex="%05d"' % index)
                     continue
-                buffer.write('%s="%s"' % (attr, val))
+                buffer.write(attr)
+                buffer.write('="')
+                self.serialize_text(val, quot=True)
+                buffer.write('"')
         if elem.text or len(elem) > 0:
             buffer.write('>')
             if elem.text:
-                buffer.write(encode(elem.text))
+                self.serialize_text(elem.text)
             for child in elem:
                 self.serialize_elem(child, item)
+                if child.tail:
+                    self.serialize_text(child.tail)
             buffer.write('</%s>' % tag)
         else:
             buffer.write('/>')
-        if elem.tail:
-            buffer.write(encode(elem.tail))
+
+    def serialize_text(self, text, quot=False):
+        text = text.replace('&', '&amp;')
+        text = text.replace('<', '&lt;')
+        text = text.replace('>', '&gt;')
+        if quot:
+            text = text.replace('"', '&quot;')
+        self.buffer.write(encode(text))
 
     def fixup_links(self):
         buffer = self.buffer
@@ -172,8 +222,8 @@ class Serializer(object):
 
     
 class MobiWriter(object):
-    def __init__(self, compress=None, logger=FauxLogger()):
-        self._compress = compress or UNCOMPRESSED
+    def __init__(self, compression=None, logger=FauxLogger()):
+        self._compression = compression or UNCOMPRESSED
         self._logger = logger
 
     def dump(self, oeb, path):
@@ -207,42 +257,113 @@ class MobiWriter(object):
         index = 1
         self._images = images = {}
         for item in self._oeb.manifest.values():
-            if item.media_type.startswith('image/'):
+            if item.media_type in OEB_RASTER_IMAGES:
                 images[item.href] = index
                 index += 1
 
+    def _read_text_record(self, text):
+        pos = text.tell()
+        text.seek(0, 2)
+        npos = min((pos + RECORD_SIZE, text.tell()))
+        last = ''
+        while not last.decode('utf-8', 'ignore'):
+            size = len(last) + 1
+            text.seek(npos - size)
+            last = text.read(size)
+        extra = 0
+        try:
+            last.decode('utf-8')
+        except UnicodeDecodeError:
+            prev = len(last)
+            while True:
+                text.seek(npos - prev)
+                last = text.read(len(last) + 1)
+                try:
+                    last.decode('utf-8')
+                except UnicodeDecodeError:
+                    pass
+                else:
+                    break
+            extra = len(last) - prev
+        text.seek(pos)
+        data = text.read(RECORD_SIZE)
+        overlap = text.read(extra)
+        text.seek(npos)
+        return data, overlap
+                
     def _generate_text(self):
         serializer = Serializer(self._oeb, self._images)
-        text = str(serializer)
+        breaks = serializer.breaks
+        text = serializer.text
         self._text_length = len(text)
         text = StringIO(text)
         nrecords = 0
-        data = text.read(0x1000)
+        offset = 0
+        data, overlap = self._read_text_record(text)
         while len(data) > 0:
-            nrecords += 1
-            if self._compress == PALMDOC:
+            if self._compression == PALMDOC:
                 data = compress_doc(data)
-            # Without the NUL Mobipocket Desktop 6.2 will thrash.  Why?
-            self._records.append(data + '\0')
-            data = text.read(0x1000)
+            record = StringIO()
+            record.write(data)
+            record.write(overlap)
+            record.write(pack('>B', len(overlap)))
+            nextra = 0
+            pbreak = 0
+            running = offset
+            while breaks and (breaks[0] - offset) < RECORD_SIZE:
+                pbreak = (breaks.pop(0) - running) >> 3
+                encoded = decint(pbreak, DECINT_FORWARD)
+                record.write(encoded)
+                running += pbreak << 3
+                nextra += len(encoded)
+            lsize = 1
+            while True:
+                size = decint(nextra + lsize, DECINT_BACKWARD)
+                if len(size) == lsize:
+                    break
+                lsize += 1
+            record.write(size)
+            self._records.append(record.getvalue())
+            nrecords += 1
+            offset += RECORD_SIZE
+            data, overlap = self._read_text_record(text)
         self._text_nrecords = nrecords
 
     def _rescale_image(self, data, maxsizeb, dimen=None):
+        image = Image.open(StringIO(data))
+        format = image.format
+        changed = False
+        if image.format not in ('JPEG', 'GIF'):
+            format = 'GIF'
+            changed = True
         if dimen is not None:
-            image = Image.open(StringIO(data))
             image.thumbnail(dimen, Image.ANTIALIAS)
+            changed = True
+        if changed:
             data = StringIO()
-            image.save(data, image.format)
+            image.save(data, format)
             data = data.getvalue()
-        if len(data) < maxsizeb:
+        if len(data) <= maxsizeb:
             return data
-        image = Image.open(StringIO(data))
+        image = image.convert('RGBA')
         for quality in xrange(95, -1, -1):
             data = StringIO()
             image.save(data, 'JPEG', quality=quality)
             data = data.getvalue()
             if len(data) <= maxsizeb:
-                break
+                return data
+        width, height = image.size
+        for scale in xrange(99, 0, -1):
+            scale = scale / 100.
+            data = StringIO()
+            scaled = image.copy()
+            size = (int(width * scale), (height * scale))
+            scaled.thumbnail(size, Image.ANTIALIAS)
+            scaled.save(data, 'JPEG', quality=0)
+            data = data.getvalue()
+            if len(data) <= maxsizeb:
+                return data
+        # Well, we tried?
         return data
         
     def _generate_images(self):
@@ -252,35 +373,37 @@ class MobiWriter(object):
         coverid = metadata.cover[0] if metadata.cover else None
         for _, href in images:
             item = self._oeb.manifest.hrefs[href]
-            maxsizek = 89 if coverid == item.id else 63
-            maxsizeb = maxsizek * 1024
-            data = self._rescale_image(item.data, maxsizeb)
+            data = self._rescale_image(item.data, MAX_IMAGE_SIZE)
             self._records.append(data)
     
     def _generate_record0(self):
         metadata = self._oeb.metadata
         exth = self._build_exth()
         record0 = StringIO()
-        record0.write(pack('>HHIHHHH', self._compress, 0, self._text_length,
-            self._text_nrecords, 0x1000, 0, 0))
+        record0.write(pack('>HHIHHHH', self._compression, 0,
+            self._text_length, self._text_nrecords, RECORD_SIZE, 0, 0))
         uid = random.randint(0, 0xffffffff)
         title = str(metadata.title[0])
         record0.write('MOBI')
-        record0.write(pack('>IIIII', 0xe8, 2, 65001, uid, 5))
+        record0.write(pack('>IIIII', 0xe8, 2, 65001, uid, 6))
         record0.write('\xff' * 40)
         record0.write(pack('>I', self._text_nrecords + 1))
         record0.write(pack('>II', 0xe8 + 16 + len(exth), len(title)))
         record0.write(iana2mobi(str(metadata.language[0])))
         record0.write('\0' * 8)
-        record0.write(pack('>II', 5, self._text_nrecords + 1))
+        record0.write(pack('>II', 6, self._text_nrecords + 1))
         record0.write('\0' * 16)
         record0.write(pack('>I', 0x50))
         record0.write('\0' * 32)
         record0.write(pack('>IIII', 0xffffffff, 0xffffffff, 0, 0))
-        # TODO: What the hell are these fields?
+        # The '5' is a bitmask of extra record data at the end:
+        #   - 0x1: <extra multibyte bytes><size> (?)
+        #   - 0x4: <uncrossable breaks><size>
+        # Of course, the formats aren't quite the same.
+        # TODO: What the hell are the rest of these fields?
         record0.write(pack('>IIIIIIIIIIIIIIIII',
             0, 0, 0, 0xffffffff, 0, 0xffffffff, 0, 0xffffffff, 0, 0xffffffff,
-            0, 0xffffffff, 0, 0xffffffff, 0xffffffff, 1, 0xffffffff))
+            0, 0xffffffff, 0, 0xffffffff, 0xffffffff, 5, 0xffffffff))
         record0.write(exth)
         record0.write(title)
         record0 = record0.getvalue()
@@ -294,13 +417,13 @@ class MobiWriter(object):
             if term not in EXTH_CODES: continue
             code = EXTH_CODES[term]
             for item in oeb.metadata[term]:
-                data = str(item)
+                data = unicode(item).encode('utf-8')
                 exth.write(pack('>II', code, len(data) + 8))
                 exth.write(data)
                 nrecs += 1
         if oeb.metadata.cover:
             id = str(oeb.metadata.cover[0])
-            item = oeb.manifest[id]
+            item = oeb.manifest.ids[id]
             href = item.href
             index = self._images[href] - 1
             exth.write(pack('>III', 0xc9, 0x0c, index))
@@ -315,9 +438,7 @@ class MobiWriter(object):
         return ''.join(exth)
 
     def _add_thumbnail(self, item):
-        maxsizeb = 16 * 1024
-        dimen = (180, 240)
-        data = self._rescale_image(item.data, maxsizeb, dimen)
+        data = self._rescale_image(item.data, MAX_THUMB_SIZE, MAX_THUMB_DIMEN)
         manifest = self._oeb.manifest
         id, href = manifest.generate('thumbnail', 'thumbnail.jpeg')
         manifest.add(id, href, 'image/jpeg', data=data)
@@ -346,9 +467,24 @@ class MobiWriter(object):
 
 
 def main(argv=sys.argv):
+    from calibre.ebooks.oeb.base import DirWriter
     inpath, outpath = argv[1:]
+    context = Context('Firefox', 'MobiDesktop')
     oeb = OEBBook(inpath)
-    writer = MobiWriter()
+    #writer = MobiWriter(compression=PALMDOC)
+    writer = MobiWriter(compression=UNCOMPRESSED)
+    #writer = DirWriter()
+    fbase = context.dest.fbase
+    fkey = context.dest.fnums.values()
+    flattener = CSSFlattener(
+        fbase=fbase, fkey=fkey, unfloat=True, untable=True)
+    rasterizer = SVGRasterizer()
+    trimmer = ManifestTrimmer()
+    mobimlizer = MobiMLizer()
+    flattener.transform(oeb, context)
+    rasterizer.transform(oeb, context)
+    mobimlizer.transform(oeb, context)
+    trimmer.transform(oeb, context)
     writer.dump(oeb, outpath)
     return 0
 

src/calibre/ebooks/oeb/base.py

@@ -14,10 +14,14 @@ from itertools import izip, count
 from urlparse import urldefrag, urlparse, urlunparse
 from urllib import unquote as urlunquote
 import logging
+import re
+import htmlentitydefs
+import uuid
+import copy
 from lxml import etree
 from calibre import LoggingInterface
 
-XML_PARSER = etree.XMLParser(recover=True, resolve_entities=False)
+XML_PARSER = etree.XMLParser(recover=True)
 XML_NS = 'http://www.w3.org/XML/1998/namespace'
 XHTML_NS = 'http://www.w3.org/1999/xhtml'
 OPF1_NS = 'http://openebook.org/namespaces/oeb-package/1.0/'
@@ -28,15 +32,20 @@ DC11_NS = 'http://purl.org/dc/elements/1.1/'
 XSI_NS = 'http://www.w3.org/2001/XMLSchema-instance'
 DCTERMS_NS = 'http://purl.org/dc/terms/'
 NCX_NS = 'http://www.daisy.org/z3986/2005/ncx/'
+SVG_NS = 'http://www.w3.org/2000/svg'
+XLINK_NS = 'http://www.w3.org/1999/xlink'
 XPNSMAP = {'h': XHTML_NS, 'o1': OPF1_NS, 'o2': OPF2_NS,
            'd09': DC09_NS, 'd10': DC10_NS, 'd11': DC11_NS,
-           'xsi': XSI_NS, 'dt': DCTERMS_NS, 'ncx': NCX_NS}
+           'xsi': XSI_NS, 'dt': DCTERMS_NS, 'ncx': NCX_NS,
+           'svg': SVG_NS, 'xl': XLINK_NS}
 
 def XML(name): return '{%s}%s' % (XML_NS, name)
 def XHTML(name): return '{%s}%s' % (XHTML_NS, name)
 def OPF(name): return '{%s}%s' % (OPF2_NS, name)
 def DC(name): return '{%s}%s' % (DC11_NS, name)
 def NCX(name): return '{%s}%s' % (NCX_NS, name)
+def SVG(name): return '{%s}%s' % (SVG_NS, name)
+def XLINK(name): return '{%s}%s' % (XLINK_NS, name)
 
 EPUB_MIME = 'application/epub+zip'
 XHTML_MIME = 'application/xhtml+xml'
@@ -46,12 +55,24 @@ OPF_MIME = 'application/oebps-package+xml'
 OEB_DOC_MIME = 'text/x-oeb1-document'
 OEB_CSS_MIME = 'text/x-oeb1-css'
 OPENTYPE_MIME = 'font/opentype'
+GIF_MIME = 'image/gif'
+JPEG_MIME = 'image/jpeg'
+PNG_MIME = 'image/png'
+SVG_MIME = 'image/svg+xml'
 
 OEB_STYLES = set([CSS_MIME, OEB_CSS_MIME, 'text/x-oeb-css'])
 OEB_DOCS = set([XHTML_MIME, 'text/html', OEB_DOC_MIME, 'text/x-oeb-document'])
+OEB_RASTER_IMAGES = set([GIF_MIME, JPEG_MIME, PNG_MIME])
+OEB_IMAGES = set([GIF_MIME, JPEG_MIME, PNG_MIME, SVG_MIME])
 
 MS_COVER_TYPE = 'other.ms-coverimage-standard'
 
+ENTITYDEFS = dict(htmlentitydefs.entitydefs)
+del ENTITYDEFS['lt']
+del ENTITYDEFS['gt']
+del ENTITYDEFS['quot']
+del ENTITYDEFS['amp']
+
 
 def element(parent, *args, **kwargs):
     if parent is not None:
@@ -101,12 +122,20 @@ def urlnormalize(href):
     return urlunparse(parts)
 
 
+class OEBError(Exception):
+    pass
+
+
 class FauxLogger(object):
     def __getattr__(self, name):
         return self
     def __call__(self, message):
         print message
 
+class Logger(LoggingInterface, object):
+    def __getattr__(self, name):
+        return object.__getattribute__(self, 'log_' + name)
+
 
 class AbstractContainer(object):
     def read_xml(self, path):
@@ -161,8 +190,9 @@ class Metadata(object):
                   'xsi': XSI_NS}
     
     class Item(object):
-        def __init__(self, term, value, fq_attrib={}):
-            self.fq_attrib = dict(fq_attrib)
+        def __init__(self, term, value, fq_attrib={}, **kwargs):
+            self.fq_attrib = fq_attrib = dict(fq_attrib)
+            fq_attrib.update(kwargs)
             if term == OPF('meta') and not value:
                 term = self.fq_attrib.pop('name')
                 value = self.fq_attrib.pop('content')
@@ -224,8 +254,8 @@ class Metadata(object):
         self.oeb = oeb
         self.items = defaultdict(list)
 
-    def add(self, term, value, attrib={}):
-        item = self.Item(term, value, attrib)
+    def add(self, term, value, attrib={}, **kwargs):
+        item = self.Item(term, value, attrib, **kwargs)
         items = self.items[barename(item.term)]
         items.append(item)
         return item
@@ -266,6 +296,9 @@ class Metadata(object):
 
 class Manifest(object):
     class Item(object):
+        ENTITY_RE = re.compile(r'&([a-zA-Z_:][a-zA-Z0-9.-_:]+);')
+        NUM_RE = re.compile('^(.*)([0-9][0-9.]*)(?=[.]|$)')
+    
         def __init__(self, id, href, media_type,
                      fallback=None, loader=str, data=None):
             self.id = id
@@ -281,19 +314,24 @@ class Manifest(object):
             return 'Item(id=%r, href=%r, media_type=%r)' \
                 % (self.id, self.href, self.media_type)
 
+        def _force_xhtml(self, data):
+            repl = lambda m: ENTITYDEFS.get(m.group(1), m.group(0))
+            data = self.ENTITY_RE.sub(repl, data)
+            data = etree.fromstring(data, parser=XML_PARSER)
+            if namespace(data.tag) != XHTML_NS:
+                data.attrib['xmlns'] = XHTML_NS
+                data = etree.tostring(data)
+                data = etree.fromstring(data, parser=XML_PARSER)
+            return data
+        
         def data():
             def fget(self):
                 if self._data is not None:
                     return self._data
                 data = self._loader(self.href)
                 if self.media_type in OEB_DOCS:
-                    data = etree.fromstring(data, parser=XML_PARSER)
-                    if namespace(data.tag) != XHTML_NS:
-                        data.attrib['xmlns'] = XHTML_NS
-                        data = etree.tostring(data)
-                        data = etree.fromstring(data, parser=XML_PARSER)
-                elif self.media_type.startswith('application/') \
-                     and self.media_type.endswith('+xml'):
+                    data = self._force_xhtml(data)
+                elif self.media_type[-4:] in ('+xml', '/xml'):
                     data = etree.fromstring(data, parser=XML_PARSER)
                 self._data = data
                 return data
@@ -310,11 +348,22 @@ class Manifest(object):
                 return xml2str(data)
             return str(data)
 
+        def __eq__(self, other):
+            return id(self) == id(other)
+
         def __cmp__(self, other):
             result = cmp(self.spine_position, other.spine_position)
             if result != 0:
                 return result
-            return cmp(self.id, other.id)
+            smatch = self.NUM_RE.search(self.href)
+            sref = smatch.group(1) if smatch else self.href
+            snum = float(smatch.group(2)) if smatch else 0.0
+            skey = (sref, snum, self.id)
+            omatch = self.NUM_RE.search(other.href)
+            oref = omatch.group(1) if omatch else other.href
+            onum = float(omatch.group(2)) if omatch else 0.0
+            okey = (oref, onum, other.id)
+            return cmp(skey, okey)
         
         def relhref(self, href):
             if '/' not in self.href:
@@ -519,8 +568,11 @@ class Guide(object):
         for type, ref in self.refs.items():
             yield type, ref
     
-    def __getitem__(self, index):
-        return self.refs[index]
+    def __getitem__(self, key):
+        return self.refs[key]
+
+    def __delitem__(self, key):
+        del self.refs[key]
     
     def __contains__(self, key):
         return key in self.refs
@@ -625,13 +677,22 @@ class OEBBook(object):
             self._all_from_opf(opf)
     
     def _convert_opf1(self, opf):
+        # Seriously, seriously wrong
+        if namespace(opf.tag) == OPF1_NS:
+            opf.tag = barename(opf.tag)
+            for elem in opf.iterdescendants():
+                if isinstance(elem.tag, basestring) \
+                   and namespace(elem.tag) == OPF1_NS:
+                    elem.tag = barename(elem.tag)
+        attrib = dict(opf.attrib)
+        attrib['version'] = '2.0'
         nroot = etree.Element(OPF('package'),
-            nsmap={None: OPF2_NS}, version="2.0", **dict(opf.attrib))
+            nsmap={None: OPF2_NS}, attrib=attrib)
         metadata = etree.SubElement(nroot, OPF('metadata'),
             nsmap={'opf': OPF2_NS, 'dc': DC11_NS,
                    'xsi': XSI_NS, 'dcterms': DCTERMS_NS})
         for prefix in ('d11', 'd10', 'd09'):
-            elements = xpath(opf, 'metadata/dc-metadata/%s:*' % prefix)
+            elements = xpath(opf, 'metadata//%s:*' % prefix)
             if elements: break
         for element in elements:
             if not element.text: continue
@@ -643,7 +704,7 @@ class OEBBook(object):
                     element.attrib[nsname] = element.attrib[name]
                     del element.attrib[name]
             metadata.append(element)
-        for element in opf.xpath('metadata/x-metadata/meta'):
+        for element in opf.xpath('metadata//meta'):
             metadata.append(element)
         for item in opf.xpath('manifest/item'):
             media_type = item.attrib['media-type'].lower()
@@ -660,31 +721,48 @@ class OEBBook(object):
     def _read_opf(self, opfpath):
         opf = self.container.read_xml(opfpath)
         version = float(opf.get('version', 1.0))
-        if version < 2.0:
+        ns = namespace(opf.tag)
+        if ns not in ('', OPF1_NS, OPF2_NS):
+            raise OEBError('Invalid namespace %r for OPF document' % ns)
+        if ns != OPF2_NS or version < 2.0:
             opf = self._convert_opf1(opf)
         return opf
     
     def _metadata_from_opf(self, opf):
-        uid = opf.attrib['unique-identifier']
+        uid = opf.get('unique-identifier', 'calibre-uuid')
+        self.uid = None
         self.metadata = metadata = Metadata(self)
-        for elem in xpath(opf, '/o2:package/o2:metadata/*'):
-            if elem.text or elem.attrib:
+        ignored = (OPF('dc-metadata'), OPF('x-metadata'))
+        for elem in xpath(opf, '/o2:package/o2:metadata//*'):
+            if elem.tag not in ignored and (elem.text or elem.attrib):
                 metadata.add(elem.tag, elem.text, elem.attrib)
+        haveuuid = haveid = False
+        for ident in metadata.identifier:
+            if unicode(ident).startswith('urn:uuid:'):
+                haveuuid = True
+            if 'id' in ident.attrib:
+                haveid = True
+        if not haveuuid and haveid:
+            bookid = "urn:uuid:%s" % str(uuid.uuid4())
+            metadata.add('identifier', bookid, id='calibre-uuid')
         for item in metadata.identifier:
             if item.id == uid:
                 self.uid = item
                 break
         else:
-            self.logger.log_warn(u'Unique-identifier %r not found.' % uid)
+            self.logger.warn(u'Unique-identifier %r not found.' % uid)
+            for ident in metadata.identifier:
+                if 'id' in ident.attrib:
                     self.uid = metadata.identifier[0]
+                    break
         if not metadata.language:
-            self.logger.log_warn(u'Language not specified.')
+            self.logger.warn(u'Language not specified.')
             metadata.add('language', 'en')
         if not metadata.creator:
-            self.logger.log_warn(u'Creator not specified.')
+            self.logger.warn(u'Creator not specified.')
             metadata.add('creator', 'Unknown')
         if not metadata.title:
-            self.logger.log_warn(u'Title not specified.')
+            self.logger.warn(u'Title not specified.')
             metadata.add('title', 'Unknown')
     
     def _manifest_from_opf(self, opf):
@@ -692,7 +770,7 @@ class OEBBook(object):
         for elem in xpath(opf, '/o2:package/o2:manifest/o2:item'):
             href = elem.get('href')
             if not self.container.exists(href):
-                self.logger.log_warn(u'Manifest item %r not found.' % href)
+                self.logger.warn(u'Manifest item %r not found.' % href)
                 continue
             manifest.add(elem.get('id'), href, elem.get('media-type'),
                          elem.get('fallback'))
@@ -702,7 +780,7 @@ class OEBBook(object):
         for elem in xpath(opf, '/o2:package/o2:spine/o2:itemref'):
             idref = elem.get('idref')
             if idref not in self.manifest:
-                self.logger.log_warn(u'Spine item %r not found.' % idref)
+                self.logger.warn(u'Spine item %r not found.' % idref)
                 continue
             item = self.manifest[idref]
             spine.add(item, elem.get('linear'))
@@ -721,7 +799,7 @@ class OEBBook(object):
             href = elem.get('href')
             path, frag = urldefrag(href)
             if path not in self.manifest.hrefs:
-                self.logger.log_warn(u'Guide reference %r not found' % href)
+                self.logger.warn(u'Guide reference %r not found' % href)
                 continue
             guide.add(elem.get('type'), elem.get('title'), href)
 
@@ -826,20 +904,27 @@ class OEBBook(object):
 
     def _ensure_cover_image(self):
         cover = None
+        spine0 = self.spine[0]
+        html = spine0.data
         if self.metadata.cover:
             id = str(self.metadata.cover[0])
-            cover = self.manifest[id]
+            cover = self.manifest.ids[id]
         elif MS_COVER_TYPE in self.guide:
             href = self.guide[MS_COVER_TYPE].href
             cover = self.manifest.hrefs[href]
-        elif 'cover' in self.guide:
-            href = self.guide['cover'].href
+        elif xpath(html, '//h:img[position()=1]'):
+            img = xpath(html, '//h:img[position()=1]')[0]
+            href = img.get('src')
             cover = self.manifest.hrefs[href]
-        else:
-            html = self.spine[0].data
-            imgs = xpath(html, '//h:img[position()=1]')
-            href = imgs[0].get('src') if imgs else None
-            cover = self.manifest.hrefs[href] if href else None
+        elif xpath(html, '//h:object[position()=1]'):
+            object = xpath(html, '//h:object[position()=1]')[0]
+            href = object.get('data')
+            cover = self.manifest.hrefs[href]
+        elif xpath(html, '//svg:svg[position()=1]'):
+            svg = copy.deepcopy(xpath(html, '//svg:svg[position()=1]')[0])
+            href = os.path.splitext(spine0.href)[0] + '.svg'
+            id, href = self.manifest.generate(spine0.id, href)
+            cover = self.manifest.add(id, href, SVG_MIME, data=svg)
         if cover and not self.metadata.cover:
             self.metadata.add('cover', cover.id)
             
@@ -913,7 +998,6 @@ class OEBBook(object):
                 NCX_MIME: (href, ncx)}
 
 
-
 def main(argv=sys.argv):
     for arg in argv[1:]:
         oeb = OEBBook(arg)

src/calibre/ebooks/oeb/html.css

@@ -35,7 +35,8 @@
  *
  * ***** END LICENSE BLOCK ***** */
 
-@namespace url(http://www.w3.org/1999/xhtml); /* set default namespace to HTML */
+@namespace url(http://www.w3.org/1999/xhtml);
+@namespace svg url(http://www.w3.org/2000/svg);
 
 /* blocks */
 
@@ -399,3 +400,8 @@ br {
   display: block;
 }
 
+/* Images, embedded object, and SVG size defaults */
+img, object, svg|svg {
+    width: auto;
+    height: auto;
+}

src/calibre/ebooks/oeb/profile.py

@@ -0,0 +1,65 @@
+'''
+Device profiles.
+'''
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
+
+from itertools import izip
+
+FONT_SIZES = [('xx-small', 1),
+              ('x-small',  None),
+              ('small',    2),
+              ('medium',   3),
+              ('large',    4),
+              ('x-large',  5),
+              ('xx-large', 6),
+              (None,       7)]
+
+
+class Profile(object):
+    def __init__(self, width, height, dpi, fbase, fsizes):
+        self.width = (float(width) / dpi) * 72.
+        self.height = (float(height) / dpi) * 72.
+        self.dpi = float(dpi)
+        self.fbase = float(fbase)
+        self.fsizes = []
+        for (name, num), size in izip(FONT_SIZES, fsizes):
+            self.fsizes.append((name, num, float(size)))
+        self.fnames = dict((name, sz) for name, _, sz in self.fsizes if name)
+        self.fnums = dict((num, sz) for _, num, sz in self.fsizes if num)
+
+
+PROFILES = {
+    'PRS505':
+        Profile(width=584, height=754, dpi=168.451, fbase=12,
+                fsizes=[7.5, 9, 10, 12, 15.5, 20, 22, 24]),
+
+    'MSReader':
+        Profile(width=480, height=652, dpi=100.0, fbase=13,
+                fsizes=[10, 11, 13, 16, 18, 20, 22, 26]),
+
+    # Not really, but let's pretend
+    'MobiDesktop':
+        Profile(width=280, height=300, dpi=96, fbase=18,
+                fsizes=[14, 14, 16, 18, 20, 22, 22, 24]),
+    
+    # No clue on usable screen size and DPI
+    'CybookG3':
+        Profile(width=584, height=754, dpi=168.451, fbase=12,
+                fsizes=[9, 10, 11, 12, 14, 17, 20, 24]),
+
+    'Firefox':
+        Profile(width=800, height=600, dpi=100.0, fbase=12,
+                fsizes=[5, 7, 9, 12, 13.5, 17, 20, 22, 24])
+    }
+
+
+class Context(object):
+    def __init__(self, source, dest):
+        if source in PROFILES:
+            source = PROFILES[source]
+        if dest in PROFILES:
+            dest = PROFILES[dest]
+        self.source = source
+        self.dest = dest

src/calibre/ebooks/oeb/stylizer.py

@@ -24,6 +24,7 @@ from lxml import etree
 from lxml.cssselect import css_to_xpath, ExpressionError
 from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES
 from calibre.ebooks.oeb.base import barename, urlnormalize
+from calibre.ebooks.oeb.profile import PROFILES
 from calibre.resources import html_css
 
 XHTML_CSS_NAMESPACE = '@namespace "%s";\n' % XHTML_NS
@@ -75,7 +76,7 @@ DEFAULTS = {'azimuth': 'center', 'background-attachment': 'scroll',
             '50', 'right': 'auto', 'speak': 'normal', 'speak-header': 'once',
             'speak-numeral': 'continuous', 'speak-punctuation': 'none',
             'speech-rate': 'medium', 'stress': '50', 'table-layout': 'auto',
-            'text-align': 'left', 'text-decoration': 'none', 'text-indent':
+            'text-align': 'auto', 'text-decoration': 'none', 'text-indent':
             0, 'text-transform': 'none', 'top': 'auto', 'unicode-bidi':
             'normal', 'vertical-align': 'baseline', 'visibility': 'visible',
             'voice-family': 'default', 'volume': 'medium', 'white-space':
@@ -85,23 +86,19 @@ DEFAULTS = {'azimuth': 'center', 'background-attachment': 'scroll',
 FONT_SIZE_NAMES = set(['xx-small', 'x-small', 'small', 'medium', 'large',
                        'x-large', 'xx-large'])
 
-FONT_SIZES = [('xx-small', 1),
-              ('x-small',  None),
-              ('small',    2),
-              ('medium',   3),
-              ('large',    4),
-              ('x-large',  5),
-              ('xx-large', 6),
-              (None,       7)]
-
 
 XPNSMAP = {'h': XHTML_NS,}
 def xpath(elem, expr):
     return elem.xpath(expr, namespaces=XPNSMAP)
 
 class CSSSelector(etree.XPath):
+    MIN_SPACE_RE = re.compile(r' *([>~+]) *')
+    LOCAL_NAME_RE = re.compile(r"(?<!local-)name[(][)] *= *'[^:]+:")
+    
     def __init__(self, css, namespaces=XPNSMAP):
+        css = self.MIN_SPACE_RE.sub(r'\1', css)
         path = css_to_xpath(css)
+        path = self.LOCAL_NAME_RE.sub(r"local-name() = '", path)
         etree.XPath.__init__(self, path, namespaces=namespaces)
         self.css = css
 
@@ -112,28 +109,11 @@ class CSSSelector(etree.XPath):
             self.css)
 
 
-class Page(object):
-    def __init__(self, width, height, dpi, fbase, fsizes):
-        self.width = (float(width) / dpi) * 72.
-        self.height = (float(height) / dpi) * 72.
-        self.dpi = float(dpi)
-        self.fbase = float(fbase)
-        self.fsizes = []
-        for (name, num), size in izip(FONT_SIZES, fsizes):
-            self.fsizes.append((name, num, float(size)))
-        self.fnames = dict((name, sz) for name, _, sz in self.fsizes if name)
-        self.fnums = dict((num, sz) for _, num, sz in self.fsizes if num)
-
-class Profiles(object):
-    PRS505 = Page(584, 754, 168.451, 12, [7.5, 9, 10, 12, 15.5, 20, 22, 24])
-    MSLIT = Page(652, 480, 100.0, 13, [10, 11, 13, 16, 18, 20, 22, 26])
-
-    
 class Stylizer(object):    
     STYLESHEETS = {}
     
-    def __init__(self, tree, path, oeb, page=Profiles.PRS505):
-        self.page = page
+    def __init__(self, tree, path, oeb, profile=PROFILES['PRS505']):
+        self.profile = profile
         base = os.path.dirname(path)
         basename = os.path.basename(path)
         cssname = os.path.splitext(basename)[0] + '.css'
@@ -183,9 +163,8 @@ class Stylizer(object):
                 continue
             for elem in selector(tree):
                 self.style(elem)._update_cssdict(cssdict)
-        for elem in tree.xpath('//*[@style]'):
-            self.style(elem)._apply_style_tag()
-        
+        for elem in xpath(tree, '//h:*[@style]'):
+            self.style(elem)._apply_style_attr()
     
     def flatten_rule(self, rule, href, index):
         results = []
@@ -215,7 +194,7 @@ class Stylizer(object):
             size = style['font-size']
             if size == 'normal': size = 'medium'
             if size in FONT_SIZE_NAMES:
-                style['font-size'] = "%dpt" % self.page.fnames[size]
+                style['font-size'] = "%dpt" % self.profile.fnames[size]
         return style
     
     def _normalize_edge(self, cssvalue, name):
@@ -284,24 +263,31 @@ class Stylizer(object):
 class Style(object):
     def __init__(self, element, stylizer):
         self._element = element
-        self._page = stylizer.page
+        self._profile = stylizer.profile
         self._stylizer = stylizer
         self._style = {}
+        self._fontSize = None
+        self._width = None
+        self._height = None
         stylizer._styles[element] = self
 
     def _update_cssdict(self, cssdict):
         self._style.update(cssdict)
         
-    def _apply_style_tag(self):
+    def _apply_style_attr(self):
         attrib = self._element.attrib
         if 'style' in attrib:
             style = CSSStyleDeclaration(attrib['style'])
             self._style.update(self._stylizer.flatten_style(style))
 
     def _has_parent(self):
-        parent = self._element.getparent()
-        return (parent is not None) \
-            and (parent in self._stylizer._styles)
+        return (self._element.getparent() is not None)
+
+    def _get_parent(self):
+        elem = self._element.getparent()
+        if elem is None:
+            return None
+        return self._stylizer.style(elem)
     
     def __getitem__(self, name):
         domname = cssproperties._toDOMname(name)
@@ -316,8 +302,8 @@ class Style(object):
         if (result == 'inherit'
             or (result is None and name in INHERITED
                 and self._has_parent())):
-            styles = self._stylizer._styles
-            result = styles[self._element.getparent()]._get(name)
+            stylizer = self._stylizer
+            result = stylizer.style(self._element.getparent())._get(name)
         if result is None:
             result = DEFAULTS[name]
         return result
@@ -340,7 +326,7 @@ class Style(object):
                 base = base or self.width
                 result = (value/100.0) * base
             elif unit == 'px':
-                result = value * 72.0 / self._page.dpi
+                result = value * 72.0 / self._profile.dpi
             elif unit == 'in':
                 result = value * 72.0
             elif unit == 'pt':
@@ -358,23 +344,22 @@ class Style(object):
 
     @property
     def fontSize(self):
-        def normalize_fontsize(value, base=None):
+        def normalize_fontsize(value, base):
             result = None
             factor = None
             if value == 'inherit':
-                # We should only see this if the root element
-                value = self._page.fbase
+                value = base
             if value in FONT_SIZE_NAMES:
-                result = self._page.fnames[value]
+                result = self._profile.fnames[value]
             elif value == 'smaller':
                 factor = 1.0/1.2
-                for _, _, size in self._page.fsizes:
+                for _, _, size in self._profile.fsizes:
                     if base <= size: break
                     factor = None
                     result = size
             elif value == 'larger':
                 factor = 1.2
-                for _, _, size in reversed(self._page.fsizes):
+                for _, _, size in reversed(self._profile.fsizes):
                     if base >= size: break
                     factor = None
                     result = size
@@ -385,39 +370,62 @@ class Style(object):
             if factor:
                 result = factor * base
             return result
+        if self._fontSize is None:
             result = None
-        if self._has_parent():
-            styles = self._stylizer._styles
-            base = styles[self._element.getparent()].fontSize
+            parent = self._get_parent()
+            if parent is not None:
+                base = parent.fontSize
             else:
-            base = self._page.fbase
+                base = self._profile.fbase
             if 'font-size' in self._style:
                 size = self._style['font-size']
                 result = normalize_fontsize(size, base)
             else:
                 result = base
-        self.__dict__['fontSize'] = result
-        return result
+            self._fontSize = result
+        return self._fontSize
 
     @property
     def width(self):
+        if self._width is None:
             result = None
             base = None
-        if self._has_parent():
-            styles = self._stylizer._styles
-            base = styles[self._element.getparent()].width
+            parent = self._get_parent()
+            if parent is not None:
+                base = parent.width
             else:
-            base = self._page.width
-        if 'width' in self._style:
+                base = self._profile.width
+            if 'width' is self._element.attrib:
+                width = self._element.attrib['width']
+            elif 'width' in self._style:
                 width = self._style['width']
-            if width == 'auto':
-                result = base
             else:
+                result = base
+            if not result:
                 result = self._unit_convert(width, base=base)
+            self._width = result
+        return self._width
+    
+    @property
+    def height(self):
+        if self._height is None:
+            result = None
+            base = None
+            parent = self._get_parent()
+            if parent is not None:
+                base = parent.height
+            else:
+                base = self._profile.height
+            if 'height' is self._element.attrib:
+                height = self._element.attrib['height']
+            elif 'height' in self._style:
+                height = self._style['height']
             else:
                 result = base
-        self.__dict__['width'] = result
-        return result
+            if not result:
+                result = self._unit_convert(height, base=base)
+            self._height = result
+        return self._height
     
     def __str__(self):
         items = self._style.items()

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -0,0 +1,265 @@
+'''
+CSS flattening transform.
+'''
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
+
+import sys
+import os
+import re
+import operator
+import math
+from itertools import chain
+from collections import defaultdict
+from lxml import etree
+from calibre.ebooks.oeb.base import XHTML, XHTML_NS
+from calibre.ebooks.oeb.base import CSS_MIME, OEB_STYLES
+from calibre.ebooks.oeb.base import namespace, barename
+from calibre.ebooks.oeb.base import OEBBook
+from calibre.ebooks.oeb.stylizer import Stylizer
+
+COLLAPSE = re.compile(r'[ \t\r\n\v]+')
+STRIPNUM = re.compile(r'[-0-9]+$')
+
+class KeyMapper(object):
+    def __init__(self, sbase, dbase, dkey):
+        self.sbase = float(sbase)
+        self.dprop = [(self.relate(x, dbase), float(x)) for x in dkey]
+        self.cache = {}
+
+    @staticmethod
+    def relate(size, base):
+        size = float(size)
+        base = float(base)
+        if size == base: return 0
+        sign = -1 if size < base else 1
+        endp = 0 if size < base else 36
+        diff = (abs(base - size) * 3) + ((36 - size) / 100)
+        logb = abs(base - endp) 
+        return sign * math.log(diff, logb)
+        
+    def __getitem__(self, ssize):
+        if ssize in self.cache:
+            return self.cache[ssize]
+        dsize = self.map(ssize)
+        self.cache[ssize] = dsize
+        return dsize
+
+    def map(self, ssize):
+        sbase = self.sbase
+        prop = self.relate(ssize, sbase)
+        diff = [(abs(prop - p), s) for p, s in self.dprop]
+        dsize = min(diff)[1]
+        return dsize
+
+class ScaleMapper(object):
+    def __init__(self, sbase, dbase):
+        self.dscale = float(dbase) / float(sbase)
+
+    def __getitem__(self, ssize):
+        dsize = ssize * self.dscale
+        return dsize
+
+class NullMapper(object):
+    def __init__(self):
+        pass
+
+    def __getitem__(self, ssize):
+        return ssize
+    
+def FontMapper(sbase=None, dbase=None, dkey=None):
+    if sbase and dbase and dkey:
+        return KeyMapper(sbase, dbase, dkey)
+    elif sbase and dbase:
+        return ScaleMapper(sbase, dbase)
+    else:
+        return NullMapper()
+
+
+class CSSFlattener(object):
+    def __init__(self, fbase=None, fkey=None, lineh=None, unfloat=False,
+                 untable=False):
+        self.fbase = fbase
+        self.fkey = fkey
+        self.lineh = lineh
+        self.unfloat = unfloat
+        self.untable = untable
+
+    def transform(self, oeb, context):
+        oeb.logger.info('Flattening CSS and remapping font sizes...')
+        self.oeb = oeb
+        self.context = context
+        self.stylize_spine()
+        self.sbase = self.baseline_spine() if self.fbase else None
+        self.fmap = FontMapper(self.sbase, self.fbase, self.fkey)
+        self.flatten_spine()
+
+    def stylize_spine(self):
+        self.stylizers = {}
+        profile = self.context.source
+        for item in self.oeb.spine:
+            html = item.data
+            stylizer = Stylizer(html, item.href, self.oeb, profile)
+            self.stylizers[item] = stylizer
+
+    def baseline_node(self, node, stylizer, sizes, csize):
+        csize = stylizer.style(node)['font-size']
+        if node.text:
+            sizes[csize] += len(COLLAPSE.sub(' ', node.text))
+        for child in node:
+            self.baseline_node(child, stylizer, sizes, csize)
+            if child.tail:
+                sizes[csize] += len(COLLAPSE.sub(' ', child.tail))
+    
+    def baseline_spine(self):
+        sizes = defaultdict(float)
+        for item in self.oeb.spine:
+            html = item.data
+            stylizer = self.stylizers[item]
+            body = html.find(XHTML('body'))
+            fsize = self.context.source.fbase
+            self.baseline_node(body, stylizer, sizes, fsize)
+        sbase = max(sizes.items(), key=operator.itemgetter(1))[0]
+        return sbase
+
+    def clean_edges(self, cssdict, style, fsize):
+        slineh = self.sbase * 1.26
+        dlineh = self.lineh
+        for kind in ('margin', 'padding'):
+            for edge in ('bottom', 'top'):
+                property = "%s-%s" % (kind, edge)
+                if property not in cssdict: continue
+                if '%' in cssdict[property]: continue
+                value = style[property]
+                if value == 0:
+                    continue
+                elif value <= slineh:
+                    cssdict[property] = "%0.5fem" % (dlineh / fsize)
+                else:
+                    value = round(value / slineh) * dlineh
+                    cssdict[property] = "%0.5fem" % (value / fsize)
+    
+    def flatten_node(self, node, stylizer, names, styles, psize, left=0):
+        if not isinstance(node.tag, basestring) \
+           or namespace(node.tag) != XHTML_NS:
+            return
+        tag = barename(node.tag)
+        style = stylizer.style(node)
+        cssdict = style.cssdict()
+        if 'align' in node.attrib:
+            cssdict['text-align'] = node.attrib['align']
+            del node.attrib['align']
+        if node.tag == XHTML('font'):
+            node.tag = XHTML('span')
+            if 'size' in node.attrib:
+                size = node.attrib['size']
+                if size.startswith('+'):
+                    cssdict['font-size'] = 'larger'
+                elif size.startswith('-'):
+                    cssdict['font-size'] = 'smaller'
+                else:
+                    fnums = self.context.source.fnums
+                    cssdict['font-size'] = fnums[int(size)]
+                del node.attrib['size']
+        if 'color' in node.attrib:
+            cssdict['color'] = node.attrib['color']
+            del node.attrib['color']
+        if 'bgcolor' in node.attrib:
+            cssdict['background-color'] = node.attrib['bgcolor']
+            del node.attrib['bgcolor']
+        if cssdict:
+            if 'font-size' in cssdict:
+                fsize = self.fmap[style['font-size']]
+                cssdict['font-size'] = "%0.5fem" % (fsize / psize)
+                psize = fsize
+            if self.lineh and self.fbase and tag != 'body':
+                self.clean_edges(cssdict, style, psize)
+            margin = style['margin-left']
+            left += margin if isinstance(margin, float) else 0
+            if (left + style['text-indent']) < 0:
+                percent = (margin - style['text-indent']) / style['width']
+                cssdict['margin-left'] = "%d%%" % (percent * 100)
+                left -= style['text-indent']
+            if self.unfloat and 'float' in cssdict \
+               and tag not in ('img', 'object') \
+               and cssdict.get('display', 'none') != 'none':
+                    del cssdict['display']
+            if self.untable and 'display' in cssdict \
+               and cssdict['display'].startswith('table'):
+                display = cssdict['display']
+                if display == 'table-cell':
+                    cssdict['display'] = 'inline'
+                else:
+                    cssdict['display'] = 'block'
+            if 'vertical-align' in cssdict \
+               and cssdict['vertical-align'] == 'sup':
+                cssdict['vertical-align'] = 'super'
+        if self.lineh and 'line-height' not in cssdict:
+            lineh = self.lineh / psize
+            cssdict['line-height'] = "%0.5fem" % lineh
+        if cssdict:
+            items = cssdict.items()
+            items.sort()
+            css = u';\n'.join(u'%s: %s' % (key, val) for key, val in items)
+            klass = STRIPNUM.sub('', node.get('class', 'calibre').split()[0])
+            if css in styles:
+                match = styles[css]
+            else:
+                match = klass + str(names[klass] or '')
+                styles[css] = match
+                names[klass] += 1
+            node.attrib['class'] = match
+        elif 'class' in node.attrib:
+            del node.attrib['class']
+        if 'style' in node.attrib:
+            del node.attrib['style']
+        for child in node:
+            self.flatten_node(child, stylizer, names, styles, psize, left)
+
+    def flatten_head(self, head, stylizer, href):
+        for node in head:
+            if node.tag == XHTML('link') \
+               and node.get('rel', 'stylesheet') == 'stylesheet' \
+               and node.get('type', CSS_MIME) in OEB_STYLES:
+                head.remove(node)
+            elif node.tag == XHTML('style') \
+                 and node.get('type', CSS_MIME) in OEB_STYLES:
+                head.remove(node)
+        etree.SubElement(head, XHTML('link'),
+            rel='stylesheet', type=CSS_MIME, href=href)
+        if stylizer.page_rule:
+            items = stylizer.page_rule.items()
+            items.sort()
+            css = '; '.join("%s: %s" % (key, val) for key, val in items)
+            style = etree.SubElement(head, XHTML('style'), type=CSS_MIME)
+            style.text = "@page { %s; }" % css
+
+    def replace_css(self, css):
+        manifest = self.oeb.manifest
+        id, href = manifest.generate('css', 'stylesheet.css')
+        for item in manifest.values():
+            if item.media_type in OEB_STYLES:
+                manifest.remove(item)
+        item = manifest.add(id, href, CSS_MIME, data=css)
+        return href
+            
+    def flatten_spine(self):
+        names = defaultdict(int)
+        styles = {}
+        for item in self.oeb.spine:
+            html = item.data
+            stylizer = self.stylizers[item]
+            body = html.find(XHTML('body'))
+            fsize = self.context.dest.fbase
+            self.flatten_node(body, stylizer, names, styles, fsize)
+        items = [(key, val) for (val, key) in styles.items()]
+        items.sort()
+        css = ''.join(".%s {\n%s;\n}\n\n" % (key, val) for key, val in items)
+        href = self.replace_css(css)
+        for item in self.oeb.spine:
+            html = item.data
+            stylizer = self.stylizers[item]
+            head = html.find(XHTML('head'))
+            self.flatten_head(head, stylizer, href)

src/calibre/ebooks/oeb/transforms/rasterize.py

@@ -0,0 +1,189 @@
+'''
+SVG rasterization transform.
+'''
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
+
+import sys
+import os
+from urlparse import urldefrag
+import base64
+from lxml import etree
+from PyQt4.QtCore import Qt
+from PyQt4.QtCore import QByteArray
+from PyQt4.QtCore import QBuffer
+from PyQt4.QtCore import QIODevice
+from PyQt4.QtGui import QColor
+from PyQt4.QtGui import QImage
+from PyQt4.QtGui import QPainter
+from PyQt4.QtSvg import QSvgRenderer
+from PyQt4.QtGui import QApplication
+from calibre.ebooks.oeb.base import XHTML_NS, XHTML, SVG_NS, SVG, XLINK
+from calibre.ebooks.oeb.base import SVG_MIME, PNG_MIME
+from calibre.ebooks.oeb.base import xml2str, xpath, namespace, barename
+from calibre.ebooks.oeb.stylizer import Stylizer
+
+IMAGE_TAGS = set([XHTML('img'), XHTML('object')])
+
+class SVGRasterizer(object):
+    def __init__(self):
+        if QApplication.instance() is None:
+            QApplication([])
+
+    def transform(self, oeb, context):
+        oeb.logger.info('Rasterizing SVG images...')
+        self.oeb = oeb
+        self.profile = context.dest
+        self.images = {}
+        self.dataize_manifest()
+        self.rasterize_spine()
+        self.rasterize_cover()
+
+    def rasterize_svg(self, elem, width=0, height=0):
+        data = QByteArray(xml2str(elem))
+        svg = QSvgRenderer(data)
+        size = svg.defaultSize()
+        if size.width() == 100 and size.height() == 100 \
+           and 'viewBox' in elem.attrib:
+            box = [float(x) for x in elem.attrib['viewBox'].split()]
+            size.setWidth(box[2] - box[0])
+            size.setHeight(box[3] - box[1])
+        if width or height:
+            size.scale(width, height, Qt.KeepAspectRatio)
+        image = QImage(size, QImage.Format_ARGB32_Premultiplied)
+        image.fill(QColor("white").rgb())
+        painter = QPainter(image)
+        svg.render(painter)
+        painter.end()
+        array = QByteArray()
+        buffer = QBuffer(array)
+        buffer.open(QIODevice.WriteOnly)
+        image.save(buffer, 'PNG')
+        return str(array)
+
+    def dataize_manifest(self):
+        for item in self.oeb.manifest.values():
+            if item.media_type == SVG_MIME:
+                self.dataize_svg(item)
+
+    def dataize_svg(self, item, svg=None):
+        if svg is None:
+            svg = item.data
+        hrefs = self.oeb.manifest.hrefs
+        for elem in xpath(svg, '//svg:*[@xl:href]'):
+            href = elem.attrib[XLINK('href')]
+            path, frag = urldefrag(href)
+            if not path:
+                continue
+            abshref = item.abshref(path)
+            if abshref not in hrefs:
+                continue
+            linkee = hrefs[abshref]
+            data = base64.encodestring(str(linkee))
+            data = "data:%s;base64,%s" % (linkee.media_type, data)
+            elem.attrib[XLINK('href')] = data
+        return svg
+            
+    def rasterize_spine(self):
+        for item in self.oeb.spine:
+            html = item.data
+            stylizer = Stylizer(html, item.href, self.oeb, self.profile)
+            self.rasterize_item(item, stylizer)
+
+    def rasterize_item(self, item, stylizer):
+        html = item.data
+        hrefs = self.oeb.manifest.hrefs
+        for elem in xpath(html, '//h:img'):
+            src = elem.get('src', None)
+            image = hrefs.get(item.abshref(src), None) if src else None
+            if image and image.media_type == SVG_MIME:
+                style = stylizer.style(elem)
+                self.rasterize_external(elem, style, item, image)
+        for elem in xpath(html, '//h:object[@type="%s"]' % SVG_MIME):
+            data = elem.get('data', None)
+            image = hrefs.get(item.abshref(data), None) if data else None
+            if image and image.media_type == SVG_MIME:
+                style = stylizer.style(elem)
+                self.rasterize_external(elem, style, item, image)
+        for elem in xpath(html, '//svg:svg'):
+            style = stylizer.style(elem)
+            self.rasterize_inline(elem, style, item)
+
+    def rasterize_inline(self, elem, style, item):
+        width = style['width']
+        if width == 'auto':
+            width = self.profile.width
+        height = style['height']
+        if height == 'auto':
+            height = self.profile.height
+        width = (width / 72) * self.profile.dpi
+        height = (height / 72) * self.profile.dpi
+        elem = self.dataize_svg(item, elem)
+        data = self.rasterize_svg(elem, width, height)
+        manifest = self.oeb.manifest
+        href = os.path.splitext(item.href)[0] + '.png'
+        id, href = manifest.generate(item.id, href)
+        manifest.add(id, href, PNG_MIME, data=data)
+        img = etree.Element(XHTML('img'), src=item.relhref(href))
+        elem.getparent().replace(elem, img)
+        for prop in ('width', 'height'):
+            if prop in elem.attrib:
+                img.attrib[prop] = elem.attrib[prop]
+
+    def rasterize_external(self, elem, style, item, svgitem):
+        width = style['width']
+        if width == 'auto':
+            width = self.profile.width
+        height = style['height']
+        if height == 'auto':
+            height = self.profile.height
+        width = (width / 72) * self.profile.dpi
+        height = (height / 72) * self.profile.dpi
+        data = QByteArray(str(svgitem))
+        svg = QSvgRenderer(data)
+        size = svg.defaultSize()
+        size.scale(width, height, Qt.KeepAspectRatio)
+        key = (svgitem.href, size.width(), size.height())
+        if key in self.images:
+            href = self.images[key]
+        else:
+            logger = self.oeb.logger
+            logger.info('Rasterizing %r to %dx%d'
+                        % (svgitem.href, size.width(), size.height()))
+            image = QImage(size, QImage.Format_ARGB32_Premultiplied)
+            image.fill(QColor("white").rgb())
+            painter = QPainter(image)
+            svg.render(painter)
+            painter.end()
+            array = QByteArray()
+            buffer = QBuffer(array)
+            buffer.open(QIODevice.WriteOnly)
+            image.save(buffer, 'PNG')
+            data = str(array)
+            manifest = self.oeb.manifest
+            href = os.path.splitext(svgitem.href)[0] + '.png'
+            id, href = manifest.generate(svgitem.id, href)
+            manifest.add(id, href, PNG_MIME, data=data)
+            self.images[key] = href
+        elem.tag = XHTML('img')
+        elem.attrib['src'] = item.relhref(href)
+        elem.text = None
+        for child in elem:
+            elem.remove(child)
+            
+    def rasterize_cover(self):
+        covers = self.oeb.metadata.cover
+        if not covers:
+            return
+        cover = self.oeb.manifest.ids[str(covers[0])]
+        if not cover.media_type == SVG_MIME:
+            return
+        logger = self.oeb.logger
+        logger.info('Rasterizing %r to %dx%d' % (cover.href, 600, 800))
+        data = self.rasterize_svg(cover.data, 600, 800)
+        href = os.path.splitext(cover.href)[0] + '.png'
+        id, href = self.oeb.manifest.generate(cover.id, href)
+        self.oeb.manifest.add(id, href, PNG_MIME, data=data)
+        covers[0].value = id

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -0,0 +1,40 @@
+'''
+OPF manifest trimming transform.
+'''
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
+
+import sys
+import os
+from lxml import etree
+from calibre.ebooks.oeb.base import XPNSMAP, CSS_MIME
+
+LINK_SELECTORS = []
+for expr in ('//h:link/@href', '//h:img/@src', '//h:object/@data',
+             '//*/@xl:href'):
+    LINK_SELECTORS.append(etree.XPath(expr, namespaces=XPNSMAP))
+
+class ManifestTrimmer(object):
+    def transform(self, oeb, context):
+        oeb.logger.info('Trimming unused files from manifest...')
+        used = set()
+        for item in oeb.spine:
+            used.add(item.href)
+            for selector in LINK_SELECTORS:
+                for href in selector(item.data):
+                    used.add(item.abshref(href))
+        # TODO: Things mentioned in CSS
+        # TODO: Things mentioned in SVG
+        # Who knows what people will do...
+        for term in oeb.metadata:
+            for item in oeb.metadata[term]:
+                if item.value in oeb.manifest.hrefs:
+                    used.add(item.value)
+                elif item.value in oeb.manifest.ids:
+                    used.add(oeb.manifest.ids[item.value].href)
+        for item in oeb.manifest.values():
+            if item.href not in used:
+                oeb.logger.info('Trimming %r from manifest' % item.href)
+                oeb.manifest.remove(item)

src/calibre/gui2/__init__.py

@@ -54,8 +54,12 @@ def _config():
     c.add_opt('autolaunch_server', default=False, help=_('Automatically launch content server on application startup'))
     c.add_opt('oldest_news', default=60, help=_('Oldest news kept in database'))
     c.add_opt('systray_icon', default=True, help=_('Show system tray icon'))
-    c.add_opt('upload_news_to_device', default=True, help=_('Upload downloaded news to device'))
-    c.add_opt('delete_news_from_library_on_upload', default=False, help=_('Delete books from library after uploading to device'))
+    c.add_opt('upload_news_to_device', default=True, 
+              help=_('Upload downloaded news to device'))
+    c.add_opt('delete_news_from_library_on_upload', default=False, 
+              help=_('Delete books from library after uploading to device'))
+    c.add_opt('separate_cover_flow', default=False, 
+              help=_('Show the cover flow in a separate window instead of in the main calibre window'))
     return ConfigProxy(c)
     
 config = _config()

src/calibre/gui2/cover_flow.py

@@ -69,11 +69,11 @@ if pictureflow is not None:
             
     class CoverFlow(pictureflow.PictureFlow):
         
-        def __init__(self, height=300, parent=None):
+        def __init__(self, height=300, parent=None, text_height=25):
             pictureflow.PictureFlow.__init__(self, parent, 
                                 config['cover_flow_queue_length']+1)
             self.setSlideSize(QSize(int(2/3. * height), height))
-            self.setMinimumSize(QSize(int(2.35*0.67*height), (5/3.)*height+25))
+            self.setMinimumSize(QSize(int(2.35*0.67*height), (5/3.)*height+text_height))
             self.setFocusPolicy(Qt.WheelFocus)
             self.setSizePolicy(QSizePolicy(QSizePolicy.Minimum, QSizePolicy.Minimum))
             

src/calibre/gui2/dialogs/config.py

@@ -20,7 +20,7 @@ from calibre.ebooks.epub.iterator import is_supported
 from calibre.library import server_config
 from calibre.customize.ui import initialized_plugins, is_disabled, enable_plugin, \
                                  disable_plugin, customize_plugin, \
-                                 plugin_customization, add_plugin
+                                 plugin_customization, add_plugin, remove_plugin
 
 class PluginModel(QAbstractItemModel):
     
@@ -186,7 +186,6 @@ class ConfigDialog(QDialog, Ui_Dialog):
         single_format = config['save_to_disk_single_format']
         self.single_format.setCurrentIndex(BOOK_EXTENSIONS.index(single_format))
         self.cover_browse.setValue(config['cover_flow_queue_length'])
-        self.confirm_delete.setChecked(config['confirm_delete'])
         from calibre.translations.compiled import translations
         from calibre.translations import language_codes
         from calibre.startup import get_lang
@@ -242,8 +241,10 @@ class ConfigDialog(QDialog, Ui_Dialog):
         self.plugin_view.setModel(self._plugin_model)
         self.connect(self.toggle_plugin, SIGNAL('clicked()'), lambda : self.modify_plugin(op='toggle'))
         self.connect(self.customize_plugin, SIGNAL('clicked()'), lambda : self.modify_plugin(op='customize'))
+        self.connect(self.remove_plugin, SIGNAL('clicked()'), lambda : self.modify_plugin(op='remove'))
         self.connect(self.button_plugin_browse, SIGNAL('clicked()'), self.find_plugin)
         self.connect(self.button_plugin_add, SIGNAL('clicked()'), self.add_plugin)
+        self.separate_cover_flow.setChecked(config['separate_cover_flow'])
     
     def add_plugin(self):
         path = unicode(self.plugin_path.text())
@@ -287,6 +288,13 @@ class ConfigDialog(QDialog, Ui_Dialog):
                 if ok:
                     customize_plugin(plugin, unicode(text))
                     self._plugin_model.refresh_plugin(plugin)
+            if op == 'remove':
+                if remove_plugin(plugin):
+                    self._plugin_model.populate()
+                    self._plugin_model.reset()
+                else:
+                    error_dialog(self, _('Cannot remove builtin plugin'), 
+                                 plugin.name + _(' cannot be removed. It is a builtin plugin. Try disabling it instead.')).exec_()
             
     
     def up_column(self):
@@ -385,7 +393,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
         config['column_map'] = cols
         config['toolbar_icon_size'] = self.ICON_SIZES[self.toolbar_button_size.currentIndex()]
         config['show_text_in_toolbar'] = bool(self.show_toolbar_text.isChecked())
-        config['confirm_delete'] =  bool(self.confirm_delete.isChecked())
+        config['separate_cover_flow'] = bool(self.separate_cover_flow.isChecked())
         pattern = self.filename_pattern.commit()
         prefs['filename_pattern'] = pattern
         p = {0:'normal', 1:'high', 2:'low'}[self.priority.currentIndex()]

src/calibre/gui2/dialogs/config.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>800</width>
-    <height>563</height>
+    <height>570</height>
    </rect>
   </property>
   <property name="windowTitle" >
@@ -127,17 +127,10 @@
            </property>
           </widget>
          </item>
-         <item>
-          <widget class="QCheckBox" name="confirm_delete" >
-           <property name="text" >
-            <string>Ask for &amp;confirmation before deleting files</string>
-           </property>
-          </widget>
-         </item>
          <item>
           <widget class="QCheckBox" name="pdf_metadata" >
            <property name="toolTip" >
-            <string>If you disable this setting, metadatas is guessed from the filename instead. This can be configured in the Advanced section.</string>
+            <string>If you disable this setting, metadata is guessed from the filename instead. This can be configured in the Advanced section.</string>
            </property>
            <property name="text" >
             <string>Read &amp;metadata from files</string>
@@ -363,7 +356,7 @@
            </item>
           </layout>
          </item>
-         <item row="5" column="0" >
+         <item row="7" column="0" >
           <widget class="QGroupBox" name="groupBox_2" >
            <property name="title" >
             <string>Toolbar</string>
@@ -411,7 +404,7 @@
            </layout>
           </widget>
          </item>
-         <item row="6" column="0" >
+         <item row="8" column="0" >
           <widget class="QGroupBox" name="groupBox" >
            <property name="title" >
             <string>Select visible &amp;columns in library view</string>
@@ -499,20 +492,27 @@
            </property>
           </widget>
          </item>
-         <item row="3" column="0" >
+         <item row="4" column="0" >
           <widget class="QCheckBox" name="sync_news" >
            <property name="text" >
             <string>Automatically send downloaded &amp;news to ebook reader</string>
            </property>
           </widget>
          </item>
-         <item row="4" column="0" >
+         <item row="5" column="0" >
           <widget class="QCheckBox" name="delete_news" >
            <property name="text" >
             <string>&amp;Delete news from library when it is sent to reader</string>
            </property>
           </widget>
          </item>
+         <item row="3" column="0" >
+          <widget class="QCheckBox" name="separate_cover_flow" >
+           <property name="text" >
+            <string>Show cover &amp;browser in a separate window (needs restart)</string>
+           </property>
+          </widget>
+         </item>
         </layout>
        </widget>
        <widget class="QWidget" name="page_2" >
@@ -811,6 +811,13 @@
              </property>
             </widget>
            </item>
+           <item>
+            <widget class="QPushButton" name="remove_plugin" >
+             <property name="text" >
+              <string>&amp;Remove plugin</string>
+             </property>
+            </widget>
+           </item>
           </layout>
          </item>
          <item>

src/calibre/gui2/dialogs/confirm_delete.py

@@ -0,0 +1,31 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+from calibre.gui2 import dynamic
+from calibre.gui2.dialogs.confirm_delete_ui import Ui_Dialog
+from PyQt4.Qt import QDialog, SIGNAL
+
+def _config_name(name):
+    return name + '_again'
+
+class Dialog(QDialog, Ui_Dialog):
+    
+    def __init__(self, msg, name, parent):
+        QDialog.__init__(self, parent)
+        self.setupUi(self)
+        
+        self.msg.setText(msg)
+        self.name = name
+        self.connect(self.again, SIGNAL('stateChanged(int)'), self.toggle)
+        
+
+    def toggle(self, x):
+        dynamic[_config_name(self.name)] = self.again.isChecked()
+        
+def confirm(msg, name, parent=None):
+    if not dynamic.get(_config_name(name), True):
+        return True
+    d = Dialog(msg, name, parent)
+    return d.exec_() == d.Accepted

src/calibre/gui2/dialogs/confirm_delete.ui

@@ -0,0 +1,100 @@
+<ui version="4.0" >
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog" >
+  <property name="geometry" >
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>439</width>
+    <height>300</height>
+   </rect>
+  </property>
+  <property name="windowTitle" >
+   <string>Are you sure?</string>
+  </property>
+  <property name="windowIcon" >
+   <iconset resource="../images.qrc" >
+    <normaloff>:/images/dialog_warning.svg</normaloff>:/images/dialog_warning.svg</iconset>
+  </property>
+  <layout class="QGridLayout" name="gridLayout" >
+   <item row="0" column="0" >
+    <layout class="QHBoxLayout" name="horizontalLayout" >
+     <item>
+      <widget class="QLabel" name="label" >
+       <property name="pixmap" >
+        <pixmap resource="../images.qrc" >:/images/dialog_warning.svg</pixmap>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QLabel" name="msg" >
+       <property name="text" >
+        <string>TextLabel</string>
+       </property>
+       <property name="wordWrap" >
+        <bool>true</bool>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="1" column="0" >
+    <widget class="QCheckBox" name="again" >
+     <property name="text" >
+      <string>&amp;Show this warning again</string>
+     </property>
+     <property name="checked" >
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item row="2" column="0" >
+    <widget class="QDialogButtonBox" name="buttonBox" >
+     <property name="orientation" >
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons" >
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../images.qrc" />
+ </resources>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/dialogs/lrf_single.ui

@@ -6,7 +6,7 @@
     <x>0</x>
     <y>0</y>
     <width>866</width>
-    <height>671</height>
+    <height>679</height>
    </rect>
   </property>
   <property name="windowTitle" >
@@ -115,7 +115,7 @@
           <item row="0" column="0" >
            <widget class="QStackedWidget" name="stack" >
             <property name="currentIndex" >
-             <number>0</number>
+             <number>1</number>
             </property>
             <widget class="QWidget" name="metadata_page" >
              <layout class="QHBoxLayout" >
@@ -652,7 +652,7 @@
                    </property>
                   </widget>
                  </item>
-                 <item row="1" column="0" >
+                 <item row="2" column="0" >
                   <widget class="QLabel" name="label_10" >
                    <property name="text" >
                     <string>&amp;Header format:</string>
@@ -662,9 +662,26 @@
                    </property>
                   </widget>
                  </item>
-                 <item row="1" column="1" >
+                 <item row="2" column="1" >
                   <widget class="QLineEdit" name="gui_headerformat" />
                  </item>
+                 <item row="1" column="1" >
+                  <widget class="QSpinBox" name="gui_header_separation" >
+                   <property name="suffix" >
+                    <string> px</string>
+                   </property>
+                  </widget>
+                 </item>
+                 <item row="1" column="0" >
+                  <widget class="QLabel" name="label_29" >
+                   <property name="text" >
+                    <string>Header &amp;separation:</string>
+                   </property>
+                   <property name="buddy" >
+                    <cstring>gui_header_separation</cstring>
+                   </property>
+                  </widget>
+                 </item>
                 </layout>
                </widget>
               </item>
@@ -985,8 +1002,8 @@
              <string>&lt;!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0//EN" "http://www.w3.org/TR/REC-html40/strict.dtd">
 &lt;html>&lt;head>&lt;meta name="qrichtext" content="1" />&lt;style type="text/css">
 p, li { white-space: pre-wrap; }
-&lt;/style>&lt;/head>&lt;body style=" font-family:'DejaVu Sans'; font-size:10pt; font-weight:400; font-style:normal;">
-&lt;p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;">&lt;/p>&lt;/body>&lt;/html></string>
+&lt;/style>&lt;/head>&lt;body style=" font-family:'Sans Serif'; font-size:10pt; font-weight:400; font-style:normal;">
+&lt;p style="-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-family:'DejaVu Sans';">&lt;/p>&lt;/body>&lt;/html></string>
             </property>
            </widget>
           </item>
@@ -1078,12 +1095,12 @@ p, li { white-space: pre-wrap; }
    <slot>setDisabled(bool)</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>650</x>
-     <y>122</y>
+     <x>308</x>
+     <y>74</y>
     </hint>
     <hint type="destinationlabel" >
-     <x>788</x>
-     <y>140</y>
+     <x>308</x>
+     <y>74</y>
     </hint>
    </hints>
   </connection>
@@ -1094,12 +1111,12 @@ p, li { white-space: pre-wrap; }
    <slot>setDisabled(bool)</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>543</x>
-     <y>122</y>
+     <x>308</x>
+     <y>74</y>
     </hint>
     <hint type="destinationlabel" >
-     <x>544</x>
-     <y>211</y>
+     <x>308</x>
+     <y>74</y>
     </hint>
    </hints>
   </connection>
@@ -1110,12 +1127,12 @@ p, li { white-space: pre-wrap; }
    <slot>setEnabled(bool)</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>298</x>
-     <y>398</y>
+     <x>308</x>
+     <y>74</y>
     </hint>
     <hint type="destinationlabel" >
-     <x>660</x>
-     <y>435</y>
+     <x>308</x>
+     <y>74</y>
     </hint>
    </hints>
   </connection>
@@ -1126,12 +1143,12 @@ p, li { white-space: pre-wrap; }
    <slot>setEnabled(bool)</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>330</x>
-     <y>367</y>
+     <x>345</x>
+     <y>363</y>
     </hint>
     <hint type="destinationlabel" >
-     <x>823</x>
-     <y>372</y>
+     <x>837</x>
+     <y>435</y>
     </hint>
    </hints>
   </connection>
@@ -1142,12 +1159,28 @@ p, li { white-space: pre-wrap; }
    <slot>setDisabled(bool)</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>344</x>
-     <y>107</y>
+     <x>308</x>
+     <y>74</y>
     </hint>
     <hint type="destinationlabel" >
-     <x>489</x>
-     <y>465</y>
+     <x>308</x>
+     <y>74</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>gui_header</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>gui_header_separation</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>261</x>
+     <y>346</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>379</x>
+     <y>378</y>
     </hint>
    </hints>
   </connection>

src/calibre/gui2/dialogs/metadata_single.py

@@ -7,7 +7,7 @@ add/remove formats
 import os
 
 from PyQt4.QtCore import SIGNAL, QObject, QCoreApplication, Qt
-from PyQt4.QtGui import QPixmap, QListWidgetItem, QErrorMessage, QDialog
+from PyQt4.QtGui import QPixmap, QListWidgetItem, QErrorMessage, QDialog, QCompleter
 
 
 from calibre.gui2 import qstring_to_unicode, error_dialog, file_icon_provider, \
@@ -20,6 +20,7 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.metadata import authors_to_sort_string, string_to_authors, authors_to_string
 from calibre.ebooks.metadata.library_thing import login, cover_from_isbn, LibraryThingError
 from calibre import islinux
+from calibre.ebooks.metadata.meta import get_metadata
 from calibre.utils.config import prefs
 from calibre.customize.ui import run_plugins_on_import
 
@@ -32,6 +33,13 @@ class Format(QListWidgetItem):
         QListWidgetItem.__init__(self, file_icon_provider().icon_from_ext(ext), 
                                  text, parent, QListWidgetItem.UserType)
 
+class AuthorCompleter(QCompleter):
+    
+    def __init__(self, db):
+        all_authors = db.all_authors()
+        all_authors.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        QCompleter.__init__(self, [x[1] for x in all_authors])
+
 class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
     
     def do_reset_cover(self, *args):
@@ -102,6 +110,39 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
             self.formats.takeItem(row.row())
             self.formats_changed = True
     
+    def set_cover(self):
+        row = self.formats.currentRow()
+        fmt = self.formats.item(row)
+        ext = fmt.ext.lower()
+        if fmt.path is None:
+            stream = self.db.format(self.row, ext, as_file=True)
+        else:
+            stream = open(fmt.path, 'r+b')
+        try:
+            mi = get_metadata(stream, ext)
+        except:
+            error_dialog(self, _('Could not read metadata'), 
+                         _('Could not read metadata from %s format')%ext).exec_()
+            return
+        cdata = None
+        if mi.cover and os.access(mi.cover, os.R_OK):
+            cdata = open(mi.cover).read()
+        elif mi.cover_data[1] is not None:
+            cdata = mi.cover_data[1]
+        if cdata is None:
+            error_dialog(self, _('Could not read cover'), 
+                         _('Could not read cover from %s format')%ext).exec_()
+            return
+        pix = QPixmap()
+        pix.loadFromData(cdata)
+        if pix.isNull():
+            error_dialog(self, _('Could not read cover'), 
+                         _('The cover in the %s format is invalid')%ext).exec_()
+            return
+        self.cover.setPixmap(pix)
+        self.cover_changed = True
+        self.cpixmap = pix
+    
     def sync_formats(self):
         old_extensions, new_extensions, paths = set(), set(), {}
         for row in range(self.formats.count()):
@@ -137,6 +178,8 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
         self.cover_changed = False
         self.cpixmap = None
         self.cover.setAcceptDrops(True)
+        self._author_completer = AuthorCompleter(self.db)
+        self.authors.setCompleter(self._author_completer)
         self.connect(self.cover, SIGNAL('cover_changed()'), self.cover_dropped)
         QObject.connect(self.cover_button, SIGNAL("clicked(bool)"), \
                                                     self.select_cover)
@@ -155,6 +198,7 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
                         self.remove_unused_series)
         QObject.connect(self.auto_author_sort, SIGNAL('clicked()'),
                         self.deduce_author_sort)
+        self.connect(self.button_set_cover, SIGNAL('clicked()'), self.set_cover)
         self.connect(self.reset_cover, SIGNAL('clicked()'), self.do_reset_cover)
         self.connect(self.swap_button, SIGNAL('clicked()'), self.swap_title_author)
         self.timeout = float(prefs['network_timeout'])
@@ -171,8 +215,6 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
             self.authors.setText('')
         aus = self.db.author_sort(row)
         self.author_sort.setText(aus if aus else '')
-        pub = self.db.publisher(row)
-        self.publisher.setText(pub if pub else '')
         tags = self.db.tags(row)
         self.tags.setText(tags if tags else '')
         rating = self.db.rating(row)
@@ -191,7 +233,8 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
                 size = self.db.sizeof_format(row, ext)
                 Format(self.formats, ext, size)
         
-        self.initialize_series()
+            
+        self.initialize_series_and_publisher()
             
         self.series_index.setValue(self.db.series_index(row))
         QObject.connect(self.series, SIGNAL('currentIndexChanged(int)'), self.enable_series_index)
@@ -224,7 +267,7 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
     def cover_dropped(self):
         self.cover_changed = True
     
-    def initialize_series(self):
+    def initialize_series_and_publisher(self):
         all_series = self.db.all_series()
         all_series.sort(cmp=lambda x, y : cmp(x[1], y[1]))
         series_id = self.db.series_id(self.row)
@@ -248,6 +291,22 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
                 l.invalidate()
                 l.activate()
         
+        all_publishers = self.db.all_publishers()
+        all_publishers.sort(cmp=lambda x, y : cmp(x[1], y[1]))
+        publisher_id = self.db.publisher_id(self.row)
+        idx, c = None, 0
+        for i in all_publishers:
+            id, name = i
+            if id == publisher_id:
+                idx = c
+            self.publisher.addItem(name)
+            c += 1
+        
+        self.publisher.setEditText('')
+        if idx is not None:
+            self.publisher.setCurrentIndex(idx)
+        
+                
         self.layout().activate()
     
     def edit_tags(self):
@@ -302,7 +361,7 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
         isbn   = qstring_to_unicode(self.isbn.text())
         title  = qstring_to_unicode(self.title.text())
         author = string_to_authors(unicode(self.authors.text()))[0]
-        publisher = qstring_to_unicode(self.publisher.text()) 
+        publisher = qstring_to_unicode(self.publisher.currentText()) 
         if isbn or title or author or publisher:
             d = FetchMetadata(self, isbn, title, author, publisher, self.timeout)
             d.exec_()
@@ -312,7 +371,7 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
                     self.title.setText(book.title)
                     self.authors.setText(authors_to_string(book.authors))
                     if book.author_sort: self.author_sort.setText(book.author_sort)
-                    if book.publisher: self.publisher.setText(book.publisher)
+                    if book.publisher: self.publisher.setEditText(book.publisher)
                     if book.isbn: self.isbn.setText(book.isbn)
                     summ = book.comments
                     if summ:
@@ -351,7 +410,7 @@ class MetadataSingleDialog(QDialog, Ui_MetadataSingleDialog):
             self.db.set_author_sort(self.id, aus, notify=False)
         self.db.set_isbn(self.id, qstring_to_unicode(self.isbn.text()), notify=False)
         self.db.set_rating(self.id, 2*self.rating.value(), notify=False)
-        self.db.set_publisher(self.id, qstring_to_unicode(self.publisher.text()), notify=False)
+        self.db.set_publisher(self.id, qstring_to_unicode(self.publisher.currentText()), notify=False)
         self.db.set_tags(self.id, qstring_to_unicode(self.tags.text()).split(','), notify=False)
         self.db.set_series(self.id, qstring_to_unicode(self.series.currentText()), notify=False)
         self.db.set_series_index(self.id, self.series_index.value(), notify=False)

src/calibre/gui2/dialogs/metadata_single.ui

@@ -95,13 +95,6 @@
             </property>
            </widget>
           </item>
-          <item row="1" column="1" >
-           <widget class="QLineEdit" name="authors" >
-            <property name="toolTip" >
-             <string>Change the author(s) of this book. Multiple authors should be separated by an &amp;. If the author name contains an &amp;, use &amp;&amp; to represent it.</string>
-            </property>
-           </widget>
-          </item>
           <item row="2" column="0" >
            <widget class="QLabel" name="label_8" >
             <property name="text" >
@@ -111,7 +104,7 @@
              <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
             </property>
             <property name="buddy" >
-             <cstring>authors</cstring>
+             <cstring>author_sort</cstring>
             </property>
            </widget>
           </item>
@@ -185,13 +178,6 @@
             </property>
            </widget>
           </item>
-          <item row="4" column="1" colspan="2" >
-           <widget class="QLineEdit" name="publisher" >
-            <property name="toolTip" >
-             <string>Change the publisher of this book</string>
-            </property>
-           </widget>
-          </item>
           <item row="5" column="0" >
            <widget class="QLabel" name="label_4" >
             <property name="text" >
@@ -330,6 +316,16 @@
           <item row="8" column="1" colspan="2" >
            <widget class="QLineEdit" name="isbn" />
           </item>
+          <item row="4" column="1" >
+           <widget class="QComboBox" name="publisher" >
+            <property name="editable" >
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
+          <item row="1" column="1" >
+           <widget class="QLineEdit" name="authors" />
+          </item>
          </layout>
         </widget>
        </item>
@@ -370,7 +366,7 @@
          <layout class="QVBoxLayout" name="verticalLayout" >
           <item>
            <layout class="QGridLayout" name="gridLayout" >
-            <item rowspan="2" row="0" column="0" >
+            <item rowspan="3" row="0" column="0" >
              <widget class="QListWidget" name="formats" >
               <property name="sizePolicy" >
                <sizepolicy vsizetype="Minimum" hsizetype="Minimum" >
@@ -412,7 +408,7 @@
               </property>
              </widget>
             </item>
-            <item row="1" column="1" >
+            <item row="2" column="1" >
              <widget class="QToolButton" name="remove_format_button" >
               <property name="toolTip" >
                <string>Remove the selected formats for this book from the database.</string>
@@ -432,6 +428,26 @@
               </property>
              </widget>
             </item>
+            <item row="1" column="1" >
+             <widget class="QToolButton" name="button_set_cover" >
+              <property name="toolTip" >
+               <string>Set the cover for the book from the selected format</string>
+              </property>
+              <property name="text" >
+               <string>...</string>
+              </property>
+              <property name="icon" >
+               <iconset resource="../images.qrc" >
+                <normaloff>:/images/book.svg</normaloff>:/images/book.svg</iconset>
+              </property>
+              <property name="iconSize" >
+               <size>
+                <width>32</width>
+                <height>32</height>
+               </size>
+              </property>
+             </widget>
+            </item>
            </layout>
           </item>
          </layout>
@@ -584,6 +600,33 @@
    <header>widgets.h</header>
   </customwidget>
  </customwidgets>
+ <tabstops>
+  <tabstop>title</tabstop>
+  <tabstop>swap_button</tabstop>
+  <tabstop>authors</tabstop>
+  <tabstop>author_sort</tabstop>
+  <tabstop>auto_author_sort</tabstop>
+  <tabstop>rating</tabstop>
+  <tabstop>publisher</tabstop>
+  <tabstop>tags</tabstop>
+  <tabstop>tag_editor_button</tabstop>
+  <tabstop>series</tabstop>
+  <tabstop>remove_series_button</tabstop>
+  <tabstop>series_index</tabstop>
+  <tabstop>isbn</tabstop>
+  <tabstop>comments</tabstop>
+  <tabstop>fetch_metadata_button</tabstop>
+  <tabstop>fetch_cover_button</tabstop>
+  <tabstop>password_button</tabstop>
+  <tabstop>cover_button</tabstop>
+  <tabstop>reset_cover</tabstop>
+  <tabstop>cover_path</tabstop>
+  <tabstop>add_format_button</tabstop>
+  <tabstop>button_set_cover</tabstop>
+  <tabstop>remove_format_button</tabstop>
+  <tabstop>formats</tabstop>
+  <tabstop>button_box</tabstop>
+ </tabstops>
  <resources>
   <include location="../images.qrc" />
  </resources>
@@ -595,8 +638,8 @@
    <slot>accept()</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>257</x>
-     <y>646</y>
+     <x>261</x>
+     <y>710</y>
     </hint>
     <hint type="destinationlabel" >
      <x>157</x>
@@ -611,8 +654,8 @@
    <slot>reject()</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>325</x>
-     <y>646</y>
+     <x>329</x>
+     <y>710</y>
     </hint>
     <hint type="destinationlabel" >
      <x>286</x>

src/calibre/gui2/dialogs/scheduler.py

@@ -155,10 +155,11 @@ class RecipeModel(QAbstractListModel, SearchQueryParser):
             return recipe
         elif role == Qt.DecorationRole:
             icon = self.default_icon
+            icon_path = (':/images/news/%s.png'%recipe.id).replace('recipe_', '') 
             if not recipe.builtin:
                 icon = self.custom_icon
-            elif QFile(':/images/news/%s.png'%recipe.id).exists():
-                icon = QIcon(':/images/news/%s.png'%recipe.id)
+            elif QFile().exists(icon_path):
+                icon = QIcon(icon_path)
             return QVariant(icon)
         
         return NONE

src/calibre/gui2/jobs2.py

@@ -128,7 +128,10 @@ class JobManager(QAbstractTableModel):
         self.emit(SIGNAL('layoutChanged()'))
     
     def _status_update(self, job):
+        try:
             row = self.jobs.index(job)
+        except ValueError: # Job has been stopped
+            return
         self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'),
                   self.index(row, 0), self.index(row, 3))
         

src/calibre/gui2/library.py

@@ -8,14 +8,15 @@ from math import cos, sin, pi
 from PyQt4.QtGui import QTableView, QAbstractItemView, QColor, \
                         QItemDelegate, QPainterPath, QLinearGradient, QBrush, \
                         QPen, QStyle, QPainter, QLineEdit, \
-                        QPalette, QImage, QApplication, QMenu
+                        QPalette, QImage, QApplication, QMenu, QStyledItemDelegate
 from PyQt4.QtCore import QAbstractTableModel, QVariant, Qt, QString, \
-                         SIGNAL, QObject, QSize, QModelIndex
+                         SIGNAL, QObject, QSize, QModelIndex, QDate
 
 from calibre import strftime
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.library.database2 import FIELD_MAP
-from calibre.gui2 import NONE, TableView, qstring_to_unicode, config
+from calibre.gui2 import NONE, TableView, qstring_to_unicode, config, \
+                         error_dialog
 from calibre.utils.search_query_parser import SearchQueryParser
 
 class LibraryDelegate(QItemDelegate):
@@ -81,6 +82,17 @@ class LibraryDelegate(QItemDelegate):
         sb.setMaximum(5)
         return sb
 
+class DateDelegate(QStyledItemDelegate):
+    
+    def displayText(self, val, locale):
+        d = val.toDate()
+        return d.toString('dd MMM yyyy')
+        if d.isNull():
+            return ''
+        d = datetime(d.year(), d.month(), d.day())
+        return strftime(BooksView.TIME_FMT, d.timetuple())
+        
+
 class BooksModel(QAbstractTableModel):
     coding = zip(
     [1000,900,500,400,100,90,50,40,10,9,5,4,1],
@@ -113,7 +125,8 @@ class BooksModel(QAbstractTableModel):
         QAbstractTableModel.__init__(self, parent)
         self.db = None
         self.column_map = config['column_map']
-        self.editable_cols = ['title', 'authors', 'rating', 'publisher', 'tags', 'series']
+        self.editable_cols = ['title', 'authors', 'rating', 'publisher', 
+                              'tags', 'series', 'timestamp']
         self.default_image = QImage(':/images/book.svg')
         self.sorted_on = ('timestamp', Qt.AscendingOrder)
         self.last_search = '' # The last search performed on this model
@@ -135,7 +148,12 @@ class BooksModel(QAbstractTableModel):
                 idx = self.column_map.index('rating')
             except ValueError:
                 idx = -1
-            self.emit(SIGNAL('columns_sorted(int)'), idx)
+            try:
+                tidx = self.column_map.index('timestamp')
+            except ValueError:
+                tidx = -1
+            
+            self.emit(SIGNAL('columns_sorted(int,int)'), idx, tidx)
         
     
     def set_database(self, db):
@@ -442,7 +460,7 @@ class BooksModel(QAbstractTableModel):
             dt = self.db.data[r][tmdx]
             if dt:
                 dt = dt - timedelta(seconds=time.timezone) + timedelta(hours=time.daylight)
-                return strftime(BooksView.TIME_FMT, dt.timetuple())
+                return QDate(dt.year, dt.month, dt.day)
         
         def rating(r):
             r = self.db.data[r][ridx]
@@ -507,35 +525,40 @@ class BooksModel(QAbstractTableModel):
         return flags
 
     def setData(self, index, value, role):
-        done = False
         if role == Qt.EditRole:
             row, col = index.row(), index.column()
             column = self.column_map[col]
             if column not in self.editable_cols:
                 return False
-            val = unicode(value.toString().toUtf8(), 'utf-8').strip() if column != 'rating' else \
-                  int(value.toInt()[0])
+            val = int(value.toInt()[0]) if column == 'rating' else \
+                  value.toDate() if column == 'timestamp' else \
+                  unicode(value.toString())
+            id = self.db.id(row)  
             if column == 'rating':
                 val = 0 if val < 0 else 5 if val > 5 else val
                 val *= 2
-            if column == 'series':
+            elif column == 'series':
                 pat = re.compile(r'\[(\d+)\]')
                 match = pat.search(val)
-                id = self.db.id(row)
                 if match is not None:
                     self.db.set_series_index(id, int(match.group(1)))
                     val = pat.sub('', val)
                 val = val.strip()
                 if val:
                     self.db.set_series(id, val)
+            elif column == 'timestamp':
+                if val.isNull() or not val.isValid():
+                    return False
+                dt = datetime(val.year(), val.month(), val.day()) + timedelta(seconds=time.timezone) - timedelta(hours=time.daylight)
+                self.db.set_timestamp(id, dt)
             else:
                 self.db.set(row, column, val)
             self.emit(SIGNAL("dataChanged(QModelIndex, QModelIndex)"), \
                                 index, index)
             if column == self.sorted_on[0]:
                 self.resort()
-            done = True
-        return done
+            
+        return True
 
 class BooksView(TableView):
     TIME_FMT = '%d %b %Y'
@@ -554,25 +577,29 @@ class BooksView(TableView):
     def __init__(self, parent, modelcls=BooksModel):
         TableView.__init__(self, parent)
         self.rating_delegate = LibraryDelegate(self)
+        self.timestamp_delegate = DateDelegate(self)
         self.display_parent = parent
         self._model = modelcls(self)
         self.setModel(self._model)
         self.setSelectionBehavior(QAbstractItemView.SelectRows)
         self.setSortingEnabled(True)
         try:
-            self.columns_sorted(self._model.column_map.index('rating'))
+            self.columns_sorted(self._model.column_map.index('rating'),
+                                self._model.column_map.index('timestamp'))
         except ValueError:
             pass
         QObject.connect(self.selectionModel(), SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
                         self._model.current_changed)
-        self.connect(self._model, SIGNAL('columns_sorted(int)'), self.columns_sorted, Qt.QueuedConnection)
+        self.connect(self._model, SIGNAL('columns_sorted(int, int)'), self.columns_sorted, Qt.QueuedConnection)
         
-    def columns_sorted(self, col):
+    def columns_sorted(self, rating_col, timestamp_col):
         for i in range(self.model().columnCount(None)):
             if self.itemDelegateForColumn(i) == self.rating_delegate:
                 self.setItemDelegateForColumn(i, self.itemDelegate())
-        if col > -1:
-            self.setItemDelegateForColumn(col, self.rating_delegate)
+        if rating_col > -1:
+            self.setItemDelegateForColumn(rating_col, self.rating_delegate)
+        if timestamp_col > -1:
+            self.setItemDelegateForColumn(timestamp_col, self.timestamp_delegate)
             
     def set_context_menu(self, edit_metadata, send_to_device, convert, view, 
                          save, open_folder, book_details, similar_menu=None):
@@ -657,6 +684,8 @@ class DeviceBooksView(BooksView):
         self.rating_delegate = None
         for i in range(10):
             self.setItemDelegateForColumn(i, self.itemDelegate())
+        self.setDragDropMode(self.NoDragDrop)
+        self.setAcceptDrops(False)
 
     def resizeColumnsToContents(self):
         QTableView.resizeColumnsToContents(self)
@@ -668,6 +697,10 @@ class DeviceBooksView(BooksView):
     def sortByColumn(self, col, order):
         TableView.sortByColumn(self, col, order)
         
+    def dropEvent(self, *args):
+        error_dialog(self, _('Not allowed'), 
+        _('Dropping onto a device is not supported. First add the book to the calibre library.')).exec_()
+
 class OnDeviceSearch(SearchQueryParser):
     
     def __init__(self, model):

src/calibre/gui2/main.py

@@ -3,11 +3,11 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import os, sys, textwrap, collections, traceback, time
 from xml.parsers.expat import ExpatError
 from functools import partial
-from PyQt4.QtCore import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer
-from PyQt4.QtGui import QPixmap, QColor, QPainter, QMenu, QIcon, QMessageBox, \
+from PyQt4.Qt import Qt, SIGNAL, QObject, QCoreApplication, QUrl, QTimer, \
+                     QModelIndex, QPixmap, QColor, QPainter, QMenu, QIcon, \
                      QToolButton, QDialog, QDesktopServices, QFileDialog, \
                      QSystemTrayIcon, QApplication, QKeySequence, QAction, \
-                        QProgressDialog
+                     QProgressDialog, QMessageBox, QStackedLayout
 from PyQt4.QtSvg import QSvgRenderer
 
 from calibre import __version__, __appname__, islinux, sanitize_file_name, \
@@ -22,7 +22,8 @@ from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            pixmap_to_data, choose_dir, ORG_NAME, \
                            set_sidebar_directories, Dispatcher, \
                            SingleApplication, Application, available_height, \
-                           max_available_height, config, info_dialog
+                           max_available_height, config, info_dialog, \
+                           available_width
 from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
 from calibre.library.database import LibraryDatabase
 from calibre.gui2.dialogs.scheduler import Scheduler
@@ -49,6 +50,7 @@ from calibre.library.database2 import LibraryDatabase2, CoverCache
 from calibre.parallel import JobKilled
 from calibre.utils.filenames import ascii_filename
 from calibre.gui2.widgets import WarningDialog
+from calibre.gui2.dialogs.confirm_delete import confirm
 
 class Main(MainWindow, Ui_MainWindow):
 
@@ -187,8 +189,8 @@ class Main(MainWindow, Ui_MainWindow):
         self.metadata_menu = md
         self.add_menu = QMenu()
         self.add_menu.addAction(_('Add books from a single directory'))
-        self.add_menu.addAction(_('Add books recursively (One book per directory, assumes every ebook file is the same book in a different format)'))
-        self.add_menu.addAction(_('Add books recursively (Multiple books per directory, assumes every ebook file is a different book)'))
+        self.add_menu.addAction(_('Add books from directories, including sub-directories (One book per directory, assumes every ebook file is the same book in a different format)'))
+        self.add_menu.addAction(_('Add books from directories, including sub directories (Multiple books per directory, assumes every ebook file is a different book)'))
         self.action_add.setMenu(self.add_menu)
         QObject.connect(self.action_add, SIGNAL("triggered(bool)"), self.add_books)
         QObject.connect(self.add_menu.actions()[0], SIGNAL("triggered(bool)"), self.add_books)
@@ -307,7 +309,6 @@ class Main(MainWindow, Ui_MainWindow):
                 db = LibraryDatabase2(self.library_path)
         self.library_view.set_database(db)
         if self.olddb is not None:
-            from PyQt4.QtGui import QProgressDialog
             pd = QProgressDialog('', '', 0, 100, self)
             pd.setWindowModality(Qt.ApplicationModal)
             pd.setCancelButton(None)
@@ -342,8 +343,15 @@ class Main(MainWindow, Ui_MainWindow):
         ########################### Cover Flow ################################
         self.cover_flow = None
         if CoverFlow is not None:
-            self.cover_flow = CoverFlow(height=220 if available_height() > 950 else 170 if available_height() > 850 else 140)
+            text_height = 40 if config['separate_cover_flow'] else 25
+            ah = available_height()
+            cfh = ah-100
+            cfh = 3./5 * cfh - text_height
+            if not config['separate_cover_flow']:
+                cfh = 220 if ah > 950 else 170 if ah > 850 else 140
+            self.cover_flow = CoverFlow(height=cfh, text_height=text_height)
             self.cover_flow.setVisible(False)
+            if not config['separate_cover_flow']:
                 self.library.layout().addWidget(self.cover_flow)
             self.connect(self.cover_flow, SIGNAL('currentChanged(int)'), self.sync_cf_to_listview)
             self.connect(self.cover_flow, SIGNAL('itemActivated(int)'), self.show_book_info)
@@ -410,6 +418,29 @@ class Main(MainWindow, Ui_MainWindow):
                 
     
     def toggle_cover_flow(self, show):
+        if config['separate_cover_flow']:
+            if show:
+                d = QDialog(self)
+                ah, aw = available_height(), available_width()
+                d.resize(int(aw/2.), ah-60)
+                d._layout = QStackedLayout()
+                d.setLayout(d._layout)
+                d.setWindowTitle(_('Browse by covers'))
+                d.layout().addWidget(self.cover_flow)
+                self.cover_flow.setVisible(True)
+                self.cover_flow.setFocus(Qt.OtherFocusReason)
+                self.library_view.scrollTo(self.library_view.currentIndex())
+                d.show()
+                self.connect(d, SIGNAL('finished(int)'), 
+                             lambda x: self.status_bar.cover_flow_button.setChecked(False))
+                self.cf_dialog = d
+            else:
+                cfd = getattr(self, 'cf_dialog', None)
+                if cfd is not None:
+                    self.cover_flow.setVisible(False)
+                    cfd.hide()
+                    self.cf_dialog = None
+        else:
             if show:
                 self.library_view.setCurrentIndex(self.library_view.currentIndex())
                 self.cover_flow.setVisible(True)
@@ -583,6 +614,7 @@ class Main(MainWindow, Ui_MainWindow):
         try:
             duplicates = self.library_view.model().db.recursive_import(root, single, callback=callback)
         finally:
+            progress.hide()
             progress.close()            
         if duplicates:
             files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
@@ -702,7 +734,9 @@ class Main(MainWindow, Ui_MainWindow):
             else:
                 self.upload_books(paths, list(map(sanitize_file_name, names)), infos, on_card=on_card)
         finally:
-            progress.setValue(len(paths))
+            progress.setValue(progress.maximum())
+            progress.hide()
+            progress.close()
 
     def upload_books(self, files, names, metadata, on_card=False, memory=None):
         '''
@@ -758,13 +792,9 @@ class Main(MainWindow, Ui_MainWindow):
         rows = view.selectionModel().selectedRows()
         if not rows or len(rows) == 0:
             return
-        if config['confirm_delete']:
-            d = question_dialog(self, _('Confirm delete'), 
-                            _('Are you sure you want to delete these %d books?')%len(rows))
-            if d.exec_() != QMessageBox.Yes:
-                return
-            
         if self.stack.currentIndex() == 0:
+            if not confirm('<p>'+_('The selected books will be <b>permanently deleted</b> and the files removed from your computer. Are you sure?')+'</p>', 'library_delete_books', self):
+                return
             view.model().delete_books(rows)
         else:
             view = self.memory_view if self.stack.currentIndex() == 1 else self.card_view
@@ -801,6 +831,7 @@ class Main(MainWindow, Ui_MainWindow):
         Edit metadata of selected books in library.
         '''
         rows = self.library_view.selectionModel().selectedRows()
+        previous = self.library_view.currentIndex()
         if not rows or len(rows) == 0:
             d = error_dialog(self, _('Cannot edit metadata'), _('No books selected'))
             d.exec_()
@@ -817,6 +848,9 @@ class Main(MainWindow, Ui_MainWindow):
                                     self.library_view.model().db,
                                     accepted_callback=accepted)
             d.exec_()
+        if rows:
+            current = self.library_view.currentIndex()
+            self.library_view.model().current_changed(current, previous)
             
     def edit_bulk_metadata(self, checked):
         '''
@@ -1046,6 +1080,8 @@ class Main(MainWindow, Ui_MainWindow):
     def convert_single(self, checked):
         r = self.get_books_for_conversion()
         if r is None: return
+        previous = self.library_view.currentIndex()
+        rows = [x.row() for x in self.library_view.selectionModel().selectedRows()]
         comics, others = r
         jobs, changed = convert_single_ebook(self, self.library_view.model().db, comics, others)
         for func, args, desc, fmt, id, temp_files in jobs:
@@ -1054,8 +1090,9 @@ class Main(MainWindow, Ui_MainWindow):
             self.conversion_jobs[job] = (temp_files, fmt, id)
             
         if changed:
-            self.library_view.model().resort(reset=False)
-            self.library_view.model().research()
+            self.library_view.model().refresh_rows(rows)
+            current = self.library_view.currentIndex()
+            self.library_view.model().current_changed(current, previous)
                     
     def book_converted(self, job):
         temp_files, fmt, book_id = self.conversion_jobs.pop(job)
@@ -1074,6 +1111,9 @@ class Main(MainWindow, Ui_MainWindow):
                         os.remove(f.name)
                 except:
                     pass
+        if self.current_view() is self.library_view:
+            current = self.library_view.currentIndex()
+            self.library_view.model().current_changed(current, QModelIndex())
     
     #############################View book######################################
 
@@ -1206,7 +1246,6 @@ class Main(MainWindow, Ui_MainWindow):
                     newloc = d.database_location
                     if not os.path.exists(os.path.join(newloc, 'metadata.db')):
                         if os.access(self.library_path, os.R_OK):
-                            from PyQt4.QtGui import QProgressDialog
                             pd = QProgressDialog('', '', 0, 100, self)
                             pd.setWindowModality(Qt.ApplicationModal)
                             pd.setCancelButton(None)
@@ -1385,7 +1424,12 @@ in which you want to store your books files. Any existing books will be automati
             self.memory_view.write_settings()
     
     def quit(self, checked, restart=False):
-        if self.shutdown():
+        if not self.confirm_quit():
+            return
+        try:
+            self.shutdown()
+        except:
+            pass
         self.restart_after_quit = restart
         QApplication.instance().quit()
             
@@ -1418,22 +1462,26 @@ in which you want to store your books files. Any existing books will be automati
         QDesktopServices.openUrl(QUrl.fromLocalFile(pt.name))
             
     
-    def shutdown(self):
+    def confirm_quit(self):
+        if self.job_manager.has_jobs():
             msg = _('There are active jobs. Are you sure you want to quit?')
             if self.job_manager.has_device_jobs():
                 msg = '<p>'+__appname__ + _(''' is communicating with the device!<br>
                       'Quitting may cause corruption on the device.<br>
                       'Are you sure you want to quit?''')+'</p>'
-        if self.job_manager.has_jobs():
+            
             d = QMessageBox(QMessageBox.Warning, _('WARNING: Active jobs'), msg,
                             QMessageBox.Yes|QMessageBox.No, self)
             d.setIconPixmap(QPixmap(':/images/dialog_warning.svg'))
             d.setDefaultButton(QMessageBox.No)
             if d.exec_() != QMessageBox.Yes:
                 return False
+        return True
 
-        self.job_manager.terminate_all_jobs()
+    
+    def shutdown(self):
         self.write_settings()
+        self.job_manager.terminate_all_jobs()
         self.device_manager.keep_going = False
         self.cover_cache.stop()
         self.hide()
@@ -1459,7 +1507,11 @@ in which you want to store your books files. Any existing books will be automati
             self.hide()
             e.ignore()
         else:
-            if self.shutdown():
+            if self.confirm_quit():
+                try:
+                    self.shutdown()
+                except:
+                    pass
                 e.accept()
             else:
                 e.ignore()

src/calibre/gui2/status.py

@@ -199,7 +199,10 @@ class StatusBar(QStatusBar):
         ret = QStatusBar.showMessage(self, msg, timeout)
         if self.systray is not None:
             if isosx and isinstance(msg, unicode):
+                try:
                     msg = msg.encode(preferred_encoding)
+                except UnicodeEncodeError:
+                    msg = msg.encode('utf-8')
             self.systray.showMessage('calibre', msg, self.systray.Information, 10000)
         return ret
     

src/calibre/gui2/tools.py

@@ -13,7 +13,7 @@ from calibre.utils.config import prefs
 from calibre.gui2.dialogs.lrf_single import LRFSingleDialog, LRFBulkDialog
 from calibre.gui2.dialogs.epub import Config as EPUBConvert
 import calibre.gui2.dialogs.comicconf as ComicConf
-from calibre.gui2 import warning_dialog, dynamic
+from calibre.gui2 import warning_dialog
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.ebooks.lrf import preferred_source_formats as LRF_PREFERRED_SOURCE_FORMATS
 from calibre.ebooks.metadata.opf import OPFCreator
@@ -22,7 +22,9 @@ from calibre.ebooks.epub.from_any import SOURCE_FORMATS as EPUB_PREFERRED_SOURCE
 def convert_single_epub(parent, db, comics, others):
     changed = False
     jobs = []
-    for row in others:
+    others_ids = [db.id(row) for row in others]
+    comics_ids = [db.id(row) for row in comics]
+    for row, row_id in zip(others, others_ids):
         temp_files = []
         d = EPUBConvert(parent, db, row)
         if d.source_format is not None:
@@ -44,10 +46,10 @@ def convert_single_epub(parent, db, comics, others):
                     opts.cover = d.cover_file.name
                 temp_files.extend([d.opf_file, pt, of])
                 jobs.append(('any2epub', args, _('Convert book: ')+d.mi.title, 
-                             'EPUB', db.id(row), temp_files))
+                             'EPUB', row_id, temp_files))
                 changed = True
                 
-    for row in comics:
+    for row, row_id in zip(comics, comics_ids):
         mi = db.get_metadata(row)
         title = author = _('Unknown')
         if mi.title:
@@ -76,7 +78,7 @@ def convert_single_epub(parent, db, comics, others):
         args = [pt.name, opts]
         changed = True
         jobs.append(('comic2epub', args, _('Convert comic: ')+opts.title, 
-                     'EPUB', db.id(row), [pt, of]))
+                     'EPUB', row_id, [pt, of]))
         
     return jobs, changed
     
@@ -85,7 +87,9 @@ def convert_single_epub(parent, db, comics, others):
 def convert_single_lrf(parent, db, comics, others):
     changed = False
     jobs = []
-    for row in others:
+    others_ids = [db.id(row) for row in others]
+    comics_ids = [db.id(row) for row in comics]
+    for row, row_id in zip(others, others_ids):
         temp_files = []
         d = LRFSingleDialog(parent, db, row)
         if d.selected_format:
@@ -104,10 +108,10 @@ def convert_single_lrf(parent, db, comics, others):
                     temp_files.append(d.cover_file)
                 temp_files.extend([pt, of])
                 jobs.append(('any2lrf', [cmdline], _('Convert book: ')+d.title(), 
-                             'LRF', db.id(row), temp_files))
+                             'LRF', row_id, temp_files))
                 changed = True
                 
-    for row in comics:
+    for row, row_id in zip(comics, comics_ids):
         mi = db.get_metadata(row)
         title = author = _('Unknown')
         if mi.title:
@@ -138,7 +142,7 @@ def convert_single_lrf(parent, db, comics, others):
         args = [pt.name, opts]
         changed = True
         jobs.append(('comic2lrf', args, _('Convert comic: ')+opts.title, 
-                     'LRF', db.id(row), [pt, of]))
+                     'LRF', row_id, [pt, of]))
         
     return jobs, changed
 
@@ -162,6 +166,7 @@ def convert_bulk_epub(parent, db, comics, others):
     parent.status_bar.showMessage(_('Starting Bulk conversion of %d books')%total, 2000)
     
     for i, row in enumerate(others+comics):
+        row_id = db.id(row)
         if row in others:
             data = None
             for fmt in EPUB_PREFERRED_SOURCE_FORMATS:
@@ -198,7 +203,7 @@ def convert_bulk_epub(parent, db, comics, others):
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
             temp_files = [cf] if cf is not None else []
             temp_files.extend([opf_file, pt, of])
-            jobs.append(('any2epub', args, desc, 'EPUB', db.id(row), temp_files))
+            jobs.append(('any2epub', args, desc, 'EPUB', row_id, temp_files))
         else:
             options = comic_opts.copy()
             mi = db.get_metadata(row)
@@ -224,7 +229,7 @@ def convert_bulk_epub(parent, db, comics, others):
             options.verbose = 1
             args = [pt.name, options]
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
-            jobs.append(('comic2epub', args, desc, 'EPUB', db.id(row), [pt, of]))        
+            jobs.append(('comic2epub', args, desc, 'EPUB', row_id, [pt, of]))        
         
     if bad_rows:
         res = []
@@ -255,6 +260,7 @@ def convert_bulk_lrf(parent, db, comics, others):
     parent.status_bar.showMessage(_('Starting Bulk conversion of %d books')%total, 2000)
     
     for i, row in enumerate(others+comics):
+        row_id = db.id(row)
         if row in others:
             cmdline = list(d.cmdline)
             mi = db.get_metadata(row)
@@ -294,7 +300,7 @@ def convert_bulk_lrf(parent, db, comics, others):
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
             temp_files = [cf] if cf is not None else []
             temp_files.extend([pt, of])
-            jobs.append(('any2lrf', [cmdline], desc, 'LRF', db.id(row), temp_files))
+            jobs.append(('any2lrf', [cmdline], desc, 'LRF', row_id, temp_files))
         else:
             options = comic_opts.copy()
             mi = db.get_metadata(row)
@@ -320,7 +326,7 @@ def convert_bulk_lrf(parent, db, comics, others):
             options.verbose = 1
             args = [pt.name, options]
             desc = _('Convert book %d of %d (%s)')%(i+1, total, repr(mi.title))
-            jobs.append(('comic2lrf', args, desc, 'LRF', db.id(row), [pt, of]))        
+            jobs.append(('comic2lrf', args, desc, 'LRF', row_id, [pt, of]))        
         
     if bad_rows:
         res = []

src/calibre/gui2/viewer/documentview.py

@@ -523,7 +523,10 @@ class DocumentView(QWebView):
                     self.manager.previous_document()
                     event.accept()
                     return
-        return QWebView.wheelEvent(self, event)
+        ret = QWebView.wheelEvent(self, event)
+        if self.manager is not None:
+            self.manager.scrolled(self.scroll_fraction)
+        return ret
     
     def keyPressEvent(self, event):
         key = event.key()

src/calibre/gui2/viewer/main.py

@@ -244,6 +244,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
                      lambda x:self.find(unicode(self.search.text()), True, repeat=True))
         self.connect(self.action_full_screen, SIGNAL('triggered(bool)'),
                      self.toggle_fullscreen)
+        self.action_full_screen.setShortcuts([Qt.Key_F11, Qt.CTRL+Qt.SHIFT+Qt.Key_F])
         self.connect(self.action_back, SIGNAL('triggered(bool)'), self.back)
         self.connect(self.action_bookmark, SIGNAL('triggered(bool)'), self.bookmark)
         self.connect(self.action_forward, SIGNAL('triggered(bool)'), self.forward)
@@ -511,7 +512,7 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
             self.metadata.show_opf(self.iterator.opf)
             title = self.iterator.opf.title
             if not title:
-                title = os.path.splitext(os.path.basename(pathtoebook))
+                title = os.path.splitext(os.path.basename(pathtoebook))[0]
             self.action_table_of_contents.setDisabled(not self.iterator.toc)
             if self.iterator.toc:
                 self.toc_model = TOC(self.iterator.toc)

src/calibre/library/cli.py

@@ -76,13 +76,25 @@ STANZA_TEMPLATE='''\
   <entry>
       <title>${record['title']}</title>
       <id>urn:calibre:${record['id']}</id>
-      <author><name>${record['authors']}</name></author>
+      <author><name>${record['author_sort']}</name></author>
       <updated>${record['timestamp'].strftime('%Y-%m-%dT%H:%M:%SZ')}</updated>
       <link type="application/epub+zip" href="${quote(record['fmt_epub'].replace(sep, '/')).replace('http%3A', 'http:')}" />
       <link py:if="record['cover']" rel="x-stanza-cover-image" type="image/png" href="${quote(record['cover'].replace(sep, '/')).replace('http%3A', 'http:')}" />
       <link py:if="record['cover']" rel="x-stanza-cover-image-thumbnail" type="image/png" href="${quote(record['cover'].replace(sep, '/')).replace('http%3A', 'http:')}" />
       <content type="xhtml">
-          <div xmlns="http://www.w3.org/1999/xhtml">${record['comments']}</div>
+          <div xmlns="http://www.w3.org/1999/xhtml">
+              <py:for each="f in ('authors', 'publisher', 'rating', 'tags', 'series', 'isbn')">
+              <py:if test="record[f]">
+              ${f.capitalize()}:${unicode(', '.join(record[f]) if f=='tags' else record[f])}
+              <py:if test="f =='series'"># ${str(record['series_index'])}</py:if>
+              <br/>
+              </py:if>
+              </py:for>
+              <py:if test="record['comments']">
+              <br/>
+              ${record['comments']}
+              </py:if>
+          </div>
       </content>
   </entry>
   </py:for>
@@ -221,7 +233,7 @@ NULL = DevNull()
 
 def do_add(db, paths, one_book_per_directory, recurse, add_duplicates):
     orig = sys.stdout
-    sys.stdout = NULL
+    #sys.stdout = NULL
     try:
         files, dirs = [], []
         for path in paths:

src/calibre/library/database.py

@@ -944,6 +944,10 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
             return self.conn.get('SELECT publisher FROM meta WHERE id=?', (index,), all=False)
         return self.data[index][3]
     
+    def publisher_id(self, index, index_is_id=False):
+        id  = index if index_is_id else self.id(index)
+        return self.conn.get('SELECT publisher from books_publishers_link WHERE book=?', (id,), all=False)
+    
     def rating(self, index, index_is_id=False):
         if index_is_id:
             return self.conn.get('SELECT rating FROM meta WHERE id=?', (index,), all=False)
@@ -1042,6 +1046,14 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
         return [ (i[0], i[1]) for i in \
                 self.conn.get('SELECT id, name FROM series')]
         
+    def all_authors(self):
+        return [ (i[0], i[1]) for i in \
+                self.conn.get('SELECT id, name FROM authors')]
+    
+    def all_publishers(self):
+        return [ (i[0], i[1]) for i in \
+                self.conn.get('SELECT id, name FROM publishers')]
+
     def all_tags(self):
         return [i[0].strip() for i in self.conn.get('SELECT name FROM tags') if i[0].strip()]
 

src/calibre/library/database2.py

@@ -21,13 +21,12 @@ from calibre.library.sqlite import connect, IntegrityError
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.ebooks.metadata import string_to_authors, authors_to_string
 from calibre.ebooks.metadata.meta import get_metadata
-from calibre.constants import preferred_encoding, iswindows, isosx
+from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import run_plugins_on_import
+from calibre import sanitize_file_name
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
-filesystem_encoding = sys.getfilesystemencoding()
-if filesystem_encoding is None: filesystem_encoding = 'utf-8'
 iscaseinsensitive = iswindows or isosx
 
 def normpath(x):
@@ -37,23 +36,6 @@ def normpath(x):
         x = x.lower()
     return x
 
-_filename_sanitize = re.compile(r'[\xae\0\\|\?\*<":>\+\[\]/]')
-
-def sanitize_file_name(name, substitute='_'):
-    '''
-    Sanitize the filename `name`. All invalid characters are replaced by `substitute`.
-    The set of invalid characters is the union of the invalid characters in Windows,
-    OS X and Linux. Also removes leading an trailing whitespace.
-    **WARNING:** This function also replaces path separators, so only pass file names
-    and not full paths to it.
-    *NOTE:* This function always returns byte strings, not unicode objects. The byte strings
-    are encoded in the filesystem encoding of the platform, or UTF-8. 
-    '''
-    if isinstance(name, unicode):
-        name = name.encode(filesystem_encoding, 'ignore')
-    one = _filename_sanitize.sub(substitute, name)
-    one = re.sub(r'\s', ' ', one).strip()
-    return re.sub(r'^\.+$', '_', one)
 
 FIELD_MAP = {'id':0, 'title':1, 'authors':2, 'publisher':3, 'rating':4, 'timestamp':5, 
              'size':6, 'tags':7, 'comments':8, 'series':9, 'series_index':10,
@@ -489,8 +471,8 @@ class LibraryDatabase2(LibraryDatabase):
         authors = self.authors(id, index_is_id=True)
         if not authors:
             authors = _('Unknown')
-        author = sanitize_file_name(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding)
-        title  = sanitize_file_name(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding)
+        author = sanitize_file_name(authors.split(',')[0][:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
+        title  = sanitize_file_name(self.title(id, index_is_id=True)[:self.PATH_LIMIT]).decode(filesystem_encoding, 'replace')
         name   = title + ' - ' + author
         return name
     
@@ -896,6 +878,14 @@ class LibraryDatabase2(LibraryDatabase):
         if notify:
             self.notify('metadata', [id])
             
+    def set_timestamp(self, id, dt, notify=True):
+        if dt:
+            self.conn.execute('UPDATE books SET timestamp=? WHERE id=?', (dt, id))
+            self.data.set(id, FIELD_MAP['timestamp'], dt, row_is_id=True)
+            self.conn.commit()
+            if notify:
+                self.notify('metadata', [id])
+    
     def set_publisher(self, id, publisher, notify=True):
         self.conn.execute('DELETE FROM books_publishers_link WHERE book=?',(id,))
         self.conn.execute('DELETE FROM publishers WHERE (SELECT COUNT(id) FROM books_publishers_link WHERE publisher=publishers.id) < 1')
@@ -1118,8 +1108,13 @@ class LibraryDatabase2(LibraryDatabase):
                 continue
             series_index = 1 if mi.series_index is None else mi.series_index
             aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
+            title = mi.title
+            if isinstance(aus, str):
+                aus = aus.decode(preferred_encoding, 'replace')
+            if isinstance(title, str):
+                title = title.decode(preferred_encoding)
             obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)', 
-                              (mi.title, uri, series_index, aus))
+                              (title, uri, series_index, aus))
             id = obj.lastrowid
             self.data.books_added([id], self.conn)
             ids.append(id)

src/calibre/linux.py

@@ -2,10 +2,10 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 ''' Post installation script for linux '''
 import sys, os, re, shutil
-
 from subprocess import check_call, call
-from calibre import __version__, __appname__
+from tempfile import NamedTemporaryFile
 
+from calibre import __version__, __appname__
 from calibre.devices import devices
 
 DEVICES = devices()
@@ -401,9 +401,9 @@ def install_man_pages(fatal_errors):
     import subprocess
     print 'Installing MAN pages...'
     manpath = '/usr/share/man/man1'
-    f = open_file('/tmp/man_extra', 'wb')
+    f = NamedTemporaryFile()
     f.write('[see also]\nhttp://%s.kovidgoyal.net\n'%__appname__)
-    f.close()
+    f.flush()
     manifest = []
     os.environ['PATH'] += ':'+os.path.expanduser('~/bin')
     for src in entry_points['console_scripts']:

src/calibre/manual/customize.rst

@@ -46,7 +46,7 @@ Create a file name :file:`my_plugin.py` (the file name must end with plugin.py)
       ext  = os.path.splitext(path_to_ebook)[-1][1:].lower()
       mi = get_metadata(file, ext)
       mi.publisher = 'Hello World'
-      set_metadata(file, ext, mi)
+      set_metadata(file, mi, ext)
       return path_to_ebook
 
 That's all. To add this code to |app| as a plugin, simply create a zip file with::

src/calibre/manual/faq.rst

@@ -143,6 +143,15 @@ Where are the book files stored?
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 When you first run |app|, it will ask you for a folder in which to store your books. Whenever you add a book to |app|, it will copy the book into that folder. Books in the folder are nicely arranged into sub-folders by Author and Title. Metadata about the books is stored in the file ``metadata.db`` (which is a sqlite database).
 
+Why doesn't |app| let me store books in my own directory structure?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+The whole point if |app|'s library management features is that they provide an interface for locating books that is *much* more efficient than any possible directory scheme you could come up with for your collection. Indeed, once you become comfortable using |app|'s interface to find, sort and browse your collection, you wont ever feel the need to hunt through the files on your disk to find a book again. By managing books in its own directory struture of Author -> Title -> Book files, |app| is able to achieve a high level of reliability and standardization.  
+
+Why doesn't |app| have a column for foo?
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+|app| is designed to have columns for the most frequently and widely used fields. If it does not have a coulmn for your favorite field, you can always add a tag to the book for that piece of information. |app| also supports a general purpose "comments" fields for longer items.
+
 
 Content From The Web
 ---------------------

src/calibre/manual/news.rst

@@ -133,7 +133,7 @@ to the recipe. Finally, lets replace some of the :term:`CSS` that we disabled ea
 
 With these additions, our recipe has become "production quality", indeed it is very close to the actual recipe used by |app| for the *BBC*, shown below:
 
-.. literalinclude:: ../web/feeds/recipes/bbc.py
+.. literalinclude:: ../web/feeds/recipes/recipe_bbc.py
 
 This :term:`recipe` explores only the tip of the iceberg when it comes to the power of |app|. To explore more of the abilities of |app| we'll examine a more complex real life example in the next section.
 

src/calibre/parallel.py

@@ -31,7 +31,7 @@ from threading import RLock, Thread, Event
 from math import ceil
 
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre import iswindows, detect_ncpus, isosx
+from calibre import iswindows, detect_ncpus, isosx, preferred_encoding
 from calibre.utils.config import prefs
 
 DEBUG = False
@@ -615,7 +615,9 @@ class Job(object):
                 self.log = unicode(self.log, 'utf-8', 'replace')
             ans.extend(self.log.split('\n'))
         
-        return '<br>'.join(ans)
+        ans = [x.decode(preferred_encoding, 'replace') if isinstance(x, str) else x for x in ans]
+        
+        return u'<br>'.join(ans)
 
 
 class ParallelJob(Job):

src/calibre/trac/donations/server.py

@@ -21,7 +21,7 @@ from lxml import etree
 
 def range_for_month(year, month):
     ty, tm = date.today().year, date.today().month
-    min = date(year=year, month=month, day=1)
+    min = max = date(year=year, month=month, day=1)
     x = date.today().day if ty == year and tm == month else 31
     while x > 1:
         try:
@@ -102,6 +102,8 @@ class Stats:
 
     def get_deviation(self, amounts):
         l = float(len(amounts))
+        if l == 0:
+            return 0
         mean = sum(amounts)/l
         return sqrt( sum([i**2 for i in amounts])/l - mean**2  )
 
@@ -199,7 +201,7 @@ class Server(object):
         x = list(range(days-1, -1, -1))
         y = stats.daily_totals
         ax.plot(x, y)#, align='center', width=20, color='g')
-        ax.set_xlabel('Day')
+        ax.set_xlabel('Days ago')
         ax.set_ylabel('Income ($)')
         ax.hlines([stats.daily_average], 0, days-1)
         ax.set_xlim([0, days-1])

src/calibre/utils/config.py

@@ -17,7 +17,9 @@ from calibre.constants import terminal_controller, iswindows, isosx, \
 from calibre.utils.lock import LockError, ExclusiveFile 
 from collections import defaultdict
 
-if iswindows:
+if os.environ.has_key('CALIBRE_CONFIG_DIRECTORY'):
+    config_dir = os.path.abspath(os.environ['CALIBRE_CONFIG_DIRECTORY'])
+elif iswindows:
     config_dir = plugins['winutil'][0].special_folder_path(plugins['winutil'][0].CSIDL_APPDATA)
     if not os.access(config_dir, os.W_OK|os.X_OK):
         config_dir = os.path.expanduser('~')

src/calibre/utils/fontconfig.py

@@ -22,7 +22,7 @@ match to a given font specification. The main functions in this module are:
 .. autofunction:: match
 '''
 
-import sys, os, locale, codecs
+import sys, os, locale, codecs, subprocess, re
 from ctypes import cdll, c_void_p, Structure, c_int, POINTER, c_ubyte, c_char, util, \
                    pointer, byref, create_string_buffer, Union, c_char_p, c_double
 
@@ -34,6 +34,7 @@ except:
 
 iswindows = 'win32' in sys.platform or 'win64' in sys.platform
 isosx     = 'darwin' in sys.platform
+isbsd     = 'bsd' in sys.platform
 DISABLED  = False 
 #if isosx:
 #    libc = ctypes.cdll.LoadLibrary(ctypes.util.find_library('c'))
@@ -57,6 +58,13 @@ def load_library():
         return cdll.LoadLibrary(lib)
     elif iswindows:
         return cdll.LoadLibrary('libfontconfig-1')
+    elif isbsd:
+        raw = subprocess.Popen('pkg-config --libs-only-L fontconfig'.split(), 
+                         stdout=subprocess.PIPE).stdout.read().strip()
+        match = re.search(r'-L([^\s,]+)', raw)
+        if not match:
+            return cdll.LoadLibrary('libfontconfig.so')
+        return cdll.LoadLibrary(match.group(1)+'/libfontconfig.so')
     else:
         try:
             return cdll.LoadLibrary(util.find_library('fontconfig'))

src/calibre/utils/msdes/spr.h

@@ -1,59 +1,24 @@
-/* crypto/des/spr.h */
-/* Copyright (C) 1995-1997 Eric Young (eay@cryptsoft.com)
- * All rights reserved.
- *
- * This package is an SSL implementation written
- * by Eric Young (eay@cryptsoft.com).
- * The implementation was written so as to conform with Netscapes SSL.
- * 
- * This library is free for commercial and non-commercial use as long as
- * the following conditions are aheared to.  The following conditions
- * apply to all code found in this distribution, be it the RC4, RSA,
- * lhash, DES, etc., code; not just the SSL code.  The SSL documentation
- * included with this distribution is covered by the same copyright terms
- * except that the holder is Tim Hudson (tjh@cryptsoft.com).
- * 
- * Copyright remains Eric Young's, and as such any Copyright notices in
- * the code are not to be removed.
- * If this package is used in a product, Eric Young should be given attribution
- * as the author of the parts of the library used.
- * This can be in the form of a textual message at program startup or
- * in documentation (online or textual) provided with the package.
- * 
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *    "This product includes cryptographic software written by
- *     Eric Young (eay@cryptsoft.com)"
- *    The word 'cryptographic' can be left out if the rouines from the library
- *    being used are not cryptographic related :-).
- * 4. If you include any Windows specific code (or a derivative thereof) from 
- *    the apps directory (application code) you must include an acknowledgement:
- *    "This product includes software written by Tim Hudson (tjh@cryptsoft.com)"
- * 
- * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- * 
- * The licence and distribution terms for any publically available version or
- * derivative of this code cannot be changed.  i.e. this code cannot simply be
- * copied and put under another distribution licence
- * [including the GNU Public Licence.]
+/*--[crypto/des/spr.h]---------------------------------------------------------
+ | Copyright (C) 2002 Dan A. Jackson 
+ |
+ | This file is part of the "openclit" library for processing .LIT files.
+ |
+ | "Openclit" is free software; you can redistribute it and/or modify
+ | it under the terms of the GNU General Public License as published by
+ | the Free Software Foundation; either version 2 of the License, or
+ | (at your option) any later version.
+ |
+ | This program is distributed in the hope that it will be useful,
+ | but WITHOUT ANY WARRANTY; without even the implied warranty of
+ | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ | GNU General Public License for more details.
+ |
+ | You should have received a copy of the GNU General Public License
+ | along with this program; if not, write to the Free Software
+ | Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ | 
+ | The GNU General Public License may also be available at the following
+ | URL: http://www.gnu.org/licenses/gpl.html
 */
 
 static unsigned long SP1[64] = {

src/calibre/web/feeds/recipes/__init__.py

@@ -4,7 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
 Builtin recipes.
 '''
-recipe_modules = [
+recipe_modules = ['recipe_' + r for r in (
            'newsweek', 'atlantic', 'economist', 'portfolio',
            'nytimes', 'usatoday', 'outlook_india', 'bbc', 'greader', 'wsj',
            'wired', 'globe_and_mail', 'smh', 'espn', 'business_week',
@@ -19,8 +19,10 @@ recipe_modules = [
            'clarin', 'financial_times', 'heise', 'le_monde', 'harpers', 'science_aas',
            'science_news', 'the_nation', 'lrb', 'harpers_full', 'liberation',
            'linux_magazine', 'telegraph_uk', 'utne', 'sciencedaily', 'forbes',
-           'time_magazine', 'endgadget', 'fudzilla',
-          ]
+           'time_magazine', 'endgadget', 'fudzilla', 'nspm_int', 'nspm', 'pescanik',
+           'spiegel_int', 'themarketticker', 'tomshardware', 'xkcd', 'ftd', 'zdnet',
+           'joelonsoftware',
+          )]
 
 import re, imp, inspect, time, os
 from calibre.web.feeds.news import BasicNewsRecipe, CustomIndexRecipe, AutomaticNewsRecipe

src/calibre/web/feeds/recipes/b92.py

@@ -1,40 +0,0 @@
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
-'''
-b92.net
-'''
-import string,re
-from calibre.web.feeds.news import BasicNewsRecipe
-
-class B92(BasicNewsRecipe):
-    title                 = u'B92'
-    __author__            = 'Darko Miletic'
-    description           = 'Dnevne vesti iz Srbije i sveta'    
-    oldest_article        = 7
-    max_articles_per_feed = 100
-    no_stylesheets        = True
-    use_embedded_content  = False
-    
-    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
-
-    remove_tags_after  = dict(name='div', attrs={'class':'gas'})
-    remove_tags = [
-                     dict(name='div'  , attrs={'class':'interaction clearfix' })
-                    ,dict(name='div'  , attrs={'class':'gas'                  })
-                    ,dict(name='ul'   , attrs={'class':'comment-nav'          })
-                    ,dict(name='table', attrs={'class':'pages-navigation-form'})
-                  ]
-
-    feeds          = [
-                        (u'Vesti'     , u'http://www.b92.net/info/rss/vesti.xml'     )
-                       ,(u'Kultura'   , u'http://www.b92.net/info/rss/kultura.xml'   )                      
-                       ,(u'Automobili', u'http://www.b92.net/info/rss/automobili.xml')                      
-                       ,(u'Zivot'     , u'http://www.b92.net/info/rss/zivot.xml'     )
-                       ,(u'Tehnopolis', u'http://www.b92.net/info/rss/tehnopolis.xml')
-                       ,(u'Biz'       , u'http://www.b92.net/info/rss/biz.xml'       )
-                     ]
-
-    def print_version(self, url):
-        return url + '&version=print'

src/calibre/web/feeds/recipes/danas.py

@@ -1,34 +0,0 @@
-#!/usr/bin/env  python
-
-__license__   = 'GPL v3'
-__copyright__ = '2008, Darko Miletic <darko.miletic at gmail.com>'
-'''
-danas.rs
-'''
-import string,re
-from calibre.web.feeds.news import BasicNewsRecipe
-
-class Danas(BasicNewsRecipe):
-    title                 = u'Danas'
-    __author__            = 'Darko Miletic'
-    description           = 'Vesti'    
-    oldest_article        = 7
-    max_articles_per_feed = 100
-    no_stylesheets        = True
-    use_embedded_content  = False
-    timefmt               = ' [%A, %d %B, %Y]' 
-
-    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
-
-    keep_only_tags     = [ dict(name='div', attrs={'id':'left'}) ]
-    remove_tags_after  = dict(name='div', attrs={'id':'comments'})
-    remove_tags = [
-                     dict(name='div', attrs={'class':'width_1_4'})
-                    ,dict(name='div', attrs={'class':'metaClanka'})
-                    ,dict(name='div', attrs={'id':'comments'})
-                    ,dict(name='div', attrs={'class':'baner'})
-                  ]
-
-
-
-    feeds          = [ (u'Vesti', u'http://www.danas.rs/rss/rss.asp')]