Sync to trunk

src/calibre/__init__.py

@@ -441,14 +441,18 @@ def entity_to_unicode(match, exceptions=[], encoding='cp1252'):
 
 if isosx:
     fdir = os.path.expanduser('~/.fonts')
-    if not os.path.exists(fdir):
+    try:
-        os.makedirs(fdir)
+        if not os.path.exists(fdir):
-    if not os.path.exists(os.path.join(fdir, 'LiberationSans_Regular.ttf')):
+            os.makedirs(fdir)
-        from calibre.ebooks.lrf.fonts.liberation import __all__ as fonts
+        if not os.path.exists(os.path.join(fdir, 'LiberationSans_Regular.ttf')):
-        for font in fonts:
+            from calibre.ebooks.lrf.fonts.liberation import __all__ as fonts
-            l = {}
+            for font in fonts:
-            exec 'from calibre.ebooks.lrf.fonts.liberation.'+font+' import font_data' in l
+                l = {}
-            open(os.path.join(fdir, font+'.ttf'), 'wb').write(l['font_data'])
+                exec 'from calibre.ebooks.lrf.fonts.liberation.'+font+' import font_data' in l
+                open(os.path.join(fdir, font+'.ttf'), 'wb').write(l['font_data'])
+    except:
+        import traceback
+        traceback.print_exc()
                 
 # Migrate from QSettings based config system
 from calibre.utils.config import migrate

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.131'
+__version__   = '0.4.132'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/devices/usbms/device.py

@@ -181,11 +181,11 @@ class Device(_Device):
             elif self.windows_match_device(str(drive.PNPDeviceID), self.WINDOWS_CARD_MEM):
                 drives['card'] = self.windows_get_drive_prefix(drive)
 
-            if 'main' and 'card' in drives.keys():
+            if 'main' in drives.keys() and 'card' in drives.keys():
                 break
 
-        self._main_prefix = drives.get('main', None)
+        self._main_prefix = drives.get('main')
-        self._card_prefix = drives.get('card', None)
+        self._card_prefix = drives.get('card')
         
         if not self._main_prefix:
             raise DeviceError(_('Unable to detect the %s disk drive. Try rebooting.') % self.__class__.__name__)

src/calibre/ebooks/epub/__init__.py

@@ -142,6 +142,8 @@ to auto-generate a Table of Contents.
         help=_('XPath expression that specifies all tags that should be added to the Table of Contents at level one. If this is specified, it takes precedence over other forms of auto-detection.'))
     toc('level2_toc', ['--level2-toc'], default=None,
         help=_('XPath expression that specifies all tags that should be added to the Table of Contents at level two. Each entry is added under the previous level one entry.'))
+    toc('level3_toc', ['--level3-toc'], default=None,
+        help=_('XPath expression that specifies all tags that should be added to the Table of Contents at level three. Each entry is added under the previous level two entry.'))
     toc('from_ncx', ['--from-ncx'], default=None,
         help=_('Path to a .ncx file that contains the table of contents to use for this ebook. The NCX file should contain links relative to the directory it is placed in. See http://www.niso.org/workrooms/daisy/Z39-86-2005.html#NCX for an overview of the NCX format.'))
     toc('use_auto_toc', ['--use-auto-toc'], default=False,

src/calibre/ebooks/epub/from_html.py

@@ -377,16 +377,13 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
         mi = merge_metadata(htmlfile, opf, opts)
     opts.chapter = XPath(opts.chapter, 
                     namespaces={'re':'http://exslt.org/regular-expressions'})
-    if opts.level1_toc:
+    for x in (1, 2, 3):
-        opts.level1_toc = XPath(opts.level1_toc, 
+        attr = 'level%d_toc'%x
-                            namespaces={'re':'http://exslt.org/regular-expressions'})
+        if getattr(opts, attr):
-    else:
+            setattr(opts, attr, XPath(getattr(opts, attr), 
-        opts.level1_toc = None
+                      namespaces={'re':'http://exslt.org/regular-expressions'}))
-    if opts.level2_toc:
+        else:
-        opts.level2_toc = XPath(opts.level2_toc, 
+            setattr(opts, attr, None) 
-                            namespaces={'re':'http://exslt.org/regular-expressions'})
-    else:
-        opts.level2_toc = None 
     
     with TemporaryDirectory(suffix='_html2epub', keep=opts.keep_intermediate) as tdir:
         if opts.keep_intermediate:

src/calibre/ebooks/html.py

@@ -558,30 +558,21 @@ class Processor(Parser):
     
     def detect_chapters(self):
         self.detected_chapters = self.opts.chapter(self.root)
+        chapter_mark = self.opts.chapter_mark
+        page_break_before = 'display: block; page-break-before: always'
+        page_break_after = 'display: block; page-break-after: always'
         for elem in self.detected_chapters:
             text = u' '.join([t.strip() for t in elem.xpath('descendant::text()')])
             self.log_info('\tDetected chapter: %s', text[:50])
-            if self.opts.chapter_mark != 'none':
+            if chapter_mark == 'none':
-                hr = etree.Element('hr')
+                continue
-                if elem.getprevious() is None:
+            elif chapter_mark == 'rule':
-                    elem.getparent()[:0] = [hr]
+                mark = etree.Element('hr')
-                elif elem.getparent() is not None:
+            elif chapter_mark == 'pagebreak':
-                    insert = None
+                mark = etree.Element('div', style=page_break_after)
-                    for i, c in enumerate(elem.getparent()):
+            else: # chapter_mark == 'both':
-                        if c is elem:
+                mark = etree.Element('hr', style=page_break_before)
-                            insert = i
+            elem.addprevious(mark)
-                            break
-                    elem.getparent()[insert:insert] = [hr]
-                if self.opts.chapter_mark != 'rule':
-                    hr.set('style', 'width:0pt;page-break-before:always')
-                    if self.opts.chapter_mark == 'both':
-                        hr2 = etree.Element('hr')
-                        hr2.tail = u'\u00a0'
-                        p = hr.getparent()
-                        i = p.index(hr)
-                        p[i:i] = [hr2]
-                
-                    
     
     def save(self):
         style_path = os.path.splitext(os.path.basename(self.save_path()))[0]
@@ -647,6 +638,7 @@ class Processor(Parser):
                         added[elem] = add_item(_href, frag, text, toc, type='chapter')
                         add_item(_href, frag, 'Top', added[elem], type='chapter')
                 if self.opts.level2_toc is not None:
+                    added2 = {}
                     level2 = list(self.opts.level2_toc(self.root))
                     for elem in level2:
                         level1 = None
@@ -657,7 +649,21 @@ class Processor(Parser):
                                 text, _href, frag = elem_to_link(elem, href, counter)
                                 counter += 1
                                 if text:
+                                    added2[elem] = \
                                     add_item(_href, frag, text, level1, type='chapter')
+                    if self.opts.level3_toc is not None:
+                        level3 = list(self.opts.level3_toc(self.root))
+                        for elem in level3:
+                            level2 = None
+                            for item in self.root.iterdescendants():
+                                if item in added2.keys():
+                                    level2 = added2[item]
+                                elif item == elem and level2 is not None:
+                                    text, _href, frag = elem_to_link(elem, href, counter)
+                                    counter += 1
+                                    if text:
+                                        add_item(_href, frag, text, level2, type='chapter')
+                
                     
             if len(toc) > 0:
                 return
@@ -892,7 +898,7 @@ def config(defaults=None, config_name='html',
     metadata('title', ['-t', '--title'], default=None,
              help=_('Set the title. Default is to autodetect.'))
     metadata('authors', ['-a', '--authors'], default=None,
-             help=_('The author(s) of the ebook, as a comma separated list.'))
+             help=_('The author(s) of the ebook, as a & separated list.'))
     metadata('tags', ['--subjects'], default=None,
              help=_('The subject(s) of this book, as a comma separated list.'))
     metadata('publisher', ['--publisher'], default=None,
@@ -988,7 +994,9 @@ def merge_metadata(htmlfile, opf, opts):
         val = getattr(opts, attr, None)
         if val is None or val == _('Unknown') or val == [_('Unknown')]:
             continue
-        if attr in ('authors', 'tags'):
+        if attr =='authors':
+            val = [i.strip() for i in val.split('&') if i.strip()]
+        elif attr == 'tags':
             val = [i.strip() for i in val.split(',') if i.strip()]
         setattr(mi, attr, val)
         
@@ -997,7 +1005,10 @@ def merge_metadata(htmlfile, opf, opts):
         mi.cover = os.path.abspath(cover)
         
     if not mi.title:
-        mi.title = os.path.splitext(os.path.basename(htmlfile))[0]
+        if htmlfile:
+            mi.title = os.path.splitext(os.path.basename(htmlfile))[0]
+        else:
+            mi.title = _('Unknown')
     if not mi.authors:
         mi.authors = [_('Unknown')]
     return mi

src/calibre/ebooks/lrf/rtf/convert_from.py

@@ -170,7 +170,7 @@ def generate_html(rtfpath, tdir):
         f.write(res)
         f.close()
         try:
-            mi = get_metadata(open(rtfpath, 'rb'))
+            mi = get_metadata(open(rtfpath, 'rb'), 'rtf')
         except:
             mi = MetaInformation(None, None)
         if not mi.title:

src/calibre/ebooks/metadata/cli.py

@@ -0,0 +1,185 @@
+from __future__ import with_statement
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+ebook-meta
+'''
+USAGE='%%prog ebook_file [' + _('options') + ']\n' + \
+_('''
+Read/Write metadata from/to ebook files.
+
+Supported formats for reading metadata: %s
+
+Supported formats for writing metadata: %s
+
+Different file types support different kinds of metadata. If you try to set
+some metadata on a file type that does not support it, the metadata will be
+silently ignored.
+''')
+
+import sys, os
+
+from calibre.utils.config import StringConfig
+from calibre.customize.ui import metadata_readers, metadata_writers
+from calibre.ebooks.metadata.meta import get_metadata, set_metadata
+from calibre.ebooks.metadata import string_to_authors, authors_to_sort_string, \
+                    title_sort, MetaInformation
+from calibre import prints
+
+def config():
+    c = StringConfig('')
+    c.add_opt('title', ['-t', '--title'], 
+              help=_('Set the title.'))
+    c.add_opt('authors', ['-a', '--authors'],
+              help=_('Set the authors. Multiple authors should be separated '
+                     'by the & character. Author names should be in the order '
+                     'Firstname Lastname.'))
+    c.add_opt('title_sort', ['--title-sort'], 
+              help=_('The version of the title to be used for sorting. '
+                     'If unspecified, and the title is specified, it will ' 
+                     'be auto-generated from the title.'))
+    c.add_opt('author_sort', ['--author-sort'], 
+              help=_('String to be used when sorting by author. '
+                     'If unspecified, and the author(s) are specified, it will ' 
+                     'be auto-generated from the author(s).'))
+    c.add_opt('cover', ['--cover'], 
+              help=_('Set the cover to the specified file.'))
+    c.add_opt('comments', ['-c', '--comments'],
+              help=_('Set the ebook description.'))
+    c.add_opt('publisher', ['-p', '--publisher'],
+              help=_('Set the ebook publisher.'))
+    c.add_opt('series', ['-s', '--series'],
+              help=_('Set the series this ebook belongs to.'))
+    c.add_opt('series_index', ['-i', '--index'],
+              help=_('Set the index of the book in this series.'))
+    c.add_opt('rating', ['-r', '--rating'],
+              help=_('Set the rating. Should be a number between 1 and 5.'))
+    c.add_opt('isbn', ['--isbn'],
+              help=_('Set the ISBN of the book.'))
+    c.add_opt('tags', ['--tags'],
+              help=_('Set the tags for the book. Should be a comma separated list.'))
+    c.add_opt('book_producer', ['-k', '--book-producer'],
+              help=_('Set the book producer.'))
+    c.add_opt('language', ['-l', '--language'],
+              help=_('Set the language.'))
+    
+    c.add_opt('get_cover', ['--get-cover'],
+              help=_('Get the cover from the ebook and save it at as the '
+                     'specified file.'))
+    c.add_opt('to_opf', ['--to-opf'],
+              help=_('Specify the name of an OPF file. The metadata will '
+                     'be written to the OPF file.'))
+    c.add_opt('from_opf', ['--from-opf'],
+              help=_('Read metadata from the specified OPF file and use it to '
+                     'set metadata in the ebook. Metadata specified on the'
+                     'command line will override metadata read from the OPF file'))
+    return c
+
+def filetypes():
+    readers = set([])
+    for r in metadata_readers():
+        readers = readers.union(set(r.file_types))
+    return readers
+
+def option_parser():
+    writers = set([])
+    for w in metadata_writers():
+        writers = writers.union(set(w.file_types))
+    return config().option_parser(USAGE%(list(filetypes()), list(writers)))
+
+def do_set_metadata(opts, mi, stream, stream_type):
+    mi = MetaInformation(mi)
+    for x in ('guide', 'toc', 'manifest', 'spine'):
+        setattr(mi, x, None)
+    
+    from_opf = getattr(opts, 'from_opf', None)
+    if from_opf is not None:
+        from calibre.ebooks.metadata.opf2 import OPF
+        opf_mi = MetaInformation(OPF(open(from_opf, 'rb')))
+        mi.smart_update(opf_mi)
+        
+    for pref in config().option_set.preferences:
+        if pref.name in ('to_opf', 'from_opf', 'authors', 'title_sort', 
+                         'author_sort', 'get_cover', 'cover', 'tags'):
+            continue
+        val = getattr(opts, pref.name, None)
+        if val is not None:
+            setattr(mi, pref.name, getattr())
+    
+    if getattr(opts, 'authors', None) is not None:
+        mi.authors = string_to_authors(opts.authors)
+        mi.author_sort = authors_to_sort_string(mi.authors)
+    if getattr(opts, 'author_sort', None) is not None:
+        mi.author_sort = opts.author_sort
+    if getattr(opts, 'title_sort', None) is not None:
+        mi.title_sort = opts.title_sort
+    elif getattr(opts, 'title', None) is not None:
+        mi.title_sort = title_sort(opts.title)
+    if getattr(opts, 'tags', None) is not None:
+        mi.tags = [t.strip() for t in opts.tags.split(',')]
+    
+    if getattr(opts, 'cover', None) is not None:
+        ext = os.path.splitext(opts.cover)[1].replace('.', '').upper()
+        mi.cover_data = (ext, open(opts.cover, 'rb').read())
+    
+    set_metadata(stream, mi, stream_type)
+    
+
+def main(args=sys.argv):
+    parser = option_parser()
+    opts, args = parser.parse_args(args)
+    if len(args) < 2:
+        parser.print_help()
+        prints(_('No file specified'), file=sys.stderr)
+        return 1
+    path = args[1]
+    stream = open(path, 'r+b')
+    stream_type = os.path.splitext(path)[1].replace('.', '').lower()
+    
+    trying_to_set = False
+    for pref in config().option_set.preferences:
+        if pref.name in ('to_opf', 'get_cover'):
+            continue
+        if getattr(opts, pref.name) is not None:
+            trying_to_set = True
+            break
+    mi = get_metadata(stream, stream_type)
+    if trying_to_set:
+        prints(_('Original metadata')+'::')
+    metadata = unicode(mi)
+    if trying_to_set:
+        metadata = '\t'+'\n\t'.join(metadata.split('\n'))
+    prints(metadata)
+    
+    if trying_to_set:
+        stream.seek(0)
+        do_set_metadata(opts, mi, stream, stream_type)
+        stream.seek(0)
+        stream.flush()
+        mi = get_metadata(stream, stream_type)
+        prints(_('Changed metadata')+'::')
+        metadata = unicode(mi)
+        metadata = '\t'+'\n\t'.join(metadata.split('\n'))
+        prints(metadata)
+        
+    if opts.to_opf is not None:
+        from calibre.ebooks.metadata.opf2 import OPFCreator
+        opf = OPFCreator(os.getcwdu(), mi)
+        with open(opts.opf, 'wb') as f:
+            opf.render(f)
+        prints(_('OPF created in'), opts.opf)
+    
+    if opts.get_cover is not None:
+        if mi.cover_data and mi.cover_data[1]:
+            with open(opts.get_cover, 'wb') as f:
+                f.write(mi.cover_data[1])
+                prints(_('Cover saved to'), f.name)
+        else:
+            prints(_('No cover found'), file=sys.stderr)
+    
+    return 0
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/ebooks/metadata/epub.py

@@ -106,9 +106,11 @@ class CoverRenderer(QObject):
     WIDTH  = 600
     HEIGHT = 800
     
-    def __init__(self, url, size, loop):
+    def __init__(self, path):
+        if QApplication.instance() is None:
+            QApplication([])
         QObject.__init__(self)
-        self.loop = loop
+        self.loop = QEventLoop()
         self.page = QWebPage()
         pal = self.page.palette()
         pal.setBrush(QPalette.Background, Qt.white)
@@ -117,32 +119,42 @@ class CoverRenderer(QObject):
         self.page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
         self.page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
         QObject.connect(self.page, SIGNAL('loadFinished(bool)'), self.render_html)
-        self.image_data = None
+        self._image_data = None
         self.rendered = False
+        url = QUrl.fromLocalFile(os.path.normpath(path))
         self.page.mainFrame().load(url)
         
     def render_html(self, ok):
-        self.rendered = True
         try:
             if not ok:
+                self.rendered = True
                 return
-            #size = self.page.mainFrame().contentsSize()
-            #width, height = fit_image(size.width(), size.height(), self.WIDTH, self.HEIGHT)[1:]
-            #self.page.setViewportSize(QSize(width, height))
             image = QImage(self.page.viewportSize(), QImage.Format_ARGB32)
             image.setDotsPerMeterX(96*(100/2.54))
             image.setDotsPerMeterY(96*(100/2.54))
             painter = QPainter(image)
             self.page.mainFrame().render(painter)
             painter.end()
-            
             ba = QByteArray()
             buf = QBuffer(ba)
             buf.open(QBuffer.WriteOnly)
             image.save(buf, 'JPEG')
-            self.image_data = str(ba.data())
+            self._image_data = str(ba.data())
         finally:
             self.loop.exit(0)
+        self.rendered = True
+
+    def image_data():
+        def fget(self):
+            if not self.rendered:
+                self.loop.exec_()
+                count = 0
+                while count < 50 and not self.rendered:
+                    time.sleep(0.1)
+                    count += 1
+            return self._image_data
+        return property(fget=fget)
+    image_data = image_data()
 
 
 def get_cover(opf, opf_path, stream):
@@ -155,20 +167,11 @@ def get_cover(opf, opf_path, stream):
             stream.seek(0)
             ZipFile(stream).extractall()
             opf_path = opf_path.replace('/', os.sep)
-            cpage = os.path.join(tdir, os.path.dirname(opf_path), *cpage.split('/'))
+            cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage)
             if not os.path.exists(cpage):
                 return
-            if QApplication.instance() is None:
+            cr = CoverRenderer(cpage)
-                QApplication([])
+            return cr.image_data
-            url = QUrl.fromLocalFile(cpage)
-            loop = QEventLoop()
-            cr = CoverRenderer(url, os.stat(cpage).st_size, loop)
-            loop.exec_()
-            count = 0
-            while count < 50 and not cr.rendered:
-                time.sleep(0.1)
-                count += 1
-    return cr.image_data 
     
 def get_metadata(stream, extract_cover=True):
     """ Return metadata as a :class:`MetaInformation` object """

src/calibre/ebooks/metadata/opf.xml

@@ -5,7 +5,7 @@
          unique-identifier="${__appname__}_id"
          
 >
-    <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf">
+    <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata">
         <dc:title py:with="attrs={'opf:files-as':mi.title_sort}" py:attrs="attrs">${mi.title}</dc:title>
         <dc:creator opf:role="aut" py:for="i, author in enumerate(mi.authors)" py:attrs="{'opf:file-as':mi.author_sort} if mi.author_sort and i == 0 else {}">${author}</dc:creator>
         <dc:contributor opf:role="bkp" py:with="attrs={'opf:files-as':__appname__}" py:attrs="attrs">${'%s (%s)'%(__appname__, __version__)} [http://${__appname__}.kovidgoyal.net]</dc:contributor> 
@@ -16,9 +16,9 @@
         <dc:description py:if="mi.comments">${mi.comments}</dc:description>
         <dc:publisher py:if="mi.publisher">${mi.publisher}</dc:publisher>
         <dc:identifier opf:scheme="ISBN" py:if="mi.isbn">${mi.isbn}</dc:identifier>
-        <series py:if="mi.series">${mi.series}</series>
+        <meta py:if="mi.series is not None" name="calibre:series" content="${mi.series}"/>
-        <series_index py:if="mi.series_index is not None">${mi.series_index}</series_index>
+        <meta py:if="mi.series_index is not None" name="calibre:series_index" content="${mi.series_index}"/>
-        <rating py:if="mi.rating is not None">${mi.rating}</rating>
+        <meta py:if="mi.rating is not None" name="calibre:rating" content="${mi.rating}"/>
         <py:for each="tag in mi.tags">
         <dc:subject py:if="mi.tags is not None">${tag}</dc:subject>
         </py:for>

src/calibre/ebooks/metadata/opf2.py

@@ -392,8 +392,8 @@ class MetadataField(object):
     def __set__(self, obj, val):
         elem = obj.get_metadata_element(self.name)
         if elem is None:
-            elem = obj.create_metadata_element(self.name, ns='dc' if self.is_dc else 'opf')
+            elem = obj.create_metadata_element(self.name, is_dc=self.is_dc)
-        elem.text = unicode(val)
+        obj.set_text(elem, unicode(val))
 
 class OPF(object):
     MIMETYPE         = 'application/oebps-package+xml'
@@ -403,16 +403,17 @@ class OPF(object):
                         'dc'  : "http://purl.org/dc/elements/1.1/",
                         'opf' : "http://www.idpf.org/2007/opf",
                        }
+    META             = '{%s}meta' % NAMESPACES['opf']
     xpn = NAMESPACES.copy()
     xpn.pop(None)
     xpn['re'] = 'http://exslt.org/regular-expressions'
     XPath = functools.partial(etree.XPath, namespaces=xpn)
+    CONTENT          = XPath('self::*[re:match(name(), "meta$", "i")]/@content')
     TEXT             = XPath('string()')
     
     
     metadata_path   = XPath('descendant::*[re:match(name(), "metadata", "i")]')
-    metadata_elem_path = XPath('descendant::*[re:match(name(), $name, "i")]')
+    metadata_elem_path = XPath('descendant::*[re:match(name(), concat($name, "$"), "i") or (re:match(name(), "meta$", "i") and re:match(@name, concat("^calibre:", $name, "$"), "i"))]')
-    series_path     = XPath('descendant::*[re:match(name(), "series$", "i")]')
     authors_path    = XPath('descendant::*[re:match(name(), "creator", "i") and (@role="aut" or @opf:role="aut" or (not(@role) and not(@opf:role)))]')
     bkp_path        = XPath('descendant::*[re:match(name(), "contributor", "i") and (@role="bkp" or @opf:role="bkp")]')
     tags_path       = XPath('descendant::*[re:match(name(), "subject", "i")]')
@@ -431,6 +432,7 @@ class OPF(object):
     language        = MetadataField('language')
     comments        = MetadataField('description')
     category        = MetadataField('category')
+    series          = MetadataField('series', is_dc=False)
     series_index    = MetadataField('series_index', is_dc=False, formatter=int, none_is=1)
     rating          = MetadataField('rating', is_dc=False, formatter=int)
     
@@ -497,7 +499,13 @@ class OPF(object):
     
         
     def get_text(self, elem):
-        return u''.join(self.TEXT(elem))
+        return u''.join(self.CONTENT(elem) or self.TEXT(elem))
+
+    def set_text(self, elem, content):
+        if elem.tag == self.META:
+            elem.attib['content'] = content
+        else:
+            elem.text = content
     
     def itermanifest(self):
         return self.manifest_path(self.root)
@@ -611,9 +619,9 @@ class OPF(object):
             for elem in remove:
                 self.metadata.remove(elem)
             for author in val:
-                elem = self.create_metadata_element('creator', ns='dc',
+                attrib = {'{%s}role'%self.NAMESPACES['opf']: 'aut'}
-                                                    attrib={'{%s}role'%self.NAMESPACES['opf']:'aut'})
+                elem = self.create_metadata_element('creator', attrib=attrib)
-                elem.text = author
+                self.set_text(elem, author)
         
         return property(fget=fget, fset=fset)
     
@@ -650,8 +658,8 @@ class OPF(object):
             for tag in list(self.tags_path(self.metadata)):
                 self.metadata.remove(tag)
             for tag in val:
-                elem = self.create_metadata_element('subject', ns='dc')
+                elem = self.create_metadata_element('subject')
-                elem.text = unicode(tag)
+                self.set_text(elem, unicode(tag))
         
         return property(fget=fget, fset=fset)
     
@@ -660,14 +668,15 @@ class OPF(object):
         
         def fget(self):
             for match in self.isbn_path(self.metadata):
-                return match.text if match.text else None
+                return self.get_text(match) or None
             
         def fset(self, val):
             matches = self.isbn_path(self.metadata)
             if not matches:
-                matches = [self.create_metadata_element('identifier', ns='dc',
+                attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'ISBN'}
-                                                attrib={'{%s}scheme'%self.NAMESPACES['opf']:'ISBN'})]
+                matches = [self.create_metadata_element('identifier',
-            matches[0].text = unicode(val)
+                                                        attrib=attrib)]
+            self.set_text(matches[0], unicode(val))
 
         return property(fget=fget, fset=fset)
 
@@ -676,48 +685,32 @@ class OPF(object):
         
         def fget(self):
             for match in self.application_id_path(self.metadata):
-                return match.text if match.text else None
+                return self.get_text(match) or None
             
         def fset(self, val):
             matches = self.application_id_path(self.metadata)
             if not matches:
-                matches = [self.create_metadata_element('identifier', ns='dc',
+                attrib = {'{%s}scheme'%self.NAMESPACES['opf']: 'calibre'}
-                                                attrib={'{%s}scheme'%self.NAMESPACES['opf']:'calibre'})]
+                matches = [self.create_metadata_element('identifier',
-            matches[0].text = unicode(val)
+                                                        attrib=attrib)]
+            self.set_text(matches[0], unicode(val))
 
         return property(fget=fget, fset=fset)
     
-    
-    @apply
-    def series():
-        
-        def fget(self):
-            for match in self.series_path(self.metadata):
-                return match.text if match.text else None
-        
-        def fset(self, val):
-            matches = self.series_path(self.metadata)
-            if not matches:
-                matches = [self.create_metadata_element('series')]
-            matches[0].text = unicode(val)
-        
-        return property(fget=fget, fset=fset)
-    
-    
-    
     @apply
     def book_producer():
         
         def fget(self):
             for match in self.bkp_path(self.metadata):
-                return match.text if match.text else None
+                return self.get_text(match) or None
             
         def fset(self, val):
             matches = self.bkp_path(self.metadata)
             if not matches:
-                matches = [self.create_metadata_element('contributor', ns='dc',
+                attrib = {'{%s}role'%self.NAMESPACES['opf']: 'bkp'}
-                                                attrib={'{%s}role'%self.NAMESPACES['opf']:'bkp'})]
+                matches = [self.create_metadata_element('contributor',
-            matches[0].text = unicode(val)
+                                                        attrib=attrib)]
+            self.set_text(matches[0], unicode(val))
         return property(fget=fget, fset=fset)
     
     
@@ -783,9 +776,15 @@ class OPF(object):
         if matches:
             return matches[-1]
         
-    def create_metadata_element(self, name, attrib=None, ns='opf'):
+    def create_metadata_element(self, name, attrib=None, is_dc=True):
-        elem = etree.SubElement(self.metadata, '{%s}%s'%(self.NAMESPACES[ns], name), 
+        if is_dc:
-                                attrib=attrib, nsmap=self.NAMESPACES)
+            name = '{%s}%s' % (self.NAMESPACES['dc'], name)
+        else:
+            attrib = attrib or {}
+            attrib['name'] = 'calibre:' + name
+            name = '{%s}%s' % (self.NAMESPACES['opf'], 'meta')
+        elem = etree.SubElement(self.metadata, name, attrib=attrib,
+                                nsmap=self.NAMESPACES)
         elem.tail = '\n'
         return elem
         

src/calibre/ebooks/mobi/mobiml.py

@@ -148,10 +148,6 @@ class MobiMLizer(object):
             if bstate.pbreak:
                 etree.SubElement(body, MBP('pagebreak'))
                 bstate.pbreak = False
-            if istate.ids:
-                for id in istate.ids:
-                    etree.SubElement(body, XHTML('a'), attrib={'id': id})
-                istate.ids.clear()
             bstate.istate = None
             bstate.anchor = None
             parent = bstate.nested[-1] if bstate.nested else bstate.body
@@ -186,14 +182,17 @@ class MobiMLizer(object):
                 wrapper.attrib['height'] = self.mobimlize_measure(vspace)
                 para.attrib['width'] = self.mobimlize_measure(indent)
             elif tag == 'table' and vspace > 0:
-                body = bstate.body
                 vspace = int(round(vspace / self.profile.fbase))
-                index = max((0, len(body) - 1))
                 while vspace > 0:
-                    body.insert(index, etree.Element(XHTML('br')))
+                    wrapper.addprevious(etree.Element(XHTML('br')))
                     vspace -= 1
             if istate.halign != 'auto':
                 para.attrib['align'] = istate.halign
+        if istate.ids:
+            last = bstate.body[-1]
+            for id in istate.ids:
+                last.addprevious(etree.Element(XHTML('a'), attrib={'id': id}))
+            istate.ids.clear()
         pstate = bstate.istate
         if tag in CONTENT_TAGS:
             bstate.inline = para

src/calibre/ebooks/mobi/reader.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Read data from .mobi files
 '''
 
-import sys, struct, os, cStringIO, re, atexit, shutil, tempfile
+import sys, struct, os, cStringIO, re
 
 try:
     from PIL import Image as PILImage
@@ -14,7 +14,7 @@ except ImportError:
 
 from lxml import html, etree
 
-from calibre import __appname__, entity_to_unicode
+from calibre import entity_to_unicode
 from calibre.ebooks import DRMError
 from calibre.ebooks.chardet import ENCODING_PATS
 from calibre.ebooks.mobi import MobiError
@@ -28,7 +28,7 @@ from calibre import sanitize_file_name
 
 class EXTHHeader(object):
     
-    def __init__(self, raw, codec):
+    def __init__(self, raw, codec, title):
         self.doctype = raw[:4]
         self.length, self.num_items = struct.unpack('>LL', raw[4:12])
         raw = raw[12:]
@@ -45,21 +45,15 @@ class EXTHHeader(object):
             elif id == 203:
                 self.has_fake_cover = bool(struct.unpack('>L', content)[0]) 
             elif id == 201:
-                self.cover_offset, = struct.unpack('>L', content)
+                co, = struct.unpack('>L', content)
+                if co < 1e7:
+                    self.cover_offset = co 
             elif id == 202:
                 self.thumbnail_offset, = struct.unpack('>L', content)
             #else:
             #    print 'unknown record', id, repr(content)
-        title = re.search(r'\0+([^\0]+)\0+', raw[pos:])
         if title:
-            title = title.group(1).decode(codec, 'replace')
+            self.mi.title = title
-            if len(title) > 2:
-                self.mi.title = title
-            else:
-                title = re.search(r'\0+([^\0]+)\0+', ''.join(reversed(raw[pos:])))
-                if title:
-                    self.mi.title = ''.join(reversed(title.group(1).decode(codec, 'replace')))
-            
     
     def process_metadata(self, id, content, codec):
         if id == 100:
@@ -119,6 +113,9 @@ class BookHeader(object):
             if self.compression_type == 'DH':
                 self.huff_offset, self.huff_number = struct.unpack('>LL', raw[0x70:0x78]) 
             
+            toff, tlen = struct.unpack('>II', raw[0x54:0x5c])
+            tend = toff + tlen
+            self.title = raw[toff:tend] if tend < len(raw) else _('Unknown')
             langcode  = struct.unpack('!L', raw[0x5C:0x60])[0]
             langid    = langcode & 0xFF
             sublangid = (langcode >> 10) & 0xFF
@@ -129,7 +126,7 @@ class BookHeader(object):
             self.exth_flag, = struct.unpack('>L', raw[0x80:0x84])
             self.exth = None
             if self.exth_flag & 0x40:
-                self.exth = EXTHHeader(raw[16+self.length:], self.codec)
+                self.exth = EXTHHeader(raw[16+self.length:], self.codec, self.title)
                 self.exth.mi.uid = self.unique_id
                 self.exth.mi.language = self.language
             
@@ -480,7 +477,7 @@ def get_metadata(stream):
         try:
             if hasattr(mr.book_header.exth, 'cover_offset'):
                 cover_index = mr.book_header.first_image_index + mr.book_header.exth.cover_offset
-                data  = mr.sections[cover_index][0]
+                data  = mr.sections[int(cover_index)][0]
             else:
                 data  = mr.sections[mr.book_header.first_image_index][0]
             buf = cStringIO.StringIO(data)

src/calibre/ebooks/mobi/writer.py

@@ -23,6 +23,7 @@ from PIL import Image
 from calibre.ebooks.oeb.base import XML_NS, XHTML, XHTML_NS, OEB_DOCS, \
     OEB_RASTER_IMAGES
 from calibre.ebooks.oeb.base import xpath, barename, namespace, prefixname
+from calibre.ebooks.oeb.base import urlnormalize
 from calibre.ebooks.oeb.base import Logger, OEBBook
 from calibre.ebooks.oeb.profile import Context
 from calibre.ebooks.oeb.transforms.flatcss import CSSFlattener
@@ -178,7 +179,7 @@ class Serializer(object):
 
     def serialize_href(self, href, base=None):
         hrefs = self.oeb.manifest.hrefs
-        path, frag = urldefrag(href)
+        path, frag = urldefrag(urlnormalize(href))
         if path and base:
             path = base.abshref(path)
         if path and path not in hrefs:
@@ -196,6 +197,7 @@ class Serializer(object):
         
     def serialize_body(self):
         buffer = self.buffer
+        self.anchor_offset = buffer.tell()
         buffer.write('<body>')
         # CybookG3 'Start Reading' link
         if 'text' in self.oeb.guide:
@@ -224,14 +226,17 @@ class Serializer(object):
            or namespace(elem.tag) not in nsrmap:
             return
         tag = prefixname(elem.tag, nsrmap)
-        for attr in ('name', 'id'):
+        # Previous layers take care of @name
-            if attr in elem.attrib:
+        id = elem.attrib.pop('id', None)
-                href = '#'.join((item.href, elem.attrib[attr]))
+        if id is not None:
-                self.id_offsets[href] = buffer.tell()
+            href = '#'.join((item.href, id))
-                del elem.attrib[attr]
+            offset = self.anchor_offset or buffer.tell()
-        if tag == 'a' and not elem.attrib \
+            self.id_offsets[href] = offset
-           and not len(elem) and not elem.text:
+        if self.anchor_offset is not None and \
+           tag == 'a' and not elem.attrib and \
+           not len(elem) and not elem.text:
             return
+        self.anchor_offset = buffer.tell()
         buffer.write('<')
         buffer.write(tag)
         if elem.attrib:
@@ -256,10 +261,12 @@ class Serializer(object):
         if elem.text or len(elem) > 0:
             buffer.write('>')
             if elem.text:
+                self.anchor_offset = None
                 self.serialize_text(elem.text)
             for child in elem:
                 self.serialize_elem(child, item)
                 if child.tail:
+                    self.anchor_offset = None
                     self.serialize_text(child.tail)
             buffer.write('</%s>' % tag)
         else:

src/calibre/ebooks/oeb/base.py

@@ -23,12 +23,15 @@ from calibre import LoggingInterface
 from calibre.translations.dynamic import translate
 from calibre.startup import get_lang
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
+from calibre.ebooks.metadata.epub import CoverRenderer
+from calibre.ptempfile import TemporaryDirectory
 
 XML_NS = 'http://www.w3.org/XML/1998/namespace'
 XHTML_NS = 'http://www.w3.org/1999/xhtml'
 OEB_DOC_NS = 'http://openebook.org/namespaces/oeb-document/1.0/'
 OPF1_NS = 'http://openebook.org/namespaces/oeb-package/1.0/'
 OPF2_NS = 'http://www.idpf.org/2007/opf'
+OPF_NSES = set([OPF1_NS, OPF2_NS])
 DC09_NS = 'http://purl.org/metadata/dublin_core'
 DC10_NS = 'http://purl.org/dc/elements/1.0/'
 DC11_NS = 'http://purl.org/dc/elements/1.1/'
@@ -38,6 +41,7 @@ DCTERMS_NS = 'http://purl.org/dc/terms/'
 NCX_NS = 'http://www.daisy.org/z3986/2005/ncx/'
 SVG_NS = 'http://www.w3.org/2000/svg'
 XLINK_NS = 'http://www.w3.org/1999/xlink'
+CALIBRE_NS = 'http://calibre.kovidgoyal.net/2009/metadata'
 XPNSMAP = {'h': XHTML_NS, 'o1': OPF1_NS, 'o2': OPF2_NS,
            'd09': DC09_NS, 'd10': DC10_NS, 'd11': DC11_NS,
            'xsi': XSI_NS, 'dt': DCTERMS_NS, 'ncx': NCX_NS,
@@ -51,6 +55,7 @@ def DC(name): return '{%s}%s' % (DC11_NS, name)
 def NCX(name): return '{%s}%s' % (NCX_NS, name)
 def SVG(name): return '{%s}%s' % (SVG_NS, name)
 def XLINK(name): return '{%s}%s' % (XLINK_NS, name)
+def CALIBRE(name): return '{%s}%s' % (CALIBRE_NS, name)
 
 EPUB_MIME = 'application/epub+zip'
 XHTML_MIME = 'application/xhtml+xml'
@@ -75,6 +80,8 @@ MS_COVER_TYPE = 'other.ms-coverimage-standard'
 
 ENTITY_RE = re.compile(r'&([a-zA-Z_:][a-zA-Z0-9.-_:]+);')
 COLLAPSE_RE = re.compile(r'[ \t\r\n\v]+')
+QNAME_RE = re.compile(r'^[{][^{}]+[}][^{}]+$')
+PREFIXNAME_RE = re.compile(r'^[^:]+[:][^:]+')
 
 def element(parent, *args, **kwargs):
     if parent is not None:
@@ -92,11 +99,30 @@ def barename(name):
     return name
 
 def prefixname(name, nsrmap):
-    prefix = nsrmap[namespace(name)]
+    if not isqname(name):
+        return name
+    ns = namespace(name)
+    if ns not in nsrmap:
+        return name
+    prefix = nsrmap[ns]
     if not prefix:
         return barename(name)
     return ':'.join((prefix, barename(name)))
 
+def isprefixname(name):
+    return name and PREFIXNAME_RE.match(name) is not None
+
+def qname(name, nsmap):
+    if not isprefixname(name):
+        return name
+    prefix, local = name.split(':', 1)
+    if prefix not in nsmap:
+        return name
+    return '{%s}%s' % (nsmap[prefix], local)
+
+def isqname(name):
+    return name and QNAME_RE.match(name) is not None
+
 def XPath(expr):
     return etree.XPath(expr, namespaces=XPNSMAP)
 
@@ -185,48 +211,65 @@ class DirWriter(object):
 
 
 class Metadata(object):
-    TERMS = set(['contributor', 'coverage', 'creator', 'date', 'description',
+    DC_TERMS = set(['contributor', 'coverage', 'creator', 'date', 'description',
-                 'format', 'identifier', 'language', 'publisher', 'relation',
+                    'format', 'identifier', 'language', 'publisher', 'relation',
-                 'rights', 'source', 'subject', 'title', 'type'])
+                    'rights', 'source', 'subject', 'title', 'type'])
-    ATTRS = set(['role', 'file-as', 'scheme'])
+    CALIBRE_TERMS = set(['series', 'series_index', 'rating'])
+    OPF_ATTRS = set(['role', 'file-as', 'scheme', 'event'])
     OPF1_NSMAP = {'dc': DC11_NS, 'oebpackage': OPF1_NS}
     OPF2_NSMAP = {'opf': OPF2_NS, 'dc': DC11_NS, 'dcterms': DCTERMS_NS,
-                  'xsi': XSI_NS}
+                  'xsi': XSI_NS, 'calibre': CALIBRE_NS}
     
     class Item(object):
-        def __init__(self, term, value, fq_attrib={}, **kwargs):
+        def __init__(self, term, value, attrib={}, nsmap={}, **kwargs):
-            self.fq_attrib = fq_attrib = dict(fq_attrib)
+            self.attrib = attrib = dict(attrib)
-            fq_attrib.update(kwargs)
+            self.nsmap = nsmap = dict(nsmap)
-            if barename(term).lower() in Metadata.TERMS and \
+            attrib.update(kwargs)
-               (not namespace(term) or namespace(term) in DC_NSES):
+            if namespace(term) == OPF2_NS:
-                # Anything looking like Dublin Core is coerced
-                term = DC(barename(term).lower())
-            elif namespace(term) == OPF2_NS:
                 term = barename(term)
+            ns = namespace(term)
+            local = barename(term).lower()
+            if local in Metadata.DC_TERMS and (not ns or ns in DC_NSES):
+                # Anything looking like Dublin Core is coerced
+                term = DC(local)
+            elif local in Metadata.CALIBRE_TERMS and ns in (CALIBRE_NS, ''):
+                # Ditto for Calibre-specific metadata
+                term = CALIBRE(local)
             self.term = term
             self.value = value
-            self.attrib = attrib = {}
+            for attr, value in attrib.items():
-            for fq_attr in fq_attrib:
+                if isprefixname(value):
-                if fq_attr in Metadata.ATTRS:
+                    attrib[attr] = qname(value, nsmap)
-                    attr = fq_attr
+                if attr in Metadata.OPF_ATTRS:
-                    fq_attr = OPF(fq_attr)
+                    attrib[OPF(attr)] = attrib.pop(attr)
-                    fq_attrib[fq_attr] = fq_attrib.pop(attr)
+            self.__setattr__ = self._setattr
-                else:
-                    attr = barename(fq_attr)
-                attrib[attr] = fq_attrib[fq_attr]
         
         def __getattr__(self, name):
-            name = name.replace('_', '-')
+            attr = name.replace('_', '-')
+            if attr in Metadata.OPF_ATTRS:
+                attr = OPF(attr)
             try:
-                return self.attrib[name]
+                return self.attrib[attr]
             except KeyError:
                 raise AttributeError(
                     '%r object has no attribute %r' \
                         % (self.__class__.__name__, name))
             
+        def _setattr(self, name, value):
+            attr = name.replace('_', '-')
+            if attr in Metadata.OPF_ATTRS:
+                attr = OPF(attr)
+            if attr in self.attrib:
+                self.attrib[attr] = value
+                return
+            super(Item, self).__setattr__(self, name, value)
+            
         def __getitem__(self, key):
             return self.attrib[key]
         
+        def __setitem__(self, key, value):
+            self.attrib[key] = value
+        
         def __contains__(self, key):
             return key in self.attrib
         
@@ -243,33 +286,41 @@ class Metadata(object):
         def __unicode__(self):
             return unicode(self.value)
 
-        def to_opf1(self, dcmeta=None, xmeta=None):
+        def to_opf1(self, dcmeta=None, xmeta=None, nsrmap={}):
+            attrib = {}
+            for key, value in self.attrib.items():
+                if namespace(key) == OPF2_NS:
+                    key = barename(key)
+                attrib[key] = prefixname(value, nsrmap)
             if namespace(self.term) == DC11_NS:
                 name = DC(barename(self.term).title())
-                elem = element(dcmeta, name, attrib=self.attrib)
+                elem = element(dcmeta, name, attrib=attrib)
                 elem.text = self.value
             else:
-                elem = element(xmeta, 'meta', attrib=self.attrib)
+                elem = element(xmeta, 'meta', attrib=attrib)
-                elem.attrib['name'] = self.term
+                elem.attrib['name'] = prefixname(self.term, nsrmap)
-                elem.attrib['content'] = self.value
+                elem.attrib['content'] = prefixname(self.value, nsrmap)
             return elem
         
-        def to_opf2(self, parent=None):
+        def to_opf2(self, parent=None, nsrmap={}):
+            attrib = {}
+            for key, value in self.attrib.items():
+                attrib[key] = prefixname(value, nsrmap)
             if namespace(self.term) == DC11_NS:
-                elem = element(parent, self.term, attrib=self.fq_attrib)
+                elem = element(parent, self.term, attrib=attrib)
                 elem.text = self.value
             else:
-                elem = element(parent, OPF('meta'), attrib=self.fq_attrib)
+                elem = element(parent, OPF('meta'), attrib=attrib)
-                elem.attrib['name'] = self.term
+                elem.attrib['name'] = prefixname(self.term, nsrmap)
-                elem.attrib['content'] = self.value
+                elem.attrib['content'] = prefixname(self.value, nsrmap)
             return elem
     
     def __init__(self, oeb):
         self.oeb = oeb
         self.items = defaultdict(list)
 
-    def add(self, term, value, attrib={}, **kwargs):
+    def add(self, term, value, attrib={}, nsmap={}, **kwargs):
-        item = self.Item(term, value, attrib, **kwargs)
+        item = self.Item(term, value, attrib, nsmap, **kwargs)
         items = self.items[barename(item.term)]
         items.append(item)
         return item
@@ -288,23 +339,55 @@ class Metadata(object):
     def __getattr__(self, term):
         return self.items[term]
 
+    def _nsmap():
+        def fget(self):
+            nsmap = {}
+            for term in self.items:
+                for item in self.items[term]:
+                    nsmap.update(item.nsmap)
+            return nsmap
+        return property(fget=fget)
+    _nsmap = _nsmap()        
+    
+    def _opf1_nsmap():
+        def fget(self):
+            nsmap = self._nsmap
+            for key, value in nsmap.items():
+                if value in OPF_NSES or value in DC_NSES:
+                    del nsmap[key]
+            return nsmap
+        return property(fget=fget)
+    _opf1_nsmap = _opf1_nsmap()
+    
+    def _opf2_nsmap():
+        def fget(self):
+            nsmap = self._nsmap
+            nsmap.update(self.OPF2_NSMAP)
+            return nsmap
+        return property(fget=fget)
+    _opf2_nsmap = _opf2_nsmap()
+    
     def to_opf1(self, parent=None):
-        elem = element(parent, 'metadata')
+        nsmap = self._opf1_nsmap
+        nsrmap = dict((value, key) for key, value in nsmap.items())
+        elem = element(parent, 'metadata', nsmap=nsmap)
         dcmeta = element(elem, 'dc-metadata', nsmap=self.OPF1_NSMAP)
         xmeta = element(elem, 'x-metadata')
         for term in self.items:
             for item in self.items[term]:
-                item.to_opf1(dcmeta, xmeta)
+                item.to_opf1(dcmeta, xmeta, nsrmap=nsrmap)
         if 'ms-chaptertour' not in self.items:
             chaptertour = self.Item('ms-chaptertour', 'chaptertour')
-            chaptertour.to_opf1(dcmeta, xmeta)
+            chaptertour.to_opf1(dcmeta, xmeta, nsrmap=nsrmap)
         return elem
         
     def to_opf2(self, parent=None):
-        elem = element(parent, OPF('metadata'), nsmap=self.OPF2_NSMAP)
+        nsmap = self._opf2_nsmap
+        nsrmap = dict((value, key) for key, value in nsmap.items())
+        elem = element(parent, OPF('metadata'), nsmap=nsmap)
         for term in self.items:
             for item in self.items[term]:
-                item.to_opf2(elem)
+                item.to_opf2(elem, nsrmap=nsrmap)
         return elem
 
 
@@ -351,9 +434,13 @@ class Manifest(object):
                 try:
                     data = etree.fromstring(data)
                 except etree.XMLSyntaxError:
+                    # TODO: Factor out HTML->XML coercion
                     self.oeb.logger.warn('Parsing file %r as HTML' % self.href)
                     data = html.fromstring(data)
                     data.attrib.pop('xmlns', None)
+                    for elem in data.iter(tag=etree.Comment):
+                        if elem.text:
+                            elem.text = elem.text.strip('-')
                     data = etree.tostring(data, encoding=unicode)
                     data = etree.fromstring(data)
             # Force into the XHTML namespace
@@ -447,7 +534,7 @@ class Manifest(object):
             return cmp(skey, okey)
         
         def relhref(self, href):
-            if '/' not in self.href:
+            if '/' not in self.href or ':' in href:
                 return href
             base = os.path.dirname(self.href).split('/')
             target, frag = urldefrag(href)
@@ -463,7 +550,7 @@ class Manifest(object):
             return relhref
 
         def abshref(self, href):
-            if '/' not in self.href:
+            if '/' not in self.href or ':' in href:
                 return href
             dirname = os.path.dirname(self.href)
             href = os.path.join(dirname, href)
@@ -546,7 +633,7 @@ class Manifest(object):
             elif media_type in OEB_STYLES:
                 media_type = CSS_MIME
             attrib = {'id': item.id, 'href': item.href,
-                      'media-type': item.media_type}
+                      'media-type': media_type}
             if item.fallback:
                 attrib['fallback'] = item.fallback
             element(elem, OPF('item'), attrib=attrib)
@@ -796,6 +883,9 @@ class TOC(object):
 
     
 class OEBBook(object):
+    COVER_SVG_XP = XPath('h:body//svg:svg[position() = 1]')
+    COVER_OBJECT_XP = XPath('h:body//h:object[@data][position() = 1]')
+
     def __init__(self, opfpath=None, container=None, encoding=None,
                  logger=FauxLogger()):
         if opfpath and not container:
@@ -809,27 +899,27 @@ class OEBBook(object):
             self._all_from_opf(opf)
     
     def _clean_opf(self, opf):
-        for elem in opf.iter():
+        nsmap = {}
-            if isinstance(elem.tag, basestring) \
+        for elem in opf.iter(tag=etree.Element):
-               and namespace(elem.tag) in ('', OPF1_NS):
+            nsmap.update(elem.nsmap)
+        for elem in opf.iter(tag=etree.Element):
+            if namespace(elem.tag) in ('', OPF1_NS):
                 elem.tag = OPF(barename(elem.tag))
+        nsmap.update(Metadata.OPF2_NSMAP)
         attrib = dict(opf.attrib)
         nroot = etree.Element(OPF('package'),
             nsmap={None: OPF2_NS}, attrib=attrib)
-        metadata = etree.SubElement(nroot, OPF('metadata'),
+        metadata = etree.SubElement(nroot, OPF('metadata'), nsmap=nsmap)
-            nsmap={'opf': OPF2_NS, 'dc': DC11_NS,
+        ignored = (OPF('dc-metadata'), OPF('x-metadata'))
-                   'xsi': XSI_NS, 'dcterms': DCTERMS_NS})
+        for elem in xpath(opf, 'o2:metadata//*'):
-        dc = lambda prefix: xpath(opf, 'o2:metadata//%s:*' % prefix)
+            if namespace(elem.tag) in DC_NSES:
-        for element in chain(*(dc(prefix) for prefix in DC_PREFIXES)):
+                tag = barename(elem.tag).lower()
-            if not element.text: continue
+                elem.tag = '{%s}%s' % (DC11_NS, tag)
-            tag = barename(element.tag).lower()
+            for name in elem.attrib:
-            element.tag = '{%s}%s' % (DC11_NS, tag)
+                if name in ('role', 'file-as', 'scheme', 'event'):
-            for name in element.attrib:
-                if name in ('role', 'file-as', 'scheme'):
                     nsname = '{%s}%s' % (OPF2_NS, name)
-                    element.attrib[nsname] = element.attrib[name]
+                    elem.attrib[nsname] = elem.attrib.pop(name)
-                    del element.attrib[name]
+            metadata.append(elem)
-            metadata.append(element)
         for element in xpath(opf, 'o2:metadata//o2:meta'):
             metadata.append(element)
         for tag in ('o2:manifest', 'o2:spine', 'o2:tours', 'o2:guide'):
@@ -856,18 +946,18 @@ class OEBBook(object):
         uid = opf.get('unique-identifier', 'calibre-uuid')
         self.uid = None
         self.metadata = metadata = Metadata(self)
-        ignored = (OPF('dc-metadata'), OPF('x-metadata'))
         for elem in xpath(opf, '/o2:package/o2:metadata//*'):
-            if elem.tag in ignored: continue
             term = elem.tag
             value = elem.text
+            attrib = dict(elem.attrib)
+            nsmap = elem.nsmap
             if term == OPF('meta'):
-                term = elem.attrib.pop('name', None)
+                term = qname(attrib.pop('name', None), nsmap)
-                value = elem.attrib.pop('content', None)
+                value = attrib.pop('content', None)
             if value:
                 value = COLLAPSE_RE.sub(' ', value.strip())
-            if term and (value or elem.attrib):
+            if term and (value or attrib):
-                metadata.add(term, value, elem.attrib)
+                metadata.add(term, value, attrib, nsmap=nsmap)
         haveuuid = haveid = False
         for ident in metadata.identifier:
             if unicode(ident).startswith('urn:uuid:'):
@@ -928,7 +1018,7 @@ class OEBBook(object):
             spine.add(item, elem.get('linear'))
         extras = []
         for item in self.manifest.values():
-            if item.media_type == XHTML_MIME \
+            if item.media_type in OEB_DOCS \
                and item not in spine:
                 extras.append(item)
         extras.sort()
@@ -971,7 +1061,7 @@ class OEBBook(object):
         ncx = item.data
         self.manifest.remove(item)
         title = xpath(ncx, 'ncx:docTitle/ncx:text/text()')
-        title = title[0].strip() if title else unicode(self.metadata.title)
+        title = title[0].strip() if title else unicode(self.metadata.title[0])
         self.toc = toc = TOC(title)
         navmaps = xpath(ncx, 'ncx:navMap')
         for navmap in navmaps:
@@ -1051,41 +1141,58 @@ class OEBBook(object):
         if self._toc_from_html(opf): return
         self._toc_from_spine(opf)
 
-    def _ensure_cover_image(self):
+    def _cover_from_html(self, hcover):
-        cover = None
+        with TemporaryDirectory('_html_cover') as tdir:
+            writer = DirWriter()
+            writer.dump(self, tdir)
+            path = os.path.join(tdir, hcover.href)
+            renderer = CoverRenderer(path)
+            data = renderer.image_data
+        id, href = self.manifest.generate('cover', 'cover.jpeg')
+        item = self.manifest.add(id, href, JPEG_MIME, data=data)
+        return item
+        
+    def _locate_cover_image(self):
+        if self.metadata.cover:
+            id = str(self.metadata.cover[0])
+            item = self.manifest.ids.get(id, None)
+            if item is not None:
+                return item
         hcover = self.spine[0]
         if 'cover' in self.guide:
             href = self.guide['cover'].href
             item = self.manifest.hrefs[href]
             media_type = item.media_type
-            if media_type in OEB_RASTER_IMAGES:
+            if media_type in OEB_IMAGES:
-                cover = item
+                return item
             elif media_type in OEB_DOCS:
                 hcover = item
         html = hcover.data
-        if cover is not None:
+        if MS_COVER_TYPE in self.guide:
-            pass
-        elif self.metadata.cover:
-            id = str(self.metadata.cover[0])
-            cover = self.manifest.ids[id]
-        elif MS_COVER_TYPE in self.guide:
             href = self.guide[MS_COVER_TYPE].href
-            cover = self.manifest.hrefs[href]
+            item = self.manifest.hrefs.get(href, None)
-        elif xpath(html, '//h:img[position()=1]'):
+            if item is not None and item.media_type in OEB_IMAGES:
-            img = xpath(html, '//h:img[position()=1]')[0]
+                return item
-            href = hcover.abshref(img.get('src'))
+        if self.COVER_SVG_XP(html):
-            cover = self.manifest.hrefs[href]
+            svg = copy.deepcopy(self.COVER_SVG_XP(html)[0])
-        elif xpath(html, '//h:object[position()=1]'):
-            object = xpath(html, '//h:object[position()=1]')[0]
-            href = hcover.abshref(object.get('data'))
-            cover = self.manifest.hrefs[href]
-        elif xpath(html, '//svg:svg[position()=1]'):
-            svg = copy.deepcopy(xpath(html, '//svg:svg[position()=1]')[0])
             href = os.path.splitext(hcover.href)[0] + '.svg'
             id, href = self.manifest.generate(hcover.id, href)
-            cover = self.manifest.add(id, href, SVG_MIME, data=svg)
+            item = self.manifest.add(id, href, SVG_MIME, data=svg)
-        if cover and not self.metadata.cover:
+            return item
-            self.metadata.add('cover', cover.id)
+        if self.COVER_OBJECT_XP(html):
+            object = self.COVER_OBJECT_XP(html)[0]
+            href = hcover.abshref(object.get('data'))
+            item = self.manifest.hrefs.get(href, None)
+            if item is not None and item.media_type in OEB_IMAGES:
+                return item
+        return self._cover_from_html(hcover)
+        
+    def _ensure_cover_image(self):
+        cover = self._locate_cover_image()
+        if self.metadata.cover:
+            self.metadata.cover[0].value = cover.id
+            return
+        self.metadata.add('cover', cover.id)
     
     def _all_from_opf(self, opf):
         self._metadata_from_opf(opf)

src/calibre/ebooks/oeb/stylizer.py

@@ -265,6 +265,8 @@ class Stylizer(object):
 
 
 class Style(object):
+    UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|px|mm|cm|in|pt|pc)$')
+    
     def __init__(self, element, stylizer):
         self._element = element
         self._profile = stylizer.profile
@@ -319,13 +321,11 @@ class Style(object):
         if isinstance(value, (int, long, float)):
             return value
         try:
-            if float(value) == 0:
+            return float(value) * 72.0 / self._profile.dpi
-                return 0.0
         except:
             pass
         result = value
-        m = re.search(
+        m = self.UNIT_RE.match(value)
-            r"^(-*[0-9]*\.?[0-9]*)\s*(%|em|px|mm|cm|in|pt|pc)$", value)
         if m is not None and m.group(1):
             value = float(m.group(1))
             unit = m.group(2)

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -23,6 +23,12 @@ from calibre.ebooks.oeb.stylizer import Stylizer
 COLLAPSE = re.compile(r'[ \t\r\n\v]+')
 STRIPNUM = re.compile(r'[-0-9]+$')
 
+def asfloat(value, default):
+    if not isinstance(value, (int, long, float)):
+        value = default
+    return float(value)
+
+
 class KeyMapper(object):
     def __init__(self, sbase, dbase, dkey):
         self.sbase = float(sbase)
@@ -179,12 +185,13 @@ class CSSFlattener(object):
         if cssdict:
             if self.lineh and self.fbase and tag != 'body':
                 self.clean_edges(cssdict, style, psize)
-            margin = style['margin-left']
+            margin = asfloat(style['margin-left'], 0)
-            left += margin if isinstance(margin, float) else 0
+            indent = asfloat(style['text-indent'], 0)
-            if (left + style['text-indent']) < 0:
+            left += margin
-                percent = (margin - style['text-indent']) / style['width']
+            if (left + indent) < 0:
+                percent = (margin - indent) / style['width']
                 cssdict['margin-left'] = "%d%%" % (percent * 100)
-                left -= style['text-indent']
+                left -= indent
             if 'display' in cssdict and cssdict['display'] == 'in-line':
                 cssdict['display'] = 'inline'
             if self.unfloat and 'float' in cssdict \

src/calibre/ebooks/oeb/transforms/rasterize.py

@@ -23,6 +23,7 @@ from PyQt4.QtGui import QApplication
 from calibre.ebooks.oeb.base import XHTML_NS, XHTML, SVG_NS, SVG, XLINK
 from calibre.ebooks.oeb.base import SVG_MIME, PNG_MIME, JPEG_MIME
 from calibre.ebooks.oeb.base import xml2str, xpath, namespace, barename
+from calibre.ebooks.oeb.base import urlnormalize
 from calibre.ebooks.oeb.stylizer import Stylizer
 
 IMAGE_TAGS = set([XHTML('img'), XHTML('object')])
@@ -78,7 +79,7 @@ class SVGRasterizer(object):
             svg = item.data
         hrefs = self.oeb.manifest.hrefs
         for elem in xpath(svg, '//svg:*[@xl:href]'):
-            href = elem.attrib[XLINK('href')]
+            href = urlnormalize(elem.attrib[XLINK('href')])
             path, frag = urldefrag(href)
             if not path:
                 continue
@@ -100,15 +101,15 @@ class SVGRasterizer(object):
     def rasterize_item(self, item, stylizer):
         html = item.data
         hrefs = self.oeb.manifest.hrefs
-        for elem in xpath(html, '//h:img'):
+        for elem in xpath(html, '//h:img[@src]'):
-            src = elem.get('src', None)
+            src = urlnormalize(elem.attrib['src'])
-            image = hrefs.get(item.abshref(src), None) if src else None
+            image = hrefs.get(item.abshref(src), None)
             if image and image.media_type == SVG_MIME:
                 style = stylizer.style(elem)
                 self.rasterize_external(elem, style, item, image)
-        for elem in xpath(html, '//h:object[@type="%s"]' % SVG_MIME):
+        for elem in xpath(html, '//h:object[@type="%s" and @data]' % SVG_MIME):
-            data = elem.get('data', None)
+            data = urlnormalize(elem.attrib['data'])
-            image = hrefs.get(item.abshref(data), None) if data else None
+            image = hrefs.get(item.abshref(data), None)
             if image and image.media_type == SVG_MIME:
                 style = stylizer.style(elem)
                 self.rasterize_external(elem, style, item, image)

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -54,7 +54,7 @@ class ManifestTrimmer(object):
                                 new.add(found)
                 elif item.media_type == CSS_MIME:
                     def replacer(uri):
-                        absuri = item.abshref(uri)
+                        absuri = item.abshref(urlnormalize(uri))
                         if absuri in oeb.manifest.hrefs:
                             found = oeb.manifest.hrefs[href]
                             if found not in used:

src/calibre/ebooks/rtf2xml/copy.py

@@ -15,7 +15,8 @@
 #                                                                       #
 #                                                                       #
 #########################################################################
-import sys, os
+import sys, os, shutil
+
 class Copy:
     """Copy each changed file to a directory for debugging purposes"""
     __dir = ""
@@ -64,25 +65,7 @@ class Copy:
         of cp. Otherwise, use a safe python method.
         """
         write_file = os.path.join(Copy.__dir,new_file)
-        platform = sys.platform
+        shutil.copyfile(file, write_file)
-        if platform[:5] == 'linux':
+    
-            command = 'cp %(file)s %(write_file)s' % vars()
-            os.system(command)
-        else:
-            read_obj = open(file,'r')
-            write_obj = open(write_file, 'w')
-            line = "dummy"
-            while line:
-                    line = read_obj.read(1000)
-                    write_obj.write(line )
-            read_obj.close()
-            write_obj.close()
     def rename(self, source, dest):
-        read_obj = open(source, 'r')
+        shutil.copyfile(source, dest)
-        write_obj = open(dest, 'w')
-        line = 1
-        while line:
-            line = read_obj.readline()
-            write_obj.write(line)
-        read_obj.close()
-        write_obj.close()

src/calibre/gui2/dialogs/epub.py

@@ -252,7 +252,7 @@ class Config(ResizableDialog, Ui_Dialog):
                     self.source_format = d.format()
                 
     def accept(self):
-        for opt in ('chapter', 'level1_toc', 'level2_toc'):
+        for opt in ('chapter', 'level1_toc', 'level2_toc', 'level3_toc'):
             text = unicode(getattr(self, 'opt_'+opt).text())
             if text:
                 try:

src/calibre/gui2/dialogs/epub.ui

@@ -93,7 +93,7 @@
        <item>
         <widget class="QStackedWidget" name="stack" >
          <property name="currentIndex" >
-          <number>1</number>
+          <number>0</number>
          </property>
          <widget class="QWidget" name="metadata_page" >
           <layout class="QGridLayout" name="gridLayout_4" >
@@ -105,36 +105,6 @@
                 <string>Book Cover</string>
                </property>
                <layout class="QGridLayout" name="_2" >
-                <item row="0" column="0" >
-                 <layout class="QHBoxLayout" name="_3" >
-                  <item>
-                   <widget class="ImageView" name="cover" >
-                    <property name="text" >
-                     <string/>
-                    </property>
-                    <property name="pixmap" >
-                     <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
-                    </property>
-                    <property name="scaledContents" >
-                     <bool>true</bool>
-                    </property>
-                    <property name="alignment" >
-                     <set>Qt::AlignCenter</set>
-                    </property>
-                   </widget>
-                  </item>
-                 </layout>
-                </item>
-                <item row="2" column="0" >
-                 <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
-                  <property name="text" >
-                   <string>Use cover from &amp;source file</string>
-                  </property>
-                  <property name="checked" >
-                   <bool>true</bool>
-                  </property>
-                 </widget>
-                </item>
                 <item row="1" column="0" >
                  <layout class="QVBoxLayout" name="_4" >
                   <property name="spacing" >
@@ -186,6 +156,36 @@
                   </item>
                  </layout>
                 </item>
+                <item row="2" column="0" >
+                 <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
+                  <property name="text" >
+                   <string>Use cover from &amp;source file</string>
+                  </property>
+                  <property name="checked" >
+                   <bool>true</bool>
+                  </property>
+                 </widget>
+                </item>
+                <item row="0" column="0" >
+                 <layout class="QHBoxLayout" name="_3" >
+                  <item>
+                   <widget class="ImageView" name="cover" >
+                    <property name="text" >
+                     <string/>
+                    </property>
+                    <property name="pixmap" >
+                     <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
+                    </property>
+                    <property name="scaledContents" >
+                     <bool>true</bool>
+                    </property>
+                    <property name="alignment" >
+                     <set>Qt::AlignCenter</set>
+                    </property>
+                   </widget>
+                  </item>
+                 </layout>
+                </item>
                </layout>
                <zorder>opt_prefer_metadata_cover</zorder>
                <zorder></zorder>
@@ -777,10 +777,10 @@ p, li { white-space: pre-wrap; }
               <item row="5" column="1" >
                <widget class="QLineEdit" name="opt_level2_toc" />
               </item>
-              <item row="6" column="1" >
+              <item row="7" column="1" >
                <widget class="QLineEdit" name="opt_toc_title" />
               </item>
-              <item row="6" column="0" >
+              <item row="7" column="0" >
                <widget class="QLabel" name="toc_title_label" >
                 <property name="text" >
                  <string>&amp;Title for generated TOC</string>
@@ -790,6 +790,19 @@ p, li { white-space: pre-wrap; }
                 </property>
                </widget>
               </item>
+              <item row="6" column="1" >
+               <widget class="QLineEdit" name="opt_level3_toc" />
+              </item>
+              <item row="6" column="0" >
+               <widget class="QLabel" name="label_11" >
+                <property name="text" >
+                 <string>Level &amp;3 TOC</string>
+                </property>
+                <property name="buddy" >
+                 <cstring>opt_level3_toc</cstring>
+                </property>
+               </widget>
+              </item>
              </layout>
             </widget>
            </item>

src/calibre/gui2/dialogs/lrf_single.py

@@ -253,6 +253,7 @@ class LRFSingleDialog(QDialog, Ui_LRFSingleDialog):
                 state = Qt.Checked if default else Qt.Unchecked
                 obj.setCheckState(state)
         self.gui_headerformat.setDisabled(True)
+        self.gui_header_separation.setDisabled(True)
         self.gui_use_metadata_cover.setCheckState(Qt.Checked)
         self.preprocess.addItem('No preprocessing')
         for opt in self.PREPROCESS_OPTIONS:

src/calibre/gui2/dialogs/lrf_single.ui

@@ -1055,5 +1055,37 @@ p, li { white-space: pre-wrap; }
     </hint>
    </hints>
   </connection>
+  <connection>
+   <sender>gui_header</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>gui_header_separation</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>235</x>
+     <y>298</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>361</x>
+     <y>321</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>gui_header</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>gui_headerformat</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>307</x>
+     <y>300</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>363</x>
+     <y>362</y>
+    </hint>
+   </hints>
+  </connection>
  </connections>
 </ui>

src/calibre/gui2/dialogs/scheduler.py

@@ -8,7 +8,7 @@ Scheduler for automated recipe downloads
 '''
 
 import sys, copy, time
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
 from PyQt4.Qt import QDialog, QApplication, QLineEdit, QPalette, SIGNAL, QBrush, \
                      QColor, QAbstractListModel, Qt, QVariant, QFont, QIcon, \
                      QFile, QObject, QTimer, QMutex, QMenu, QAction, QTime
@@ -289,7 +289,8 @@ class SchedulerDialog(QDialog, Ui_Dialog):
                 recipe.last_downloaded = datetime.fromordinal(1)
                 recipes.append(recipe)
             if recipe.needs_subscription and not config['recipe_account_info_%s'%recipe.id]:
-                error_dialog(self, _('Must set account information'), _('This recipe requires a username and password')).exec_()
+                error_dialog(self, _('Must set account information'), 
+                             _('This recipe requires a username and password')).exec_()
                 self.schedule.setCheckState(Qt.Unchecked)
                 return
             if self.interval_button.isChecked():
@@ -350,9 +351,11 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             self.username.blockSignals(False)
             self.password.blockSignals(False)
         d = datetime.utcnow() - recipe.last_downloaded
-        ld = '%.2f'%(d.days + d.seconds/(24.*3600))
+        def hm(x): return (x-x%3600)//3600, (x%3600 - (x%3600)%60)//60
+        hours, minutes = hm(d.seconds)
+        tm = _('%d days, %d hours and %d minutes ago')%(d.days, hours, minutes)
         if d < timedelta(days=366):
-            self.last_downloaded.setText(_('Last downloaded: %s days ago')%ld)
+            self.last_downloaded.setText(_('Last downloaded')+': '+tm)
         else:
             self.last_downloaded.setText(_('Last downloaded: never'))
             
@@ -431,7 +434,7 @@ class Scheduler(QObject):
                     day_matches = day > 6 or day == now.tm_wday
                     tnow = now.tm_hour*60 + now.tm_min
                     matches = day_matches and (hour*60+minute) < tnow
-                    if matches and delta >= timedelta(days=1):
+                    if matches and nowt.toordinal() < date.today().toordinal():
                         needs_downloading.add(recipe)
                 
             self.debug('Needs downloading:', needs_downloading)

src/calibre/gui2/dialogs/scheduler.ui

@@ -5,7 +5,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>726</width>
+    <width>738</width>
     <height>575</height>
    </rect>
   </property>
@@ -194,6 +194,9 @@
           <property name="text" >
            <string/>
           </property>
+          <property name="wordWrap" >
+           <bool>true</bool>
+          </property>
          </widget>
         </item>
         <item>

src/calibre/gui2/lrf_renderer/bookview.py

@@ -21,3 +21,4 @@ class BookView(QGraphicsView):
     def resize_for(self, width, height):
         self.preferred_size = QSize(width, height)
     
+    

src/calibre/gui2/lrf_renderer/main.py

@@ -80,8 +80,8 @@ class Main(MainWindow, Ui_MainWindow):
         
         QObject.connect(self.search, SIGNAL('search(PyQt_PyObject, PyQt_PyObject)'), self.find)
         
-        self.action_next_page.setShortcuts(QKeySequence.MoveToNextPage)
+        self.action_next_page.setShortcuts([QKeySequence.MoveToNextPage, QKeySequence(Qt.Key_Space)])
-        self.action_previous_page.setShortcuts(QKeySequence.MoveToPreviousPage)
+        self.action_previous_page.setShortcuts([QKeySequence.MoveToPreviousPage, QKeySequence(Qt.Key_Backspace)])
         self.action_next_match.setShortcuts(QKeySequence.FindNext)
         self.addAction(self.action_next_match)
         QObject.connect(self.action_next_page, SIGNAL('triggered(bool)'), self.next) 
@@ -191,6 +191,7 @@ class Main(MainWindow, Ui_MainWindow):
             self.spin_box.setSuffix(' of %d'%(self.document.num_of_pages,))
             self.spin_box.updateGeometry()
             self.stack.setCurrentIndex(0)
+            self.graphics_view.setFocus(Qt.OtherFocusReason)
         elif self.renderer.exception is not None:
             exception = self.renderer.exception
             print >>sys.stderr, 'Error rendering document'

src/calibre/gui2/main_window.py

@@ -35,7 +35,9 @@ class DebugWindow(ConversionErrorDialog):
 
 class MainWindow(QMainWindow):
 
-    _menu_bar = None
+    ___menu_bar = None
+    ___menu     = None
+    __actions   = [] 
     
     @classmethod
     def create_application_menubar(cls):
@@ -43,9 +45,11 @@ class MainWindow(QMainWindow):
         menu = QMenu()
         for action in cls.get_menubar_actions():
             menu.addAction(action)
+            cls.__actions.append(action)
             yield action
         mb.addMenu(menu)
-        cls._menu_bar = mb
+        cls.___menu_bar = mb
+        cls.___menu = menu
         
     
     @classmethod

src/calibre/manual/news.rst

@@ -299,10 +299,10 @@ To learn more about writing advanced recipes using some of the facilities, avail
     :ref:`API Documentation <news_recipe>`
         Documentation of the ``BasicNewsRecipe`` class and all its important methods and fields.
 
-    `BasicNewsRecipe <http://bazaar.launchpad.net/~kovid/calibre/trunk/annotate/kovid%40kovidgoyal.net-20080509231359-le3xf7ynwc6eew90?file_id=1245%40b0dd1a5d-880a-0410-ada5-a57097536bc1%3Alibprs500%252Ftrunk%3Asrc%252Flibprs500%252Fweb%252Ffeeds%252Fnews.py>`_
+    `BasicNewsRecipe <http://bazaar.launchpad.net/~kovid/calibre/trunk/annotate/head:/src/calibre/web/feeds/news.py>`_
         The source code of ``BasicNewsRecipe``
 
-    `Built-in recipes <http://bazaar.launchpad.net/~kovid/calibre/trunk/files/kovid%40kovidgoyal.net-20080509231359-le3xf7ynwc6eew90?file_id=1298%40b0dd1a5d-880a-0410-ada5-a57097536bc1%3Alibprs500%252Ftrunk%3Asrc%252Flibprs500%252Fweb%252Ffeeds%252Frecipes>`_
+    `Built-in recipes <http://bazaar.launchpad.net/~kovid/calibre/trunk/files/head:/src/calibre/web/feeds/recipes/>`_
         The source code for the built-in recipes that come with |app|
 
 Migrating old style profiles to recipes

src/calibre/web/feeds/recipes/recipe_lrb.py

@@ -32,3 +32,8 @@ class LondonReviewOfBooks(BasicNewsRecipe):
     def print_version(self, url):
         main, split, rest = url.rpartition('/')
         return main + '/print/' + rest
+    
+    def postprocess_html(self, soup, first_fetch):
+        for t in soup.findAll(['table', 'tr', 'td']):
+            t.name = 'div'
+        return soup

src/odf/grammar.py

@@ -624,7 +624,7 @@ allowed_children = {
 		(TEXTNS,u'object-index'),
 		(TEXTNS,u'p'),
 		(TEXTNS,u'section'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TEXTNS,u'table-index'),
 		(TEXTNS,u'table-of-content'),
 		(TEXTNS,u'user-index'),
@@ -1073,7 +1073,7 @@ allowed_children = {
 		(TEXTNS,u'page-sequence'),
 		(TEXTNS,u'section'),
 		(TEXTNS,u'sequence-decls'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TEXTNS,u'table-index'),
 		(TEXTNS,u'table-of-content'),
 		(TEXTNS,u'tracked-changes'),
@@ -1529,7 +1529,7 @@ allowed_children = {
 		(TEXTNS,u'object-index'),
 		(TEXTNS,u'p'),
 		(TEXTNS,u'section'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TEXTNS,u'table-index'),
 		(TEXTNS,u'table-of-content'),
 		(TEXTNS,u'user-index'),
@@ -1759,7 +1759,7 @@ allowed_children = {
 	(TABLENS,u'table') : (
 		(OFFICENS,u'dde-source'),
 		(OFFICENS,u'forms'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TABLENS,u'scenario'),
 		(TABLENS,u'shapes'),
 		(TABLENS,u'table-column'),
@@ -1807,7 +1807,7 @@ allowed_children = {
 		(TEXTNS,u'object-index'),
 		(TEXTNS,u'p'),
 		(TEXTNS,u'section'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TEXTNS,u'table-index'),
 		(TEXTNS,u'table-of-content'),
 		(TEXTNS,u'user-index'),
@@ -1828,7 +1828,7 @@ allowed_children = {
 	),
 	(TABLENS,u'table-header-rows') : (
 		(TABLENS,u'table-row'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 	),
 	(TABLENS,u'table-row') : (
 		(TABLENS,u'covered-table-cell'),
@@ -1842,7 +1842,7 @@ allowed_children = {
 	),
 	(TABLENS,u'table-rows') : (
 		(TABLENS,u'table-row'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 	),
 	(TABLENS,u'table-source') : (
 	),
@@ -2060,6 +2060,8 @@ allowed_children = {
 	),
 	(TEXTNS,u'chapter') : (
 	),
+	(TEXTNS,u'character-count') : (
+	),
 	(TEXTNS,u'conditional-text') : (
 	),
 	(TEXTNS,u'creation-date') : (
@@ -2126,7 +2128,7 @@ allowed_children = {
 		(TEXTNS,u'object-index'),
 		(TEXTNS,u'p'),
 		(TEXTNS,u'section'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TEXTNS,u'table-index'),
 		(TEXTNS,u'table-of-content'),
 		(TEXTNS,u'user-index'),
@@ -2254,7 +2256,7 @@ allowed_children = {
 		(TEXTNS,u'sequence'),
 		(TEXTNS,u'sequence-ref'),
 		(TEXTNS,u'sheet-name'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TEXTNS,u'span'),
 		(TEXTNS,u'subject'),
 		(TEXTNS,u'tab'),
@@ -2294,6 +2296,8 @@ allowed_children = {
 		(TEXTNS,u'illustration-index-entry-template'),
 		(TEXTNS,u'index-title-template'),
 	),
+	(TEXTNS,u'image-count') : (
+	),
 # allowed_children
 	(TEXTNS,u'index-body') : (
 		(DR3DNS,u'scene'),
@@ -2328,7 +2332,7 @@ allowed_children = {
 		(TEXTNS,u'object-index'),
 		(TEXTNS,u'p'),
 		(TEXTNS,u'section'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TEXTNS,u'table-index'),
 		(TEXTNS,u'table-of-content'),
 		(TEXTNS,u'user-index'),
@@ -2423,7 +2427,7 @@ allowed_children = {
 		(TEXTNS,u'list'),
 		(TEXTNS,u'number'),
 		(TEXTNS,u'p'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 	),
 	(TEXTNS,u'list-level-style-bullet') : (
 		(STYLENS,u'list-level-properties'),
@@ -2635,7 +2639,7 @@ allowed_children = {
 		(TEXTNS,u'sequence'),
 		(TEXTNS,u'sequence-ref'),
 		(TEXTNS,u'sheet-name'),
-                (TEXTNS,u'soft-page-break'),
+		(TEXTNS,u'soft-page-break'),
 		(TEXTNS,u'span'),
 		(TEXTNS,u'subject'),
 		(TEXTNS,u'tab'),
@@ -2661,6 +2665,8 @@ allowed_children = {
 	),
 	(TEXTNS,u'page') : (
 	),
+	(TEXTNS,u'page-count') : (
+	),
 	(TEXTNS,u'page-continuation') : (
 	),
 	(TEXTNS,u'page-number') : (
@@ -2672,6 +2678,8 @@ allowed_children = {
 	),
 	(TEXTNS,u'page-variable-set') : (
 	),
+	(TEXTNS,u'paragraph-count') : (
+	),
 	(TEXTNS,u'placeholder') : (
 	),
 	(TEXTNS,u'print-date') : (
@@ -2686,6 +2694,8 @@ allowed_children = {
 	),
 	(TEXTNS,u'reference-mark-start') : (
 	),
+	(TEXTNS,u'reference-ref') : (
+	),
 	(TEXTNS,u'ruby') : (
 		(TEXTNS,u'ruby-base'),
 		(TEXTNS,u'ruby-text'),
@@ -3035,6 +3045,8 @@ allowed_children = {
 	),
 	(TEXTNS,u'tab') : (
 	),
+	(TEXTNS,u'table-count') : (
+	),
 	(TEXTNS,u'table-formula') : (
 	),
 	(TEXTNS,u'table-index') : (
@@ -3132,6 +3144,8 @@ allowed_children = {
 	),
 	(TEXTNS,u'variable-set') : (
 	),
+	(TEXTNS,u'word-count') : (
+	),
 }
 
 struct_elements = (  # Unused?
@@ -3182,6 +3196,7 @@ allows_text = (
 	(PRESENTATIONNS,u'footer-decl'),
 	(PRESENTATIONNS,u'header-decl'),
 	(SVGNS,u'desc'),
+	(SVGNS,u'title'),
 	(TEXTNS,u'a'),
 	(TEXTNS,u'author-initials'),
 	(TEXTNS,u'author-name'),
@@ -3403,7 +3418,6 @@ required_attributes = {
 	(DRAWNS,u'glue-point'): (
 		(SVGNS,u'y'),
 		(SVGNS,u'x'),
-		(DRAWNS,u'align'),
 		(DRAWNS,u'id'),
 	),
 	(DRAWNS,u'gradient'): (
@@ -4297,35 +4311,47 @@ allowed_attributes = {
 		(ANIMNS,u'value'),
 	),
 	(ANIMNS,u'seq'):(
-		(PRESENTATIONNS,u'node-type'),
-		(SMILNS,u'decelerate'),
-		(SMILNS,u'begin'),
-		(SMILNS,u'end'),
-		(PRESENTATIONNS,u'group-id'),
-		(SMILNS,u'accelerate'),
-		(SMILNS,u'repeatDur'),
-		(SMILNS,u'endsync'),
-		(SMILNS,u'restartDefault'),
-		(PRESENTATIONNS,u'preset-class'),
-		(SMILNS,u'fillDefault'),
-		(PRESENTATIONNS,u'preset-id'),
-		(SMILNS,u'autoReverse'),
-		(PRESENTATIONNS,u'preset-sub-type'),
-		(SMILNS,u'repeatCount'),
-		(SMILNS,u'dur'),
-		(SMILNS,u'fill'),
 		(ANIMNS,u'id'),
-		(SMILNS,u'restart'),
+		(PRESENTATIONNS,u'group-id'),
 		(PRESENTATIONNS,u'master-element'),
+		(PRESENTATIONNS,u'node-type'),
+		(PRESENTATIONNS,u'preset-class'),
+		(PRESENTATIONNS,u'preset-id'),
+		(PRESENTATIONNS,u'preset-sub-type'),
+		(SMILNS,u'accelerate'),
+		(SMILNS,u'autoReverse'),
+		(SMILNS,u'begin'),
+		(SMILNS,u'decelerate'),
+		(SMILNS,u'dur'),
+		(SMILNS,u'end'),
+		(SMILNS,u'endsync'),
+		(SMILNS,u'fill'),
+		(SMILNS,u'fillDefault'),
+		(SMILNS,u'repeatCount'),
+		(SMILNS,u'repeatDur'),
+		(SMILNS,u'restart'),
+		(SMILNS,u'restartDefault'),
 	),
 	(ANIMNS,u'set'):(
+		(ANIMNS,u'sub-item'),
+		(SMILNS,u'accelerate'),
+		(SMILNS,u'accumulate'),
+		(SMILNS,u'autoReverse'),
 		(SMILNS,u'additive'),
 		(SMILNS,u'attributeName'),
-		(SMILNS,u'to'),
+		(SMILNS,u'begin'),
-		(ANIMNS,u'sub-item'),
+		(SMILNS,u'decelerate'),
-		(SMILNS,u'targetElement'),
+		(SMILNS,u'dur'),
-		(SMILNS,u'accumulate'),
+		(SMILNS,u'end'),
 		(SMILNS,u'fill'),
+		(SMILNS,u'fillDefault'),
+		(SMILNS,u'repeatCount'),
+		(SMILNS,u'repeatDur'),
+		(SMILNS,u'restart'),
+		(SMILNS,u'restartDefault'),
+		(SMILNS,u'targetElement'),
+		(SMILNS,u'to'),
+
 	),
 	(ANIMNS,u'transitionFilter'):(
 		(SMILNS,u'direction'),
@@ -5789,6 +5815,8 @@ allowed_attributes = {
 		(MANIFESTNS,'salt'),
 		(MANIFESTNS,'iteration-count'),
 	),
+	(MANIFESTNS,u'manifest'):(
+	),
 # allowed_attributes
 	(METANS,u'auto-reload'):(
 		(METANS,u'delay'),
@@ -6096,7 +6124,7 @@ allowed_attributes = {
 		(CHARTNS,u'gap-width'),
 		(CHARTNS,u'interpolation'),
 		(CHARTNS,u'interval-major'),
-		(CHARTNS,u'interval-minor'),
+		(CHARTNS,u'interval-minor-divisor'),
 		(CHARTNS,u'japanese-candle-stick'),
 		(CHARTNS,u'label-arrangement'),
 		(CHARTNS,u'lines'),
@@ -6117,6 +6145,7 @@ allowed_attributes = {
 		(CHARTNS,u'spline-resolution'),
 		(CHARTNS,u'stacked'),
 		(CHARTNS,u'symbol-height'),
+		(CHARTNS,u'symbol-name'),
 		(CHARTNS,u'symbol-type'),
 		(CHARTNS,u'symbol-width'),
 		(CHARTNS,u'text-overlap'),
@@ -6236,11 +6265,9 @@ allowed_attributes = {
 	),
 	(STYLENS,u'footer'):(
 		(STYLENS,u'display'),
-		(STYLENS,u'dynamic-spacing'),
 	),
 	(STYLENS,u'footer-left'):(
 		(STYLENS,u'display'),
-		(STYLENS,u'dynamic-spacing'),
 	),
 	(STYLENS,u'footer-style'):(
 	),
@@ -6437,7 +6464,6 @@ allowed_attributes = {
 	),
 	(STYLENS,u'header'):(
 		(STYLENS,u'display'),
-		(STYLENS,u'dynamic-spacing'),
 	),
 	(STYLENS,u'header-footer-properties'): (
 		(FONS,u'background-color'),
@@ -6468,7 +6494,6 @@ allowed_attributes = {
 	),
 	(STYLENS,u'header-left'):(
 		(STYLENS,u'display'),
-		(STYLENS,u'dynamic-spacing'),
 	),
 	(STYLENS,u'header-style'):(
 	),
@@ -6480,6 +6505,7 @@ allowed_attributes = {
 		(STYLENS,u'font-name'),
 		(STYLENS,u'vertical-pos'),
 		(STYLENS,u'vertical-rel'),
+		(SVGNS,u'y'),
 		(TEXTNS,u'min-label-distance'),
 		(TEXTNS,u'min-label-width'),
 		(TEXTNS,u'space-before'),
@@ -6534,12 +6560,10 @@ allowed_attributes = {
 		(STYLENS,u'layout-grid-print'),
 		(STYLENS,u'layout-grid-ruby-below'),
 		(STYLENS,u'layout-grid-ruby-height'),
-		(STYLENS,u'name'),
 		(STYLENS,u'num-format'),
 		(STYLENS,u'num-letter-sync'),
 		(STYLENS,u'num-prefix'),
 		(STYLENS,u'num-suffix'),
-		(STYLENS,u'page-usage'),
 		(STYLENS,u'paper-tray-name'),
 		(STYLENS,u'print'),
 		(STYLENS,u'print-orientation'),
@@ -7156,7 +7180,6 @@ allowed_attributes = {
 		(TABLENS,u'cell-range-address'),
 	),
 	(TABLENS,u'null-date'):(
-		(TABLENS,u'date-value-type'),
 		(TABLENS,u'value-type'),
 	),
 	(TABLENS,u'odd-columns'):(

src/odf/manifest.py

@@ -39,15 +39,3 @@ def Algorithm(**args):
 def KeyDerivation(**args):
     return Element(qname = (MANIFESTNS,'key-derivation'), **args)
 
-
-
-if __name__ == "__main__":
-    import cStringIO
-    xml=cStringIO.StringIO()
-    m = Manifest()
-    f = FileEntry(mediatype="text/xml", fullpath="content.xml")
-    m.addElement(f)
-
-    m.toXml(0,xml)
-    print xml.getvalue()
-

src/odf/namespaces.py

@@ -17,7 +17,7 @@
 #
 # Contributor(s):
 #
-TOOLSVERSION = u"ODFPY/0.8.1dev"
+TOOLSVERSION = u"ODFPY/0.8.2dev"
 
 ANIMNS         = u"urn:oasis:names:tc:opendocument:xmlns:animation:1.0"
 DBNS           = u"urn:oasis:names:tc:opendocument:xmlns:database:1.0"
@@ -49,7 +49,7 @@ TABLENS        = u"urn:oasis:names:tc:opendocument:xmlns:table:1.0"
 TEXTNS         = u"urn:oasis:names:tc:opendocument:xmlns:text:1.0"
 XFORMSNS       = u"http://www.w3.org/2002/xforms"
 XLINKNS        = u"http://www.w3.org/1999/xlink"
-XMLNS          = "http://www.w3.org/XML/1998/namespace"
+XMLNS          = u"http://www.w3.org/XML/1998/namespace"
 
 
 nsdict = {

src/odf/opendocument.py

@@ -64,6 +64,12 @@ odmimetypes = {
  'application/vnd.oasis.opendocument.text-web':              '.oth',
 }
 
+class OpaqueObject:
+    def __init__(self, filename, mediatype, content=None):
+       self.mediatype = mediatype
+       self.filename = filename
+       self.content = content
+
 class OpenDocument:
     """ A class to hold the content of an OpenDocument document
         Use the xml method to write the XML
@@ -76,6 +82,7 @@ class OpenDocument:
     def __init__(self, mimetype, add_generator=True):
         self.mimetype = mimetype
         self.childobjects = []
+        self._extra = []
         self.folder = "" # Always empty for toplevel documents
         self.topnode = Document(mimetype=self.mimetype)
         self.topnode.ownerDocument = self
@@ -303,12 +310,15 @@ class OpenDocument:
         else:
             self.thumbnail = filecontent
 
-    def addObject(self, document):
+    def addObject(self, document, objectname=None):
         """ Add an object. The object must be an OpenDocument class
             The return value will be the folder in the zipfile the object is stored in
         """
         self.childobjects.append(document)
-        document.folder = "%s/Object %d" % (self.folder, len(self.childobjects))
+        if objectname is None:
+            document.folder = "%s/Object %d" % (self.folder, len(self.childobjects))
+        else:
+            document.folder = objectname
         return ".%s" % document.folder
 
     def _savePictures(self, object, folder):
@@ -348,7 +358,7 @@ class OpenDocument:
         else:
             if addsuffix:
                 outputfile = outputfile + odmimetypes.get(self.mimetype,'.xxx')
-            outputfp = zipfile.ZipFile(outputfile,"w")
+            outputfp = zipfile.ZipFile(outputfile, "w")
         self._zipwrite(outputfp)
         outputfp.close()
 
@@ -382,6 +392,14 @@ class OpenDocument:
             zi.external_attr = UNIXPERMS
             self._z.writestr(zi, self.thumbnail)
 
+        # Write any extra files
+        for op in self._extra:
+            self.manifest.addElement(manifest.FileEntry(fullpath=op.filename, mediatype=op.mediatype))
+            zi = zipfile.ZipInfo(op.filename.encode('utf-8'), self._now)
+            zi.compress_type = zipfile.ZIP_DEFLATED
+            zi.external_attr = UNIXPERMS
+            if op.content is not None:
+                self._z.writestr(zi, op.content)
         # Write manifest
         zi = zipfile.ZipInfo("META-INF/manifest.xml", self._now)
         zi.compress_type = zipfile.ZIP_DEFLATED
@@ -528,15 +546,20 @@ def load(odffile):
             parser.parse(inpsrc)
             del doc._parsing
         except KeyError, v: pass
-    # Add the thumbnail here
+    # FIXME: Add subobjects correctly here
-    # Add the images here
     for mentry,mvalue in manifest.items():
         if mentry[:9] == "Pictures/" and len(mentry) > 9:
             doc.addPicture(mvalue['full-path'], mvalue['media-type'], z.read(mentry))
         elif mentry == "Thumbnails/thumbnail.png":
             doc.addThumbnail(z.read(mentry))
+        elif mentry in ('settings.xml', 'meta.xml', 'content.xml', 'styles.xml'):
+            pass
         else:
-            pass  # Add the SUN junk here to the struct somewhere
+            if mvalue['full-path'][-1] == '/':
+                doc._extra.append(OpaqueObject(mvalue['full-path'], mvalue['media-type'], None))
+            else:
+                doc._extra.append(OpaqueObject(mvalue['full-path'], mvalue['media-type'], z.read(mentry)))
+            # Add the SUN junk here to the struct somewhere
             # It is cached data, so it can be out-of-date
     z.close()
     b = doc.getElementsByType(Body)

src/odf/userfield.py

@@ -231,6 +231,19 @@ class ODFContentParser(xml.sax.saxutils.XMLGenerator):
         self._callback_func = callback_func
         xml.sax.saxutils.XMLGenerator.__init__(self, out, encoding)
 
+    def _qname(self, name):
+        """Builds a qualified name from a (ns_url, localname) pair"""
+        if name[0]:
+            if name[0] == u'http://www.w3.org/XML/1998/namespace':
+                return u'xml' + ":" + name[1]
+            # The name is in a non-empty namespace
+            prefix = self._current_context[name[0]]
+            if prefix:
+                # If it is not the default namespace, prepend the prefix
+                return prefix + ":" + name[1]
+        # Return the unqualified name
+        return name[1]
+
     def startElementNS(self, name, qname, attrs):
         if name == (TEXTNS, u'user-field-decl'):
             field_name = attrs.get((TEXTNS, u'name'))