Sync to trunk

src/calibre/debug.py

@@ -83,6 +83,23 @@ def debug_device_driver():
     s = DeviceScanner()
     s.scan()
     print 'USB devices on system:', repr(s.devices)
+    if iswindows:
+        wmi = __import__('wmi', globals(), locals(), [], -1) 
+        drives = []
+        print 'Drives detected:'
+        print '\t', '(ID, Partitions, Drive letter)' 
+        for drive in wmi.WMI().Win32_DiskDrive():
+            if drive.Partitions == 0:
+                continue
+            try:
+                partition = drive.associators("Win32_DiskDriveToDiskPartition")[0]
+                logical_disk = partition.associators('Win32_LogicalDiskToPartition')[0]
+                prefix = logical_disk.DeviceID+os.sep
+                drives.append((str(drive.PNPDeviceID), drive.Index, prefix))
+            except IndexError:
+                drives.append(str(drive.PNPDeviceID))
+        for drive in drives:
+            print '\t', drive
     from calibre.devices import devices
     for dev in devices():
         print 'Looking for', dev.__name__

src/calibre/ebooks/epub/__init__.py

@@ -142,6 +142,8 @@ to auto-generate a Table of Contents.
         help=_('XPath expression that specifies all tags that should be added to the Table of Contents at level one. If this is specified, it takes precedence over other forms of auto-detection.'))
     toc('level2_toc', ['--level2-toc'], default=None,
         help=_('XPath expression that specifies all tags that should be added to the Table of Contents at level two. Each entry is added under the previous level one entry.'))
+    toc('level3_toc', ['--level3-toc'], default=None,
+        help=_('XPath expression that specifies all tags that should be added to the Table of Contents at level three. Each entry is added under the previous level two entry.'))
     toc('from_ncx', ['--from-ncx'], default=None,
         help=_('Path to a .ncx file that contains the table of contents to use for this ebook. The NCX file should contain links relative to the directory it is placed in. See http://www.niso.org/workrooms/daisy/Z39-86-2005.html#NCX for an overview of the NCX format.'))
     toc('use_auto_toc', ['--use-auto-toc'], default=False,

src/calibre/ebooks/epub/from_html.py

@@ -377,16 +377,13 @@ def convert(htmlfile, opts, notification=None, create_epub=True,
         mi = merge_metadata(htmlfile, opf, opts)
     opts.chapter = XPath(opts.chapter, 
                     namespaces={'re':'http://exslt.org/regular-expressions'})
-    if opts.level1_toc:
+    for x in (1, 2, 3):
-        opts.level1_toc = XPath(opts.level1_toc, 
+        attr = 'level%d_toc'%x
-                            namespaces={'re':'http://exslt.org/regular-expressions'})
+        if getattr(opts, attr):
-    else:
+            setattr(opts, attr, XPath(getattr(opts, attr), 
-        opts.level1_toc = None
+                      namespaces={'re':'http://exslt.org/regular-expressions'}))
-    if opts.level2_toc:
+        else:
-        opts.level2_toc = XPath(opts.level2_toc, 
+            setattr(opts, attr, None) 
-                            namespaces={'re':'http://exslt.org/regular-expressions'})
-    else:
-        opts.level2_toc = None 
     
     with TemporaryDirectory(suffix='_html2epub', keep=opts.keep_intermediate) as tdir:
         if opts.keep_intermediate:

src/calibre/ebooks/epub/split.py

@@ -307,7 +307,11 @@ class Splitter(LoggingInterface):
         Search order is:
             * Heading tags
             * <div> tags
+            * <pre> tags
+            * <hr> tags
             * <p> tags
+            * <br> tags
+            * <li> tags
             
         We try to split in the "middle" of the file (as defined by tag counts.
         '''
@@ -327,6 +331,7 @@ class Splitter(LoggingInterface):
                      '//hr', 
                      '//p',
                      '//br',
+                     '//li',
                      ):
             elems = root.xpath(path, namespaces={'re':'http://exslt.org/regular-expressions'})
             elem = pick_elem(elems)

src/calibre/ebooks/html.py

@@ -558,30 +558,21 @@ class Processor(Parser):
     
     def detect_chapters(self):
         self.detected_chapters = self.opts.chapter(self.root)
+        chapter_mark = self.opts.chapter_mark
+        page_break_before = 'display: block; page-break-before: always'
+        page_break_after = 'display: block; page-break-after: always'
         for elem in self.detected_chapters:
             text = u' '.join([t.strip() for t in elem.xpath('descendant::text()')])
             self.log_info('\tDetected chapter: %s', text[:50])
-            if self.opts.chapter_mark != 'none':
+            if chapter_mark == 'none':
-                hr = etree.Element('hr')
+                continue
-                if elem.getprevious() is None:
+            elif chapter_mark == 'rule':
-                    elem.getparent()[:0] = [hr]
+                mark = etree.Element('hr')
-                elif elem.getparent() is not None:
+            elif chapter_mark == 'pagebreak':
-                    insert = None
+                mark = etree.Element('div', style=page_break_after)
-                    for i, c in enumerate(elem.getparent()):
+            else: # chapter_mark == 'both':
-                        if c is elem:
+                mark = etree.Element('hr', style=page_break_before)
-                            insert = i
+            elem.addprevious(mark)
-                            break
-                    elem.getparent()[insert:insert] = [hr]
-                if self.opts.chapter_mark != 'rule':
-                    hr.set('style', 'width:0pt;page-break-before:always')
-                    if self.opts.chapter_mark == 'both':
-                        hr2 = etree.Element('hr')
-                        hr2.tail = u'\u00a0'
-                        p = hr.getparent()
-                        i = p.index(hr)
-                        p[i:i] = [hr2]
-                
-                    
     
     def save(self):
         style_path = os.path.splitext(os.path.basename(self.save_path()))[0]
@@ -647,6 +638,7 @@ class Processor(Parser):
                         added[elem] = add_item(_href, frag, text, toc, type='chapter')
                         add_item(_href, frag, 'Top', added[elem], type='chapter')
                 if self.opts.level2_toc is not None:
+                    added2 = {}
                     level2 = list(self.opts.level2_toc(self.root))
                     for elem in level2:
                         level1 = None
@@ -657,7 +649,21 @@ class Processor(Parser):
                                 text, _href, frag = elem_to_link(elem, href, counter)
                                 counter += 1
                                 if text:
+                                    added2[elem] = \
                                     add_item(_href, frag, text, level1, type='chapter')
+                    if self.opts.level3_toc is not None:
+                        level3 = list(self.opts.level3_toc(self.root))
+                        for elem in level3:
+                            level2 = None
+                            for item in self.root.iterdescendants():
+                                if item in added2.keys():
+                                    level2 = added2[item]
+                                elif item == elem and level2 is not None:
+                                    text, _href, frag = elem_to_link(elem, href, counter)
+                                    counter += 1
+                                    if text:
+                                        add_item(_href, frag, text, level2, type='chapter')
+                
                     
             if len(toc) > 0:
                 return

src/calibre/ebooks/metadata/epub.py

@@ -106,9 +106,11 @@ class CoverRenderer(QObject):
     WIDTH  = 600
     HEIGHT = 800
     
-    def __init__(self, url, size, loop):
+    def __init__(self, path):
+        if QApplication.instance() is None:
+            QApplication([])
         QObject.__init__(self)
-        self.loop = loop
+        self.loop = QEventLoop()
         self.page = QWebPage()
         pal = self.page.palette()
         pal.setBrush(QPalette.Background, Qt.white)
@@ -117,32 +119,42 @@ class CoverRenderer(QObject):
         self.page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
         self.page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
         QObject.connect(self.page, SIGNAL('loadFinished(bool)'), self.render_html)
-        self.image_data = None
+        self._image_data = None
         self.rendered = False
+        url = QUrl.fromLocalFile(os.path.normpath(path))
         self.page.mainFrame().load(url)
         
     def render_html(self, ok):
-        self.rendered = True
         try:
             if not ok:
+                self.rendered = True
                 return
-            #size = self.page.mainFrame().contentsSize()
-            #width, height = fit_image(size.width(), size.height(), self.WIDTH, self.HEIGHT)[1:]
-            #self.page.setViewportSize(QSize(width, height))
             image = QImage(self.page.viewportSize(), QImage.Format_ARGB32)
             image.setDotsPerMeterX(96*(100/2.54))
             image.setDotsPerMeterY(96*(100/2.54))
             painter = QPainter(image)
             self.page.mainFrame().render(painter)
             painter.end()
-            
             ba = QByteArray()
             buf = QBuffer(ba)
             buf.open(QBuffer.WriteOnly)
             image.save(buf, 'JPEG')
-            self.image_data = str(ba.data())
+            self._image_data = str(ba.data())
         finally:
             self.loop.exit(0)
+        self.rendered = True
+
+    def image_data():
+        def fget(self):
+            if not self.rendered:
+                self.loop.exec_()
+                count = 0
+                while count < 50 and not self.rendered:
+                    time.sleep(0.1)
+                    count += 1
+            return self._image_data
+        return property(fget=fget)
+    image_data = image_data()
 
 
 def get_cover(opf, opf_path, stream):
@@ -155,20 +167,11 @@ def get_cover(opf, opf_path, stream):
             stream.seek(0)
             ZipFile(stream).extractall()
             opf_path = opf_path.replace('/', os.sep)
-            cpage = os.path.join(tdir, os.path.dirname(opf_path), *cpage.split('/'))
+            cpage = os.path.join(tdir, os.path.dirname(opf_path), cpage)
             if not os.path.exists(cpage):
                 return
-            if QApplication.instance() is None:
+            cr = CoverRenderer(cpage)
-                QApplication([])
+            return cr.image_data
-            url = QUrl.fromLocalFile(cpage)
-            loop = QEventLoop()
-            cr = CoverRenderer(url, os.stat(cpage).st_size, loop)
-            loop.exec_()
-            count = 0
-            while count < 50 and not cr.rendered:
-                time.sleep(0.1)
-                count += 1
-    return cr.image_data 
     
 def get_metadata(stream, extract_cover=True):
     """ Return metadata as a :class:`MetaInformation` object """

src/calibre/ebooks/mobi/mobiml.py

@@ -148,10 +148,6 @@ class MobiMLizer(object):
             if bstate.pbreak:
                 etree.SubElement(body, MBP('pagebreak'))
                 bstate.pbreak = False
-            if istate.ids:
-                for id in istate.ids:
-                    etree.SubElement(body, XHTML('a'), attrib={'id': id})
-                istate.ids.clear()
             bstate.istate = None
             bstate.anchor = None
             parent = bstate.nested[-1] if bstate.nested else bstate.body
@@ -186,14 +182,17 @@ class MobiMLizer(object):
                 wrapper.attrib['height'] = self.mobimlize_measure(vspace)
                 para.attrib['width'] = self.mobimlize_measure(indent)
             elif tag == 'table' and vspace > 0:
-                body = bstate.body
                 vspace = int(round(vspace / self.profile.fbase))
-                index = max((0, len(body) - 1))
                 while vspace > 0:
-                    body.insert(index, etree.Element(XHTML('br')))
+                    wrapper.addprevious(etree.Element(XHTML('br')))
                     vspace -= 1
             if istate.halign != 'auto':
                 para.attrib['align'] = istate.halign
+        if istate.ids:
+            last = bstate.body[-1]
+            for id in istate.ids:
+                last.addprevious(etree.Element(XHTML('a'), attrib={'id': id}))
+            istate.ids.clear()
         pstate = bstate.istate
         if tag in CONTENT_TAGS:
             bstate.inline = para

src/calibre/ebooks/mobi/reader.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Read data from .mobi files
 '''
 
-import sys, struct, os, cStringIO, re, atexit, shutil, tempfile
+import sys, struct, os, cStringIO, re
 
 try:
     from PIL import Image as PILImage
@@ -14,7 +14,7 @@ except ImportError:
 
 from lxml import html, etree
 
-from calibre import __appname__, entity_to_unicode
+from calibre import entity_to_unicode
 from calibre.ebooks import DRMError
 from calibre.ebooks.chardet import ENCODING_PATS
 from calibre.ebooks.mobi import MobiError
@@ -28,7 +28,7 @@ from calibre import sanitize_file_name
 
 class EXTHHeader(object):
     
-    def __init__(self, raw, codec):
+    def __init__(self, raw, codec, title):
         self.doctype = raw[:4]
         self.length, self.num_items = struct.unpack('>LL', raw[4:12])
         raw = raw[12:]
@@ -45,21 +45,15 @@ class EXTHHeader(object):
             elif id == 203:
                 self.has_fake_cover = bool(struct.unpack('>L', content)[0]) 
             elif id == 201:
-                self.cover_offset, = struct.unpack('>L', content)
+                co, = struct.unpack('>L', content)
+                if co < 1e7:
+                    self.cover_offset = co 
             elif id == 202:
                 self.thumbnail_offset, = struct.unpack('>L', content)
             #else:
             #    print 'unknown record', id, repr(content)
-        title = re.search(r'\0+([^\0]+)\0+', raw[pos:])
         if title:
-            title = title.group(1).decode(codec, 'replace')
+            self.mi.title = title
-            if len(title) > 2:
-                self.mi.title = title
-            else:
-                title = re.search(r'\0+([^\0]+)\0+', ''.join(reversed(raw[pos:])))
-                if title:
-                    self.mi.title = ''.join(reversed(title.group(1).decode(codec, 'replace')))
-            
     
     def process_metadata(self, id, content, codec):
         if id == 100:
@@ -119,6 +113,9 @@ class BookHeader(object):
             if self.compression_type == 'DH':
                 self.huff_offset, self.huff_number = struct.unpack('>LL', raw[0x70:0x78]) 
             
+            toff, tlen = struct.unpack('>II', raw[0x54:0x5c])
+            tend = toff + tlen
+            self.title = raw[toff:tend] if tend < len(raw) else _('Unknown')
             langcode  = struct.unpack('!L', raw[0x5C:0x60])[0]
             langid    = langcode & 0xFF
             sublangid = (langcode >> 10) & 0xFF
@@ -129,7 +126,7 @@ class BookHeader(object):
             self.exth_flag, = struct.unpack('>L', raw[0x80:0x84])
             self.exth = None
             if self.exth_flag & 0x40:
-                self.exth = EXTHHeader(raw[16+self.length:], self.codec)
+                self.exth = EXTHHeader(raw[16+self.length:], self.codec, self.title)
                 self.exth.mi.uid = self.unique_id
                 self.exth.mi.language = self.language
             
@@ -480,7 +477,7 @@ def get_metadata(stream):
         try:
             if hasattr(mr.book_header.exth, 'cover_offset'):
                 cover_index = mr.book_header.first_image_index + mr.book_header.exth.cover_offset
-                data  = mr.sections[cover_index][0]
+                data  = mr.sections[int(cover_index)][0]
             else:
                 data  = mr.sections[mr.book_header.first_image_index][0]
             buf = cStringIO.StringIO(data)

src/calibre/ebooks/mobi/writer.py

@@ -23,6 +23,7 @@ from PIL import Image
 from calibre.ebooks.oeb.base import XML_NS, XHTML, XHTML_NS, OEB_DOCS, \
     OEB_RASTER_IMAGES
 from calibre.ebooks.oeb.base import xpath, barename, namespace, prefixname
+from calibre.ebooks.oeb.base import urlnormalize
 from calibre.ebooks.oeb.base import Logger, OEBBook
 from calibre.ebooks.oeb.profile import Context
 from calibre.ebooks.oeb.transforms.flatcss import CSSFlattener
@@ -178,7 +179,7 @@ class Serializer(object):
 
     def serialize_href(self, href, base=None):
         hrefs = self.oeb.manifest.hrefs
-        path, frag = urldefrag(href)
+        path, frag = urldefrag(urlnormalize(href))
         if path and base:
             path = base.abshref(path)
         if path and path not in hrefs:
@@ -196,6 +197,7 @@ class Serializer(object):
         
     def serialize_body(self):
         buffer = self.buffer
+        self.anchor_offset = buffer.tell()
         buffer.write('<body>')
         # CybookG3 'Start Reading' link
         if 'text' in self.oeb.guide:
@@ -224,14 +226,17 @@ class Serializer(object):
            or namespace(elem.tag) not in nsrmap:
             return
         tag = prefixname(elem.tag, nsrmap)
-        for attr in ('name', 'id'):
+        # Previous layers take care of @name
-            if attr in elem.attrib:
+        id = elem.attrib.pop('id', None)
-                href = '#'.join((item.href, elem.attrib[attr]))
+        if id is not None:
-                self.id_offsets[href] = buffer.tell()
+            href = '#'.join((item.href, id))
-                del elem.attrib[attr]
+            offset = self.anchor_offset or buffer.tell()
-        if tag == 'a' and not elem.attrib \
+            self.id_offsets[href] = offset
-           and not len(elem) and not elem.text:
+        if self.anchor_offset is not None and \
+           tag == 'a' and not elem.attrib and \
+           not len(elem) and not elem.text:
             return
+        self.anchor_offset = buffer.tell()
         buffer.write('<')
         buffer.write(tag)
         if elem.attrib:
@@ -256,10 +261,12 @@ class Serializer(object):
         if elem.text or len(elem) > 0:
             buffer.write('>')
             if elem.text:
+                self.anchor_offset = None
                 self.serialize_text(elem.text)
             for child in elem:
                 self.serialize_elem(child, item)
                 if child.tail:
+                    self.anchor_offset = None
                     self.serialize_text(child.tail)
             buffer.write('</%s>' % tag)
         else:

src/calibre/ebooks/oeb/base.py

@@ -23,6 +23,8 @@ from calibre import LoggingInterface
 from calibre.translations.dynamic import translate
 from calibre.startup import get_lang
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
+from calibre.ebooks.metadata.epub import CoverRenderer
+from calibre.ptempfile import TemporaryDirectory
 
 XML_NS = 'http://www.w3.org/XML/1998/namespace'
 XHTML_NS = 'http://www.w3.org/1999/xhtml'
@@ -351,9 +353,13 @@ class Manifest(object):
                 try:
                     data = etree.fromstring(data)
                 except etree.XMLSyntaxError:
+                    # TODO: Factor out HTML->XML coercion
                     self.oeb.logger.warn('Parsing file %r as HTML' % self.href)
                     data = html.fromstring(data)
                     data.attrib.pop('xmlns', None)
+                    for elem in data.iter(tag=etree.Comment):
+                        if elem.text:
+                            elem.text = elem.text.strip('-')
                     data = etree.tostring(data, encoding=unicode)
                     data = etree.fromstring(data)
             # Force into the XHTML namespace
@@ -447,7 +453,7 @@ class Manifest(object):
             return cmp(skey, okey)
         
         def relhref(self, href):
-            if '/' not in self.href:
+            if '/' not in self.href or ':' in href:
                 return href
             base = os.path.dirname(self.href).split('/')
             target, frag = urldefrag(href)
@@ -463,7 +469,7 @@ class Manifest(object):
             return relhref
 
         def abshref(self, href):
-            if '/' not in self.href:
+            if '/' not in self.href or ':' in href:
                 return href
             dirname = os.path.dirname(self.href)
             href = os.path.join(dirname, href)
@@ -546,7 +552,7 @@ class Manifest(object):
             elif media_type in OEB_STYLES:
                 media_type = CSS_MIME
             attrib = {'id': item.id, 'href': item.href,
-                      'media-type': item.media_type}
+                      'media-type': media_type}
             if item.fallback:
                 attrib['fallback'] = item.fallback
             element(elem, OPF('item'), attrib=attrib)
@@ -796,6 +802,9 @@ class TOC(object):
 
     
 class OEBBook(object):
+    COVER_SVG_XP = XPath('h:body//svg:svg[position() = 1]')
+    COVER_OBJECT_XP = XPath('h:body//h:object[@data][position() = 1]')
+
     def __init__(self, opfpath=None, container=None, encoding=None,
                  logger=FauxLogger()):
         if opfpath and not container:
@@ -928,7 +937,7 @@ class OEBBook(object):
             spine.add(item, elem.get('linear'))
         extras = []
         for item in self.manifest.values():
-            if item.media_type == XHTML_MIME \
+            if item.media_type in OEB_DOCS \
                and item not in spine:
                 extras.append(item)
         extras.sort()
@@ -971,7 +980,7 @@ class OEBBook(object):
         ncx = item.data
         self.manifest.remove(item)
         title = xpath(ncx, 'ncx:docTitle/ncx:text/text()')
-        title = title[0].strip() if title else unicode(self.metadata.title)
+        title = title[0].strip() if title else unicode(self.metadata.title[0])
         self.toc = toc = TOC(title)
         navmaps = xpath(ncx, 'ncx:navMap')
         for navmap in navmaps:
@@ -1051,41 +1060,58 @@ class OEBBook(object):
         if self._toc_from_html(opf): return
         self._toc_from_spine(opf)
 
-    def _ensure_cover_image(self):
+    def _cover_from_html(self, hcover):
-        cover = None
+        with TemporaryDirectory('_html_cover') as tdir:
+            writer = DirWriter()
+            writer.dump(self, tdir)
+            path = os.path.join(tdir, hcover.href)
+            renderer = CoverRenderer(path)
+            data = renderer.image_data
+        id, href = self.manifest.generate('cover', 'cover.jpeg')
+        item = self.manifest.add(id, href, JPEG_MIME, data=data)
+        return item
+        
+    def _locate_cover_image(self):
+        if self.metadata.cover:
+            id = str(self.metadata.cover[0])
+            item = self.manifest.ids.get(id, None)
+            if item is not None:
+                return item
         hcover = self.spine[0]
         if 'cover' in self.guide:
             href = self.guide['cover'].href
             item = self.manifest.hrefs[href]
             media_type = item.media_type
-            if media_type in OEB_RASTER_IMAGES:
+            if media_type in OEB_IMAGES:
-                cover = item
+                return item
             elif media_type in OEB_DOCS:
                 hcover = item
         html = hcover.data
-        if cover is not None:
+        if MS_COVER_TYPE in self.guide:
-            pass
-        elif self.metadata.cover:
-            id = str(self.metadata.cover[0])
-            cover = self.manifest.ids[id]
-        elif MS_COVER_TYPE in self.guide:
             href = self.guide[MS_COVER_TYPE].href
-            cover = self.manifest.hrefs[href]
+            item = self.manifest.hrefs.get(href, None)
-        elif xpath(html, '//h:img[position()=1]'):
+            if item is not None and item.media_type in OEB_IMAGES:
-            img = xpath(html, '//h:img[position()=1]')[0]
+                return item
-            href = hcover.abshref(img.get('src'))
+        if self.COVER_SVG_XP(html):
-            cover = self.manifest.hrefs[href]
+            svg = copy.deepcopy(self.COVER_SVG_XP(html)[0])
-        elif xpath(html, '//h:object[position()=1]'):
-            object = xpath(html, '//h:object[position()=1]')[0]
-            href = hcover.abshref(object.get('data'))
-            cover = self.manifest.hrefs[href]
-        elif xpath(html, '//svg:svg[position()=1]'):
-            svg = copy.deepcopy(xpath(html, '//svg:svg[position()=1]')[0])
             href = os.path.splitext(hcover.href)[0] + '.svg'
             id, href = self.manifest.generate(hcover.id, href)
-            cover = self.manifest.add(id, href, SVG_MIME, data=svg)
+            item = self.manifest.add(id, href, SVG_MIME, data=svg)
-        if cover and not self.metadata.cover:
+            return item
-            self.metadata.add('cover', cover.id)
+        if self.COVER_OBJECT_XP(html):
+            object = self.COVER_OBJECT_XP(html)[0]
+            href = hcover.abshref(object.get('data'))
+            item = self.manifest.hrefs.get(href, None)
+            if item is not None and item.media_type in OEB_IMAGES:
+                return item
+        return self._cover_from_html(hcover)
+        
+    def _ensure_cover_image(self):
+        cover = self._locate_cover_image()
+        if self.metadata.cover:
+            self.metadata.cover[0].value = cover.id
+            return
+        self.metadata.add('cover', cover.id)
     
     def _all_from_opf(self, opf):
         self._metadata_from_opf(opf)

src/calibre/ebooks/oeb/stylizer.py

@@ -265,6 +265,8 @@ class Stylizer(object):
 
 
 class Style(object):
+    UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|px|mm|cm|in|pt|pc)$')
+    
     def __init__(self, element, stylizer):
         self._element = element
         self._profile = stylizer.profile
@@ -319,13 +321,11 @@ class Style(object):
         if isinstance(value, (int, long, float)):
             return value
         try:
-            if float(value) == 0:
+            return float(value) * 72.0 / self._profile.dpi
-                return 0.0
         except:
             pass
         result = value
-        m = re.search(
+        m = self.UNIT_RE.match(value)
-            r"^(-*[0-9]*\.?[0-9]*)\s*(%|em|px|mm|cm|in|pt|pc)$", value)
         if m is not None and m.group(1):
             value = float(m.group(1))
             unit = m.group(2)

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -23,6 +23,12 @@ from calibre.ebooks.oeb.stylizer import Stylizer
 COLLAPSE = re.compile(r'[ \t\r\n\v]+')
 STRIPNUM = re.compile(r'[-0-9]+$')
 
+def asfloat(value, default):
+    if not isinstance(value, (int, long, float)):
+        value = default
+    return float(value)
+
+
 class KeyMapper(object):
     def __init__(self, sbase, dbase, dkey):
         self.sbase = float(sbase)
@@ -179,12 +185,13 @@ class CSSFlattener(object):
         if cssdict:
             if self.lineh and self.fbase and tag != 'body':
                 self.clean_edges(cssdict, style, psize)
-            margin = style['margin-left']
+            margin = asfloat(style['margin-left'], 0)
-            left += margin if isinstance(margin, float) else 0
+            indent = asfloat(style['text-indent'], 0)
-            if (left + style['text-indent']) < 0:
+            left += margin
-                percent = (margin - style['text-indent']) / style['width']
+            if (left + indent) < 0:
+                percent = (margin - indent) / style['width']
                 cssdict['margin-left'] = "%d%%" % (percent * 100)
-                left -= style['text-indent']
+                left -= indent
             if 'display' in cssdict and cssdict['display'] == 'in-line':
                 cssdict['display'] = 'inline'
             if self.unfloat and 'float' in cssdict \

src/calibre/ebooks/oeb/transforms/rasterize.py

@@ -23,6 +23,7 @@ from PyQt4.QtGui import QApplication
 from calibre.ebooks.oeb.base import XHTML_NS, XHTML, SVG_NS, SVG, XLINK
 from calibre.ebooks.oeb.base import SVG_MIME, PNG_MIME, JPEG_MIME
 from calibre.ebooks.oeb.base import xml2str, xpath, namespace, barename
+from calibre.ebooks.oeb.base import urlnormalize
 from calibre.ebooks.oeb.stylizer import Stylizer
 
 IMAGE_TAGS = set([XHTML('img'), XHTML('object')])
@@ -78,7 +79,7 @@ class SVGRasterizer(object):
             svg = item.data
         hrefs = self.oeb.manifest.hrefs
         for elem in xpath(svg, '//svg:*[@xl:href]'):
-            href = elem.attrib[XLINK('href')]
+            href = urlnormalize(elem.attrib[XLINK('href')])
             path, frag = urldefrag(href)
             if not path:
                 continue
@@ -100,15 +101,15 @@ class SVGRasterizer(object):
     def rasterize_item(self, item, stylizer):
         html = item.data
         hrefs = self.oeb.manifest.hrefs
-        for elem in xpath(html, '//h:img'):
+        for elem in xpath(html, '//h:img[@src]'):
-            src = elem.get('src', None)
+            src = urlnormalize(elem.attrib['src'])
-            image = hrefs.get(item.abshref(src), None) if src else None
+            image = hrefs.get(item.abshref(src), None)
             if image and image.media_type == SVG_MIME:
                 style = stylizer.style(elem)
                 self.rasterize_external(elem, style, item, image)
-        for elem in xpath(html, '//h:object[@type="%s"]' % SVG_MIME):
+        for elem in xpath(html, '//h:object[@type="%s" and @data]' % SVG_MIME):
-            data = elem.get('data', None)
+            data = urlnormalize(elem.attrib['data'])
-            image = hrefs.get(item.abshref(data), None) if data else None
+            image = hrefs.get(item.abshref(data), None)
             if image and image.media_type == SVG_MIME:
                 style = stylizer.style(elem)
                 self.rasterize_external(elem, style, item, image)

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -54,7 +54,7 @@ class ManifestTrimmer(object):
                                 new.add(found)
                 elif item.media_type == CSS_MIME:
                     def replacer(uri):
-                        absuri = item.abshref(uri)
+                        absuri = item.abshref(urlnormalize(uri))
                         if absuri in oeb.manifest.hrefs:
                             found = oeb.manifest.hrefs[href]
                             if found not in used:

src/calibre/gui2/dialogs/epub.py

@@ -252,7 +252,7 @@ class Config(ResizableDialog, Ui_Dialog):
                     self.source_format = d.format()
                 
     def accept(self):
-        for opt in ('chapter', 'level1_toc', 'level2_toc'):
+        for opt in ('chapter', 'level1_toc', 'level2_toc', 'level3_toc'):
             text = unicode(getattr(self, 'opt_'+opt).text())
             if text:
                 try:

src/calibre/gui2/dialogs/epub.ui

@@ -93,7 +93,7 @@
        <item>
         <widget class="QStackedWidget" name="stack" >
          <property name="currentIndex" >
-          <number>1</number>
+          <number>0</number>
          </property>
          <widget class="QWidget" name="metadata_page" >
           <layout class="QGridLayout" name="gridLayout_4" >
@@ -105,36 +105,6 @@
                 <string>Book Cover</string>
                </property>
                <layout class="QGridLayout" name="_2" >
-                <item row="0" column="0" >
-                 <layout class="QHBoxLayout" name="_3" >
-                  <item>
-                   <widget class="ImageView" name="cover" >
-                    <property name="text" >
-                     <string/>
-                    </property>
-                    <property name="pixmap" >
-                     <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
-                    </property>
-                    <property name="scaledContents" >
-                     <bool>true</bool>
-                    </property>
-                    <property name="alignment" >
-                     <set>Qt::AlignCenter</set>
-                    </property>
-                   </widget>
-                  </item>
-                 </layout>
-                </item>
-                <item row="2" column="0" >
-                 <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
-                  <property name="text" >
-                   <string>Use cover from &amp;source file</string>
-                  </property>
-                  <property name="checked" >
-                   <bool>true</bool>
-                  </property>
-                 </widget>
-                </item>
                 <item row="1" column="0" >
                  <layout class="QVBoxLayout" name="_4" >
                   <property name="spacing" >
@@ -186,6 +156,36 @@
                   </item>
                  </layout>
                 </item>
+                <item row="2" column="0" >
+                 <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
+                  <property name="text" >
+                   <string>Use cover from &amp;source file</string>
+                  </property>
+                  <property name="checked" >
+                   <bool>true</bool>
+                  </property>
+                 </widget>
+                </item>
+                <item row="0" column="0" >
+                 <layout class="QHBoxLayout" name="_3" >
+                  <item>
+                   <widget class="ImageView" name="cover" >
+                    <property name="text" >
+                     <string/>
+                    </property>
+                    <property name="pixmap" >
+                     <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
+                    </property>
+                    <property name="scaledContents" >
+                     <bool>true</bool>
+                    </property>
+                    <property name="alignment" >
+                     <set>Qt::AlignCenter</set>
+                    </property>
+                   </widget>
+                  </item>
+                 </layout>
+                </item>
                </layout>
                <zorder>opt_prefer_metadata_cover</zorder>
                <zorder></zorder>
@@ -777,10 +777,10 @@ p, li { white-space: pre-wrap; }
               <item row="5" column="1" >
                <widget class="QLineEdit" name="opt_level2_toc" />
               </item>
-              <item row="6" column="1" >
+              <item row="7" column="1" >
                <widget class="QLineEdit" name="opt_toc_title" />
               </item>
-              <item row="6" column="0" >
+              <item row="7" column="0" >
                <widget class="QLabel" name="toc_title_label" >
                 <property name="text" >
                  <string>&amp;Title for generated TOC</string>
@@ -790,6 +790,19 @@ p, li { white-space: pre-wrap; }
                 </property>
                </widget>
               </item>
+              <item row="6" column="1" >
+               <widget class="QLineEdit" name="opt_level3_toc" />
+              </item>
+              <item row="6" column="0" >
+               <widget class="QLabel" name="label_11" >
+                <property name="text" >
+                 <string>Level &amp;3 TOC</string>
+                </property>
+                <property name="buddy" >
+                 <cstring>opt_level3_toc</cstring>
+                </property>
+               </widget>
+              </item>
              </layout>
             </widget>
            </item>

src/calibre/gui2/dialogs/metadata_single.ui

@@ -638,6 +638,31 @@
   </customwidget>
  </customwidgets>
  <tabstops>
+  <tabstop>title</tabstop>
+  <tabstop>swap_button</tabstop>
+  <tabstop>authors</tabstop>
+  <tabstop>author_sort</tabstop>
+  <tabstop>auto_author_sort</tabstop>
+  <tabstop>rating</tabstop>
+  <tabstop>publisher</tabstop>
+  <tabstop>tags</tabstop>
+  <tabstop>series</tabstop>
+  <tabstop>tag_editor_button</tabstop>
+  <tabstop>remove_series_button</tabstop>
+  <tabstop>series_index</tabstop>
+  <tabstop>isbn</tabstop>
+  <tabstop>comments</tabstop>
+  <tabstop>fetch_metadata_button</tabstop>
+  <tabstop>fetch_cover_button</tabstop>
+  <tabstop>password_button</tabstop>
+  <tabstop>formats</tabstop>
+  <tabstop>add_format_button</tabstop>
+  <tabstop>remove_format_button</tabstop>
+  <tabstop>button_set_cover</tabstop>
+  <tabstop>cover_path</tabstop>
+  <tabstop>cover_button</tabstop>
+  <tabstop>reset_cover</tabstop>
+  <tabstop>scrollArea</tabstop>
   <tabstop>button_box</tabstop>
  </tabstops>
  <resources>

src/calibre/gui2/dialogs/scheduler.py

@@ -8,7 +8,7 @@ Scheduler for automated recipe downloads
 '''
 
 import sys, copy, time
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, date
 from PyQt4.Qt import QDialog, QApplication, QLineEdit, QPalette, SIGNAL, QBrush, \
                      QColor, QAbstractListModel, Qt, QVariant, QFont, QIcon, \
                      QFile, QObject, QTimer, QMutex, QMenu, QAction, QTime
@@ -289,7 +289,8 @@ class SchedulerDialog(QDialog, Ui_Dialog):
                 recipe.last_downloaded = datetime.fromordinal(1)
                 recipes.append(recipe)
             if recipe.needs_subscription and not config['recipe_account_info_%s'%recipe.id]:
-                error_dialog(self, _('Must set account information'), _('This recipe requires a username and password')).exec_()
+                error_dialog(self, _('Must set account information'), 
+                             _('This recipe requires a username and password')).exec_()
                 self.schedule.setCheckState(Qt.Unchecked)
                 return
             if self.interval_button.isChecked():
@@ -350,9 +351,11 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             self.username.blockSignals(False)
             self.password.blockSignals(False)
         d = datetime.utcnow() - recipe.last_downloaded
-        ld = '%.2f'%(d.days + d.seconds/(24.*3600))
+        def hm(x): return (x-x%3600)//3600, (x%3600 - (x%3600)%60)//60
+        hours, minutes = hm(d.seconds)
+        tm = _('%d days, %d hours and %d minutes ago')%(d.days, hours, minutes)
         if d < timedelta(days=366):
-            self.last_downloaded.setText(_('Last downloaded: %s days ago')%ld)
+            self.last_downloaded.setText(_('Last downloaded')+': '+tm)
         else:
             self.last_downloaded.setText(_('Last downloaded: never'))
             
@@ -431,7 +434,7 @@ class Scheduler(QObject):
                     day_matches = day > 6 or day == now.tm_wday
                     tnow = now.tm_hour*60 + now.tm_min
                     matches = day_matches and (hour*60+minute) < tnow
-                    if matches and delta >= timedelta(days=1):
+                    if matches and nowt.toordinal() < date.today().toordinal():
                         needs_downloading.add(recipe)
                 
             self.debug('Needs downloading:', needs_downloading)

src/calibre/gui2/dialogs/scheduler.ui

@@ -5,7 +5,7 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>726</width>
+    <width>738</width>
     <height>575</height>
    </rect>
   </property>
@@ -194,6 +194,9 @@
           <property name="text" >
            <string/>
           </property>
+          <property name="wordWrap" >
+           <bool>true</bool>
+          </property>
          </widget>
         </item>
         <item>

src/calibre/gui2/lrf_renderer/bookview.py

@@ -21,3 +21,4 @@ class BookView(QGraphicsView):
     def resize_for(self, width, height):
         self.preferred_size = QSize(width, height)
     
+    

src/calibre/gui2/lrf_renderer/main.py

@@ -80,8 +80,8 @@ class Main(MainWindow, Ui_MainWindow):
         
         QObject.connect(self.search, SIGNAL('search(PyQt_PyObject, PyQt_PyObject)'), self.find)
         
-        self.action_next_page.setShortcuts(QKeySequence.MoveToNextPage)
+        self.action_next_page.setShortcuts([QKeySequence.MoveToNextPage, QKeySequence(Qt.Key_Space)])
-        self.action_previous_page.setShortcuts(QKeySequence.MoveToPreviousPage)
+        self.action_previous_page.setShortcuts([QKeySequence.MoveToPreviousPage, QKeySequence(Qt.Key_Backspace)])
         self.action_next_match.setShortcuts(QKeySequence.FindNext)
         self.addAction(self.action_next_match)
         QObject.connect(self.action_next_page, SIGNAL('triggered(bool)'), self.next) 
@@ -191,6 +191,7 @@ class Main(MainWindow, Ui_MainWindow):
             self.spin_box.setSuffix(' of %d'%(self.document.num_of_pages,))
             self.spin_box.updateGeometry()
             self.stack.setCurrentIndex(0)
+            self.graphics_view.setFocus(Qt.OtherFocusReason)
         elif self.renderer.exception is not None:
             exception = self.renderer.exception
             print >>sys.stderr, 'Error rendering document'

src/calibre/library/server.py

@@ -312,7 +312,8 @@ class LibraryServer(object):
         
         book, books = MarkupTemplate(self.BOOK), []
         for record in items[start:start+num]:
-            authors = '|'.join([i.replace('|', ',') for i in record[2].split(',')])
+            aus = record[2] if record[2] else _('Unknown')
+            authors = '|'.join([i.replace('|', ',') for i in aus.split(',')])
             books.append(book.generate(r=record, authors=authors).render('xml').decode('utf-8'))
         updated = self.db.last_modified()
         

src/calibre/manual/news.rst

@@ -299,10 +299,10 @@ To learn more about writing advanced recipes using some of the facilities, avail
     :ref:`API Documentation <news_recipe>`
         Documentation of the ``BasicNewsRecipe`` class and all its important methods and fields.
 
-    `BasicNewsRecipe <http://bazaar.launchpad.net/~kovid/calibre/trunk/annotate/kovid%40kovidgoyal.net-20080509231359-le3xf7ynwc6eew90?file_id=1245%40b0dd1a5d-880a-0410-ada5-a57097536bc1%3Alibprs500%252Ftrunk%3Asrc%252Flibprs500%252Fweb%252Ffeeds%252Fnews.py>`_
+    `BasicNewsRecipe <http://bazaar.launchpad.net/~kovid/calibre/trunk/annotate/head:/src/calibre/web/feeds/news.py>`_
         The source code of ``BasicNewsRecipe``
 
-    `Built-in recipes <http://bazaar.launchpad.net/~kovid/calibre/trunk/files/kovid%40kovidgoyal.net-20080509231359-le3xf7ynwc6eew90?file_id=1298%40b0dd1a5d-880a-0410-ada5-a57097536bc1%3Alibprs500%252Ftrunk%3Asrc%252Flibprs500%252Fweb%252Ffeeds%252Frecipes>`_
+    `Built-in recipes <http://bazaar.launchpad.net/~kovid/calibre/trunk/files/head:/src/calibre/web/feeds/recipes/>`_
         The source code for the built-in recipes that come with |app|
 
 Migrating old style profiles to recipes

src/calibre/web/feeds/recipes/recipe_lrb.py

@@ -32,3 +32,8 @@ class LondonReviewOfBooks(BasicNewsRecipe):
     def print_version(self, url):
         main, split, rest = url.rpartition('/')
         return main + '/print/' + rest
+    
+    def postprocess_html(self, soup, first_fetch):
+        for t in soup.findAll(['table', 'tr', 'td']):
+            t.name = 'div'
+        return soup