Sync to trunk

installer/osx/freeze.py

@@ -3,7 +3,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 ''' Create an OSX installer '''
 
-import sys, re, os, shutil, subprocess, stat, glob, zipfile
+import sys, re, os, shutil, subprocess, stat, glob, zipfile, plistlib
 l = {}
 exec open('setup.py').read() in l
 VERSION = l['VERSION']
@@ -36,7 +36,7 @@ loader = open(loader_path, 'w')
 site_packages = glob.glob(resources_dir+'/lib/python*/site-packages.zip')[0]
 print >>loader, '#!'+python
 print >>loader, 'import sys'
-print >>loader, 'sys.path.remove('+repr(dirpath)+')'
+print >>loader, 'if', repr(dirpath), 'in sys.path: sys.path.remove(', repr(dirpath), ')'
 print >>loader, 'sys.path.append(', repr(site_packages), ')'
 print >>loader, 'sys.frozen = "macosx_app"'
 print >>loader, 'sys.frameworks_dir =', repr(frameworks_dir)
@@ -294,11 +294,26 @@ sys.frameworks_dir = os.path.join(os.path.dirname(os.environ['RESOURCEPATH']), '
         f.close()
         print 
         print 'Adding main scripts to site-packages'
-        f = zipfile.ZipFile(os.path.join(self.dist_dir, APPNAME+'.app', 'Contents', 'Resources', 'lib', 'python2.6', 'site-packages.zip'), 'a', zipfile.ZIP_DEFLATED)
+        f = zipfile.ZipFile(os.path.join(self.dist_dir, APPNAME+'.app', 'Contents', 'Resources', 'lib', 'python'+sys.version[:3], 'site-packages.zip'), 'a', zipfile.ZIP_DEFLATED)
         for script in scripts['gui']+scripts['console']:
             f.write(script, script.partition('/')[-1])
         f.close()
         print 
+        print 'Creating console.app'
+        contents_dir = os.path.dirname(resource_dir)
+        cc_dir = os.path.join(contents_dir, 'console.app', 'Contents')
+        os.makedirs(cc_dir)
+        for x in os.listdir(contents_dir):
+            if x == 'console.app':
+                continue
+            if x == 'Info.plist':
+                plist = plistlib.readPlist(os.path.join(contents_dir, x))
+                plist['LSUIElement'] = '1'
+                plistlib.writePlist(plist, os.path.join(cc_dir, x))
+            else:
+                os.symlink(os.path.join('../..', x), 
+                           os.path.join(cc_dir, x))
+        print
         print 'Building disk image'
         BuildAPP.makedmg(os.path.join(self.dist_dir, APPNAME+'.app'), APPNAME+'-'+VERSION)
 

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.130'
+__version__   = '0.4.131'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/builtins.py

@@ -132,7 +132,7 @@ class HTMLMetadataReader(MetadataReaderPlugin):
 class MOBIMetadataReader(MetadataReaderPlugin):
     
     name        = 'Read MOBI metadata'
-    file_types  = set(['mobi'])
+    file_types  = set(['mobi', 'prc'])
     description = _('Read metadata from %s files')%'MOBI'
     
     def get_metadata(self, stream, ftype):
@@ -204,7 +204,7 @@ class EPUBMetadataWriter(MetadataWriterPlugin):
     
     name = 'Set EPUB metadata'
     file_types = set(['epub'])
-    description = _('Set metadata in EPUB files')
+    description = _('Set metadata in %s files')%'EPUB'
     
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.epub import set_metadata
@@ -214,7 +214,7 @@ class LRFMetadataWriter(MetadataWriterPlugin):
     
     name = 'Set LRF metadata'
     file_types = set(['lrf'])
-    description = _('Set metadata in LRF files')
+    description = _('Set metadata in %s files')%'LRF'
     
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.lrf.meta import set_metadata
@@ -224,12 +224,24 @@ class RTFMetadataWriter(MetadataWriterPlugin):
     
     name = 'Set RTF metadata'
     file_types = set(['rtf'])
-    description = _('Set metadata in RTF files')
+    description = _('Set metadata in %s files')%'RTF'
     
     def set_metadata(self, stream, mi, type):
         from calibre.ebooks.metadata.rtf import set_metadata
         set_metadata(stream, mi)
 
+class MOBIMetadataWriter(MetadataWriterPlugin):
+    
+    name        = 'Set MOBI metadata'
+    file_types  = set(['mobi', 'prc'])
+    description = _('Set metadata in %s files')%'MOBI'
+    author      = 'Marshall T. Vandegrift'
+    
+    def set_metadata(self, stream, mi, type):
+        from calibre.ebooks.metadata.mobi import set_metadata
+        set_metadata(stream, mi)
+
+
 plugins = [HTML2ZIP]
 plugins += [x for x in list(locals().values()) if isinstance(x, type) and \
                                         x.__name__.endswith('MetadataReader')]

src/calibre/debug.py

@@ -21,6 +21,9 @@ Run an embedded python interpreter.
     'Module specifications are of the form full.name.of.module,path_to_module.py', default=None
     )
     parser.add_option('-c', '--command', help='Run python code.', default=None)
+    parser.add_option('-e', '--exec-file', default=None, help='Run the python code in file.')
+    parser.add_option('-d', '--debug-device-driver', default=False, action='store_true', 
+                      help='Debug the specified device driver.')
     parser.add_option('-g', '--gui',  default=False, action='store_true',
                       help='Run the GUI',)
     parser.add_option('--migrate', action='store_true', default=False, 
@@ -75,6 +78,23 @@ def migrate(old, new):
     prefs['library_path'] = os.path.abspath(new)
     print 'Database migrated to', os.path.abspath(new)
     
+def debug_device_driver():
+    from calibre.devices.scanner import DeviceScanner
+    s = DeviceScanner()
+    s.scan()
+    print 'USB devices on system:', repr(s.devices)
+    from calibre.devices import devices
+    for dev in devices():
+        print 'Looking for', dev.__name__
+        connected = s.is_device_connected(dev)
+        if connected:
+            print 'Device Connected:', dev
+            print 'Trying to open device...'
+            d = dev()
+            d.open()
+            print 'Total space:', d.total_space()
+            break
+        
 
 def main(args=sys.argv):
     opts, args = option_parser().parse_args(args)
@@ -87,6 +107,11 @@ def main(args=sys.argv):
     elif opts.command:
         sys.argv = args[:1]
         exec opts.command
+    elif opts.exec_file:
+        sys.argv = args[:1]
+        execfile(opts.exec_file)
+    elif opts.debug_device_driver:
+        debug_device_driver()
     elif opts.migrate:
         if len(args) < 3:
             print 'You must specify the path to library1.db and the path to the new library folder'

src/calibre/ebooks/epub/from_html.py

@@ -165,8 +165,11 @@ class HTMLProcessor(Processor, Rationalizer):
                 br.tag = 'p'
                 br.text = u'\u00a0'
                 if (br.tail and br.tail.strip()) or sibling is None or \
-                    getattr(sibling, 'tag', '') != 'br':
+                   getattr(sibling, 'tag', '') != 'br':
-                    br.set('style', br.get('style', '')+'; margin: 0pt; border:0pt; height:0pt')
+                    style = br.get('style', '').split(';')
+                    style = filter(None, map(lambda x: x.strip(), style))
+                    style.append('margin: 0pt; border:0pt; height:0pt')
+                    br.set('style', '; '.join(style))
                 else:
                     sibling.getparent().remove(sibling)
                     if sibling.tail:
@@ -191,6 +194,8 @@ class HTMLProcessor(Processor, Rationalizer):
             if not tag.text and not tag.get('src', False):
                 tag.getparent().remove(tag)
                 
+        
+    
     def save(self):
         for meta in list(self.root.xpath('//meta')):
             meta.getparent().remove(meta)

src/calibre/ebooks/epub/split.py

@@ -50,6 +50,7 @@ class Splitter(LoggingInterface):
         self.split_size = 0
         
         # Split on page breaks
+        self.splitting_on_page_breaks = True
         if not opts.dont_split_on_page_breaks:
             self.log_info('\tSplitting on page breaks...')
             if self.path in stylesheet_map:
@@ -61,6 +62,7 @@ class Splitter(LoggingInterface):
             trees = list(self.trees)
         
         # Split any remaining over-sized trees
+        self.splitting_on_page_breaks = False
         if self.opts.profile.flow_size < sys.maxint:
             lt_found = False
             self.log_info('\tLooking for large trees...')
@@ -203,7 +205,8 @@ class Splitter(LoggingInterface):
                     elem.set('style', 'display:none')
         
         def fix_split_point(sp):
-            sp.set('style', sp.get('style', '')+'page-break-before:avoid;page-break-after:avoid') 
+            if not self.splitting_on_page_breaks:
+                sp.set('style', sp.get('style', '')+'page-break-before:avoid;page-break-after:avoid') 
         
         # Tree 1
         hit_split_point = False

src/calibre/ebooks/html.py

@@ -417,39 +417,44 @@ class Parser(PreProcessor, LoggingInterface):
         self.level = self.htmlfile.level
         for f in self.htmlfiles:
             name = os.path.basename(f.path)
+            name = os.path.splitext(name)[0] + '.xhtml'
             if name in self.htmlfile_map.values():
                 name = os.path.splitext(name)[0] + '_cr_%d'%save_counter + os.path.splitext(name)[1]
                 save_counter += 1
             self.htmlfile_map[f.path] = name
         
         self.parse_html()
+        # Handle <image> tags inside embedded <svg>
+        # At least one source of EPUB files (Penguin) uses xlink:href
+        # without declaring the xlink namespace
+        for image in self.root.xpath('//image'): 
+            for attr in image.attrib.keys():
+                if attr.endswith(':href'):
+                    nhref = self.rewrite_links(image.get(attr))
+                    image.set(attr, nhref)
+        
         self.root.rewrite_links(self.rewrite_links, resolve_base_href=False)
         for bad in ('xmlns', 'lang', 'xml:lang'): # lxml also adds these attributes for XHTML documents, leading to duplicates
             if self.root.get(bad, None) is not None:
                 self.root.attrib.pop(bad)
         
+        
+        
     def save_path(self):
         return os.path.join(self.tdir, self.htmlfile_map[self.htmlfile.path])
     
-    def declare_xhtml_namespace(self, match):
-        if not match.group('raw'):
-            return '<html xmlns="http://www.w3.org/1999/xhtml">'
-        raw = match.group('raw')
-        m = re.search(r'(?i)xmlns\s*=\s*[\'"](?P<uri>[^"\']*)[\'"]', raw)
-        if not m:
-            return '<html xmlns="http://www.w3.org/1999/xhtml" %s>'%raw
-        else:
-            return  match.group().replace(m.group('uri'), "http://www.w3.org/1999/xhtml")
-    
     def save(self):
         '''
         Save processed HTML into the content directory.
         Should be called after all HTML processing is finished.
         '''
-        ans = tostring(self.root, pretty_print=self.opts.pretty_print)
+        self.root.set('xmlns', 'http://www.w3.org/1999/xhtml')
-        ans = re.sub(r'(?i)<\s*html(?P<raw>\s+[^>]*){0,1}>', self.declare_xhtml_namespace, ans[:1000]) + ans[1000:]
+        self.root.set('xmlns:xlink', 'http://www.w3.org/1999/xlink')
-        ans = re.compile(r'<head>', re.IGNORECASE).sub('<head>\n\t<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n', ans[:1000])+ans[1000:]
+        for svg in self.root.xpath('//svg'):
+            svg.set('xmlns', 'http://www.w3.org/2000/svg')
         
+        ans = tostring(self.root, pretty_print=self.opts.pretty_print)
+        ans = re.compile(r'<head>', re.IGNORECASE).sub('<head>\n\t<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />\n', ans[:1000])+ans[1000:]
         with open(self.save_path(), 'wb') as f:
             f.write(ans)
             return f.name
@@ -823,21 +828,28 @@ class Processor(Parser):
             font.set('class', cn)
             font.tag = 'span'
             
+        id_css, id_css_counter = {}, 0
         for elem in self.root.xpath('//*[@style]'):
             setting = elem.get('style')
-            classname = cache.get(setting, None)
+            if elem.get('id', False) or elem.get('class', False):
-            if classname is None:
+                elem.set('id', elem.get('id', 'calibre_css_id_%d'%id_css_counter))
-                classname = 'calibre_class_%d'%class_counter
+                id_css_counter += 1
-                class_counter += 1
+                id_css[elem.tag+'#'+elem.get('id')] = setting
-                cache[setting] = classname
+            else:
-            cn = elem.get('class', '')
+                classname = cache.get(setting, None)
-            if cn: cn += ' '
+                if classname is None:
-            cn += classname
+                    classname = 'calibre_class_%d'%class_counter
-            elem.set('class', cn)
+                    class_counter += 1
+                    cache[setting] = classname
+                cn = elem.get('class', classname)
+                elem.set('class', cn)
             elem.attrib.pop('style')
         
         css = '\n'.join(['.%s {%s;}'%(cn, setting) for \
                          setting, cn in cache.items()])
+        css += '\n\n'
+        css += '\n'.join(['%s {%s;}'%(selector, setting) for \
+                         selector, setting in id_css.items()])
         sheet = self.css_parser.parseString(self.preprocess_css(css.replace(';;}', ';}')))
         for rule in sheet:
             self.stylesheet.add(rule)

src/calibre/ebooks/lit/reader.py

@@ -11,6 +11,7 @@ import sys, struct, cStringIO, os
 import functools
 import re
 from urlparse import urldefrag
+from urllib import unquote as urlunquote
 from lxml import etree
 from calibre.ebooks.lit import LitError
 from calibre.ebooks.lit.maps import OPF_MAP, HTML_MAP
@@ -611,6 +612,8 @@ class LitReader(object):
                     offset, raw = u32(raw), raw[4:]
                     internal, raw = consume_sized_utf8_string(raw)
                     original, raw = consume_sized_utf8_string(raw)
+                    # The path should be stored unquoted, but not always
+                    original = urlunquote(original)
                     # Is this last one UTF-8 or ASCIIZ?
                     mime_type, raw = consume_sized_utf8_string(raw, zpad=True)
                     self.manifest[internal] = ManifestItem(

src/calibre/ebooks/lit/writer.py

@@ -122,6 +122,8 @@ LZXC_CONTROL = \
 
 COLLAPSE = re.compile(r'[ \t\r\n\v]+')
 
+PAGE_BREAKS = set(['always', 'left', 'right'])
+
 def decint(value):
     bytes = []
     while True:
@@ -202,7 +204,7 @@ class ReBinary(object):
             self.write(FLAG_CUSTOM, len(tag)+1, tag)
         last_break = self.page_breaks[-1][0] if self.page_breaks else None
         if style and last_break != tag_offset \
-           and style['page-break-before'] not in ('avoid', 'auto'):
+           and style['page-break-before'] in PAGE_BREAKS:
             self.page_breaks.append((tag_offset, list(parents)))
         for attr, value in attrib.items():
             attr = prefixname(attr, nsrmap)

src/calibre/ebooks/metadata/mobi.py

@@ -1,23 +1,225 @@
-#!/usr/bin/env  python
+'''
+Retrieve and modify in-place Mobipocket book metadata.
+'''
+
+from __future__ import with_statement
+
 __license__   = 'GPL v3'
-__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net and ' \
+    'Marshall T. Vandegrift <llasram@gmail.com>'
 __docformat__ = 'restructuredtext en'
 
-'''
+import sys
-'''
+import os
-
+from struct import pack, unpack
-import sys, os
+from cStringIO import StringIO
-
+from calibre.ebooks.metadata import get_parser
+from calibre.ebooks.mobi import MobiError
 from calibre.ebooks.mobi.reader import get_metadata
+from calibre.ebooks.mobi.writer import rescale_image, MAX_THUMB_DIMEN
+from calibre.ebooks.mobi.langcodes import iana2mobi
+
+class StreamSlicer(object):
+    def __init__(self, stream, start=0, stop=None):
+        self._stream = stream
+        self.start = start
+        if stop is None: 
+            stream.seek(0, 2)        
+            stop = stream.tell()
+        self.stop = stop
+        self._len = stop - start
+
+    def __len__(self):
+        return self._len
+        
+    def __getitem__(self, key):
+        stream = self._stream
+        base = self.start
+        if isinstance(key, (int, long)):
+            stream.seek(base + key)
+            return stream.read(1)
+        if isinstance(key, slice):
+            start, stop, stride = key.indices(self._len)
+            if stride < 0:
+                start, stop = stop, start
+            size = stop - start
+            if size <= 0:
+                return ""
+            stream.seek(base + start)
+            data = stream.read(size)
+            if stride != 1:
+                data = data[::stride]
+            return data
+        raise TypeError("stream indices must be integers")
+
+    def __setitem__(self, key, value):
+        stream = self._stream
+        base = self.start
+        if isinstance(key, (int, long)):
+            if len(value) != 1:
+                raise ValueError("key and value lengths must match")
+            stream.seek(base + key)
+            return stream.write(value)
+        if isinstance(key, slice):
+            start, stop, stride = key.indices(self._len)
+            if stride < 0:
+                start, stop = stop, start
+            size = stop - start
+            if stride != 1:
+                value = value[::stride]
+            if len(value) != size:
+                raise ValueError("key and value lengths must match")
+            stream.seek(base + start)
+            return stream.write(value)
+        raise TypeError("stream indices must be integers")
+
+    
+class MetadataUpdater(object):    
+    def __init__(self, stream):
+        self.stream = stream
+        data = self.data = StreamSlicer(stream)
+        type = self.type = data[60:68]
+        self.nrecs, = unpack('>H', data[76:78])
+        record0 = self.record0 = self.record(0)
+        codepage, = unpack('>I', record0[28:32])
+        self.codec = 'utf-8' if codepage == 65001 else 'cp1252'
+        image_base, = unpack('>I', record0[108:112])
+        flags, = unpack('>I', record0[128:132])
+        have_exth = self.have_exth = (flags & 0x40) != 0
+        if not have_exth:
+            return
+        self.cover_record = self.thumbnail_record = None
+        exth_off = unpack('>I', record0[20:24])[0] + 16 + record0.start
+        exth = self.exth = StreamSlicer(stream, exth_off, record0.stop)
+        nitems, = unpack('>I', exth[8:12])
+        pos = 12
+        for i in xrange(nitems):
+            id, size = unpack('>II', exth[pos:pos + 8])
+            content = exth[pos + 8: pos + size]
+            pos += size
+            if id == 201:
+                rindex, = self.cover_rindex, = unpack('>I', content)
+                self.cover_record = self.record(rindex + image_base)
+            elif id == 202:
+                rindex, = self.thumbnail_rindex, = unpack('>I', content)
+                self.thumbnail_record = self.record(rindex + image_base)
+
+    def record(self, n):
+        if n >= self.nrecs:
+            raise ValueError('non-existent record %r' % n)
+        offoff = 78 + (8 * n)
+        start, = unpack('>I', self.data[offoff + 0:offoff + 4])
+        stop = None
+        if n < (self.nrecs - 1):
+            stop, = unpack('>I', self.data[offoff + 8:offoff + 12])
+        return StreamSlicer(self.stream, start, stop)
+
+    def update(self, mi):
+        recs = []
+        if mi.authors:
+            authors = '; '.join(mi.authors)
+            recs.append((100, authors.encode(self.codec, 'replace')))
+        if mi.publisher:
+            recs.append((101, mi.publisher.encode(self.codec, 'replace')))
+        if mi.comments:
+            recs.append((103, mi.comments.encode(self.codec, 'replace')))
+        if mi.isbn:
+            recs.append((104, mi.isbn.encode(self.codec, 'replace')))
+        if mi.tags:
+            subjects = '; '.join(mi.tags)
+            recs.append((105, subjects.encode(self.codec, 'replace')))
+        if self.cover_record is not None:
+            recs.append((201, pack('>I', self.cover_rindex)))
+            recs.append((203, pack('>I', 0)))
+        if self.thumbnail_record is not None:
+            recs.append((202, pack('>I', self.thumbnail_rindex)))
+        exth = StringIO()
+        for code, data in recs:
+            exth.write(pack('>II', code, len(data) + 8))
+            exth.write(data)
+        exth = exth.getvalue()
+        trail = len(exth) % 4
+        pad = '\0' * (4 - trail) # Always pad w/ at least 1 byte
+        exth = ['EXTH', pack('>II', len(exth) + 12, len(recs)), exth, pad]
+        exth = ''.join(exth)
+        title = (mi.title or _('Unknown')).encode(self.codec, 'replace')
+        title_off = (self.exth.start - self.record0.start) + len(exth)
+        title_len = len(title)
+        trail = len(self.exth) - len(exth) - len(title)
+        if trail < 0:
+            raise MobiError("Insufficient space to update metadata")
+        self.exth[:] = ''.join([exth, title, '\0' * trail])
+        self.record0[84:92] = pack('>II', title_off, title_len)
+        self.record0[92:96] = iana2mobi(mi.language)
+        if mi.cover_data[1]:
+            data =  mi.cover_data[1]
+            if self.cover_record is not None:
+                size = len(self.cover_record)
+                cover = rescale_image(data, size)
+                cover += '\0' * (size - len(cover))
+                self.cover_record[:] = cover
+            if self.thumbnail_record is not None:
+                size = len(self.thumbnail_record)
+                thumbnail = rescale_image(data, size, dimen=MAX_THUMB_DIMEN)
+                thumbnail += '\0' * (size - len(thumbnail))
+                self.thumbnail_record[:] = thumbnail
+        return
+
+def set_metadata(stream, mi):
+    mu = MetadataUpdater(stream)
+    mu.update(mi)
+    return
+
+
+def option_parser():
+    parser = get_parser('mobi')
+    parser.remove_option('--category')
+    parser.add_option('--tags', default=None,
+                      help=_('Set the subject tags'))
+    parser.add_option('--language', default=None,
+                      help=_('Set the language'))
+    parser.add_option('--publisher', default=None,
+                      help=_('Set the publisher'))
+    parser.add_option('--isbn', default=None,
+                      help=_('Set the ISBN'))
+    return parser
 
 def main(args=sys.argv):
+    parser = option_parser()
+    opts, args = parser.parse_args(args)
     if len(args) != 2:
+        parser.print_help()
         print >>sys.stderr, 'Usage: %s file.mobi' % args[0]
         return 1
     fname = args[1]
-    mi = get_metadata(open(fname, 'rb'))
+    changed = False
-    print unicode(mi)
+    with open(fname, 'r+b') as stream:
-    if mi.cover_data[1]:
+        mi = get_metadata(stream)
+        if opts.title:
+            mi.title = opts.title
+            changed = True
+        if opts.authors:
+            mi.authors = opts.authors.split(',')
+            changed = True
+        if opts.comment:
+            mi.comments = opts.comment
+            changed = True
+        if opts.tags is not None:
+            mi.tags = opts.tags.split(',')
+            changed = True
+        if opts.language is not None:
+            mi.language = opts.language
+            changed = True
+        if opts.publisher is not None:
+            mi.publisher = opts.publisher
+            changed = True
+        if opts.isbn is not None:
+            mi.isbn = opts.isbn
+            changed = True
+        if changed:
+            set_metadata(stream, mi)
+        print unicode(get_metadata(stream))
+    if not changed and mi.cover_data[1]:
         cover = os.path.abspath(
             '.'.join((os.path.splitext(os.path.basename(fname))[0],
                       mi.cover_data[0].lower())))

src/calibre/ebooks/mobi/mobiml.py

@@ -27,7 +27,7 @@ TABLE_TAGS = set(['table', 'tr', 'td', 'th'])
 SPECIAL_TAGS = set(['hr', 'br'])
 CONTENT_TAGS = set(['img', 'hr', 'br'])
 
-PAGE_BREAKS = set(['always', 'odd', 'even'])
+PAGE_BREAKS = set(['always', 'left', 'right'])
 
 COLLAPSE = re.compile(r'[ \t\r\n\v]+')
 
@@ -79,6 +79,9 @@ class FormatState(object):
 
 
 class MobiMLizer(object):
+    def __init__(self, ignore_tables=False):
+        self.ignore_tables = ignore_tables
+    
     def transform(self, oeb, context):
         oeb.logger.info('Converting XHTML to Mobipocket markup...')
         self.oeb = oeb
@@ -341,6 +344,8 @@ class MobiMLizer(object):
             tag = 'tr'
         elif display == 'table-cell':
             tag = 'td'
+        if tag in TABLE_TAGS and self.ignore_tables:
+            tag = 'span' if tag == 'td' else 'div'
         if tag in TABLE_TAGS:
             for attr in ('rowspan', 'colspan'):
                 if attr in elem.attrib:

src/calibre/ebooks/mobi/writer.py

@@ -87,6 +87,49 @@ def decint(value, direction):
         bytes[-1] |= 0x80
     return ''.join(chr(b) for b in reversed(bytes))
 
+def rescale_image(data, maxsizeb, dimen=None):
+    image = Image.open(StringIO(data))
+    format = image.format
+    changed = False
+    if image.format not in ('JPEG', 'GIF'):
+        width, height = image.size
+        area = width * height
+        if area <= 40000:
+            format = 'GIF'
+        else:
+            image = image.convert('RGBA')
+            format = 'JPEG'
+        changed = True
+    if dimen is not None:
+        image.thumbnail(dimen, Image.ANTIALIAS)
+        changed = True
+    if changed:
+        data = StringIO()
+        image.save(data, format)
+        data = data.getvalue()
+    if len(data) <= maxsizeb:
+        return data
+    image = image.convert('RGBA')
+    for quality in xrange(95, -1, -1):
+        data = StringIO()
+        image.save(data, 'JPEG', quality=quality)
+        data = data.getvalue()
+        if len(data) <= maxsizeb:
+            return data
+    width, height = image.size
+    for scale in xrange(99, 0, -1):
+        scale = scale / 100.
+        data = StringIO()
+        scaled = image.copy()
+        size = (int(width * scale), (height * scale))
+        scaled.thumbnail(size, Image.ANTIALIAS)
+        scaled.save(data, 'JPEG', quality=0)
+        data = data.getvalue()
+        if len(data) <= maxsizeb:
+            return data
+    # Well, we tried?
+    return data
+
 
 class Serializer(object):
     NSRMAP = {'': None, XML_NS: 'xml', XHTML_NS: '', MBP_NS: 'mbp'}
@@ -356,49 +399,6 @@ class MobiWriter(object):
             data, overlap = self._read_text_record(text)
         self._text_nrecords = nrecords
     
-    def _rescale_image(self, data, maxsizeb, dimen=None):
-        image = Image.open(StringIO(data))
-        format = image.format
-        changed = False
-        if image.format not in ('JPEG', 'GIF'):
-            width, height = image.size
-            area = width * height
-            if area <= 40000:
-                format = 'GIF'
-            else:
-                image = image.convert('RGBA')
-                format = 'JPEG'
-            changed = True
-        if dimen is not None:
-            image.thumbnail(dimen, Image.ANTIALIAS)
-            changed = True
-        if changed:
-            data = StringIO()
-            image.save(data, format)
-            data = data.getvalue()
-        if len(data) <= maxsizeb:
-            return data
-        image = image.convert('RGBA')
-        for quality in xrange(95, -1, -1):
-            data = StringIO()
-            image.save(data, 'JPEG', quality=quality)
-            data = data.getvalue()
-            if len(data) <= maxsizeb:
-                return data
-        width, height = image.size
-        for scale in xrange(99, 0, -1):
-            scale = scale / 100.
-            data = StringIO()
-            scaled = image.copy()
-            size = (int(width * scale), (height * scale))
-            scaled.thumbnail(size, Image.ANTIALIAS)
-            scaled.save(data, 'JPEG', quality=0)
-            data = data.getvalue()
-            if len(data) <= maxsizeb:
-                return data
-        # Well, we tried?
-        return data
-        
     def _generate_images(self):
         self._oeb.logger.info('Serializing images...')
         images = [(index, href) for href, index in self._images.items()]
@@ -407,7 +407,7 @@ class MobiWriter(object):
         coverid = metadata.cover[0] if metadata.cover else None
         for _, href in images:
             item = self._oeb.manifest.hrefs[href]
-            data = self._rescale_image(item.data, self._imagemax)
+            data = rescale_image(item.data, self._imagemax)
             self._records.append(data)
     
     def _generate_record0(self):
@@ -480,7 +480,7 @@ class MobiWriter(object):
         return ''.join(exth)
 
     def _add_thumbnail(self, item):
-        data = self._rescale_image(item.data, MAX_THUMB_SIZE, MAX_THUMB_DIMEN)
+        data = rescale_image(item.data, MAX_THUMB_SIZE, MAX_THUMB_DIMEN)
         manifest = self._oeb.manifest
         id, href = manifest.generate('thumbnail', 'thumbnail.jpeg')
         manifest.add(id, href, 'image/jpeg', data=data)
@@ -524,6 +524,10 @@ def config(defaults=None):
         help=_('Modify images to meet Palm device size limitations.'))
     mobi('toc_title', ['--toc-title'], default=None, 
          help=_('Title for any generated in-line table of contents.'))
+    mobi('ignore_tables', ['--ignore-tables'], default=False,
+         help=_('Render HTML tables as blocks of text instead of actual '
+                'tables. This is neccessary if the HTML contains very large '
+                'or complex tables.'))
     profiles = c.add_group('profiles', _('Device renderer profiles. '
         'Affects conversion of font sizes, image rescaling and rasterization '
         'of tables. Valid profiles are: %s.') % ', '.join(_profiles))
@@ -581,7 +585,7 @@ def oeb2mobi(opts, inpath):
     rasterizer.transform(oeb, context)
     trimmer = ManifestTrimmer()
     trimmer.transform(oeb, context)
-    mobimlizer = MobiMLizer()
+    mobimlizer = MobiMLizer(ignore_tables=opts.ignore_tables)
     mobimlizer.transform(oeb, context)
     writer = MobiWriter(compression=compression, imagemax=imagemax)
     writer.dump(oeb, outpath)

src/calibre/ebooks/oeb/base.py

@@ -10,7 +10,7 @@ import os
 import sys
 from collections import defaultdict
 from types import StringTypes
-from itertools import izip, count
+from itertools import izip, count, chain
 from urlparse import urldefrag, urlparse, urlunparse
 from urllib import unquote as urlunquote
 import logging
@@ -22,9 +22,11 @@ from lxml import html
 from calibre import LoggingInterface
 from calibre.translations.dynamic import translate
 from calibre.startup import get_lang
+from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
 
 XML_NS = 'http://www.w3.org/XML/1998/namespace'
 XHTML_NS = 'http://www.w3.org/1999/xhtml'
+OEB_DOC_NS = 'http://openebook.org/namespaces/oeb-document/1.0/'
 OPF1_NS = 'http://openebook.org/namespaces/oeb-package/1.0/'
 OPF2_NS = 'http://www.idpf.org/2007/opf'
 DC09_NS = 'http://purl.org/metadata/dublin_core'
@@ -40,6 +42,7 @@ XPNSMAP = {'h': XHTML_NS, 'o1': OPF1_NS, 'o2': OPF2_NS,
            'd09': DC09_NS, 'd10': DC10_NS, 'd11': DC11_NS,
            'xsi': XSI_NS, 'dt': DCTERMS_NS, 'ncx': NCX_NS,
            'svg': SVG_NS, 'xl': XLINK_NS}
+DC_PREFIXES = ('d11', 'd10', 'd09')
 
 def XML(name): return '{%s}%s' % (XML_NS, name)
 def XHTML(name): return '{%s}%s' % (XHTML_NS, name)
@@ -61,6 +64,7 @@ GIF_MIME = 'image/gif'
 JPEG_MIME = 'image/jpeg'
 PNG_MIME = 'image/png'
 SVG_MIME = 'image/svg+xml'
+BINARY_MIME = 'application/octet-stream'
 
 OEB_STYLES = set([CSS_MIME, OEB_CSS_MIME, 'text/x-oeb-css'])
 OEB_DOCS = set([XHTML_MIME, 'text/html', OEB_DOC_MIME, 'text/x-oeb-document'])
@@ -69,6 +73,8 @@ OEB_IMAGES = set([GIF_MIME, JPEG_MIME, PNG_MIME, SVG_MIME])
 
 MS_COVER_TYPE = 'other.ms-coverimage-standard'
 
+ENTITY_RE = re.compile(r'&([a-zA-Z_:][a-zA-Z0-9.-_:]+);')
+COLLAPSE_RE = re.compile(r'[ \t\r\n\v]+')
 
 def element(parent, *args, **kwargs):
     if parent is not None:
@@ -191,11 +197,8 @@ class Metadata(object):
         def __init__(self, term, value, fq_attrib={}, **kwargs):
             self.fq_attrib = fq_attrib = dict(fq_attrib)
             fq_attrib.update(kwargs)
-            if term == OPF('meta') and not value:
+            if barename(term).lower() in Metadata.TERMS and \
-                term = self.fq_attrib.pop('name')
+               (not namespace(term) or namespace(term) in DC_NSES):
-                value = self.fq_attrib.pop('content')
-            elif barename(term).lower() in Metadata.TERMS and \
-                 (not namespace(term) or namespace(term) in DC_NSES):
                 # Anything looking like Dublin Core is coerced
                 term = DC(barename(term).lower())
             elif namespace(term) == OPF2_NS:
@@ -329,20 +332,74 @@ class Manifest(object):
                 % (self.id, self.href, self.media_type)
 
         def _force_xhtml(self, data):
+            # Possibly decode in user-specified encoding
             if self.oeb.encoding is not None:
                 data = data.decode(self.oeb.encoding, 'replace')
+            # Handle broken XHTML w/ SVG (ugh)
+            if 'svg:' in data and SVG_NS not in data:
+                data = data.replace(
+                    '<html', '<html xmlns:svg="%s"' % SVG_NS, 1)
+            if 'xlink:' in data and XLINK_NS not in data:
+                data = data.replace(
+                    '<html', '<html xmlns:xlink="%s"' % XLINK_NS, 1)
+            # Try with more & more drastic measures to parse
             try:
                 data = etree.fromstring(data)
             except etree.XMLSyntaxError:
-                data = html.fromstring(data)
+                repl = lambda m: ENTITYDEFS.get(m.group(1), m.group(0))
-                data = etree.tostring(data, encoding=unicode)
+                data = ENTITY_RE.sub(repl, data)
-                data = etree.fromstring(data)
+                try:
-            if namespace(data.tag) != XHTML_NS:
+                    data = etree.fromstring(data)
+                except etree.XMLSyntaxError:
+                    self.oeb.logger.warn('Parsing file %r as HTML' % self.href)
+                    data = html.fromstring(data)
+                    data.attrib.pop('xmlns', None)
+                    data = etree.tostring(data, encoding=unicode)
+                    data = etree.fromstring(data)
+            # Force into the XHTML namespace
+            if barename(data.tag) != 'html':
+                raise OEBError(
+                    'File %r does not appear to be (X)HTML' % self.href)
+            elif not namespace(data.tag):
                 data.attrib['xmlns'] = XHTML_NS
                 data = etree.tostring(data, encoding=unicode)
                 data = etree.fromstring(data)
+            elif namespace(data.tag) != XHTML_NS:
+                # OEB_DOC_NS, but possibly others
+                ns = namespace(data.tag)
+                attrib = dict(data.attrib)
+                nroot = etree.Element(XHTML('html'),
+                    nsmap={None: XHTML_NS}, attrib=attrib)
+                for elem in data.iterdescendants():
+                    if isinstance(elem.tag, basestring) and \
+                       namespace(elem.tag) == ns:
+                        elem.tag = XHTML(barename(elem.tag))
+                for elem in data:
+                    nroot.append(elem)
+                data = nroot
+            # Remove any encoding-specifying <meta/> elements
             for meta in self.META_XP(data):
                 meta.getparent().remove(meta)
+            # Ensure has a <head/>
+            head = xpath(data, '/h:html/h:head')
+            head = head[0] if head else None
+            if head is None:
+                self.oeb.logger.warn(
+                    'File %r missing <head/> element' % self.href)
+                head = etree.Element(XHTML('head'))
+                data.insert(0, head)
+                title = etree.SubElement(head, XHTML('title'))
+                title.text = self.oeb.translate(__('Unknown'))
+            elif not xpath(data, '/h:html/h:head/h:title'):
+                self.oeb.logger.warn(
+                    'File %r missing <title/> element' % self.href)
+                title = etree.SubElement(head, XHTML('title'))
+                title.text = self.oeb.translate(__('Unknown'))
+            # Ensure has a <body/>
+            if not xpath(data, '/h:html/h:body'):
+                self.oeb.logger.warn(
+                    'File %r missing <body/> element' % self.href)
+                etree.SubElement(data, XHTML('body'))
             return data
         
         def data():
@@ -469,9 +526,9 @@ class Manifest(object):
         elem = element(parent, 'manifest')
         for item in self.ids.values():
             media_type = item.media_type
-            if media_type == XHTML_MIME:
+            if media_type in OEB_DOCS:
                 media_type = OEB_DOC_MIME
-            elif media_type == CSS_MIME:
+            elif media_type in OEB_STYLES:
                 media_type = OEB_CSS_MIME
             attrib = {'id': item.id, 'href': item.href,
                       'media-type': media_type}
@@ -483,6 +540,11 @@ class Manifest(object):
     def to_opf2(self, parent=None):
         elem = element(parent, OPF('manifest'))
         for item in self.ids.values():
+            media_type = item.media_type
+            if media_type in OEB_DOCS:
+                media_type = XHTML_MIME
+            elif media_type in OEB_STYLES:
+                media_type = CSS_MIME
             attrib = {'id': item.id, 'href': item.href,
                       'media-type': item.media_type}
             if item.fallback:
@@ -746,25 +808,19 @@ class OEBBook(object):
             opf = self._read_opf(opfpath)
             self._all_from_opf(opf)
     
-    def _convert_opf1(self, opf):
+    def _clean_opf(self, opf):
-        # Seriously, seriously wrong
+        for elem in opf.iter():
-        if namespace(opf.tag) == OPF1_NS:
+            if isinstance(elem.tag, basestring) \
-            opf.tag = barename(opf.tag)
+               and namespace(elem.tag) in ('', OPF1_NS):
-            for elem in opf.iterdescendants():
+                elem.tag = OPF(barename(elem.tag))
-                if isinstance(elem.tag, basestring) \
-                   and namespace(elem.tag) == OPF1_NS:
-                    elem.tag = barename(elem.tag)
         attrib = dict(opf.attrib)
-        attrib['version'] = '2.0'
         nroot = etree.Element(OPF('package'),
             nsmap={None: OPF2_NS}, attrib=attrib)
         metadata = etree.SubElement(nroot, OPF('metadata'),
             nsmap={'opf': OPF2_NS, 'dc': DC11_NS,
                    'xsi': XSI_NS, 'dcterms': DCTERMS_NS})
-        for prefix in ('d11', 'd10', 'd09'):
+        dc = lambda prefix: xpath(opf, 'o2:metadata//%s:*' % prefix)
-            elements = xpath(opf, 'metadata//%s:*' % prefix)
+        for element in chain(*(dc(prefix) for prefix in DC_PREFIXES)):
-            if elements: break
-        for element in elements:
             if not element.text: continue
             tag = barename(element.tag).lower()
             element.tag = '{%s}%s' % (DC11_NS, tag)
@@ -774,28 +830,26 @@ class OEBBook(object):
                     element.attrib[nsname] = element.attrib[name]
                     del element.attrib[name]
             metadata.append(element)
-        for element in opf.xpath('metadata//meta'):
+        for element in xpath(opf, 'o2:metadata//o2:meta'):
             metadata.append(element)
-        for item in opf.xpath('manifest/item'):
+        for tag in ('o2:manifest', 'o2:spine', 'o2:tours', 'o2:guide'):
-            media_type = item.attrib['media-type'].lower()
+            for element in xpath(opf, tag):
-            if media_type in OEB_DOCS:
-                media_type = XHTML_MIME
-            elif media_type in OEB_STYLES:
-                media_type = CSS_MIME
-            item.attrib['media-type'] = media_type
-        for tag in ('manifest', 'spine', 'tours', 'guide'):
-            for element in opf.xpath(tag):
                 nroot.append(element)
-        return etree.fromstring(etree.tostring(nroot))
+        return nroot
     
     def _read_opf(self, opfpath):
-        opf = self.container.read_xml(opfpath)
+        opf = self.container.read(opfpath)
-        version = float(opf.get('version', 1.0))
+        try:
+            opf = etree.fromstring(opf)
+        except etree.XMLSyntaxError:
+            repl = lambda m: ENTITYDEFS.get(m.group(1), m.group(0))
+            opf = ENTITY_RE.sub(repl, opf)
+            opf = etree.fromstring(opf)
+            self.logger.warn('OPF contains invalid HTML named entities')
         ns = namespace(opf.tag)
         if ns not in ('', OPF1_NS, OPF2_NS):
             raise OEBError('Invalid namespace %r for OPF document' % ns)
-        if ns != OPF2_NS or version < 2.0:
+        opf = self._clean_opf(opf)
-            opf = self._convert_opf1(opf)
         return opf
     
     def _metadata_from_opf(self, opf):
@@ -804,8 +858,16 @@ class OEBBook(object):
         self.metadata = metadata = Metadata(self)
         ignored = (OPF('dc-metadata'), OPF('x-metadata'))
         for elem in xpath(opf, '/o2:package/o2:metadata//*'):
-            if elem.tag not in ignored and (elem.text or elem.attrib):
+            if elem.tag in ignored: continue
-                metadata.add(elem.tag, elem.text, elem.attrib)
+            term = elem.tag
+            value = elem.text
+            if term == OPF('meta'):
+                term = elem.attrib.pop('name', None)
+                value = elem.attrib.pop('content', None)
+            if value:
+                value = COLLAPSE_RE.sub(' ', value.strip())
+            if term and (value or elem.attrib):
+                metadata.add(term, value, elem.attrib)
         haveuuid = haveid = False
         for ident in metadata.identifier:
             if unicode(ident).startswith('urn:uuid:'):
@@ -820,36 +882,38 @@ class OEBBook(object):
                 self.uid = item
                 break
         else:
-            self.logger.warn(u'Unique-identifier %r not found.' % uid)
+            self.logger.warn(u'Unique-identifier %r not found' % uid)
             for ident in metadata.identifier:
                 if 'id' in ident.attrib:
                     self.uid = metadata.identifier[0]
                     break
         if not metadata.language:
-            self.logger.warn(u'Language not specified.')
+            self.logger.warn(u'Language not specified')
             metadata.add('language', get_lang())
         if not metadata.creator:
-            self.logger.warn(u'Creator not specified.')
+            self.logger.warn('Creator not specified')
-            metadata.add('creator', _('Unknown'))
+            metadata.add('creator', self.translate(__('Unknown')))
         if not metadata.title:
-            self.logger.warn(u'Title not specified.')
+            self.logger.warn('Title not specified')
-            metadata.add('title', _('Unknown'))
+            metadata.add('title', self.translate(__('Unknown')))
     
     def _manifest_from_opf(self, opf):
         self.manifest = manifest = Manifest(self)
         for elem in xpath(opf, '/o2:package/o2:manifest/o2:item'):
             id = elem.get('id')
             href = elem.get('href')
-            media_type = elem.get('media-type')
+            media_type = elem.get('media-type', None)
+            if media_type is None:
+                media_type = elem.get('mediatype', BINARY_MIME)
             fallback = elem.get('fallback')
             if href in manifest.hrefs:
-                self.logger.warn(u'Duplicate manifest entry for %r.' % href)
+                self.logger.warn(u'Duplicate manifest entry for %r' % href)
                 continue
             if not self.container.exists(href):
-                self.logger.warn(u'Manifest item %r not found.' % href)
+                self.logger.warn(u'Manifest item %r not found' % href)
                 continue
             if id in manifest.ids:
-                self.logger.warn(u'Duplicate manifest id %r.' % id)
+                self.logger.warn(u'Duplicate manifest id %r' % id)
                 id, href = manifest.generate(id, href)
             manifest.add(id, href, media_type, fallback)
     
@@ -858,7 +922,7 @@ class OEBBook(object):
         for elem in xpath(opf, '/o2:package/o2:spine/o2:itemref'):
             idref = elem.get('idref')
             if idref not in self.manifest:
-                self.logger.warn(u'Spine item %r not found.' % idref)
+                self.logger.warn(u'Spine item %r not found' % idref)
                 continue
             item = self.manifest[idref]
             spine.add(item, elem.get('linear'))
@@ -906,7 +970,8 @@ class OEBBook(object):
         item = self.manifest.ids[id]
         ncx = item.data
         self.manifest.remove(item)
-        title = xpath(ncx, 'ncx:docTitle/ncx:text/text()')[0]
+        title = xpath(ncx, 'ncx:docTitle/ncx:text/text()')
+        title = title[0].strip() if title else unicode(self.metadata.title)
         self.toc = toc = TOC(title)
         navmaps = xpath(ncx, 'ncx:navMap')
         for navmap in navmaps:
@@ -963,7 +1028,8 @@ class OEBBook(object):
             if not item.linear: continue
             html = item.data
             title = xpath(html, '/h:html/h:head/h:title/text()')
-            if title: titles.append(title[0])
+            title = title[0].strip() if title else None
+            if title: titles.append(title)
             headers.append('(unlabled)')
             for tag in ('h1', 'h2', 'h3', 'h4', 'h5', 'strong'):
                 expr = '/h:html/h:body//h:%s[position()=1]/text()' % (tag,)
@@ -987,9 +1053,19 @@ class OEBBook(object):
 
     def _ensure_cover_image(self):
         cover = None
-        spine0 = self.spine[0]
+        hcover = self.spine[0]
-        html = spine0.data
+        if 'cover' in self.guide:
-        if self.metadata.cover:
+            href = self.guide['cover'].href
+            item = self.manifest.hrefs[href]
+            media_type = item.media_type
+            if media_type in OEB_RASTER_IMAGES:
+                cover = item
+            elif media_type in OEB_DOCS:
+                hcover = item
+        html = hcover.data
+        if cover is not None:
+            pass
+        elif self.metadata.cover:
             id = str(self.metadata.cover[0])
             cover = self.manifest.ids[id]
         elif MS_COVER_TYPE in self.guide:
@@ -997,16 +1073,16 @@ class OEBBook(object):
             cover = self.manifest.hrefs[href]
         elif xpath(html, '//h:img[position()=1]'):
             img = xpath(html, '//h:img[position()=1]')[0]
-            href = spine0.abshref(img.get('src'))
+            href = hcover.abshref(img.get('src'))
             cover = self.manifest.hrefs[href]
         elif xpath(html, '//h:object[position()=1]'):
             object = xpath(html, '//h:object[position()=1]')[0]
-            href = spine0.abshref(object.get('data'))
+            href = hcover.abshref(object.get('data'))
             cover = self.manifest.hrefs[href]
         elif xpath(html, '//svg:svg[position()=1]'):
             svg = copy.deepcopy(xpath(html, '//svg:svg[position()=1]')[0])
-            href = os.path.splitext(spine0.href)[0] + '.svg'
+            href = os.path.splitext(hcover.href)[0] + '.svg'
-            id, href = self.manifest.generate(spine0.id, href)
+            id, href = self.manifest.generate(hcover.id, href)
             cover = self.manifest.add(id, href, SVG_MIME, data=svg)
         if cover and not self.metadata.cover:
             self.metadata.add('cover', cover.id)

src/calibre/ebooks/oeb/entitydefs.py

@@ -0,0 +1,256 @@
+"""
+Replacement for htmlentitydefs which uses purely numeric entities.
+"""
+
+__license__   = 'GPL v3'
+__copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
+
+ENTITYDEFS = \
+    {'AElig': '&#198;',
+     'Aacute': '&#193;',
+     'Acirc': '&#194;',
+     'Agrave': '&#192;',
+     'Alpha': '&#913;',
+     'Aring': '&#197;',
+     'Atilde': '&#195;',
+     'Auml': '&#196;',
+     'Beta': '&#914;',
+     'Ccedil': '&#199;',
+     'Chi': '&#935;',
+     'Dagger': '&#8225;',
+     'Delta': '&#916;',
+     'ETH': '&#208;',
+     'Eacute': '&#201;',
+     'Ecirc': '&#202;',
+     'Egrave': '&#200;',
+     'Epsilon': '&#917;',
+     'Eta': '&#919;',
+     'Euml': '&#203;',
+     'Gamma': '&#915;',
+     'Iacute': '&#205;',
+     'Icirc': '&#206;',
+     'Igrave': '&#204;',
+     'Iota': '&#921;',
+     'Iuml': '&#207;',
+     'Kappa': '&#922;',
+     'Lambda': '&#923;',
+     'Mu': '&#924;',
+     'Ntilde': '&#209;',
+     'Nu': '&#925;',
+     'OElig': '&#338;',
+     'Oacute': '&#211;',
+     'Ocirc': '&#212;',
+     'Ograve': '&#210;',
+     'Omega': '&#937;',
+     'Omicron': '&#927;',
+     'Oslash': '&#216;',
+     'Otilde': '&#213;',
+     'Ouml': '&#214;',
+     'Phi': '&#934;',
+     'Pi': '&#928;',
+     'Prime': '&#8243;',
+     'Psi': '&#936;',
+     'Rho': '&#929;',
+     'Scaron': '&#352;',
+     'Sigma': '&#931;',
+     'THORN': '&#222;',
+     'Tau': '&#932;',
+     'Theta': '&#920;',
+     'Uacute': '&#218;',
+     'Ucirc': '&#219;',
+     'Ugrave': '&#217;',
+     'Upsilon': '&#933;',
+     'Uuml': '&#220;',
+     'Xi': '&#926;',
+     'Yacute': '&#221;',
+     'Yuml': '&#376;',
+     'Zeta': '&#918;',
+     'aacute': '&#225;',
+     'acirc': '&#226;',
+     'acute': '&#180;',
+     'aelig': '&#230;',
+     'agrave': '&#224;',
+     'alefsym': '&#8501;',
+     'alpha': '&#945;',
+     'and': '&#8743;',
+     'ang': '&#8736;',
+     'aring': '&#229;',
+     'asymp': '&#8776;',
+     'atilde': '&#227;',
+     'auml': '&#228;',
+     'bdquo': '&#8222;',
+     'beta': '&#946;',
+     'brvbar': '&#166;',
+     'bull': '&#8226;',
+     'cap': '&#8745;',
+     'ccedil': '&#231;',
+     'cedil': '&#184;',
+     'cent': '&#162;',
+     'chi': '&#967;',
+     'circ': '&#710;',
+     'clubs': '&#9827;',
+     'cong': '&#8773;',
+     'copy': '&#169;',
+     'crarr': '&#8629;',
+     'cup': '&#8746;',
+     'curren': '&#164;',
+     'dArr': '&#8659;',
+     'dagger': '&#8224;',
+     'darr': '&#8595;',
+     'deg': '&#176;',
+     'delta': '&#948;',
+     'diams': '&#9830;',
+     'divide': '&#247;',
+     'eacute': '&#233;',
+     'ecirc': '&#234;',
+     'egrave': '&#232;',
+     'empty': '&#8709;',
+     'emsp': '&#8195;',
+     'ensp': '&#8194;',
+     'epsilon': '&#949;',
+     'equiv': '&#8801;',
+     'eta': '&#951;',
+     'eth': '&#240;',
+     'euml': '&#235;',
+     'euro': '&#8364;',
+     'exist': '&#8707;',
+     'fnof': '&#402;',
+     'forall': '&#8704;',
+     'frac12': '&#189;',
+     'frac14': '&#188;',
+     'frac34': '&#190;',
+     'frasl': '&#8260;',
+     'gamma': '&#947;',
+     'ge': '&#8805;',
+     'hArr': '&#8660;',
+     'harr': '&#8596;',
+     'hearts': '&#9829;',
+     'hellip': '&#8230;',
+     'iacute': '&#237;',
+     'icirc': '&#238;',
+     'iexcl': '&#161;',
+     'igrave': '&#236;',
+     'image': '&#8465;',
+     'infin': '&#8734;',
+     'int': '&#8747;',
+     'iota': '&#953;',
+     'iquest': '&#191;',
+     'isin': '&#8712;',
+     'iuml': '&#239;',
+     'kappa': '&#954;',
+     'lArr': '&#8656;',
+     'lambda': '&#955;',
+     'lang': '&#9001;',
+     'laquo': '&#171;',
+     'larr': '&#8592;',
+     'lceil': '&#8968;',
+     'ldquo': '&#8220;',
+     'le': '&#8804;',
+     'lfloor': '&#8970;',
+     'lowast': '&#8727;',
+     'loz': '&#9674;',
+     'lrm': '&#8206;',
+     'lsaquo': '&#8249;',
+     'lsquo': '&#8216;',
+     'macr': '&#175;',
+     'mdash': '&#8212;',
+     'micro': '&#181;',
+     'middot': '&#183;',
+     'minus': '&#8722;',
+     'mu': '&#956;',
+     'nabla': '&#8711;',
+     'nbsp': '&#160;',
+     'ndash': '&#8211;',
+     'ne': '&#8800;',
+     'ni': '&#8715;',
+     'not': '&#172;',
+     'notin': '&#8713;',
+     'nsub': '&#8836;',
+     'ntilde': '&#241;',
+     'nu': '&#957;',
+     'oacute': '&#243;',
+     'ocirc': '&#244;',
+     'oelig': '&#339;',
+     'ograve': '&#242;',
+     'oline': '&#8254;',
+     'omega': '&#969;',
+     'omicron': '&#959;',
+     'oplus': '&#8853;',
+     'or': '&#8744;',
+     'ordf': '&#170;',
+     'ordm': '&#186;',
+     'oslash': '&#248;',
+     'otilde': '&#245;',
+     'otimes': '&#8855;',
+     'ouml': '&#246;',
+     'para': '&#182;',
+     'part': '&#8706;',
+     'permil': '&#8240;',
+     'perp': '&#8869;',
+     'phi': '&#966;',
+     'pi': '&#960;',
+     'piv': '&#982;',
+     'plusmn': '&#177;',
+     'pound': '&#163;',
+     'prime': '&#8242;',
+     'prod': '&#8719;',
+     'prop': '&#8733;',
+     'psi': '&#968;',
+     'rArr': '&#8658;',
+     'radic': '&#8730;',
+     'rang': '&#9002;',
+     'raquo': '&#187;',
+     'rarr': '&#8594;',
+     'rceil': '&#8969;',
+     'rdquo': '&#8221;',
+     'real': '&#8476;',
+     'reg': '&#174;',
+     'rfloor': '&#8971;',
+     'rho': '&#961;',
+     'rlm': '&#8207;',
+     'rsaquo': '&#8250;',
+     'rsquo': '&#8217;',
+     'sbquo': '&#8218;',
+     'scaron': '&#353;',
+     'sdot': '&#8901;',
+     'sect': '&#167;',
+     'shy': '&#173;',
+     'sigma': '&#963;',
+     'sigmaf': '&#962;',
+     'sim': '&#8764;',
+     'spades': '&#9824;',
+     'sub': '&#8834;',
+     'sube': '&#8838;',
+     'sum': '&#8721;',
+     'sup': '&#8835;',
+     'sup1': '&#185;',
+     'sup2': '&#178;',
+     'sup3': '&#179;',
+     'supe': '&#8839;',
+     'szlig': '&#223;',
+     'tau': '&#964;',
+     'there4': '&#8756;',
+     'theta': '&#952;',
+     'thetasym': '&#977;',
+     'thinsp': '&#8201;',
+     'thorn': '&#254;',
+     'tilde': '&#732;',
+     'times': '&#215;',
+     'trade': '&#8482;',
+     'uArr': '&#8657;',
+     'uacute': '&#250;',
+     'uarr': '&#8593;',
+     'ucirc': '&#251;',
+     'ugrave': '&#249;',
+     'uml': '&#168;',
+     'upsih': '&#978;',
+     'upsilon': '&#965;',
+     'uuml': '&#252;',
+     'weierp': '&#8472;',
+     'xi': '&#958;',
+     'yacute': '&#253;',
+     'yen': '&#165;',
+     'yuml': '&#255;',
+     'zeta': '&#950;',
+     'zwj': '&#8205;',
+     'zwnj': '&#8204;'}

src/calibre/ebooks/oeb/stylizer.py

@@ -110,7 +110,8 @@ class Stylizer(object):
     
     def __init__(self, tree, path, oeb, profile=PROFILES['PRS505']):
         self.profile = profile
-        base = os.path.dirname(path)
+        self.logger = oeb.logger
+        item = oeb.manifest.hrefs[path]
         basename = os.path.basename(path)
         cssname = os.path.splitext(basename)[0] + '.css'
         stylesheets = [HTML_CSS_STYLESHEET]
@@ -128,8 +129,12 @@ class Stylizer(object):
                  and elem.get('rel', 'stylesheet') == 'stylesheet' \
                  and elem.get('type', CSS_MIME) in OEB_STYLES:
                 href = urlnormalize(elem.attrib['href'])
-                path = os.path.join(base, href)
+                path = item.abshref(href)
-                path = os.path.normpath(path).replace('\\', '/')
+                if path not in oeb.manifest.hrefs:
+                    self.logger.warn(
+                        'Stylesheet %r referenced by file %r not in manifest' %
+                        (path, item.href))
+                    continue
                 if path in self.STYLESHEETS:
                     stylesheet = self.STYLESHEETS[path]
                 else:
@@ -277,10 +282,9 @@ class Style(object):
     def _apply_style_attr(self):
         attrib = self._element.attrib
         if 'style' in attrib:
-            css = attrib['style'].strip()
+            css = attrib['style'].split(';')
-            if css.startswith(';'):
+            css = filter(None, map(lambda x: x.strip(), css))
-                css = css[1:]
+            style = CSSStyleDeclaration('; '.join(css))
-            style = CSSStyleDeclaration(css)
             self._style.update(self._stylizer.flatten_style(style))
 
     def _has_parent(self):

src/calibre/ebooks/oeb/transforms/flatcss.py

@@ -207,7 +207,8 @@ class CSSFlattener(object):
             items = cssdict.items()
             items.sort()
             css = u';\n'.join(u'%s: %s' % (key, val) for key, val in items)
-            klass = STRIPNUM.sub('', node.get('class', 'calibre').split()[0])
+            classes = node.get('class', None) or 'calibre'
+            klass = STRIPNUM.sub('', classes.split()[0])
             if css in styles:
                 match = styles[css]
             else:

src/calibre/ebooks/oeb/transforms/rasterize.py

@@ -46,9 +46,10 @@ class SVGRasterizer(object):
         data = QByteArray(xml2str(elem))
         svg = QSvgRenderer(data)
         size = svg.defaultSize()
+        view_box = elem.get('viewBox', elem.get('viewbox', None))
         if size.width() == 100 and size.height() == 100 \
-           and 'viewBox' in elem.attrib:
+           and view_box is not None:
-            box = [float(x) for x in elem.attrib['viewBox'].split()]
+            box = [float(x) for x in view_box.split()]
             size.setWidth(box[2] - box[0])
             size.setHeight(box[3] - box[1])
         if width or height:

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -13,6 +13,7 @@ from urlparse import urldefrag
 from lxml import etree
 import cssutils
 from calibre.ebooks.oeb.base import XPNSMAP, CSS_MIME, OEB_DOCS
+from calibre.ebooks.oeb.base import urlnormalize
 
 LINK_SELECTORS = []
 for expr in ('//h:link/@href', '//h:img/@src', '//h:object/@data',
@@ -46,7 +47,7 @@ class ManifestTrimmer(object):
                    item.data is not None:
                     hrefs = [sel(item.data) for sel in LINK_SELECTORS]
                     for href in chain(*hrefs):
-                        href = item.abshref(href)
+                        href = item.abshref(urlnormalize(href))
                         if href in oeb.manifest.hrefs:
                             found = oeb.manifest.hrefs[href]
                             if found not in used:

src/calibre/gui2/dialogs/epub.py

@@ -61,6 +61,7 @@ class Config(ResizableDialog, Ui_Dialog):
         self.opt_toc_title.setVisible(False)
         self.toc_title_label.setVisible(False)
         self.opt_rescale_images.setVisible(False)
+        self.opt_ignore_tables.setVisible(False)
         
     def initialize(self):
         self.__w = []

src/calibre/gui2/dialogs/epub.ui

@@ -93,7 +93,7 @@
        <item>
         <widget class="QStackedWidget" name="stack" >
          <property name="currentIndex" >
-          <number>0</number>
+          <number>1</number>
          </property>
          <widget class="QWidget" name="metadata_page" >
           <layout class="QGridLayout" name="gridLayout_4" >
@@ -105,6 +105,36 @@
                 <string>Book Cover</string>
                </property>
                <layout class="QGridLayout" name="_2" >
+                <item row="0" column="0" >
+                 <layout class="QHBoxLayout" name="_3" >
+                  <item>
+                   <widget class="ImageView" name="cover" >
+                    <property name="text" >
+                     <string/>
+                    </property>
+                    <property name="pixmap" >
+                     <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
+                    </property>
+                    <property name="scaledContents" >
+                     <bool>true</bool>
+                    </property>
+                    <property name="alignment" >
+                     <set>Qt::AlignCenter</set>
+                    </property>
+                   </widget>
+                  </item>
+                 </layout>
+                </item>
+                <item row="2" column="0" >
+                 <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
+                  <property name="text" >
+                   <string>Use cover from &amp;source file</string>
+                  </property>
+                  <property name="checked" >
+                   <bool>true</bool>
+                  </property>
+                 </widget>
+                </item>
                 <item row="1" column="0" >
                  <layout class="QVBoxLayout" name="_4" >
                   <property name="spacing" >
@@ -156,36 +186,6 @@
                   </item>
                  </layout>
                 </item>
-                <item row="2" column="0" >
-                 <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
-                  <property name="text" >
-                   <string>Use cover from &amp;source file</string>
-                  </property>
-                  <property name="checked" >
-                   <bool>true</bool>
-                  </property>
-                 </widget>
-                </item>
-                <item row="0" column="0" >
-                 <layout class="QHBoxLayout" name="_3" >
-                  <item>
-                   <widget class="ImageView" name="cover" >
-                    <property name="text" >
-                     <string/>
-                    </property>
-                    <property name="pixmap" >
-                     <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
-                    </property>
-                    <property name="scaledContents" >
-                     <bool>true</bool>
-                    </property>
-                    <property name="alignment" >
-                     <set>Qt::AlignCenter</set>
-                    </property>
-                   </widget>
-                  </item>
-                 </layout>
-                </item>
                </layout>
                <zorder>opt_prefer_metadata_cover</zorder>
                <zorder></zorder>
@@ -479,6 +479,13 @@
                </property>
               </widget>
              </item>
+             <item row="5" column="0" >
+              <widget class="QCheckBox" name="opt_ignore_tables" >
+               <property name="text" >
+                <string>&amp;Ignore tables</string>
+               </property>
+              </widget>
+             </item>
             </layout>
            </item>
            <item>

src/calibre/gui2/dialogs/metadata_single.ui

@@ -6,7 +6,7 @@
     <x>0</x>
     <y>0</y>
     <width>887</width>
-    <height>740</height>
+    <height>750</height>
    </rect>
   </property>
   <property name="sizePolicy" >
@@ -43,7 +43,7 @@
         <x>0</x>
         <y>0</y>
         <width>879</width>
-        <height>700</height>
+        <height>710</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_5" >
@@ -55,7 +55,7 @@
          <property name="minimumSize" >
           <size>
            <width>800</width>
-           <height>685</height>
+           <height>665</height>
           </size>
          </property>
          <layout class="QVBoxLayout" name="verticalLayout_3" >

src/calibre/gui2/dialogs/scheduler.py

@@ -7,11 +7,11 @@ __docformat__ = 'restructuredtext en'
 Scheduler for automated recipe downloads
 '''
 
-import sys, copy
+import sys, copy, time
 from datetime import datetime, timedelta
 from PyQt4.Qt import QDialog, QApplication, QLineEdit, QPalette, SIGNAL, QBrush, \
                      QColor, QAbstractListModel, Qt, QVariant, QFont, QIcon, \
-                     QFile, QObject, QTimer, QMutex, QMenu, QAction
+                     QFile, QObject, QTimer, QMutex, QMenu, QAction, QTime
 
 from calibre import english_sort
 from calibre.gui2.dialogs.scheduler_ui import Ui_Dialog
@@ -66,7 +66,10 @@ class Recipe(object):
         return self.id == getattr(other, 'id', None)
     
     def __repr__(self):
-        return u'%s|%s|%s|%s'%(self.id, self.title, self.last_downloaded.ctime(), self.schedule)
+        schedule = self.schedule
+        if schedule and schedule > 1e5:
+            schedule = decode_schedule(schedule)
+        return u'%s|%s|%s|%s'%(self.id, self.title, self.last_downloaded.ctime(), schedule)
     
 builtin_recipes = [Recipe(m, r, True) for r, m in zip(recipes, recipe_modules)]
 
@@ -170,6 +173,11 @@ class RecipeModel(QAbstractListModel, SearchQueryParser):
         
         return NONE
     
+    def update_recipe_schedule(self, recipe):
+        for srecipe in self.recipes:
+            if srecipe == recipe:
+                srecipe.schedule = recipe.schedule
+            
 
 class Search(QLineEdit):
     
@@ -210,7 +218,17 @@ class Search(QLineEdit):
             text = unicode(self.text())
             self.emit(SIGNAL('search(PyQt_PyObject)'), text)
 
+def encode_schedule(day, hour, minute):
+    day = 1e7 * (day+1)
+    hour = 1e4 * (hour+1)
+    return day + hour + minute + 1
 
+def decode_schedule(num):
+    raw = '%d'%int(num)
+    day = int(raw[0])
+    hour = int(raw[2:4])
+    minute = int(raw[-2:])
+    return day-1, hour-1, minute-1
 
 class SchedulerDialog(QDialog, Ui_Dialog):
     
@@ -228,17 +246,22 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         self.connect(self.username, SIGNAL('textEdited(QString)'), self.set_account_info)
         self.connect(self.password, SIGNAL('textEdited(QString)'), self.set_account_info)
         self.connect(self.schedule, SIGNAL('stateChanged(int)'), self.do_schedule)
-        self.connect(self.schedule, SIGNAL('stateChanged(int)'), 
-                     lambda state: self.interval.setEnabled(state == Qt.Checked))
         self.connect(self.show_password, SIGNAL('stateChanged(int)'),
                      lambda state: self.password.setEchoMode(self.password.Normal if state == Qt.Checked else self.password.Password))
         self.connect(self.interval, SIGNAL('valueChanged(double)'), self.do_schedule)
+        self.connect(self.day, SIGNAL('currentIndexChanged(int)'), self.do_schedule)
+        self.connect(self.time, SIGNAL('timeChanged(QTime)'), self.do_schedule)
+        for button in (self.daily_button, self.interval_button):
+            self.connect(button, SIGNAL('toggled(bool)'), self.do_schedule)
         self.connect(self.search, SIGNAL('search(PyQt_PyObject)'), self._model.search)
         self.connect(self._model, SIGNAL('modelReset()'), lambda : self.detail_box.setVisible(False))
         self.connect(self.download, SIGNAL('clicked()'), self.download_now)
         self.search.setFocus(Qt.OtherFocusReason)
         self.old_news.setValue(gconf['oldest_news'])
         self.rnumber.setText(_('%d recipes')%self._model.rowCount(None))
+        for day in (_('day'), _('Monday'), _('Tuesday'), _('Wednesday'), 
+                    _('Thursday'), _('Friday'), _('Saturday'), _('Sunday')):
+            self.day.addItem(day)
         
     def download_now(self):
         recipe = self._model.data(self.recipes.currentIndex(), Qt.UserRole)
@@ -252,6 +275,8 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         config[key] = (username, password) if username and password else None
         
     def do_schedule(self, *args):
+        if not getattr(self, 'allow_scheduling', False):
+            return
         recipe = self.recipes.currentIndex()
         if not recipe.isValid():
             return
@@ -263,17 +288,26 @@ class SchedulerDialog(QDialog, Ui_Dialog):
             else:
                 recipe.last_downloaded = datetime.fromordinal(1)
                 recipes.append(recipe)
-            recipe.schedule = self.interval.value()
-            if recipe.schedule < 0.1:
-                recipe.schedule = 1/24.
             if recipe.needs_subscription and not config['recipe_account_info_%s'%recipe.id]:
                 error_dialog(self, _('Must set account information'), _('This recipe requires a username and password')).exec_()
                 self.schedule.setCheckState(Qt.Unchecked)
                 return
+            if self.interval_button.isChecked():
+                recipe.schedule = self.interval.value()
+                if recipe.schedule < 0.1:
+                    recipe.schedule = 1/24.
+            else:
+                day_of_week = self.day.currentIndex() - 1
+                if day_of_week < 0:
+                    day_of_week = 7
+                t = self.time.time()
+                hour, minute = t.hour(), t.minute()
+                recipe.schedule = encode_schedule(day_of_week, hour, minute)
         else:
             if recipe in recipes:
                 recipes.remove(recipe)
         save_recipes(recipes)
+        self._model.update_recipe_schedule(recipe)
         self.emit(SIGNAL('new_schedule(PyQt_PyObject)'), recipes)
                 
     def show_recipe(self, index):
@@ -282,8 +316,26 @@ class SchedulerDialog(QDialog, Ui_Dialog):
         self.title.setText(recipe.title)
         self.author.setText(_('Created by: ') + recipe.author)
         self.description.setText(recipe.description if recipe.description else '')
+        self.allow_scheduling = False
+        schedule = -1 if recipe.schedule is None else recipe.schedule
+        if schedule < 1e5 and schedule >= 0:
+            self.interval.setValue(schedule)
+            self.interval_button.setChecked(True)
+            self.day.setEnabled(False), self.time.setEnabled(False)
+        else:
+            if schedule > 0:
+                day, hour, minute = decode_schedule(schedule)
+            else:
+                day, hour, minute = 7, 12, 0
+            if day == 7:
+                day = -1
+            self.day.setCurrentIndex(day+1)
+            self.time.setTime(QTime(hour, minute))
+            self.daily_button.setChecked(True)
+            self.interval_button.setChecked(False)
+            self.interval.setEnabled(False)
         self.schedule.setChecked(recipe.schedule is not None)
-        self.interval.setValue(recipe.schedule if recipe.schedule is not None else 1)
+        self.allow_scheduling = True    
         self.detail_box.setVisible(True)
         self.account.setVisible(recipe.needs_subscription)
         self.interval.setEnabled(self.schedule.checkState() == Qt.Checked)
@@ -365,13 +417,22 @@ class Scheduler(QObject):
                 self.dirtied = False
             needs_downloading = set([])
             self.debug('Checking...')
-            now = datetime.utcnow()
+            nowt = datetime.utcnow()
             for recipe in self.recipes:
                 if recipe.schedule is None:
                     continue
-                delta = now - recipe.last_downloaded
+                delta = nowt - recipe.last_downloaded
-                if delta > timedelta(days=recipe.schedule):
+                if recipe.schedule < 1e5:
-                    needs_downloading.add(recipe)
+                    if delta > timedelta(days=recipe.schedule):
+                        needs_downloading.add(recipe)
+                else:
+                    day, hour, minute = decode_schedule(recipe.schedule)
+                    now = time.localtime()
+                    day_matches = day > 6 or day == now.tm_wday
+                    tnow = now.tm_hour*60 + now.tm_min
+                    matches = day_matches and (hour*60+minute) < tnow
+                    if matches and delta >= timedelta(days=1):
+                        needs_downloading.add(recipe)
                 
             self.debug('Needs downloading:', needs_downloading)
         

src/calibre/gui2/dialogs/scheduler.ui

@@ -6,7 +6,7 @@
     <x>0</x>
     <y>0</y>
     <width>726</width>
-    <height>551</height>
+    <height>575</height>
    </rect>
   </property>
   <property name="windowTitle" >
@@ -42,25 +42,12 @@
    </item>
    <item row="0" column="1" >
     <layout class="QVBoxLayout" name="verticalLayout_3" >
-     <item>
-      <spacer name="horizontalSpacer_3" >
-       <property name="orientation" >
-        <enum>Qt::Horizontal</enum>
-       </property>
-       <property name="sizeHint" stdset="0" >
-        <size>
-         <width>40</width>
-         <height>0</height>
-        </size>
-       </property>
-      </spacer>
-     </item>
      <item>
       <widget class="QGroupBox" name="detail_box" >
        <property name="title" >
         <string>Schedule for download</string>
        </property>
-       <layout class="QVBoxLayout" name="verticalLayout_2" >
+       <layout class="QVBoxLayout" name="verticalLayout_4" >
         <item>
          <widget class="QLabel" name="title" >
           <property name="font" >
@@ -110,70 +97,97 @@
         <item>
          <widget class="QCheckBox" name="schedule" >
           <property name="text" >
-           <string>&amp;Schedule for download every:</string>
+           <string>&amp;Schedule for download:</string>
           </property>
          </widget>
         </item>
         <item>
-         <layout class="QHBoxLayout" name="horizontalLayout" >
+         <widget class="QWidget" native="1" name="widget" >
-          <item>
+          <property name="enabled" >
-           <spacer name="horizontalSpacer" >
+           <bool>false</bool>
-            <property name="orientation" >
+          </property>
-             <enum>Qt::Horizontal</enum>
+          <layout class="QVBoxLayout" name="verticalLayout_2" >
-            </property>
+           <item>
-            <property name="sizeHint" stdset="0" >
+            <layout class="QHBoxLayout" name="horizontalLayout_2" >
-             <size>
+             <item>
-              <width>40</width>
+              <widget class="QRadioButton" name="daily_button" >
-              <height>20</height>
+               <property name="text" >
-             </size>
+                <string>Every </string>
-            </property>
+               </property>
-           </spacer>
+              </widget>
-          </item>
+             </item>
-          <item>
+             <item>
-           <widget class="QDoubleSpinBox" name="interval" >
+              <widget class="QComboBox" name="day" />
-            <property name="sizePolicy" >
+             </item>
-             <sizepolicy vsizetype="Fixed" hsizetype="Expanding" >
+             <item>
-              <horstretch>0</horstretch>
+              <widget class="QLabel" name="label_4" >
-              <verstretch>0</verstretch>
+               <property name="text" >
-             </sizepolicy>
+                <string>at</string>
-            </property>
+               </property>
-            <property name="toolTip" >
+              </widget>
-             <string>Interval at which to download this recipe. A value of zero means that the recipe will be downloaded every hour.</string>
+             </item>
-            </property>
+             <item>
-            <property name="suffix" >
+              <widget class="QTimeEdit" name="time" />
-             <string> days</string>
+             </item>
-            </property>
+             <item>
-            <property name="decimals" >
+              <spacer name="horizontalSpacer" >
-             <number>1</number>
+               <property name="orientation" >
-            </property>
+                <enum>Qt::Horizontal</enum>
-            <property name="minimum" >
+               </property>
-             <double>0.000000000000000</double>
+               <property name="sizeHint" stdset="0" >
-            </property>
+                <size>
-            <property name="maximum" >
+                 <width>40</width>
-             <double>365.100000000000023</double>
+                 <height>20</height>
-            </property>
+                </size>
-            <property name="singleStep" >
+               </property>
-             <double>1.000000000000000</double>
+              </spacer>
-            </property>
+             </item>
-            <property name="value" >
+            </layout>
-             <double>1.000000000000000</double>
+           </item>
-            </property>
+           <item>
-           </widget>
+            <layout class="QHBoxLayout" name="horizontalLayout" >
-          </item>
+             <item>
-          <item>
+              <widget class="QRadioButton" name="interval_button" >
-           <spacer name="horizontalSpacer_2" >
+               <property name="text" >
-            <property name="orientation" >
+                <string>Every </string>
-             <enum>Qt::Horizontal</enum>
+               </property>
-            </property>
+              </widget>
-            <property name="sizeHint" stdset="0" >
+             </item>
-             <size>
+             <item>
-              <width>40</width>
+              <widget class="QDoubleSpinBox" name="interval" >
-              <height>20</height>
+               <property name="sizePolicy" >
-             </size>
+                <sizepolicy vsizetype="Fixed" hsizetype="Expanding" >
-            </property>
+                 <horstretch>0</horstretch>
-           </spacer>
+                 <verstretch>0</verstretch>
-          </item>
+                </sizepolicy>
-         </layout>
+               </property>
+               <property name="toolTip" >
+                <string>Interval at which to download this recipe. A value of zero means that the recipe will be downloaded every hour.</string>
+               </property>
+               <property name="suffix" >
+                <string> days</string>
+               </property>
+               <property name="decimals" >
+                <number>1</number>
+               </property>
+               <property name="minimum" >
+                <double>0.000000000000000</double>
+               </property>
+               <property name="maximum" >
+                <double>365.100000000000023</double>
+               </property>
+               <property name="singleStep" >
+                <double>1.000000000000000</double>
+               </property>
+               <property name="value" >
+                <double>1.000000000000000</double>
+               </property>
+              </widget>
+             </item>
+            </layout>
+           </item>
+          </layout>
+         </widget>
         </item>
         <item>
          <widget class="QLabel" name="last_downloaded" >
@@ -315,8 +329,8 @@
    <slot>accept()</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>248</x>
+     <x>613</x>
-     <y>254</y>
+     <y>824</y>
     </hint>
     <hint type="destinationlabel" >
      <x>157</x>
@@ -331,8 +345,8 @@
    <slot>reject()</slot>
    <hints>
     <hint type="sourcelabel" >
-     <x>316</x>
+     <x>681</x>
-     <y>260</y>
+     <y>824</y>
     </hint>
     <hint type="destinationlabel" >
      <x>286</x>
@@ -340,5 +354,85 @@
     </hint>
    </hints>
   </connection>
+  <connection>
+   <sender>schedule</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>widget</receiver>
+   <slot>setDisabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>454</x>
+     <y>147</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>461</x>
+     <y>168</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>schedule</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>widget</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>458</x>
+     <y>137</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>461</x>
+     <y>169</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>daily_button</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>day</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>421</x>
+     <y>186</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>500</x>
+     <y>184</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>daily_button</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>time</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>442</x>
+     <y>193</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>603</x>
+     <y>183</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>interval_button</sender>
+   <signal>toggled(bool)</signal>
+   <receiver>interval</receiver>
+   <slot>setEnabled(bool)</slot>
+   <hints>
+    <hint type="sourcelabel" >
+     <x>428</x>
+     <y>213</y>
+    </hint>
+    <hint type="destinationlabel" >
+     <x>495</x>
+     <y>218</y>
+    </hint>
+   </hints>
+  </connection>
  </connections>
 </ui>

src/calibre/gui2/dialogs/user_profiles.ui

@@ -5,8 +5,8 @@
    <rect>
     <x>0</x>
     <y>0</y>
-    <width>719</width>
+    <width>738</width>
-    <height>612</height>
+    <height>640</height>
    </rect>
   </property>
   <property name="windowTitle" >
@@ -33,8 +33,8 @@
        <rect>
         <x>0</x>
         <y>0</y>
-        <width>711</width>
+        <width>730</width>
-        <height>572</height>
+        <height>600</height>
        </rect>
       </property>
       <layout class="QVBoxLayout" name="verticalLayout_3" >

src/calibre/gui2/main.py

@@ -63,7 +63,8 @@ class Main(MainWindow, Ui_MainWindow):
         p.end()
         self.default_thumbnail = (pixmap.width(), pixmap.height(), pixmap_to_data(pixmap))
 
-    def __init__(self, single_instance, opts, parent=None):
+    def __init__(self, single_instance, opts, actions, parent=None):
+        self.preferences_action, self.quit_action = actions
         MainWindow.__init__(self, opts, parent)
         # Initialize fontconfig in a separate thread as this can be a lengthy
         # process if run for the first time on this machine
@@ -99,8 +100,8 @@ class Main(MainWindow, Ui_MainWindow):
             self.system_tray_icon.show()
         self.system_tray_menu = QMenu()
         self.restore_action = self.system_tray_menu.addAction(QIcon(':/images/page.svg'), _('&Restore'))
-        self.donate_action  = self.system_tray_menu.addAction(QIcon(':/images/donate.svg'), _('&Donate'))
+        self.donate_action  = self.system_tray_menu.addAction(QIcon(':/images/donate.svg'), _('&Donate to support calibre'))
-        self.quit_action    = QAction(QIcon(':/images/window-close.svg'), _('&Quit'), self)
+        self.donate_button.setDefaultAction(self.donate_action)
         self.addAction(self.quit_action)
         self.action_restart = QAction(_('&Restart'), self)
         self.addAction(self.action_restart)
@@ -242,6 +243,7 @@ class Main(MainWindow, Ui_MainWindow):
         self.tool_bar.setContextMenuPolicy(Qt.PreventContextMenu)
 
         QObject.connect(self.config_button, SIGNAL('clicked(bool)'), self.do_config)
+        self.connect(self.preferences_action, SIGNAL('triggered(bool)'), self.do_config)
         QObject.connect(self.advanced_search_button, SIGNAL('clicked(bool)'), self.do_advanced_search)
         
         ####################### Library view ########################
@@ -912,13 +914,14 @@ class Main(MainWindow, Ui_MainWindow):
         _files   = self.library_view.model().get_preferred_formats(rows,
                                     self.device_manager.device_class.FORMATS, paths=True)
         files = [getattr(f, 'name', None) for f in _files]
-        bad, good, gf, names = [], [], [], []
+        bad, good, gf, names, remove_ids = [], [], [], [], []
         for f in files:
             mi = metadata.next()
             id = ids.next()
             if f is None:
                 bad.append(mi['title'])
             else:
+                remove_ids.append(id)
                 aus = mi['authors'].split(',')
                 aus2 = []
                 for a in aus:
@@ -945,7 +948,7 @@ class Main(MainWindow, Ui_MainWindow):
                     prefix = prefix.decode(preferred_encoding, 'replace')
                 prefix = ascii_filename(prefix)
                 names.append('%s_%d%s'%(prefix, id, os.path.splitext(f)[1]))
-        remove = [self.library_view.model().id(r) for r in rows] if delete_from_library else []
+        remove = remove_ids if delete_from_library else []
         self.upload_books(gf, names, good, on_card, memory=(_files, remove))
         self.status_bar.showMessage(_('Sending books to device.'), 5000)
         if bad:
@@ -1420,7 +1423,7 @@ class Main(MainWindow, Ui_MainWindow):
         self.restart_after_quit = restart
         QApplication.instance().quit()
             
-    def donate(self):
+    def donate(self, *args):
         BUTTON = '''
         <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
             <input type="hidden" name="cmd" value="_s-xclick">
@@ -1546,6 +1549,7 @@ def main(args=sys.argv):
             prefs.set('library_path', opts.with_library)
             print 'Using library at', prefs['library_path']
         app = Application(args)
+        actions = tuple(Main.create_application_menubar())
         app.setWindowIcon(QIcon(':/library'))
         QCoreApplication.setOrganizationName(ORG_NAME)
         QCoreApplication.setApplicationName(APP_UID)
@@ -1560,7 +1564,7 @@ def main(args=sys.argv):
                                  '<p>%s is already running. %s</p>'%(__appname__, extra))
             return 1
         initialize_file_icon_provider()
-        main = Main(single_instance, opts)
+        main = Main(single_instance, opts, actions)
         sys.excepthook = main.unhandled_exception
         if len(args) > 1:
             main.add_filesystem_book(args[1])

src/calibre/gui2/main.ui

@@ -82,6 +82,29 @@
         </property>
        </widget>
       </item>
+      <item>
+       <widget class="QToolButton" name="donate_button" >
+        <property name="cursor" >
+         <cursorShape>PointingHandCursor</cursorShape>
+        </property>
+        <property name="text" >
+         <string>...</string>
+        </property>
+        <property name="icon" >
+         <iconset resource="images.qrc" >
+          <normaloff>:/images/donate.svg</normaloff>:/images/donate.svg</iconset>
+        </property>
+        <property name="iconSize" >
+         <size>
+          <width>64</width>
+          <height>64</height>
+         </size>
+        </property>
+        <property name="autoRaise" >
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
       <item>
        <layout class="QVBoxLayout" name="verticalLayout_3" >
         <item>

src/calibre/gui2/main_window.py

@@ -3,7 +3,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 
 import StringIO, traceback, sys
 
-from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QCoreApplication, SIGNAL
+from PyQt4.Qt import QMainWindow, QString, Qt, QFont, QCoreApplication, SIGNAL,\
+                     QAction, QMenu, QMenuBar, QIcon
 from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog
 from calibre.utils.config import OptionParser
 
@@ -34,6 +35,27 @@ class DebugWindow(ConversionErrorDialog):
 
 class MainWindow(QMainWindow):
 
+    _menu_bar = None
+    
+    @classmethod
+    def create_application_menubar(cls):
+        mb = QMenuBar(None)
+        menu = QMenu()
+        for action in cls.get_menubar_actions():
+            menu.addAction(action)
+            yield action
+        mb.addMenu(menu)
+        cls._menu_bar = mb
+        
+    
+    @classmethod
+    def get_menubar_actions(cls):
+        preferences_action = QAction(QIcon(':/images/config.svg'), _('&Preferences'), None)
+        quit_action        = QAction(QIcon(':/images/window-close.svg'), _('&Quit'), None)
+        preferences_action.setMenuRole(QAction.PreferencesRole)
+        quit_action.setMenuRole(QAction.QuitRole)
+        return preferences_action, quit_action
+    
     def __init__(self, opts, parent=None):
         QMainWindow.__init__(self, parent)
         app = QCoreApplication.instance()

src/calibre/gui2/status.py

@@ -148,6 +148,7 @@ class CoverFlowButton(QToolButton):
         self.setSizePolicy(QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding))
         self.connect(self, SIGNAL('toggled(bool)'), self.adjust_tooltip)
         self.adjust_tooltip(False)
+        self.setCursor(Qt.PointingHandCursor)
         
     def adjust_tooltip(self, on):
         tt = _('Click to turn off Cover Browsing') if on else _('Click to browse books by their covers')
@@ -165,6 +166,7 @@ class TagViewButton(QToolButton):
         self.setIcon(QIcon(':/images/tags.svg'))
         self.setToolTip(_('Click to browse books by tags'))
         self.setSizePolicy(QSizePolicy(QSizePolicy.Preferred, QSizePolicy.Expanding))
+        self.setCursor(Qt.PointingHandCursor)
         self.setCheckable(True)
         self.setChecked(False)
         self.setAutoRaise(True)

src/calibre/manual/faq.rst

@@ -46,7 +46,7 @@ What formats does |app| support conversion to/from?
 |                   |        |                  |                       |                       |
 |                   |  PDF   |       ✔          |          ✔            |          ✔            | 
 |                   |        |                  |                       |                       |
-|                   |  LRS   |                  |          ✔            |          ✔            |
+|                   |  LRS   |                  |          ✔            |                       |
 +-------------------+--------+------------------+-----------------------+-----------------------+
            
 

src/calibre/parallel.py

@@ -158,21 +158,27 @@ class WorkerMother(object):
             self.executable = os.path.join(os.path.dirname(sys.executable),
                    'calibre-parallel.exe' if isfrozen else 'Scripts\\calibre-parallel.exe')
         elif isosx:
-            self.executable = sys.executable
+            self.executable = self.gui_executable = sys.executable
             self.prefix = ''
             if isfrozen:
                 fd = getattr(sys, 'frameworks_dir')
                 contents = os.path.dirname(fd)
-                resources = os.path.join(contents, 'Resources')
+                self.gui_executable = os.path.join(contents, 'MacOS',
-                sp = os.path.join(resources, 'lib', 'python'+sys.version[:3], 'site-packages.zip')
+                                               os.path.basename(sys.executable))
+                contents = os.path.join(contents, 'console.app', 'Contents')
+                self.executable = os.path.join(contents, 'MacOS',
+                                               os.path.basename(sys.executable))
                 
+                resources = os.path.join(contents, 'Resources')
+                fd = os.path.join(contents, 'Frameworks')
+                sp = os.path.join(resources, 'lib', 'python'+sys.version[:3], 'site-packages.zip')
                 self.prefix += 'import sys; sys.frameworks_dir = "%s"; sys.frozen = "macosx_app"; '%fd
                 self.prefix += 'sys.path.insert(0, %s); '%repr(sp)
                 if fd not in os.environ['PATH']:
                     self.env['PATH']    = os.environ['PATH']+':'+fd
                 self.env['PYTHONHOME']  = resources
-                self.env['MAGICK_HOME'] = os.path.join(getattr(sys, 'frameworks_dir'), 'ImageMagick')
+                self.env['MAGICK_HOME'] = os.path.join(fd, 'ImageMagick')
-                self.env['DYLD_LIBRARY_PATH'] = os.path.join(getattr(sys, 'frameworks_dir'), 'ImageMagick', 'lib')
+                self.env['DYLD_LIBRARY_PATH'] = os.path.join(fd, 'ImageMagick', 'lib')
         else:
             self.executable = os.path.join(getattr(sys, 'frozen_path'), 'calibre-parallel') \
                                 if isfrozen else 'calibre-parallel'
@@ -219,7 +225,8 @@ class WorkerMother(object):
 
     def spawn_free_spirit_osx(self, arg, type='free_spirit'):
         script = 'from calibre.parallel import main; main(args=["calibre-parallel", %s]);'%repr(arg)
-        cmdline = [self.executable, '-c', self.prefix+script]
+        exe = self.gui_executable if type == 'free_spirit' else self.executable
+        cmdline = [exe, '-c', self.prefix+script]
         child = WorkerStatus(subprocess.Popen(cmdline, env=self.get_env()))
         atexit.register(self.cleanup_child_linux, child)
         return child

src/calibre/utils/config.py

@@ -97,7 +97,7 @@ class OptionParser(_OptionParser):
         
     def merge(self, parser):
         '''
-        Add options from parser to self. In case of conflicts, confilicting options from
+        Add options from parser to self. In case of conflicts, conflicting options from
         parser are skipped.
         '''
         opts   = list(parser.option_list)
@@ -224,6 +224,8 @@ class OptionSet(object):
     def update(self, other):
         for name in other.groups.keys():
             self.groups[name] = other.groups[name]
+            if name not in self.group_list:
+                self.group_list.append(name)
         for pref in other.preferences:
             if pref in self.preferences:
                 self.preferences.remove(pref)

src/calibre/web/feeds/recipes/__init__.py

@@ -24,6 +24,7 @@ recipe_modules = ['recipe_' + r for r in (
            'joelonsoftware', 'telepolis', 'common_dreams', 'nin', 'tomshardware_de',
            'pagina12', 'infobae', 'ambito', 'elargentino', 'sueddeutsche', 'the_age',
            'laprensa', 'amspec', 'freakonomics', 'criticadigital', 'elcronista',
+           'shacknews', 'teleread',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_shacknews.py

@@ -0,0 +1,26 @@
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Shacknews(BasicNewsRecipe):
+    __author__  = 'Docbrown00'
+    __license__   = 'GPL v3'
+    title          = u'Shacknews'
+    oldest_article = 7
+    max_articles_per_feed = 100
+    no_stylesheets = True
+    remove_tags = [dict(name='div', attrs={'class': ['nuggets', 'comments']}), 
+	        dict(name='p', attrs={'class': 'videoembed'})]
+    keep_only_tags = [dict(name='div', attrs={'class':'story'})]
+    feeds          = [
+		(u'Latest News', u'http://feed.shacknews.com/shackfeed.xml'),
+		(u'PC', u'http://feed.shacknews.com/extras/tag_rss.x/PC'),
+		(u'Wii', u'http://feed.shacknews.com/extras/tag_rss.x/Nintendo+Wii'),
+		(u'Xbox 360', u'http://feed.shacknews.com/extras/tag_rss.x/Xbox+360'),
+		(u'Playstation 3', u'http://feed.shacknews.com/extras/tag_rss.x/PlayStation+3'),
+		(u'PSP', u'http://feed.shacknews.com/extras/tag_rss.x/PSP'),
+		(u'Nintendo DS', u'http://feed.shacknews.com/extras/tag_rss.x/Nintendo+DS'),
+		(u'iPhone', u'http://feed.shacknews.com/extras/tag_rss.x/iPhone'),
+		(u'DLC', u'http://feed.shacknews.com/extras/tag_rss.x/DLC'),
+		(u'Valve', u'http://feed.shacknews.com/extras/tag_rss.x/Valve'),
+		(u'Electronic Arts', u'http://feed.shacknews.com/extras/tag_rss.x/Electronic+Arts')
+	     ]

src/calibre/web/feeds/recipes/recipe_teleread.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env  python
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal kovid@kovidgoyal.net'
+__docformat__ = 'restructuredtext en'
+
+'''
+teleread.org
+'''
+
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Teleread(BasicNewsRecipe):
+    
+    title = 'Teleread Blog'
+    description = 'News & views on e-books, libraries, publishing and related topics'
+    __author__  = 'Kovid Goyal'
+    
+    feeds = [('Entries', 'http://www.teleread.org/feed/')]
+    
+    remove_tags = [dict(attrs={'class':['sociable', 'comments', 
+                                        'wlWriterSmartContent', 'feedflare']})]
+    
+    def get_article_url(self, article):
+        return article.get('feedburner_origlink', article.get('link', article.get('guid')))