Sync to pluginize

.bzrignore

@@ -29,3 +29,4 @@ src/cssutils/scripts/
 src/cssutils/css/.svn/
 src/cssutils/stylesheets/.svn/
 src/odf/.svn
+tags

installer/linux/freeze.py

@@ -9,7 +9,7 @@ Create linux binary.
 '''
 
 def freeze():
-    import glob, sys, subprocess, tarfile, os, re, textwrap, shutil, cStringIO, bz2, codecs
+    import glob, sys, tarfile, os, textwrap, shutil
     from contextlib import closing
     from cx_Freeze import Executable, setup
     from calibre.constants import __version__, __appname__
@@ -41,7 +41,6 @@ def freeze():
                        '/usr/lib/libxslt.so.1',
                        '/usr/lib/libxslt.so.1',
                        '/usr/lib/libgthread-2.0.so.0',
-                       '/usr/lib/libglib-2.0.so.0',
                        '/usr/lib/gcc/i686-pc-linux-gnu/4.3.3/libstdc++.so.6',
                        '/usr/lib/libpng12.so.0',
                        '/usr/lib/libexslt.so.0',
@@ -81,6 +80,8 @@ def freeze():
 
     includes = [x[0] for x in executables.values()]
     includes += ['calibre.ebooks.lrf.fonts.prs500.'+x for x in FONT_MAP.values()]
+    includes += ['email.iterators', 'email.generator']
+
 
     excludes = ['matplotlib', "Tkconstants", "Tkinter", "tcl", "_imagingtk",
                 "ImageTk", "FixTk", 'wx', 'PyQt4.QtAssistant', 'PyQt4.QtOpenGL.so',
@@ -88,7 +89,7 @@ def freeze():
                 'glib', 'gobject']
 
     packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg',
-                'dateutil']
+                'dateutil', 'dns', 'email']
 
     includes += ['calibre.web.feeds.recipes.'+r for r in recipe_modules]
 

installer/osx/freeze.py

@@ -270,7 +270,7 @@ _check_symlinks_prescript()
         print 'Adding ImageMagick'
         dest = os.path.join(frameworks_dir, 'ImageMagick')
         if os.path.exists(dest):
-            sutil.rmtree(dest)
+            shutil.rmtree(dest)
         shutil.copytree(os.path.expanduser('~/ImageMagick'), dest, True)
         shutil.copyfile('/usr/local/lib/libpng12.0.dylib', os.path.join(dest, 'lib', 'libpng12.0.dylib'))
 
@@ -343,9 +343,10 @@ def main():
                                        'calibre.ebooks.lrf.any.*', 'calibre.ebooks.lrf.feeds.*',
                                        'keyword', 'codeop', 'pydoc', 'readline',
                                        'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*',
-                                       'dateutil',
+                                       'dateutil', 'email.iterators',
+                                       'email.generator',
                                        ],
-                         'packages' : ['PIL', 'Authorization', 'lxml'],
+                         'packages' : ['PIL', 'Authorization', 'lxml', 'dns'],
                          'excludes' : ['IPython'],
                          'plist'    : { 'CFBundleGetInfoString' : '''calibre, an E-book management application.'''
                                         ''' Visit http://calibre.kovidgoyal.net for details.''',

installer/windows/calibre/calibre.mpi

@@ -273,7 +273,6 @@ File ::C49805D2-C0B8-01C4-DF6F-674D9C0BFD15 -name IM_MOD_RL_viff_.dll -parent 8E
 File ::1B9F2F00-20A5-B207-5A80-8F75470286AD -name txt2lrf.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::826F1915-9F97-59DD-6637-3EEC0744A79C -name IM_MOD_RL_ps2_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::519A6618-8A1F-93A5-93B4-6EEF5A4A3DE9 -name comic2pdf.exe -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::B0CEAA35-52BF-0DE0-BAC7-7B23157E29BD -name isbndb.exe -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::A5F23791-BCDC-A997-4941-5D1F2F227E6D -name type.xml -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::0A1C107A-C0AA-3ED6-4F37-A6894386DCBE -name IM_MOD_RL_ps3_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::EEBA64E7-6509-EBAF-3E23-1A203216F39A -name epub2lrf.exe -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
@@ -284,7 +283,6 @@ File ::EA37C1C2-57BB-4E7A-C004-0010D79142C2 -name IM_MOD_RL_fits_.dll -parent 8E
 File ::05F5C10D-6988-F1F4-A486-86C96DB20302 -name pywintypes26.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::0137A2B1-EB94-EB26-7295-0C7CD941A1DF -name IM_MOD_RL_histogram_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::7F199A1F-4FA4-2ABA-DED3-36ECF3C089CA -name epub2lrf.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
-File ::F9F112C9-B61B-E041-1A9D-47641B047135 -name isbndb.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::CF6398D8-2140-53CF-1DA6-421A82E92621 -name any2epub.exe -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::8DFA6C69-360D-FA63-7FF9-860E3DB00B19 -name any2lrf.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::5BB7579D-9183-412C-81F8-B411B07C57B3 -name IM_MOD_RL_pnm_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
@@ -544,6 +542,8 @@ File ::325F545D-30A8-08DA-74F0-AC1244F6C1D9 -name IM_MOD_RL_vid_.dll -parent 8E5
 File ::24238371-77D0-0A8F-35D1-498A5FCC1B0D -name IM_MOD_RL_rla_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::6F5D62F3-5E63-0753-364C-01CAAF1002E0 -name IM_MOD_RL_magick_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 File ::9FDAC308-5D4F-A865-A09A-9FBF48162A47 -name IM_MOD_RL_djvu_.dll -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::5D748040-5973-EFF1-41FC-B424636C642E -name fetch-ebook-metadata.exe.local -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
+File ::8B8655B8-3823-AA02-1CDA-02F5AD4677C0 -name fetch-ebook-metadata.exe -parent 8E5D85A4-7608-47A1-CF7C-309060D5FF40
 Component ::F6829AB7-9F66-4CEE-CA0E-21F54C6D3609 -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows} -name Main -parent Components
 SetupType ::D9ADE41C-B744-690C-2CED-CF826BF03D2E -setup Install -active Yes -platforms {AIX-ppc FreeBSD-4-x86 FreeBSD-x86 HPUX-hppa Linux-x86 Solaris-sparc Windows} -name Typical -parent SetupTypes
 

installer/windows/freeze.py

@@ -14,12 +14,11 @@ IMAGEMAGICK_DIR  = 'C:\\ImageMagick'
 FONTCONFIG_DIR   = 'C:\\fontconfig'
 VC90             = r'C:\VC90.CRT'
 
-import sys, os, py2exe, shutil, zipfile, glob, subprocess, re
+import sys, os, py2exe, shutil, zipfile, glob, re
 from distutils.core import setup
-from distutils.filelist import FileList
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 sys.path.insert(0, BASE_DIR)
-from setup import VERSION, APPNAME, entry_points, scripts, basenames
+from setup import VERSION, APPNAME, scripts, basenames
 sys.path.remove(BASE_DIR)
 
 ICONS = [os.path.abspath(os.path.join(BASE_DIR, 'icons', i)) for i in ('library.ico', 'viewer.ico')]
@@ -145,6 +144,8 @@ def main(args=sys.argv):
                                              'sip', 'pkg_resources', 'PyQt4.QtSvg',
                                              'mechanize', 'ClientForm', 'wmi',
                                              'win32file', 'pythoncom',
+                                             'email.iterators',
+                                             'email.generator',
                                              'win32process', 'win32api', 'msvcrt',
                                              'win32event', 'calibre.ebooks.lrf.any.*',
                                              'calibre.ebooks.lrf.feeds.*',
@@ -155,7 +156,7 @@ def main(args=sys.argv):
                                              'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
                                              ],
                                   'packages'  : ['PIL', 'lxml', 'cherrypy',
-                                                 'dateutil'],
+                                                 'dateutil', 'dns'],
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",
                                                  "_imagingtk", "ImageTk", "FixTk"
                                                 ],

session.vim

@@ -0,0 +1,16 @@
+" Project wide builtins
+let g:pyflakes_builtins += ["dynamic_property", '__']
+
+python << EOFPY
+import os
+
+import vipy
+
+source_file = vipy.vipy.eval('expand("<sfile>")')
+project_dir = os.path.dirname(source_file)
+src_dir = os.path.abspath(os.path.join(project_dir, 'src'))
+base_dir = os.path.join(src_dir, 'calibre')
+
+vipy.session.initialize(project_name='calibre', src_dir=src_dir,
+            project_dir=project_dir, base_dir=base_dir)
+EOFPY

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.5.2'
+__version__   = '0.5.3'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/conversion.py

@@ -1,11 +1,9 @@
 from __future__ import with_statement
 '''
-Defines the plugin sytem for conversions.
+Defines the plugin system for conversions.
 '''
 import re, os, shutil
 
-from lxml import html
-
 from calibre import CurrentDir
 from calibre.customize import Plugin
 
@@ -121,7 +119,7 @@ class InputFormatPlugin(Plugin):
     #: (option_name, recommended_value, recommendation_level)
     recommendations = set([])
 
-    def convert(self, stream, options, file_ext, parse_cache, log, accelerators):
+    def convert(self, stream, options, file_ext, log, accelerators):
         '''
         This method must be implemented in sub-classes. It must return
         the path to the created OPF file. All output should be contained in
@@ -144,17 +142,6 @@ class InputFormatPlugin(Plugin):
                          is guaranteed to be one of the `file_types` supported
                          by this plugin.
 
-        :param parse_cache:    A dictionary that maps absolute file paths to
-                               parsed representations of their contents. For
-                               HTML the representation is an lxml element of 
-                               the root of the tree. For CSS it is a cssutils
-                               stylesheet. If this plugin parses any of the
-                               output files, it should add them to the cache
-                               so that later stages of the conversion wont
-                               have to re-parse them. If a parsed representation
-                               is in the cache, there is no need to actually 
-                               write the file to disk.
-        
         :param log: A :class:`calibre.utils.logging.Log` object. All output
                     should use this object.
 
@@ -165,7 +152,7 @@ class InputFormatPlugin(Plugin):
         '''
         raise NotImplementedError
 
-    def __call__(self, stream, options, file_ext, parse_cache, log, 
+    def __call__(self, stream, options, file_ext, log,
                  accelerators, output_dir):
         log('InputFormatPlugin: %s running'%self.name, end=' ')
         if hasattr(stream, 'name'):
@@ -176,33 +163,15 @@ class InputFormatPlugin(Plugin):
                 shutil.rmtree(x) if os.path.isdir(x) else os.remove(x)
 
 
-            ret = self.convert(stream, options, file_ext, parse_cache, 
+            ret = self.convert(stream, options, file_ext,
                                log, accelerators)
-            for key in list(parse_cache.keys()):
-                if os.path.abspath(key) != key:
-                    log.warn(('InputFormatPlugin: %s returned a '
-                             'relative path: %s')%(self.name, key)
-                             )
-                    parse_cache[os.path.abspath(key)] = parse_cache.pop(key)
-                    
         if options.debug_input is not None:
             options.debug_input = os.path.abspath(options.debug_input)
             if not os.path.exists(options.debug_input):
                 os.makedirs(options.debug_input)
             shutil.rmtree(options.debug_input)
-            for f, obj in parse_cache.items():
-                if hasattr(obj, 'cssText'):
-                    raw = obj.cssText
-                else:
-                    raw = html.tostring(obj, encoding='utf-8', method='xml', 
-                         include_meta_content_type=True, pretty_print=True)
-                if isinstance(raw, unicode):
-                    raw = raw.encode('utf-8')
-                open(f, 'wb').write(raw)
             shutil.copytree('.', options.debug_input)
 
-            
-                    
         return ret
 
 
@@ -236,6 +205,6 @@ class OutputFormatPlugin(Plugin):
     #: (option_name, recommended_value, recommendation_level)
     recommendations = set([])
 
-    def convert(self, oeb_book, input_plugin, options, parse_cache, log):
+    def convert(self, oeb_book, input_plugin, options, context, log):
         raise NotImplementedError
 

src/calibre/customize/profiles.py

@@ -4,7 +4,36 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
 import sys, re
-from calibre.customize import Plugin
+from itertools import izip
+
+from calibre.customize import Plugin as _Plugin
+
+FONT_SIZES = [('xx-small', 1),
+              ('x-small',  None),
+              ('small',    2),
+              ('medium',   3),
+              ('large',    4),
+              ('x-large',  5),
+              ('xx-large', 6),
+              (None,       7)]
+
+
+class Plugin(_Plugin):
+
+    fbase  = 12
+    fsizes = [5, 7, 9, 12, 13.5, 17, 20, 22, 24]
+    screen_size = (800, 600)
+    dpi = 100
+
+    def initialize(self):
+        self.width, self.height = self.screen_size
+        fsizes = list(self.fsizes)
+        self.fsizes = []
+        for (name, num), size in izip(FONT_SIZES, fsizes):
+            self.fsizes.append((name, num, float(size)))
+        self.fnames = dict((name, sz) for name, _, sz in self.fsizes if name)
+        self.fnums = dict((num, sz) for _, num, sz in self.fsizes if num)
+
 
 class InputProfile(Plugin):
 
@@ -13,15 +42,88 @@ class InputProfile(Plugin):
     can_be_disabled = False
     type = _('Input profile')
 
-# TODO: Add some real information to this profile. All other profiles must
-#       inherit from this profile and override as needed
-
     name        = 'Default Input Profile'
     short_name  = 'default' # Used in the CLI so dont use spaces etc. in it
     description = _('This profile tries to provide sane defaults and is useful '
                     'if you know nothing about the input document.')
 
-input_profiles = [InputProfile]
+
+class SonyReaderInput(InputProfile):
+
+    name        = 'Sony Reader'
+    short_name  = 'sony'
+    description = _('This profile is intended for the SONY PRS line. '
+                    'The 500/505/700 etc.')
+
+    screen_size               = (584, 754)
+    dpi                       = 168.451
+    fbase                     = 12
+    fsizes                    = [7.5, 9, 10, 12, 15.5, 20, 22, 24]
+
+
+class MSReaderInput(InputProfile):
+
+    name        = 'Microsoft Reader'
+    short_name  = 'msreader'
+    description = _('This profile is intended for the Microsoft Reader.')
+
+    screen_size               = (480, 652)
+    dpi                       = 96
+    fbase                     = 13
+    fsizes                    = [10, 11, 13, 16, 18, 20, 22, 26]
+
+class MobipocketInput(InputProfile):
+
+    name        = 'Mobipocket Books'
+    short_name  = 'mobipocket'
+    description = _('This profile is intended for the Mobipocket books.')
+
+    # Unfortunately MOBI books are not narrowly targeted, so this information is
+    # quite likely to be spurious
+    screen_size               = (600, 800)
+    dpi                       = 96
+    fbase                     = 18
+    fsizes                    = [14, 14, 16, 18, 20, 22, 24, 26]
+
+class HanlinV3Input(InputProfile):
+
+    name        = 'Hanlin V3'
+    short_name  = 'hanlinv3'
+    description = _('This profile is intended for the Hanlin V3 and its clones.')
+
+    # Screen size is a best guess
+    screen_size               = (584, 754)
+    dpi                       = 168.451
+    fbase                     = 16
+    fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]
+
+class CybookG3Input(InputProfile):
+
+    name        = 'Cybook G3'
+    short_name  = 'cybookg3'
+    description = _('This profile is intended for the Cybook G3.')
+
+    # Screen size is a best guess
+    screen_size               = (600, 800)
+    dpi                       = 168.451
+    fbase                     = 16
+    fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]
+
+class KindleInput(InputProfile):
+
+    name        = 'Kindle'
+    short_name  = 'kindle'
+    description = _('This profile is intended for the Amazon Kindle.')
+
+    # Screen size is a best guess
+    screen_size               = (525, 640)
+    dpi                       = 168.451
+    fbase                     = 16
+    fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]
+
+
+input_profiles = [InputProfile, SonyReaderInput, MSReaderInput,
+        MobipocketInput, HanlinV3Input, CybookG3Input, KindleInput]
 
 
 class OutputProfile(Plugin):
@@ -37,23 +139,86 @@ class OutputProfile(Plugin):
                     'if you want to produce a document intended to be read at a '
                     'computer or on a range of devices.')
 
-    epub_flow_size            = sys.maxint
+    # ADE dies an agonizing, long drawn out death if HTML files have more
-    screen_size               = None
+    # bytes than this.
-    remove_special_chars      = False
+    flow_size                 = sys.maxint
-    remove_object_tags        = False
+    # ADE runs screaming when it sees these characters
+    remove_special_chars      = re.compile(u'[\u200b\u00ad]')
+    # ADE falls to the ground in a dead faint when it sees an <object>
+    remove_object_tags        = True
 
-class SonyReader(OutputProfile):
+class SonyReaderOutput(OutputProfile):
 
     name        = 'Sony Reader'
     short_name  = 'sony'
     description = _('This profile is intended for the SONY PRS line. '
                     'The 500/505/700 etc.')
 
-    epub_flow_size            = 270000
+    flow_size                 = 270000
-    screen_size               = (590, 765)
+    screen_size               = (600, 775)
-    remove_special_chars      = re.compile(u'[\u200b\u00ad]')
+    dpi                       = 168.451
-    remove_object_tags        = True
+    fbase                     = 12
+    fsizes                    = [7.5, 9, 10, 12, 15.5, 20, 22, 24]
 
+class MSReaderOutput(OutputProfile):
 
+    name        = 'Microsoft Reader'
+    short_name  = 'msreader'
+    description = _('This profile is intended for the Microsoft Reader.')
 
-output_profiles = [OutputProfile, SonyReader]
+    screen_size               = (480, 652)
+    dpi                       = 96
+    fbase                     = 13
+    fsizes                    = [10, 11, 13, 16, 18, 20, 22, 26]
+
+class MobipocketOutput(OutputProfile):
+
+    name        = 'Mobipocket Books'
+    short_name  = 'mobipocket'
+    description = _('This profile is intended for the Mobipocket books.')
+
+    # Unfortunately MOBI books are not narrowly targeted, so this information is
+    # quite likely to be spurious
+    screen_size               = (600, 800)
+    dpi                       = 96
+    fbase                     = 18
+    fsizes                    = [14, 14, 16, 18, 20, 22, 24, 26]
+
+class HanlinV3Output(OutputProfile):
+
+    name        = 'Hanlin V3'
+    short_name  = 'hanlinv3'
+    description = _('This profile is intended for the Hanlin V3 and its clones.')
+
+    # Screen size is a best guess
+    screen_size               = (584, 754)
+    dpi                       = 168.451
+    fbase                     = 16
+    fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]
+
+class CybookG3Output(OutputProfile):
+
+    name        = 'Cybook G3'
+    short_name  = 'cybookg3'
+    description = _('This profile is intended for the Cybook G3.')
+
+    # Screen size is a best guess
+    screen_size               = (600, 800)
+    dpi                       = 168.451
+    fbase                     = 16
+    fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]
+
+class KindleOutput(OutputProfile):
+
+    name        = 'Kindle'
+    short_name  = 'kindle'
+    description = _('This profile is intended for the Amazon Kindle.')
+
+    # Screen size is a best guess
+    screen_size               = (525, 640)
+    dpi                       = 168.451
+    fbase                     = 16
+    fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]
+
+output_profiles = [OutputProfile, SonyReaderOutput, MSReaderOutput,
+        MobipocketOutput, HanlinV3Output, CybookG3Output, KindleOutput]

src/calibre/debug.py

@@ -17,8 +17,10 @@ def option_parser():
 
 Run an embedded python interpreter.
 ''')
-    parser.add_option('--update-module', help='Update the specified module in the frozen library. '+
+    parser.add_option('--update-module',
-    'Module specifications are of the form full.name.of.module,path_to_module.py', default=None
+            help='Update the specified module in the frozen library. '+
+    'Module specifications are of the form full.name.of.module,path_to_module.py',
+    default=None
     )
     parser.add_option('-c', '--command', help='Run python code.', default=None)
     parser.add_option('-e', '--exec-file', default=None, help='Run the python code in file.')
@@ -27,7 +29,8 @@ Run an embedded python interpreter.
     parser.add_option('-g', '--gui',  default=False, action='store_true',
                       help='Run the GUI',)
     parser.add_option('--migrate', action='store_true', default=False,
-                      help='Migrate old database. Needs two arguments. Path to library1.db and path to new library folder.')
+                      help='Migrate old database. Needs two arguments. Path '
+                           'to library1.db and path to new library folder.')
     return parser
 
 def update_zipfile(zipfile, mod, path):

src/calibre/devices/kindle/driver.py

@@ -4,7 +4,7 @@ __copyright__ = '2009, John Schember <john at nachtimwald.com>'
 Device driver for Amazon's Kindle
 '''
 
-import os, re
+import os, re, sys
 
 from calibre.devices.usbms.driver import USBMS, metadata_from_formats
 
@@ -51,6 +51,9 @@ class KINDLE(USBMS):
             match = cls.WIRELESS_FILE_NAME_PATTERN.match(os.path.basename(path))
             if match is not None:
                 mi.title = match.group('title')
+                if not isinstance(mi.title, unicode):
+                    mi.title = mi.title.decode(sys.getfilesystemencoding(),
+                                               'replace')
         return mi
 
 

src/calibre/devices/prs505/driver.py

@@ -112,7 +112,7 @@ class PRS505(Device):
             if not os.access(ioreg, os.X_OK):
                 ioreg = 'ioreg'
             raw = subprocess.Popen((ioreg+' -w 0 -S -c IOMedia').split(),
-                                   stdout=subprocess.PIPE).stdout.read()
+                                   stdout=subprocess.PIPE).communicate()[0]
         lines = raw.splitlines()
         names = {}
         for i, line in enumerate(lines):

src/calibre/devices/usbms/device.py

@@ -200,7 +200,7 @@ class Device(_Device):
             if not os.access(ioreg, os.X_OK):
                 ioreg = 'ioreg'
             raw = subprocess.Popen((ioreg+' -w 0 -S -c IOMedia').split(),
-                                   stdout=subprocess.PIPE).stdout.read()
+                                   stdout=subprocess.PIPE).communicate()[0]
         lines = raw.splitlines()
         names = {}
 

src/calibre/ebooks/conversion/plumber.py

@@ -8,11 +8,17 @@ import os
 from calibre.customize.conversion import OptionRecommendation
 from calibre.customize.ui import input_profiles, output_profiles, \
         plugin_for_input_format, plugin_for_output_format
+from calibre.ebooks.conversion.preprocess import HTMLPreProcessor
 
 class OptionValues(object):
     pass
 
 class Plumber(object):
+    '''
+    The `Plumber` manages the conversion pipeline. An UI should call the methods
+    :method:`merge_ui_recommendations` and then :method:`run`. The plumber will
+    take care of the rest.
+    '''
 
     metadata_option_names = [
         'title', 'authors', 'title_sort', 'author_sort', 'cover', 'comments',
@@ -21,10 +27,17 @@ class Plumber(object):
         ]
 
     def __init__(self, input, output, log):
+        '''
+        :param input: Path to input file.
+        :param output: Path to output file/directory
+        '''
         self.input = input
         self.output = output
         self.log = log
 
+        # Initialize the conversion options that are independent of input and
+        # output formats. The input and output plugins can still disable these
+        # options via recommendations.
         self.pipeline_options = [
 
 OptionRecommendation(name='verbose',
@@ -143,11 +156,15 @@ OptionRecommendation(name='language',
         self.input_fmt = input_fmt
         self.output_fmt = output_fmt
 
+        # Build set of all possible options. Two options are equal iff their
+        # names are the same.
         self.input_options  = self.input_plugin.options.union(
                                     self.input_plugin.common_options)
         self.output_options = self.output_plugin.options.union(
                                     self.output_plugin.common_options)
 
+        # Remove the options that have been disabled by recommendations from the
+        # plugins.
         self.merge_plugin_recommendations()
 
     def get_option_by_name(self, name):
@@ -165,12 +182,21 @@ OptionRecommendation(name='language',
                     rec.recommended_value = val
 
     def merge_ui_recommendations(self, recommendations):
+        '''
+        Merge recommendations from the UI. As long as the UI recommendation
+        level is >= the baseline recommended level, the UI value is used,
+        *except* if the baseline has a recommendation level of `HIGH`.
+        '''
         for name, val, level in recommendations:
             rec = self.get_option_by_name(name)
             if rec is not None and rec.level <= level and rec.level < rec.HIGH:
                 rec.recommended_value = val
 
     def read_user_metadata(self):
+        '''
+        Read all metadata specified by the user. Command line options override
+        metadata from a specified OPF file.
+        '''
         from calibre.ebooks.metadata import MetaInformation, string_to_authors
         from calibre.ebooks.metadata.opf2 import OPF
         mi = MetaInformation(None, [])
@@ -197,6 +223,9 @@ OptionRecommendation(name='language',
 
 
     def setup_options(self):
+        '''
+        Setup the `self.opts` object.
+        '''
         self.opts = OptionValues()
         for group in (self.input_options, self.pipeline_options,
                   self.output_options):
@@ -216,21 +245,31 @@ OptionRecommendation(name='language',
         self.read_user_metadata()
 
     def run(self):
+        '''
+        Run the conversion pipeline
+        '''
+        # Setup baseline option values
         self.setup_options()
+
+        # Run any preprocess plugins
         from calibre.customize.ui import run_plugins_on_preprocess
         self.input = run_plugins_on_preprocess(self.input)
 
+        # Create an OEBBook from the input file. The input plugin does all the
+        # heavy lifting.
         from calibre.ebooks.oeb.reader import OEBReader
         from calibre.ebooks.oeb.base import OEBBook
-        parse_cache, accelerators = {}, {}
+        accelerators = {}
 
         opfpath = self.input_plugin(open(self.input, 'rb'), self.opts,
-                                    self.input_fmt, parse_cache, self.log,
+                                    self.input_fmt, self.log,
                                     accelerators)
-        
+        html_preprocessor = HTMLPreProcessor()
         self.reader = OEBReader()
-        self.oeb = OEBBook(self.log, parse_cache=parse_cache) 
+        self.oeb = OEBBook(self.log, html_preprocessor=html_preprocessor)
+        # Read OEB Book into OEBBook
         self.reader(self.oeb, opfpath)
 
 
 
+

src/calibre/ebooks/conversion/preprocess.py

@@ -0,0 +1,123 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+import re, functools
+
+from calibre import entity_to_unicode
+
+XMLDECL_RE    = re.compile(r'^\s*<[?]xml.*?[?]>')
+SVG_NS       = 'http://www.w3.org/2000/svg'
+XLINK_NS     = 'http://www.w3.org/1999/xlink'
+
+convert_entities = functools.partial(entity_to_unicode, exceptions=['quot', 'apos', 'lt', 'gt', 'amp'])
+_span_pat = re.compile('<span.*?</span>', re.DOTALL|re.IGNORECASE)
+
+
+def sanitize_head(match):
+    x = match.group(1)
+    x = _span_pat.sub('', x)
+    return '<head>\n'+x+'\n</head>'
+
+
+class CSSPreProcessor(object):
+
+    PAGE_PAT   = re.compile(r'@page[^{]*?{[^}]*?}')
+
+    def __call__(self, data):
+        data = self.PAGE_PAT.sub('', data)
+        return data
+
+class HTMLPreProcessor(object):
+
+    PREPROCESS = [
+                  # Some idiotic HTML generators (Frontpage I'm looking at you)
+                  # Put all sorts of crap into <head>. This messes up lxml
+                  (re.compile(r'<head[^>]*>(.*?)</head>', re.IGNORECASE|re.DOTALL),
+                   sanitize_head),
+                  # Convert all entities, since lxml doesn't handle them well
+                  (re.compile(r'&(\S+?);'), convert_entities),
+                  # Remove the <![if/endif tags inserted by everybody's darling, MS Word
+                  (re.compile(r'</{0,1}!\[(end){0,1}if\]{0,1}>', re.IGNORECASE),
+                   lambda match: ''),
+                  ]
+
+    # Fix pdftohtml markup
+    PDFTOHTML  = [
+                  # Remove <hr> tags
+                  (re.compile(r'<hr.*?>', re.IGNORECASE), lambda match: '<br />'),
+                  # Remove page numbers
+                  (re.compile(r'\d+<br>', re.IGNORECASE), lambda match: ''),
+                  # Remove <br> and replace <br><br> with <p>
+                  (re.compile(r'<br.*?>\s*<br.*?>', re.IGNORECASE), lambda match: '<p>'),
+                  (re.compile(r'(.*)<br.*?>', re.IGNORECASE),
+                   lambda match: match.group() if \
+                           re.match('<', match.group(1).lstrip()) or \
+                           len(match.group(1)) < 40  else match.group(1)),
+                  # Remove hyphenation
+                  (re.compile(r'-\n\r?'), lambda match: ''),
+
+                  # Remove gray background
+                  (re.compile(r'<BODY[^<>]+>'), lambda match : '<BODY>'),
+
+                  # Remove non breaking spaces
+                  (re.compile(ur'\u00a0'), lambda match : ' '),
+
+                  ]
+
+    # Fix Book Designer markup
+    BOOK_DESIGNER = [
+                     # HR
+                     (re.compile('<hr>', re.IGNORECASE),
+                      lambda match : '<span style="page-break-after:always"> </span>'),
+                     # Create header tags
+                     (re.compile('<h2[^><]*?id=BookTitle[^><]*?(align=)*(?(1)(\w+))*[^><]*?>[^><]*?</h2>', re.IGNORECASE),
+                      lambda match : '<h1 id="BookTitle" align="%s">%s</h1>'%(match.group(2) if match.group(2) else 'center', match.group(3))),
+                     (re.compile('<h2[^><]*?id=BookAuthor[^><]*?(align=)*(?(1)(\w+))*[^><]*?>[^><]*?</h2>', re.IGNORECASE),
+                      lambda match : '<h2 id="BookAuthor" align="%s">%s</h2>'%(match.group(2) if match.group(2) else 'center', match.group(3))),
+                     (re.compile('<span[^><]*?id=title[^><]*?>(.*?)</span>', re.IGNORECASE|re.DOTALL),
+                      lambda match : '<h2 class="title">%s</h2>'%(match.group(1),)),
+                     (re.compile('<span[^><]*?id=subtitle[^><]*?>(.*?)</span>', re.IGNORECASE|re.DOTALL),
+                      lambda match : '<h3 class="subtitle">%s</h3>'%(match.group(1),)),
+                     ]
+
+    def is_baen(self, src):
+        return re.compile(r'<meta\s+name="Publisher"\s+content=".*?Baen.*?"',
+                          re.IGNORECASE).search(src) is not None
+
+    def is_book_designer(self, raw):
+        return re.search('<H2[^><]*id=BookTitle', raw) is not None
+
+    def is_pdftohtml(self, src):
+        return '<!-- created by calibre\'s pdftohtml -->' in src[:1000]
+
+    def __call__(self, html, remove_special_chars=None):
+        if remove_special_chars is not None:
+            html = remove_special_chars.sub('', html)
+        if self.is_baen(html):
+            rules = []
+        elif self.is_book_designer(html):
+            rules = self.BOOK_DESIGNER
+        elif self.is_pdftohtml(html):
+            rules = self.PDFTOHTML
+        else:
+            rules = []
+        for rule in self.PREPROCESS + rules:
+            html = rule[0].sub(rule[1], html)
+
+        # Handle broken XHTML w/ SVG (ugh)
+        if 'svg:' in html and SVG_NS not in html:
+            html = html.replace(
+                '<html', '<html xmlns:svg="%s"' % SVG_NS, 1)
+        if 'xlink:' in html and XLINK_NS not in html:
+            html = html.replace(
+                '<html', '<html xmlns:xlink="%s"' % XLINK_NS, 1)
+
+        html = XMLDECL_RE.sub('', html)
+
+        return html
+

src/calibre/ebooks/epub/iterator.py

@@ -38,6 +38,8 @@ class UnsupportedFormatError(Exception):
 class SpineItem(unicode):
 
     def __new__(cls, *args):
+        args = list(args)
+        args[0] = args[0].partition('#')[0]
         obj = super(SpineItem, cls).__new__(cls, *args)
         path = args[0]
         raw = open(path, 'rb').read()
@@ -67,6 +69,7 @@ class EbookIterator(object):
     CHARACTERS_PER_PAGE = 1000
 
     def __init__(self, pathtoebook):
+        pathtoebook = pathtoebook.strip()
         self.pathtoebook = os.path.abspath(pathtoebook)
         self.config = DynamicConfig(name='iterator')
         ext = os.path.splitext(pathtoebook)[1].replace('.', '').lower()

src/calibre/ebooks/html.py

@@ -354,7 +354,10 @@ class PreProcessor(object):
                   (re.compile(r'-\n\r?'), lambda match: ''),
                   
                   # Remove gray background
-                  (re.compile(r'<BODY[^<>]+>'), lambda match : '<BODY>')
+                  (re.compile(r'<BODY[^<>]+>'), lambda match : '<BODY>'),
+                  
+                  # Remove non breaking spaces
+                  (re.compile(ur'\u00a0'), lambda match : ' '),
                   
                   ]
     

src/calibre/ebooks/lit/reader.py

@@ -129,8 +129,6 @@ class UnBinary(object):
         self.tag_map, self.attr_map, self.tag_to_attr_map = map
         self.is_html = map is HTML_MAP
         self.tag_atoms, self.attr_atoms = atoms
-        self.opf = map is OPF_MAP
-        self.bin = bin
         self.dir = os.path.dirname(path)
         buf = StringIO()
         self.binary_to_text(bin, buf)
@@ -210,7 +208,8 @@ class UnBinary(object):
                         continue
                     if flags & FLAG_ATOM:
                         if not self.tag_atoms or tag not in self.tag_atoms:
-                            raise LitError("atom tag %d not in atom tag list" % tag)
+                            raise LitError(
+                                "atom tag %d not in atom tag list" % tag)
                         tag_name = self.tag_atoms[tag]
                         current_map = self.attr_atoms
                     elif tag < len(self.tag_map):
@@ -295,7 +294,7 @@ class UnBinary(object):
                             c = '&quot;'
                         elif c == '<':
                             c = '&lt;'
-                        self.buf.write(c.encode('ascii', 'xmlcharrefreplace'))
+                        buf.write(c.encode('ascii', 'xmlcharrefreplace'))
                     count -= 1
                 if count == 0:
                     if not in_censorship:
@@ -842,23 +841,6 @@ class LitFile(object):
             self._warn("damaged or invalid atoms attributes table")
         return (tags, attrs)
 
-    def get_entry_content(self, entry, pretty_print=False):
-        if 'spine' in entry.state:
-            name = '/'.join(('/data', entry.internal, 'content'))
-            path = entry.path
-            raw = self.get_file(name)
-            decl, map = (OPF_DECL, OPF_MAP) \
-                if name == '/meta' else (HTML_DECL, HTML_MAP)
-            atoms = self.get_atoms(entry)
-            content = decl + unicode(UnBinary(raw, path, self.manifest, map, atoms))
-            if pretty_print:
-                content = self._pretty_print(content)
-            content = content.encode('utf-8')
-        else:
-            internal = '/'.join(('/data', entry.internal))
-            content = self._litfile.get_file(internal)
-        return content
- 
 
 class LitContainer(object):
     """Simple Container-interface, read-only accessor for LIT files."""
@@ -879,8 +861,14 @@ class LitContainer(object):
         elif 'spine' in entry.state:
             internal = '/'.join(('/data', entry.internal, 'content'))
             raw = self._litfile.get_file(internal)
-            unbin = UnBinary(raw, name, self._litfile.manifest, HTML_MAP)
+            manifest = self._litfile.manifest
+            atoms = self._litfile.get_atoms(entry)
+            unbin = UnBinary(raw, name, manifest, HTML_MAP, atoms)
             content = HTML_DECL + str(unbin)
+        else:
+            internal = '/'.join(('/data', entry.internal))
+            content = self._litfile.get_file(internal)
+        return content
     
     def _read_meta(self):
         path = 'content.opf'

src/calibre/ebooks/lit/writer.py

@@ -27,7 +27,7 @@ from calibre.ebooks.oeb.base import OEB_DOCS, XHTML_MIME, OEB_STYLES, \
     CSS_MIME, OPF_MIME, XML_NS, XML
 from calibre.ebooks.oeb.base import namespace, barename, prefixname, \
     urlnormalize, xpath
-from calibre.ebooks.oeb.base import Logger, OEBBook
+from calibre.ebooks.oeb.base import OEBBook
 from calibre.ebooks.oeb.profile import Context
 from calibre.ebooks.oeb.stylizer import Stylizer
 from calibre.ebooks.oeb.transforms.flatcss import CSSFlattener
@@ -732,7 +732,7 @@ def option_parser():
     return parser
 
 def oeb2lit(opts, inpath):
-    logger = Logger(logging.getLogger('oeb2lit'))
+    logger = logging.getLogger('oeb2lit')
     logger.setup_cli_handler(opts.verbose)
     outpath = opts.output
     if outpath is None:

src/calibre/ebooks/metadata/fetch.py

@@ -59,6 +59,8 @@ class FetchISBNDB(Thread):
                 args.extend(['--author', self.author])
             if self.publisher:
                 args.extend(['--publisher', self.publisher])
+        if self.verbose:
+            args.extend(['--verbose'])
         args.append(self.key)
         try:
             opts, args = option_parser().parse_args(args)

src/calibre/ebooks/metadata/google_books.py

@@ -60,10 +60,12 @@ class Query(object):
             if title is not None:
                 q += build_term('title', title.split())
             if author is not None:
-                q += build_term('author', author.split())
+                q += ('+' if q else '')+build_term('author', author.split())
             if publisher is not None:
-                q += build_term('publisher', publisher.split())
+                q += ('+' if q else '')+build_term('publisher', publisher.split())
 
+        if isinstance(q, unicode):
+            q = q.encode('utf-8')
         self.url = self.BASE_URL+urlencode({
             'q':q,
             'max-results':max_results,

src/calibre/ebooks/metadata/isbndb.py

@@ -8,7 +8,7 @@ import sys, re, socket
 from urllib import urlopen, quote
 
 from calibre.utils.config import OptionParser
-from calibre.ebooks.metadata import MetaInformation, authors_to_sort_string
+from calibre.ebooks.metadata import MetaInformation
 from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
 
 BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%(key)s&page_number=1&results=subjects,authors,texts&'
@@ -28,7 +28,8 @@ def fetch_metadata(url, max=100, timeout=5.):
                 raw = urlopen(url).read()
             except Exception, err:
                 raise ISBNDBError('Could not fetch ISBNDB metadata. Error: '+str(err))
-            soup = BeautifulStoneSoup(raw)
+            soup = BeautifulStoneSoup(raw,
+                    convertEntities=BeautifulStoneSoup.XML_ENTITIES)
             book_list = soup.find('booklist')
             if book_list is None:
                 errmsg = soup.find('errormessage').string

src/calibre/ebooks/metadata/opf2.py

@@ -258,6 +258,11 @@ class Manifest(ResourceCollection):
             if i.id == id:
                 return i.path
 
+    def type_for_id(self, id):
+        for i in self:
+            if i.id == id:
+                return i.mime_type
+
 class Spine(ResourceCollection):
 
     class Item(Resource):
@@ -444,7 +449,7 @@ class OPF(object):
         if not hasattr(stream, 'read'):
             stream = open(stream, 'rb')
         self.basedir  = self.base_dir = basedir
-        self.path_to_html_toc = None
+        self.path_to_html_toc = self.html_toc_fragment = None
         raw, self.encoding = xml_to_unicode(stream.read(), strip_encoding_pats=True, resolve_entities=True)
         raw = raw[raw.find('<'):]
         self.root     = etree.fromstring(raw, self.PARSER)
@@ -487,7 +492,10 @@ class OPF(object):
 
             if toc is None: return
             self.toc = TOC(base_path=self.base_dir)
-            if toc.lower() in ('ncx', 'ncxtoc'):
+            is_ncx = getattr(self, 'manifest', None) is not None and \
+                     self.manifest.type_for_id(toc) is not None and \
+                     'dtbncx' in self.manifest.type_for_id(toc)
+            if is_ncx or toc.lower() in ('ncx', 'ncxtoc'):
                 path = self.manifest.path_for_id(toc)
                 if path:
                     self.toc.read_ncx_toc(path)
@@ -496,7 +504,8 @@ class OPF(object):
                     if f:
                         self.toc.read_ncx_toc(f[0])
             else:
-                self.path_to_html_toc = toc
+                self.path_to_html_toc, self.html_toc_fragment = \
+                    toc.partition('#')[0], toc.partition('#')[-1]
                 self.toc.read_html_toc(toc)
         except:
             pass

src/calibre/ebooks/mobi/input.py

@@ -12,19 +12,22 @@ class MOBIInput(InputFormatPlugin):
     description = 'Convert MOBI files (.mobi, .prc, .azw) to HTML'
     file_types  = set(['mobi', 'prc', 'azw'])
 
-    def convert(self, stream, options, file_ext, parse_cache, log, 
+    def convert(self, stream, options, file_ext, log,
                 accelerators):
         from calibre.ebooks.mobi.reader import MobiReader
+        from lxml import html
         mr = MobiReader(stream, log, options.input_encoding,
                         options.debug_input)
+        parse_cache = {}
         mr.extract_content('.', parse_cache)
-        raw = parse_cache.get('calibre_raw_mobi_markup', False)
+        raw = parse_cache.pop('calibre_raw_mobi_markup', False)
         if raw:
             if isinstance(raw, unicode):
                 raw = raw.encode('utf-8')
             open('debug-raw.html', 'wb').write(raw)
         for f, root in parse_cache.items():
-            if '.' in f:
+            with open(f, 'wb') as q:
-                accelerators[f] = {'pagebreaks':root.xpath(
+                q.write(html.tostring(root, encoding='utf-8', method='xml',
-                                            '//div[@class="mbp_pagebreak"]')}
+                    include_meta_content_type=False))
+            accelerators['pagebreaks'] = {f: '//div[@class="mbp_pagebreak"]'}
         return mr.created_opf_path

src/calibre/ebooks/mobi/reader.py

@@ -5,10 +5,11 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Read data from .mobi files
 '''
 
-import struct, os, cStringIO, re, functools
+import struct, os, cStringIO, re, functools, datetime
 
 try:
     from PIL import Image as PILImage
+    PILImage
 except ImportError:
     import Image as PILImage
 
@@ -52,6 +53,14 @@ class EXTHHeader(object):
                     self.cover_offset = co
             elif id == 202:
                 self.thumbnail_offset, = struct.unpack('>L', content)
+            elif id == 501:
+                # cdetype
+                pass
+            elif id == 502:
+                # last update time
+                pass
+            elif id == 503 and (not title or title == _('Unknown')):
+                title = content
             #else:
             #    print 'unknown record', id, repr(content)
         if title:
@@ -72,8 +81,14 @@ class EXTHHeader(object):
             if not self.mi.tags:
                 self.mi.tags = []
             self.mi.tags.append(content.decode(codec, 'ignore'))
+        elif id == 106:
+            try:
+                self.mi.publish_date = datetime.datetime.strptime(
+                        content, '%Y-%m-%d',).date()
+            except:
+                pass
         #else:
-        #    print 'unhandled metadata record', id, repr(content), codec 
+        #    print 'unhandled metadata record', id, repr(content)
 
 
 class BookHeader(object):
@@ -110,7 +125,6 @@ class BookHeader(object):
                 self.codec = 'cp1252' if user_encoding is None else user_encoding
                 log.warn('Unknown codepage %d. Assuming %s'%(self.codepage,
                                                             self.codec))
-            
             if ident == 'TEXTREAD' or self.length < 0xE4 or 0xE8 < self.length:
                 self.extra_flags = 0
             else:
@@ -267,6 +281,17 @@ class MobiReader(object):
                     rule = rule.encode('utf-8')
                 s.write(rule+'\n\n')
 
+
+        if self.book_header.exth is not None or self.embedded_mi is not None:
+            if self.verbose:
+                print 'Creating OPF...'
+            ncx = cStringIO.StringIO()
+            opf = self.create_opf(htmlfile, guide, root)
+            opf.render(open(os.path.splitext(htmlfile)[0]+'.opf', 'wb'), ncx)
+            ncx = ncx.getvalue()
+            if ncx:
+                open(os.path.splitext(htmlfile)[0]+'.ncx', 'wb').write(ncx)
+
     def read_embedded_metadata(self, root, elem, guide):
         raw = '<package>'+html.tostring(elem, encoding='utf-8')+'</package>'
         stream = cStringIO.StringIO(raw)
@@ -314,8 +339,8 @@ class MobiReader(object):
         mobi_version = self.book_header.mobi_version
         for i, tag in enumerate(root.iter(etree.Element)):
             if tag.tag in ('country-region', 'place', 'placetype', 'placename',
-                           'state', 'city', 'street', 'address'):
+                           'state', 'city', 'street', 'address', 'content'):
-                tag.tag = 'span'
+                tag.tag = 'div' if tag.tag == 'content' else 'span'
                 for key in tag.attrib.keys():
                     tag.attrib.pop(key)
                 continue
@@ -369,19 +394,19 @@ class MobiReader(object):
 
             if 'filepos-id' in attrib:
                 attrib['id'] = attrib.pop('filepos-id')
+                if 'name' in attrib and attrib['name'] != attrib['id']:
+                    attrib['name'] = attrib['id']
             if 'filepos' in attrib:
                 filepos = attrib.pop('filepos')
                 try:
                     attrib['href'] = "#filepos%d" % int(filepos)
                 except ValueError:
                     pass
-            
             if styles:
                 attrib['id'] = attrib.get('id', 'calibre_mr_gid%d'%i)
                 self.tag_css_rules.append('#%s {%s}'%(attrib['id'],
                                                       '; '.join(styles)))
 
-    
     def create_opf(self, htmlfile, guide=None, root=None):
         mi = getattr(self.book_header.exth, 'mi', self.embedded_mi)
         if mi is None:
@@ -583,3 +608,4 @@ def get_metadata(stream):
             log.exception()
     return mi
 
+

src/calibre/ebooks/mobi/writer.py

@@ -211,12 +211,14 @@ class Serializer(object):
 
     def serialize_item(self, item):
         buffer = self.buffer
+        #buffer.write('<mbp:section>')
         if not item.linear:
             self.breaks.append(buffer.tell() - 1)
         self.id_offsets[item.href] = buffer.tell()
         for elem in item.data.find(XHTML('body')):
             self.serialize_elem(elem, item)
-        buffer.write('<mbp:pagebreak/>')
+        #buffer.write('</mbp:section>')
+        buffer.write('</mbp:pagebreak>')
 
     def serialize_elem(self, elem, item, nsrmap=NSRMAP):
         buffer = self.buffer

src/calibre/ebooks/oeb/base.py

@@ -13,11 +13,15 @@ from collections import defaultdict
 from itertools import count
 from urlparse import urldefrag, urlparse, urlunparse
 from urllib import unquote as urlunquote
+import logging
 from lxml import etree, html
 import calibre
+from cssutils import CSSParser
 from calibre.translations.dynamic import translate
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre.ebooks.oeb.entitydefs import ENTITYDEFS
+from calibre.ebooks.conversion.preprocess import HTMLPreProcessor, \
+        CSSPreProcessor
 
 XML_NS       = 'http://www.w3.org/XML/1998/namespace'
 XHTML_NS     = 'http://www.w3.org/1999/xhtml'
@@ -99,6 +103,8 @@ PNG_MIME       = types_map['.png']
 SVG_MIME       = types_map['.svg']
 BINARY_MIME    = 'application/octet-stream'
 
+XHTML_CSS_NAMESPACE = u'@namespace "%s";\n' % XHTML_NS
+
 OEB_STYLES        = set([CSS_MIME, OEB_CSS_MIME, 'text/x-oeb-css'])
 OEB_DOCS          = set([XHTML_MIME, 'text/html', OEB_DOC_MIME,
                          'text/x-oeb-document'])
@@ -203,6 +209,10 @@ class OEBError(Exception):
     """Generic OEB-processing error."""
     pass
 
+class NotHTML(OEBError):
+    '''Raised when a file that should be HTML (as per manifest) is not'''
+    pass
+
 class NullContainer(object):
     """An empty container.
 
@@ -234,7 +244,7 @@ class DirContainer(object):
         for path in self.namelist():
             ext = os.path.splitext(path)[1].lower()
             if ext == '.opf':
-                self.opfname = fname
+                self.opfname = path
                 return
         self.opfname = None
 
@@ -284,6 +294,9 @@ class Metadata(object):
     OPF_ATTRS     = {'role': OPF('role'), 'file-as': OPF('file-as'),
                      'scheme': OPF('scheme'), 'event': OPF('event'),
                      'type': XSI('type'), 'lang': XML('lang'), 'id': 'id'}
+    OPF1_NSMAP    = {'dc': DC11_NS, 'oebpackage': OPF1_NS}
+    OPF2_NSMAP    = {'opf': OPF2_NS, 'dc': DC11_NS, 'dcterms': DCTERMS_NS,
+                     'xsi': XSI_NS, 'calibre': CALIBRE_NS}
 
     class Item(object):
         """An item of OEB data model metadata.
@@ -565,17 +578,10 @@ class Manifest(object):
             return 'Item(id=%r, href=%r, media_type=%r)' \
                 % (self.id, self.href, self.media_type)
 
-        def _force_xhtml(self, data):
+        def _parse_xhtml(self, data):
             # Convert to Unicode and normalize line endings
             data = self.oeb.decode(data)
-            data = XMLDECL_RE.sub('', data)
+            data = self.oeb.html_preprocessor(data)
-            # Handle broken XHTML w/ SVG (ugh)
-            if 'svg:' in data and SVG_NS not in data:
-                data = data.replace(
-                    '<html', '<html xmlns:svg="%s"' % SVG_NS, 1)
-            if 'xlink:' in data and XLINK_NS not in data:
-                data = data.replace(
-                    '<html', '<html xmlns:xlink="%s"' % XLINK_NS, 1)
             # Try with more & more drastic measures to parse
             try:
                 data = etree.fromstring(data)
@@ -599,12 +605,16 @@ class Manifest(object):
                         data = etree.fromstring(data, parser=RECOVER_PARSER)
             # Force into the XHTML namespace
             if barename(data.tag) != 'html':
-                raise OEBError(
+                raise NotHTML(
                     'File %r does not appear to be (X)HTML' % self.href)
             elif not namespace(data.tag):
                 data.attrib['xmlns'] = XHTML_NS
                 data = etree.tostring(data, encoding=unicode)
-                data = etree.fromstring(data)
+                try:
+                    data = etree.fromstring(data)
+                except:
+                    data=data.replace(':=', '=').replace(':>', '>')
+                    data = etree.fromstring(data)
             elif namespace(data.tag) != XHTML_NS:
                 # OEB_DOC_NS, but possibly others
                 ns = namespace(data.tag)
@@ -646,6 +656,28 @@ class Manifest(object):
                 etree.SubElement(data, XHTML('body'))
             return data
 
+        def _parse_css(self, data):
+            data = self.oeb.decode(data)
+            data = self.CSSPreProcessor(data)
+            data = XHTML_CSS_NAMESPACE + data
+            parser = CSSParser(log=self.oeb.logger, loglevel=logging.WARNING,
+                               fetcher=self._fetch_css)
+            data = parser.parseString(data, href=self.href)
+            data.namespaces['h'] = XHTML_NS
+            return data
+
+        def _fetch_css(self, path):
+            hrefs = self.oeb.manifest.hrefs
+            if path not in hrefs:
+                self.oeb.logger.warn('CSS import of missing file %r' % path)
+                return (None, None)
+            item = hrefs[path]
+            if item.media_type not in OEB_STYLES:
+                self.oeb.logger.warn('CSS import of non-CSS file %r' % path)
+                return (None, None)
+            data = item.data.cssText
+            return ('utf-8', data)
+
         @dynamic_property
         def data(self):
             doc = """Provides MIME type sensitive access to the manifest
@@ -661,15 +693,19 @@ class Manifest(object):
               special parsing.
             """
             def fget(self):
-                if self._data is not None:
+                data = self._data
-                    return self._data
+                if data is None:
-                data = self._loader(self.href)
+                    if self._loader is None:
-                if self.media_type in OEB_DOCS:
+                        return None
-                    data = self._force_xhtml(data)
+                    data = self._loader(self.href)
+                if not isinstance(data, basestring):
+                    pass # already parsed
+                elif self.media_type in OEB_DOCS:
+                    data = self._parse_xhtml(data)
                 elif self.media_type[-4:] in ('+xml', '/xml'):
                     data = etree.fromstring(data)
                 elif self.media_type in OEB_STYLES:
-                    data = self.oeb.decode(data)
+                    data = self._parse_css(data)
                 self._data = data
                 return data
             def fset(self, value):
@@ -757,7 +793,7 @@ class Manifest(object):
         MIME type which is not one of the OPS core media types.  Either the
         item's data itself may be provided with :param:`data`, or a loader
         function for the data may be provided with :param:`loader`, or the
-        item's data may latter be set manually via the :attr:`data` attribute.
+        item's data may later be set manually via the :attr:`data` attribute.
         """
         item = self.Item(
             self.oeb, id, href, media_type, fallback, loader, data)
@@ -804,6 +840,9 @@ class Manifest(object):
         for item in self.items:
             yield item
 
+    def __len__(self):
+        return len(self.items)
+
     def values(self):
         return list(self.items)
 
@@ -1216,17 +1255,25 @@ class PageList(object):
 class OEBBook(object):
     """Representation of a book in the IDPF OEB data model."""
 
-    def __init__(self, logger, parse_cache={}, encoding='utf-8', 
+    COVER_SVG_XP    = XPath('h:body//svg:svg[position() = 1]')
-                 pretty_print=False):
+    COVER_OBJECT_XP = XPath('h:body//h:object[@data][position() = 1]')
-        """Create empty book.  Optional arguments:
+
+    def __init__(self, logger,
+            html_preprocessor=HTMLPreProcessor(),
+            css_preprocessor=CSSPreProcessor(),
+            encoding='utf-8', pretty_print=False):
+        """Create empty book.  Arguments:
 
-        :param parse_cache: A cache of parsed XHTML/CSS. Keys are absolute
-            paths to te cached files and values are lxml root objects and
-            cssutils stylesheets.
         :param:`encoding`: Default encoding for textual content read
             from an external container.
         :param:`pretty_print`: Whether or not the canonical string form
             of XML markup is pretty-printed.
+        :param html_preprocessor: A callable that takes a unicode object
+            and returns a unicode object. Will be called on all html files
+            before they are parsed.
+        :param css_preprocessor: A callable that takes a unicode object
+            and returns a unicode object. Will be called on all CSS files
+            before they are parsed.
         :param:`logger`: A Log object to use for logging all messages
             related to the processing of this book.  It is accessible
             via the instance data members :attr:`logger,log`.
@@ -1245,7 +1292,10 @@ class OEBBook(object):
         :attr:`pages`: List of "pages," such as indexed to a print edition of
             the same text.
         """
+
         self.encoding = encoding
+        self.html_preprocessor = html_preprocessor
+        self.css_preprocessor = css_preprocessor
         self.pretty_print = pretty_print
         self.logger = self.log = logger
         self.version = '2.0'

src/calibre/ebooks/oeb/factory.py

@@ -8,6 +8,7 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
 import sys, os, logging
 from itertools import chain
+import calibre
 from calibre.ebooks.oeb.base import OEBError
 from calibre.ebooks.oeb.reader import OEBReader
 from calibre.ebooks.oeb.writer import OEBWriter
@@ -15,7 +16,7 @@ from calibre.ebooks.lit.reader import LitReader
 from calibre.ebooks.lit.writer import LitWriter
 from calibre.ebooks.mobi.reader import MobiReader
 from calibre.ebooks.mobi.writer import MobiWriter
-from calibre.ebooks.oeb.base import Logger, OEBBook
+from calibre.ebooks.oeb.base import OEBBook
 from calibre.ebooks.oeb.profile import Context
 from calibre.utils.config import Config
 
@@ -77,8 +78,8 @@ def main(argv=sys.argv):
     if len(args) != 0:
         parser.print_help()
         return 1
-    logger = Logger(logging.getLogger('ebook-convert'))
+    logger = logging.getLogger('ebook-convert')
-    logger.setup_cli_handler(opts.verbose)
+    calibre.setup_cli_handlers(logger, logging.DEBUG)
     encoding = opts.encoding
     pretty_print = opts.pretty_print
     oeb = OEBBook(encoding=encoding, pretty_print=pretty_print, logger=logger)

src/calibre/ebooks/oeb/output.py

@@ -12,6 +12,6 @@ class OEBOutput(OutputFormatPlugin):
     file_type = 'oeb'
 
 
-    def convert(self, oeb_book, input_plugin, options, parse_cache, log):
+    def convert(self, oeb_book, input_plugin, options, context, log):
-        pass
+       pass
 

src/calibre/ebooks/oeb/reader.py

@@ -161,10 +161,30 @@ class OEBReader(object):
             self.logger.warn('Title not specified')
             metadata.add('title', self.oeb.translate(__('Unknown')))
 
-    def _manifest_add_missing(self):
+    def _manifest_prune_invalid(self):
+        '''
+        Remove items from manifest that contain invalid data. This prevents
+        catastrophic conversion failure, when a few files contain corrupted
+        data.
+        '''
+        bad = []
+        check = OEB_DOCS+OEB_STYLES
+        for item in list(self.oeb.manifest.values()):
+            if item.media_type in check:
+                try:
+                    item.data
+                except:
+                    self.logger.exception('Failed to parse content in %s'%
+                            item.href)
+                    bad.append(item)
+                    self.oeb.manifest.remove(item)
+        return bad
+
+    def _manifest_add_missing(self, invalid):
         manifest = self.oeb.manifest
         known = set(manifest.hrefs)
         unchecked = set(manifest.values())
+        bad = []
         while unchecked:
             new = set()
             for item in unchecked:
@@ -181,7 +201,7 @@ class OEBReader(object):
                         if not scheme and href not in known:
                             new.add(href)
                 elif item.media_type in OEB_STYLES:
-                    for match in CSSURL_RE.finditer(item.data):
+                    for match in CSSURL_RE.finditer(item.data.cssText):
                         href, _ = urldefrag(match.group('url'))
                         href = item.abshref(urlnormalize(href))
                         scheme = urlparse(href).scheme
@@ -190,6 +210,13 @@ class OEBReader(object):
             unchecked.clear()
             for href in new:
                 known.add(href)
+                is_invalid = False
+                for item in invalid:
+                    if href == item.abshref(urlnormalize(href)):
+                        is_invalid = True
+                        break
+                if is_invalid:
+                    continue
                 if not self.oeb.container.exists(href):
                     self.logger.warn('Referenced file %r not found' % href)
                     continue
@@ -222,7 +249,8 @@ class OEBReader(object):
                 self.logger.warn(u'Duplicate manifest id %r' % id)
                 id, href = manifest.generate(id, href)
             manifest.add(id, href, media_type, fallback)
-        self._manifest_add_missing()
+        invalid = self._manifest_prune_invalid()
+        self._manifest_add_missing(invalid)
 
     def _spine_add_extra(self):
         manifest = self.oeb.manifest

src/calibre/ebooks/oeb/stylizer.py

@@ -115,8 +115,7 @@ class Stylizer(object):
         cssname = os.path.splitext(basename)[0] + '.css'
         stylesheets = [HTML_CSS_STYLESHEET]
         head = xpath(tree, '/h:html/h:head')[0]
-        parser = cssutils.CSSParser()
+        parser = cssutils.CSSParser(fetcher=self._fetch_css_file)
-        parser.setFetcher(self._fetch_css_file)
         for elem in head:
             if elem.tag == XHTML('style') and elem.text \
                and elem.get('type', CSS_MIME) in OEB_STYLES:
@@ -135,14 +134,7 @@ class Stylizer(object):
                         'Stylesheet %r referenced by file %r not in manifest' %
                         (path, item.href))
                     continue
-                if sitem in self.STYLESHEETS:
+                stylesheets.append(sitem.data)
-                    stylesheet = self.STYLESHEETS[sitem]
-                else:
-                    data = self._fetch_css_file(path)[1]
-                    stylesheet = parser.parseString(data, href=path)
-                    stylesheet.namespaces['h'] = XHTML_NS
-                    self.STYLESHEETS[sitem] = stylesheet
-                stylesheets.append(stylesheet)
         rules = []
         index = 0
         self.stylesheets = set()
@@ -159,9 +151,9 @@ class Stylizer(object):
         for _, _, cssdict, text, _ in rules:
             try:
                 selector = CSSSelector(text)
-            except (AssertionError, ExpressionError, etree.XPathSyntaxError,\
+            except (AssertionError, ExpressionError, etree.XPathSyntaxError,
-                NameError, # gets thrown on OS X instead of SelectorSyntaxError
+                    NameError, # thrown on OS X instead of SelectorSyntaxError
-                SelectorSyntaxError):
+                    SelectorSyntaxError):
                 continue
             for elem in selector(tree):
                 self.style(elem)._update_cssdict(cssdict)
@@ -171,9 +163,13 @@ class Stylizer(object):
     def _fetch_css_file(self, path):
         hrefs = self.oeb.manifest.hrefs
         if path not in hrefs:
+            self.logger.warn('CSS import of missing file %r' % path)
             return (None, None)
-        data = hrefs[path].data
+        item = hrefs[path]
-        data = XHTML_CSS_NAMESPACE + data
+        if item.media_type not in OEB_STYLES:
+            self.logger.warn('CSS import of non-CSS file %r' % path)
+            return (None, None)
+        data = item.data.cssText
         return ('utf-8', data)
     
     def flatten_rule(self, rule, href, index):

src/calibre/ebooks/oeb/transforms/__init__.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import with_statement
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+

src/calibre/ebooks/oeb/transforms/trimmanifest.py

@@ -24,7 +24,6 @@ class ManifestTrimmer(object):
     def __call__(self, oeb, context):
         oeb.logger.info('Trimming unused files from manifest...')
         used = set()
-        hrefs = oeb.manifest.hrefs
         for term in oeb.metadata:
             for item in oeb.metadata[term]:
                 if item.value in oeb.manifest.hrefs:
@@ -53,7 +52,7 @@ class ManifestTrimmer(object):
                             if found not in used:
                                 new.add(found)
                 elif item.media_type == CSS_MIME:
-                    for match in CSSURL_RE.finditer(item.data):
+                    for match in CSSURL_RE.finditer(item.data.cssText):
                         href = match.group('url')
                         href = item.abshref(urlnormalize(href))
                         if href in oeb.manifest.hrefs:

src/calibre/ebooks/oeb/writer.py

@@ -8,7 +8,7 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>'
 
 import sys, os, logging
 from calibre.ebooks.oeb.base import OPF_MIME, xml2str
-from calibre.ebooks.oeb.base import Logger, DirContainer, OEBBook
+from calibre.ebooks.oeb.base import DirContainer, OEBBook
 
 __all__ = ['OEBWriter']
 

src/calibre/gui2/__init__.py

@@ -1,16 +1,15 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 """ The GUI """
-import sys, os, re, StringIO, traceback, time
+import os
 from PyQt4.QtCore import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \
-                         QByteArray, QLocale, QUrl, QTranslator, QCoreApplication, \
+                         QByteArray, QUrl, QTranslator, QCoreApplication
-                         QModelIndex
 from PyQt4.QtGui import QFileDialog, QMessageBox, QPixmap, QFileIconProvider, \
-                        QIcon, QTableView, QDialogButtonBox, QApplication, QDialog
+                        QIcon, QTableView, QApplication, QDialog
 
 ORG_NAME = 'KovidsBrain'
 APP_UID  = 'libprs500'
-from calibre import __author__, islinux, iswindows, isosx
+from calibre import islinux, iswindows
 from calibre.startup import get_lang
 from calibre.utils.config import Config, ConfigProxy, dynamic
 import calibre.resources as resources
@@ -65,6 +64,9 @@ def _config():
               help=_('Show the cover flow in a separate window instead of in the main calibre window'))
     c.add_opt('disable_tray_notification', default=False,
               help=_('Disable notifications from the system tray icon'))
+    c.add_opt('default_send_to_device_action', default=None,
+            help=_('Default action to perform when send to device button is '
+                'clicked'))
     return ConfigProxy(c)
 
 config = _config()
@@ -139,15 +141,15 @@ def human_readable(size):
 
 class Dispatcher(QObject):
     '''Convenience class to ensure that a function call always happens in the GUI thread'''
+    SIGNAL = SIGNAL('dispatcher(PyQt_PyObject,PyQt_PyObject)')
 
     def __init__(self, func):
         QObject.__init__(self)
         self.func = func
-        self.connect(self, SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject)'), 
+        self.connect(self, self.SIGNAL, self.dispatch, Qt.QueuedConnection)
-                     self.dispatch, Qt.QueuedConnection)
 
     def __call__(self, *args, **kwargs):
-        self.emit(SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject)'), args, kwargs)
+        self.emit(self.SIGNAL, args, kwargs)
 
     def dispatch(self, args, kwargs):
         self.func(*args, **kwargs)
@@ -447,6 +449,7 @@ class ResizableDialog(QDialog):
 
 try:
     from calibre.utils.single_qt_application import SingleApplication
+    SingleApplication
 except:
     SingleApplication = None
 

src/calibre/gui2/device.py

@@ -1,12 +1,29 @@
+from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import os, traceback, Queue, time
+import os, traceback, Queue, time, socket
-from threading import Thread
+from threading import Thread, RLock
+from itertools import repeat
+from functools import partial
+from binascii import unhexlify
+
+from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, QPixmap, \
+                     Qt
 
 from calibre.devices import devices
+from calibre.gui2.dialogs.choose_format import ChooseFormatDialog
 from calibre.parallel import Job
 from calibre.devices.scanner import DeviceScanner
-
+from calibre.gui2 import config, error_dialog, Dispatcher, dynamic, \
+                                   pixmap_to_data, warning_dialog
+from calibre.ebooks.metadata import authors_to_string
+from calibre.gui2.dialogs.conversion_error import ConversionErrorDialog
+from calibre.devices.interface import Device
+from calibre import sanitize_file_name, preferred_encoding
+from calibre.utils.filenames import ascii_filename
+from calibre.devices.errors import FreeSpaceError
+from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \
+        config as email_config
 
 class DeviceJob(Job):
 
@@ -26,11 +43,7 @@ class DeviceJob(Job):
 
 
 class DeviceManager(Thread):
-    '''
+
-    Worker thread that polls the USB ports for devices. Emits the
-    signal connected(PyQt_PyObject, PyQt_PyObject) on connection and
-    disconnection events.
-    '''
     def __init__(self, connected_slot, job_manager, sleep_time=2):
         '''
         @param sleep_time: Time to sleep between device probes in millisecs
@@ -104,6 +117,12 @@ class DeviceManager(Thread):
         self.jobs.put(job)
         return job
 
+    def has_card(self):
+        try:
+            return bool(self.device.card_prefix())
+        except:
+            return False
+
     def _get_device_information(self):
         info = self.device.get_device_information(end_session=False)
         info = [i.replace('\x00', '').replace('\x01', '') for i in info]
@@ -116,7 +135,6 @@ class DeviceManager(Thread):
         return self.create_job(self._get_device_information, done,
                     description=_('Get device information'))
 
-    
     def _books(self):
         '''Get metadata from device'''
         mainlist = self.device.books(oncard=False, end_session=False)
@@ -185,3 +203,487 @@ class DeviceManager(Thread):
         return self.create_job(self._view_book, done, args=[path, target],
                         description=_('View book on device'))
 
+
+class DeviceAction(QAction):
+
+    def __init__(self, dest, delete, specific, icon_path, text, parent=None):
+        if delete:
+            text += ' ' + _('and delete from library')
+        QAction.__init__(self, QIcon(icon_path), text, parent)
+        self.dest = dest
+        self.delete = delete
+        self.specific = specific
+        self.connect(self, SIGNAL('triggered(bool)'),
+                lambda x : self.emit(SIGNAL('a_s(QAction)'), self))
+
+    def __repr__(self):
+        return self.__class__.__name__ + ':%s:%s:%s'%(self.dest, self.delete,
+                self.specific)
+
+
+class DeviceMenu(QMenu):
+
+    def __init__(self, parent=None):
+        QMenu.__init__(self, parent)
+        self.group = QActionGroup(self)
+        self.actions = []
+        self._memory = []
+
+        self.set_default_menu = self.addMenu(_('Set default send to device'
+            ' action'))
+        opts = email_config().parse()
+        default_account = None
+        if opts.accounts:
+            self.email_to_menu = self.addMenu(_('Email to')+'...')
+            keys = sorted(opts.accounts.keys())
+            for account in keys:
+                formats, auto, default = opts.accounts[account]
+                dest = 'mail:'+account+';'+formats
+                if default:
+                    default_account = (dest, False, False, ':/images/mail.svg',
+                            _('Email to')+' '+account)
+                action1 = DeviceAction(dest, False, False, ':/images/mail.svg',
+                        _('Email to')+' '+account, self)
+                action2 = DeviceAction(dest, True, False, ':/images/mail.svg',
+                        _('Email to')+' '+account, self)
+                map(self.email_to_menu.addAction, (action1, action2))
+                map(self._memory.append, (action1, action2))
+                self.email_to_menu.addSeparator()
+                self.connect(action1, SIGNAL('a_s(QAction)'),
+                            self.action_triggered)
+                self.connect(action2, SIGNAL('a_s(QAction)'),
+                            self.action_triggered)
+
+
+
+
+        _actions = [
+                ('main:', False, False,  ':/images/reader.svg',
+                    _('Send to main memory')),
+                ('card:0', False, False, ':/images/sd.svg',
+                    _('Send to storage card')),
+                '-----',
+                ('main:', True, False,   ':/images/reader.svg',
+                    _('Send to main memory')),
+                ('card:0', True, False,  ':/images/sd.svg',
+                    _('Send to storage card')),
+                '-----',
+                ('main:', False, True,  ':/images/reader.svg',
+                    _('Send specific format to main memory')),
+                ('card:0', False, True, ':/images/sd.svg',
+                    _('Send specific format to storage card')),
+
+                ]
+        if default_account is not None:
+            _actions.insert(2, default_account)
+            _actions.insert(6, list(default_account))
+            _actions[6][1] = True
+        for round in (0, 1):
+            for dest, delete, specific, icon, text in _actions:
+                if dest == '-':
+                    (self.set_default_menu if round else self).addSeparator()
+                    continue
+                action = DeviceAction(dest, delete, specific, icon, text, self)
+                self._memory.append(action)
+                if round == 1:
+                    action.setCheckable(True)
+                    action.setText(action.text())
+                    self.group.addAction(action)
+                    self.set_default_menu.addAction(action)
+                else:
+                    self.connect(action, SIGNAL('a_s(QAction)'),
+                            self.action_triggered)
+                    self.actions.append(action)
+                    self.addAction(action)
+
+
+        da = config['default_send_to_device_action']
+        done = False
+        for action in self.group.actions():
+            if repr(action) == da:
+                action.setChecked(True)
+                done = True
+                break
+        if not done:
+            action = list(self.group.actions())[0]
+            action.setChecked(True)
+            config['default_send_to_device_action'] = repr(action)
+
+        self.connect(self.group, SIGNAL('triggered(QAction*)'),
+                self.change_default_action)
+        self.enable_device_actions(False)
+        if opts.accounts:
+            self.addSeparator()
+            self.addMenu(self.email_to_menu)
+
+    def change_default_action(self, action):
+        config['default_send_to_device_action'] = repr(action)
+        action.setChecked(True)
+
+    def action_triggered(self, action):
+        self.emit(SIGNAL('sync(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                action.dest, action.delete, action.specific)
+
+    def trigger_default(self, *args):
+        r = config['default_send_to_device_action']
+        for action in self.actions:
+            if repr(action) == r:
+                self.action_triggered(action)
+                break
+
+    def enable_device_actions(self, enable):
+        for action in self.actions:
+            if action.dest[:4] in ('main', 'card'):
+                action.setEnabled(enable)
+
+class Emailer(Thread):
+
+    def __init__(self, timeout=60):
+        Thread.__init__(self)
+        self.setDaemon(True)
+        self.job_lock = RLock()
+        self.jobs = []
+        self._run = True
+        self.timeout = timeout
+
+    def run(self):
+        while self._run:
+            job = None
+            with self.job_lock:
+                if self.jobs:
+                    job = self.jobs[0]
+                    self.jobs = self.jobs[1:]
+            if job is not None:
+                self._send_mails(*job)
+            time.sleep(1)
+
+    def stop(self):
+        self._run = False
+
+    def send_mails(self, jobnames, callback, attachments, to_s, subjects,
+                  texts, attachment_names):
+        job = (jobnames, callback, attachments, to_s, subjects, texts,
+                attachment_names)
+        with self.job_lock:
+            self.jobs.append(job)
+
+    def _send_mails(self, jobnames, callback, attachments,
+                    to_s, subjects, texts, attachment_names):
+        opts = email_config().parse()
+        opts.verbose = 3 if os.environ.get('CALIBRE_DEBUG_EMAIL', False) else 0
+        from_ = opts.from_
+        if not from_:
+            from_ = 'calibre <calibre@'+socket.getfqdn()+'>'
+        results = []
+        for i, jobname in enumerate(jobnames):
+            try:
+                msg = compose_mail(from_, to_s[i], texts[i], subjects[i],
+                        open(attachments[i], 'rb'),
+                        attachment_name = attachment_names[i])
+                efrom, eto = map(extract_email_address, (from_, to_s[i]))
+                eto = [eto]
+                sendmail(msg, efrom, eto, localhost=None,
+                            verbose=opts.verbose,
+                            timeout=self.timeout, relay=opts.relay_host,
+                            username=opts.relay_username,
+                            password=unhexlify(opts.relay_password), port=opts.relay_port,
+                            encryption=opts.encryption)
+                results.append([jobname, None, None])
+            except Exception, e:
+                results.append([jobname, e, traceback.format_exc()])
+        callback(results)
+
+
+class DeviceGUI(object):
+
+    def dispatch_sync_event(self, dest, delete, specific):
+        rows = self.library_view.selectionModel().selectedRows()
+        if not rows or len(rows) == 0:
+            error_dialog(self, _('No books'), _('No books')+' '+\
+                    _('selected to send')).exec_()
+            return
+
+        fmt = None
+        if specific:
+            d = ChooseFormatDialog(self, _('Choose format to send to device'),
+                                self.device_manager.device_class.FORMATS)
+            d.exec_()
+            fmt = d.format().lower()
+        dest, sub_dest = dest.split(':')
+        if dest in ('main', 'card'):
+            if not self.device_connected or not self.device_manager:
+                error_dialog(self, _('No device'),
+                        _('Cannot send: No device is connected')).exec_()
+                return
+            on_card = dest == 'card'
+            if on_card and not self.device_manager.has_card():
+                error_dialog(self, _('No card'),
+                        _('Cannot send: Device has no storage card')).exec_()
+                return
+            self.sync_to_device(on_card, delete, fmt)
+        elif dest == 'mail':
+            to, fmts = sub_dest.split(';')
+            fmts = [x.strip().lower() for x in fmts.split(',')]
+            self.send_by_mail(to, fmts, delete)
+
+    def send_by_mail(self, to, fmts, delete_from_library):
+        rows = self.library_view.selectionModel().selectedRows()
+        if not rows or len(rows) == 0:
+            return
+        ids = iter(self.library_view.model().id(r) for r in rows)
+        full_metadata = self.library_view.model().get_metadata(
+                                        rows, full_metadata=True)[-1]
+        files = self.library_view.model().get_preferred_formats(rows,
+                                    fmts, paths=True, set_metadata=True)
+        files = [getattr(f, 'name', None) for f in files]
+
+        bad, remove_ids, jobnames = [], [], []
+        texts, subjects, attachments, attachment_names = [], [], [], []
+        for f, mi, id in zip(files, full_metadata, ids):
+            t = mi.title
+            if not t:
+                t = _('Unknown')
+            if f is None:
+                bad.append(t)
+            else:
+                remove_ids.append(id)
+                jobnames.append(u'%s:%s'%(id, t))
+                attachments.append(f)
+                subjects.append(_('E-book:')+ ' '+t)
+                a = authors_to_string(mi.authors if mi.authors else \
+                        [_('Unknown')])
+                texts.append(_('Attached, you will find the e-book') + \
+                        '\n\n' + t + '\n\t' + _('by') + ' ' + a + '\n\n' + \
+                        _('in the %s format.') %
+                        os.path.splitext(f)[1][1:].upper())
+                prefix = sanitize_file_name(t+' - '+a)
+                if not isinstance(prefix, unicode):
+                    prefix = prefix.decode(preferred_encoding, 'replace')
+                attachment_names.append(prefix + os.path.splitext(f)[1])
+        remove = remove_ids if delete_from_library else []
+
+        to_s = list(repeat(to, len(attachments)))
+        if attachments:
+            self.emailer.send_mails(jobnames,
+                    Dispatcher(partial(self.emails_sent, remove=remove)),
+                    attachments, to_s, subjects, texts, attachment_names)
+            self.status_bar.showMessage(_('Sending email to')+' '+to, 3000)
+
+        if bad:
+            bad = '\n'.join('<li>%s</li>'%(i,) for i in bad)
+            d = warning_dialog(self, _('No suitable formats'),
+                '<p>'+ _('Could not email the following books '
+                'as no suitable formats were found:<br><ul>%s</ul>')%(bad,))
+            d.exec_()
+
+    def emails_sent(self, results, remove=[]):
+        errors, good = [], []
+        for jobname, exception, tb in results:
+            id = jobname.partition(':')[0]
+            title = jobname.partition(':')[-1]
+            if exception is not None:
+                errors.append([title, exception, tb])
+            else:
+                good.append(title)
+        if errors:
+            errors = '\n'.join([
+                    '<li><b>%s</b><br>%s<br>%s<br></li>' %
+                    (title, e, tb.replace('\n', '<br>')) for \
+                            title, e, tb in errors
+                    ])
+            ConversionErrorDialog(self, _('Failed to email books'),
+                    '<p>'+_('Failed to email the following books:')+\
+                            '<ul>%s</ul>'%errors,
+                        show=True)
+        else:
+            self.status_bar.showMessage(_('Sent by email:') + ', '.join(good),
+                    5000)
+
+    def cover_to_thumbnail(self, data):
+        p = QPixmap()
+        p.loadFromData(data)
+        if not p.isNull():
+            ht = self.device_manager.device_class.THUMBNAIL_HEIGHT \
+                    if self.device_manager else Device.THUMBNAIL_HEIGHT
+            p = p.scaledToHeight(ht, Qt.SmoothTransformation)
+            return (p.width(), p.height(), pixmap_to_data(p))
+
+    def email_news(self, id):
+        opts = email_config().parse()
+        accounts = [(account, [x.strip().lower() for x in x[0].split(',')])
+                for account, x in opts.accounts.items() if x[1]]
+        sent_mails = []
+        for account, fmts in accounts:
+            files = self.library_view.model().\
+                    get_preferred_formats_from_ids([id], fmts)
+            files = [f.name for f in files if f is not None]
+            if not files:
+                continue
+            attachment = files[0]
+            mi = self.library_view.model().db.get_metadata(id,
+                    index_is_id=True)
+            to_s = [account]
+            subjects = [_('News:')+' '+mi.title]
+            texts    = [_('Attached is the')+' '+mi.title]
+            attachment_names = [mi.title+os.path.splitext(attachment)[1]]
+            attachments = [attachment]
+            jobnames = ['%s:%s'%(id, mi.title)]
+            remove = [id] if config['delete_news_from_library_on_upload']\
+                    else []
+            self.emailer.send_mails(jobnames,
+                    Dispatcher(partial(self.emails_sent, remove=remove)),
+                    attachments, to_s, subjects, texts, attachment_names)
+            sent_mails.append(to_s[0])
+        if sent_mails:
+            self.status_bar.showMessage(_('Sent news to')+' '+\
+                    ', '.join(sent_mails),  3000)
+
+
+    def sync_news(self):
+        if self.device_connected:
+            ids = list(dynamic.get('news_to_be_synced', set([])))
+            ids = [id for id in ids if self.library_view.model().db.has_id(id)]
+            files = self.library_view.model().get_preferred_formats_from_ids(
+                                ids, self.device_manager.device_class.FORMATS)
+            files = [f for f in files if f is not None]
+            if not files:
+                dynamic.set('news_to_be_synced', set([]))
+                return
+            metadata = self.library_view.model().get_metadata(ids,
+                    rows_are_ids=True)
+            names = []
+            for mi in metadata:
+                prefix = sanitize_file_name(mi['title'])
+                if not isinstance(prefix, unicode):
+                    prefix = prefix.decode(preferred_encoding, 'replace')
+                prefix = ascii_filename(prefix)
+                names.append('%s_%d%s'%(prefix, id,
+                    os.path.splitext(f.name)[1]))
+                cdata = mi['cover']
+                if cdata:
+                    mi['cover'] = self.cover_to_thumbnail(cdata)
+            dynamic.set('news_to_be_synced', set([]))
+            if config['upload_news_to_device'] and files:
+                remove = ids if \
+                    config['delete_news_from_library_on_upload'] else []
+                on_card = self.location_view.model().free[0] < \
+                          self.location_view.model().free[1]
+                self.upload_books(files, names, metadata,
+                        on_card=on_card,
+                        memory=[[f.name for f in files], remove])
+                self.status_bar.showMessage(_('Sending news to device.'), 5000)
+
+
+    def sync_to_device(self, on_card, delete_from_library,
+            specific_format=None):
+        rows = self.library_view.selectionModel().selectedRows()
+        if not self.device_manager or not rows or len(rows) == 0:
+            return
+        ids = iter(self.library_view.model().id(r) for r in rows)
+        metadata = self.library_view.model().get_metadata(rows)
+        for mi in metadata:
+            cdata = mi['cover']
+            if cdata:
+                mi['cover'] = self.cover_to_thumbnail(cdata)
+        metadata = iter(metadata)
+        _files   = self.library_view.model().get_preferred_formats(rows,
+                                    self.device_manager.device_class.FORMATS,
+                                    paths=True, set_metadata=True,
+                                    specific_format=specific_format)
+        files = [getattr(f, 'name', None) for f in _files]
+        bad, good, gf, names, remove_ids = [], [], [], [], []
+        for f in files:
+            mi = metadata.next()
+            id = ids.next()
+            if f is None:
+                bad.append(mi['title'])
+            else:
+                remove_ids.append(id)
+                good.append(mi)
+                gf.append(f)
+                t = mi['title']
+                if not t:
+                    t = _('Unknown')
+                a = mi['authors']
+                if not a:
+                    a = _('Unknown')
+                prefix = sanitize_file_name(t+' - '+a)
+                if not isinstance(prefix, unicode):
+                    prefix = prefix.decode(preferred_encoding, 'replace')
+                prefix = ascii_filename(prefix)
+                names.append('%s_%d%s'%(prefix, id, os.path.splitext(f)[1]))
+        remove = remove_ids if delete_from_library else []
+        self.upload_books(gf, names, good, on_card, memory=(_files, remove))
+        self.status_bar.showMessage(_('Sending books to device.'), 5000)
+        if bad:
+            bad = '\n'.join('<li>%s</li>'%(i,) for i in bad)
+            d = warning_dialog(self, _('No suitable formats'),
+                    _('Could not upload the following books to the device, '
+                'as no suitable formats were found:<br><ul>%s</ul>')%(bad,))
+            d.exec_()
+
+    def upload_booklists(self):
+        '''
+        Upload metadata to device.
+        '''
+        self.device_manager.sync_booklists(Dispatcher(self.metadata_synced),
+                                           self.booklists())
+
+    def metadata_synced(self, job):
+        '''
+        Called once metadata has been uploaded.
+        '''
+        if job.exception is not None:
+            self.device_job_exception(job)
+            return
+        cp, fs = job.result
+        self.location_view.model().update_devices(cp, fs)
+
+    def upload_books(self, files, names, metadata, on_card=False, memory=None):
+        '''
+        Upload books to device.
+        :param files: List of either paths to files or file like objects
+        '''
+        titles = [i['title'] for i in metadata]
+        job = self.device_manager.upload_books(
+                Dispatcher(self.books_uploaded),
+                files, names, on_card=on_card,
+                metadata=metadata, titles=titles
+              )
+        self.upload_memory[job] = (metadata, on_card, memory, files)
+
+    def books_uploaded(self, job):
+        '''
+        Called once books have been uploaded.
+        '''
+        metadata, on_card, memory, files = self.upload_memory.pop(job)
+
+        if job.exception is not None:
+            if isinstance(job.exception, FreeSpaceError):
+                where = 'in main memory.' if 'memory' in str(job.exception) \
+                        else 'on the storage card.'
+                titles = '\n'.join(['<li>'+mi['title']+'</li>' \
+                                    for mi in metadata])
+                d = error_dialog(self, _('No space on device'),
+                                 _('<p>Cannot upload books to device there '
+                                 'is no more free space available ')+where+
+                                 '</p>\n<ul>%s</ul>'%(titles,))
+                d.exec_()
+            else:
+                self.device_job_exception(job)
+            return
+
+        self.device_manager.add_books_to_metadata(job.result,
+                metadata, self.booklists())
+
+        self.upload_booklists()
+
+        view = self.card_view if on_card else self.memory_view
+        view.model().resort(reset=False)
+        view.model().research()
+        for f in files:
+            getattr(f, 'close', lambda : True)()
+        if memory and memory[1]:
+            self.library_view.model().delete_books_by_id(memory[1])
+
+

src/calibre/gui2/dialogs/config.py

@@ -1,15 +1,17 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
-import os, re, time, textwrap
+import os, re, time, textwrap, sys, cStringIO
+from binascii import hexlify, unhexlify
 
 from PyQt4.Qt import    QDialog, QMessageBox, QListWidgetItem, QIcon, \
                         QDesktopServices, QVBoxLayout, QLabel, QPlainTextEdit, \
-                        QStringListModel, QAbstractItemModel, \
+                        QStringListModel, QAbstractItemModel, QFont, \
                         SIGNAL, QTimer, Qt, QSize, QVariant, QUrl, \
-                        QModelIndex, QInputDialog
+                        QModelIndex, QInputDialog, QAbstractTableModel
 
 from calibre.constants import islinux, iswindows
 from calibre.gui2.dialogs.config_ui import Ui_Dialog
+from calibre.gui2.dialogs.test_email_ui import Ui_Dialog as TE_Dialog
 from calibre.gui2 import qstring_to_unicode, choose_dir, error_dialog, config, \
                          ALL_COLUMNS, NONE, info_dialog, choose_files
 from calibre.utils.config import prefs
@@ -21,6 +23,7 @@ from calibre.library import server_config
 from calibre.customize.ui import initialized_plugins, is_disabled, enable_plugin, \
                                  disable_plugin, customize_plugin, \
                                  plugin_customization, add_plugin, remove_plugin
+from calibre.utils.smtp import config as smtp_prefs
 
 class PluginModel(QAbstractItemModel):
 
@@ -120,18 +123,160 @@ class CategoryModel(QStringListModel):
 
     def __init__(self, *args):
         QStringListModel.__init__(self, *args)
-        self.setStringList([_('General'), _('Interface'), _('Advanced'), 
+        self.setStringList([_('General'), _('Interface'), _('Email\nDelivery'),
-                            _('Content\nServer'), _('Plugins')])
+                            _('Advanced'), _('Content\nServer'), _('Plugins')])
         self.icons = list(map(QVariant, map(QIcon,
             [':/images/dialog_information.svg', ':/images/lookfeel.svg',
-             ':/images/view.svg', ':/images/network-server.svg',
+             ':/images/mail.svg', ':/images/view.svg',
-             ':/images/plugins.svg'])))
+             ':/images/network-server.svg', ':/images/plugins.svg'])))
 
     def data(self, index, role):
         if role == Qt.DecorationRole:
             return self.icons[index.row()]
         return QStringListModel.data(self, index, role)
 
+class TestEmail(QDialog, TE_Dialog):
+
+    def __init__(self, accounts, parent):
+        QDialog.__init__(self, parent)
+        TE_Dialog.__init__(self)
+        self.setupUi(self)
+        opts = smtp_prefs().parse()
+        self.test_func = parent.test_email_settings
+        self.connect(self.test_button, SIGNAL('clicked(bool)'), self.test)
+        self.from_.setText(unicode(self.from_.text())%opts.from_)
+        if accounts:
+            self.to.setText(list(accounts.keys())[0])
+        if opts.relay_host:
+            self.label.setText(_('Using: %s:%s@%s:%s and %s encryption')%
+                    (opts.relay_username, unhexlify(opts.relay_password),
+                        opts.relay_host, opts.relay_port, opts.encryption))
+
+    def test(self):
+        self.log.setPlainText(_('Sending...'))
+        self.test_button.setEnabled(False)
+        try:
+            tb = self.test_func(unicode(self.to.text()))
+            if not tb:
+                tb = _('Mail successfully sent')
+            self.log.setPlainText(tb)
+        finally:
+            self.test_button.setEnabled(True)
+
+class EmailAccounts(QAbstractTableModel):
+
+    def __init__(self, accounts):
+        QAbstractTableModel.__init__(self)
+        self.accounts = accounts
+        self.account_order = sorted(self.accounts.keys())
+        self.headers  = map(QVariant, [_('Email'), _('Formats'), _('Auto send')])
+        self.default_font = QFont()
+        self.default_font.setBold(True)
+        self.default_font = QVariant(self.default_font)
+        self.tooltips =[NONE] + map(QVariant,
+            [_('Formats to email. The first matching format will be sent.'),
+             '<p>'+_('If checked, downloaded news will be automatically '
+                     'mailed <br>to this email address '
+                     '(provided it is in one of the listed formats).')])
+
+    def rowCount(self, *args):
+        return len(self.account_order)
+
+    def columnCount(self, *args):
+        return 3
+
+    def headerData(self, section, orientation, role):
+        if role == Qt.DisplayRole and orientation == Qt.Horizontal:
+            return self.headers[section]
+        return NONE
+
+    def data(self, index, role):
+        row, col = index.row(), index.column()
+        if row < 0 or row >= self.rowCount():
+            return NONE
+        account = self.account_order[row]
+        if role == Qt.UserRole:
+            return (account, self.accounts[account])
+        if role == Qt.ToolTipRole:
+            return self.tooltips[col]
+        if role == Qt.DisplayRole:
+            if col == 0:
+                return QVariant(account)
+            if col ==  1:
+                return QVariant(self.accounts[account][0])
+        if role == Qt.FontRole and self.accounts[account][2]:
+            return self.default_font
+        if role == Qt.CheckStateRole and col == 2:
+            return QVariant(Qt.Checked if self.accounts[account][1] else Qt.Unchecked)
+        return NONE
+
+    def flags(self, index):
+        if index.column() == 2:
+            return QAbstractTableModel.flags(self, index)|Qt.ItemIsUserCheckable
+        else:
+            return QAbstractTableModel.flags(self, index)|Qt.ItemIsEditable
+
+    def setData(self, index, value, role):
+        if not index.isValid():
+            return False
+        row, col = index.row(), index.column()
+        account = self.account_order[row]
+        if col == 2:
+            self.accounts[account][1] ^= True
+        elif col == 1:
+            self.accounts[account][0] = unicode(value.toString()).upper()
+        else:
+            na = unicode(value.toString())
+            from email.utils import parseaddr
+            addr = parseaddr(na)[-1]
+            if not addr:
+                return False
+            self.accounts[na] = self.accounts.pop(account)
+            self.account_order[row] = na
+            if '@kindle.com' in addr:
+                self.accounts[na][0] = 'AZW, MOBI, TPZ, PRC, AZW1'
+
+        self.emit(SIGNAL('dataChanged(QModelIndex,QModelIndex)'),
+                self.index(index.row(), 0), self.index(index.row(), 2))
+        return True
+
+    def make_default(self, index):
+        if index.isValid():
+            row = index.row()
+            for x in self.accounts.values():
+                x[2] = False
+            self.accounts[self.account_order[row]][2] = True
+            self.reset()
+
+    def add(self):
+        x = _('new email address')
+        y = x
+        c = 0
+        while y in self.accounts:
+            c += 1
+            y = x + str(c)
+        self.accounts[y] = ['MOBI, EPUB', True,
+                                                len(self.account_order) == 0]
+        self.account_order = sorted(self.accounts.keys())
+        self.reset()
+        return self.index(self.account_order.index(y), 0)
+
+    def remove(self, index):
+        if index.isValid():
+            row = self.index.row()
+            account = self.account_order[row]
+            self.accounts.pop(account)
+            self.account_order = sorted(self.accounts.keys())
+            has_default = False
+            for account in self.account_order:
+                if self.accounts[account][2]:
+                    has_default = True
+                    break
+            if not has_default and self.account_order:
+                self.accounts[self.account_order[0]][2] = True
+
+            self.reset()
+
 
 class ConfigDialog(QDialog, Ui_Dialog):
 
@@ -142,8 +287,8 @@ class ConfigDialog(QDialog, Ui_Dialog):
         self.setupUi(self)
         self._category_model = CategoryModel()
 
-        self.connect(self.category_view, SIGNAL('activated(QModelIndex)'), lambda i: self.stackedWidget.setCurrentIndex(i.row()))
+        self.category_view.currentChanged = \
-        self.connect(self.category_view, SIGNAL('clicked(QModelIndex)'), lambda i: self.stackedWidget.setCurrentIndex(i.row()))
+            lambda n, p: self.stackedWidget.setCurrentIndex(n.row())
         self.category_view.setModel(self._category_model)
         self.db = db
         self.server = server
@@ -242,7 +387,6 @@ class ConfigDialog(QDialog, Ui_Dialog):
         self.priority.setCurrentIndex(p)
         self.priority.setVisible(iswindows)
         self.priority_label.setVisible(iswindows)
-        self.category_view.setCurrentIndex(self._category_model.index(0))
         self._plugin_model = PluginModel()
         self.plugin_view.setModel(self._plugin_model)
         self.connect(self.toggle_plugin, SIGNAL('clicked()'), lambda : self.modify_plugin(op='toggle'))
@@ -251,6 +395,105 @@ class ConfigDialog(QDialog, Ui_Dialog):
         self.connect(self.button_plugin_browse, SIGNAL('clicked()'), self.find_plugin)
         self.connect(self.button_plugin_add, SIGNAL('clicked()'), self.add_plugin)
         self.separate_cover_flow.setChecked(config['separate_cover_flow'])
+        self.setup_email_page()
+        self.category_view.setCurrentIndex(self.category_view.model().index(0))
+
+    def setup_email_page(self):
+        opts = smtp_prefs().parse()
+        if opts.from_:
+            self.email_from.setText(opts.from_)
+        self._email_accounts = EmailAccounts(opts.accounts)
+        self.email_view.setModel(self._email_accounts)
+        if opts.relay_host:
+            self.relay_host.setText(opts.relay_host)
+        self.relay_port.setValue(opts.relay_port)
+        if opts.relay_username:
+            self.relay_username.setText(opts.relay_username)
+        if opts.relay_password:
+            self.relay_password.setText(unhexlify(opts.relay_password))
+        (self.relay_tls if opts.encryption == 'TLS' else self.relay_ssl).setChecked(True)
+        self.connect(self.relay_use_gmail, SIGNAL('clicked(bool)'),
+                     self.create_gmail_relay)
+        self.connect(self.relay_show_password, SIGNAL('stateChanged(int)'),
+         lambda
+         state:self.relay_password.setEchoMode(self.relay_password.Password if
+             state == 0 else self.relay_password.Normal))
+        self.connect(self.email_add, SIGNAL('clicked(bool)'),
+                     self.add_email_account)
+        self.connect(self.email_make_default, SIGNAL('clicked(bool)'),
+             lambda c: self._email_accounts.make_default(self.email_view.currentIndex()))
+        self.email_view.resizeColumnsToContents()
+        self.connect(self.test_email_button, SIGNAL('clicked(bool)'),
+                self.test_email)
+
+    def add_email_account(self, checked):
+        index = self._email_accounts.add()
+        self.email_view.setCurrentIndex(index)
+        self.email_view.resizeColumnsToContents()
+        self.email_view.edit(index)
+
+    def create_gmail_relay(self, *args):
+        self.relay_username.setText('@gmail.com')
+        self.relay_password.setText('')
+        self.relay_host.setText('smtp.gmail.com')
+        self.relay_port.setValue(587)
+        self.relay_tls.setChecked(True)
+
+        info_dialog(self, _('Finish gmail setup'),
+            _('Dont forget to enter your gmail username and password')).exec_()
+        self.relay_username.setFocus(Qt.OtherFocusReason)
+        self.relay_username.setCursorPosition(0)
+
+    def set_email_settings(self):
+        from_ = unicode(self.email_from.text()).strip()
+        if self._email_accounts.accounts and not from_:
+            error_dialog(self, _('Bad configuration'),
+                         _('You must set the From email address')).exec_()
+            return False
+        username = unicode(self.relay_username.text()).strip()
+        password = unicode(self.relay_password.text()).strip()
+        host = unicode(self.relay_host.text()).strip()
+        if host and not (username and password):
+            error_dialog(self, _('Bad configuration'),
+                         _('You must set the username and password for '
+                           'the mail server.')).exec_()
+            return False
+        conf = smtp_prefs()
+        conf.set('from_', from_)
+        conf.set('accounts', self._email_accounts.accounts)
+        conf.set('relay_host', host if host else None)
+        conf.set('relay_port', self.relay_port.value())
+        conf.set('relay_username', username if username else None)
+        conf.set('relay_password', hexlify(password))
+        conf.set('encryption', 'TLS' if self.relay_tls.isChecked() else 'SSL')
+        return True
+
+    def test_email(self, *args):
+        if self.set_email_settings():
+          TestEmail(self._email_accounts.accounts, self).exec_()
+
+    def test_email_settings(self, to):
+        opts = smtp_prefs().parse()
+        from calibre.utils.smtp import sendmail, create_mail
+        buf = cStringIO.StringIO()
+        oout, oerr = sys.stdout, sys.stderr
+        sys.stdout = sys.stderr = buf
+        tb = None
+        try:
+            msg = create_mail(opts.from_, to, 'Test mail from calibre',
+                    'Test mail from calibre')
+            sendmail(msg, from_=opts.from_, to=[to],
+                verbose=3, timeout=30, relay=opts.relay_host,
+                username=opts.relay_username,
+                password=unhexlify(opts.relay_password),
+                encryption=opts.encryption, port=opts.relay_port)
+        except:
+            import traceback
+            tb = traceback.format_exc()
+            tb += '\n\nLog:\n' + buf.getvalue()
+        finally:
+            sys.stdout, sys.stderr = oout, oerr
+        return tb
 
     def add_plugin(self):
         path = unicode(self.plugin_path.text())
@@ -286,7 +529,7 @@ class ConfigDialog(QDialog, Ui_Dialog):
             if op == 'customize':
                 if not plugin.is_customizable():
                     info_dialog(self, _('Plugin not customizable'),
-                                _('Plugin: %s does not need customization')%plugin.name).exec_()
+                        _('Plugin: %s does not need customization')%plugin.name).exec_()
                     return
                 help = plugin.customization_help()
                 text, ok = QInputDialog.getText(self, _('Customize %s')%plugin.name,
@@ -300,7 +543,8 @@ class ConfigDialog(QDialog, Ui_Dialog):
                     self._plugin_model.reset()
                 else:
                     error_dialog(self, _('Cannot remove builtin plugin'),
-                                 plugin.name + _(' cannot be removed. It is a builtin plugin. Try disabling it instead.')).exec_()
+                         plugin.name + _(' cannot be removed. It is a '
+                         'builtin plugin. Try disabling it instead.')).exec_()
 
 
     def up_column(self):
@@ -376,7 +620,8 @@ class ConfigDialog(QDialog, Ui_Dialog):
         d.exec_()
 
     def browse(self):
-        dir = choose_dir(self, 'database location dialog', 'Select database location')
+        dir = choose_dir(self, 'database location dialog',
+                         _('Select database location'))
         if dir:
             self.location.setText(dir)
 
@@ -393,7 +638,10 @@ class ConfigDialog(QDialog, Ui_Dialog):
     def accept(self):
         mcs = unicode(self.max_cover_size.text()).strip()
         if not re.match(r'\d+x\d+', mcs):
-            error_dialog(self, _('Invalid size'), _('The size %s is invalid. must be of the form widthxheight')%mcs).exec_()
+            error_dialog(self, _('Invalid size'),
+             _('The size %s is invalid. must be of the form widthxheight')%mcs).exec_()
+            return
+        if not self.set_email_settings():
             return
         config['use_roman_numerals_for_series_number'] = bool(self.roman_numerals.isChecked())
         config['new_version_notification'] = bool(self.new_version_notification.isChecked())
@@ -432,15 +680,18 @@ class ConfigDialog(QDialog, Ui_Dialog):
 
         if not path or not os.path.exists(path) or not os.path.isdir(path):
             d = error_dialog(self, _('Invalid database location'),
-                             _('Invalid database location ')+path+_('<br>Must be a directory.'))
+                             _('Invalid database location ')+path+
+                             _('<br>Must be a directory.'))
             d.exec_()
         elif not os.access(path, os.W_OK):
             d = error_dialog(self, _('Invalid database location'),
-                             _('Invalid database location.<br>Cannot write to ')+path)
+                     _('Invalid database location.<br>Cannot write to ')+path)
             d.exec_()
         else:
             self.database_location = os.path.abspath(path)
-            self.directories = [qstring_to_unicode(self.directory_list.item(i).text()) for i in range(self.directory_list.count())]
+            self.directories = [
+              qstring_to_unicode(self.directory_list.item(i).text()) for i in \
+                    range(self.directory_list.count())]
             config['frequently_used_directories'] =  self.directories
             QDialog.accept(self)
 
@@ -448,7 +699,8 @@ class Vacuum(QMessageBox):
 
     def __init__(self, parent, db):
         self.db = db
-        QMessageBox.__init__(self, QMessageBox.Information, _('Compacting...'), _('Compacting database. This may take a while.'),
+        QMessageBox.__init__(self, QMessageBox.Information, _('Compacting...'),
+                             _('Compacting database. This may take a while.'),
                              QMessageBox.NoButton, parent)
         QTimer.singleShot(200, self.vacuum)
 
@@ -456,3 +708,11 @@ class Vacuum(QMessageBox):
         self.db.vacuum()
         self.accept()
 
+if __name__ == '__main__':
+    from calibre.library.database2 import LibraryDatabase2
+    from PyQt4.Qt import QApplication
+    app = QApplication([])
+    d=ConfigDialog(None, LibraryDatabase2('/tmp'))
+    d.category_view.setCurrentIndex(d.category_view.model().index(2))
+    d.show()
+    app.exec_()

src/calibre/gui2/dialogs/epub.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 '''
 The GUI for conversion to EPUB.
 '''
-import os
+import os, uuid
 
 from PyQt4.Qt import QDialog, QSpinBox, QDoubleSpinBox, QComboBox, QLineEdit, \
                      QTextEdit, QCheckBox, Qt, QPixmap, QIcon, QListWidgetItem, SIGNAL
@@ -272,6 +272,7 @@ class Config(ResizableDialog, Ui_Dialog):
         if self.row is not None:
             self.db.set_metadata(self.id, mi)
             self.mi = self.db.get_metadata(self.id, index_is_id=True)
+            self.mi.application_id = uuid.uuid4()
             opf = OPFCreator(os.getcwdu(), self.mi)
             self.opf_file = PersistentTemporaryFile('.opf')
             opf.render(self.opf_file)

src/calibre/gui2/dialogs/fetch_metadata.py

@@ -8,10 +8,11 @@ import time
 
 from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, QThread, \
                          QAbstractTableModel, QCoreApplication, QTimer
-from PyQt4.QtGui import QDialog, QItemSelectionModel, QWidget, QLabel, QMovie
+from PyQt4.QtGui import QDialog, QItemSelectionModel
 
 from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata
-from calibre.gui2 import error_dialog, NONE, info_dialog, warning_dialog
+from calibre.gui2 import error_dialog, NONE, info_dialog
+from calibre.gui2.widgets import ProgressIndicator
 from calibre.utils.config import prefs
 
 class Fetcher(QThread):
@@ -30,40 +31,6 @@ class Fetcher(QThread):
                                                self.publisher, self.isbn,
                                                self.key if self.key else None)
 
-class ProgressIndicator(QWidget):
-    
-    def __init__(self, *args):
-        QWidget.__init__(self, *args)
-        self.setGeometry(0, 0, 300, 350)
-        self.movie = QMovie(':/images/jobs-animated.mng')
-        self.ml = QLabel(self)
-        self.ml.setMovie(self.movie)
-        self.movie.start()
-        self.movie.setPaused(True)
-        self.status = QLabel(self)
-        self.status.setWordWrap(True)
-        self.status.setAlignment(Qt.AlignHCenter|Qt.AlignTop)
-        self.status.font().setBold(True)
-        self.status.font().setPointSize(self.font().pointSize()+6)
-        self.setVisible(False)
-        
-    def start(self, msg=''):
-        view = self.parent()
-        pwidth, pheight = view.size().width(), view.size().height()
-        self.resize(pwidth, min(pheight, 250))
-        self.move(0, (pheight-self.size().height())/2.)
-        self.ml.resize(self.ml.sizeHint())
-        self.ml.move(int((self.size().width()-self.ml.size().width())/2.), 0)
-        self.status.resize(self.size().width(), self.size().height()-self.ml.size().height()-10)
-        self.status.move(0, self.ml.size().height()+10)
-        self.status.setText(msg)
-        self.setVisible(True)
-        self.movie.setPaused(False)
-        
-    def stop(self):
-        if self.movie.state() == self.movie.Running:
-            self.movie.setPaused(True)
-            self.setVisible(False)
 
 class Matches(QAbstractTableModel):
 
@@ -137,14 +104,15 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
         self.author = author.strip()
         self.publisher = publisher
         self.previous_row = None
+        self.warning.setVisible(False)
         self.connect(self.matches, SIGNAL('activated(QModelIndex)'), self.chosen)
         self.connect(self.matches, SIGNAL('entered(QModelIndex)'),
-                     lambda index:self.matches.setCurrentIndex(index))
+                     self.show_summary)
         self.matches.setMouseTracking(True)
         self.fetch_metadata()
 
 
-    def show_summary(self, current, previous):
+    def show_summary(self, current, *args):
         row  = current.row()
         if row != self.previous_row:
             summ =  self.model.summary(row)
@@ -152,6 +120,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
             self.previous_row = row
 
     def fetch_metadata(self):
+        self.warning.setVisible(False)
         key = str(self.key.text())
         if key:
             prefs['isbndb_com_key'] =  key
@@ -173,7 +142,7 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
         self._hangcheck = QTimer(self)
         self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck)
         self.start_time = time.time()
-        self._hangcheck.start()
+        self._hangcheck.start(100)
 
     def hangcheck(self):
         if not (self.fetcher.isFinished() or time.time() - self.start_time > 75):
@@ -191,14 +160,14 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
                             self.fetcher.exceptions if x[1] is not None]
             if warnings:
                 warnings='<br>'.join(['<b>%s</b>: %s'%(name, exc) for name,exc in warnings])
-                warning_dialog(self, _('Warning'),
+                self.warning.setText('<p><b>'+ _('Warning')+':</b>'+\
-                               '<p>'+_('Could not fetch metadata from:')+\
+                               _('Could not fetch metadata from:')+\
-                               '<br><br>'+warnings+'</p>').exec_()
+                               '<br>'+warnings+'</p>')
+                self.warning.setVisible(True)
             if self.model.rowCount() < 1:
                 info_dialog(self, _('No metadata found'),
                      _('No metadata found, try adjusting the title and author '
                        'or the ISBN key.')).exec_()
-                self.reject()
                 return
 
             self.matches.setModel(self.model)
@@ -215,6 +184,16 @@ class FetchMetadata(QDialog, Ui_FetchMetadata):
             self.matches.resizeColumnsToContents()
             self.pi.stop()
 
+    def terminate(self):
+        if hasattr(self, 'fetcher') and self.fetcher.isRunning():
+            self.fetcher.terminate()
+
+
+    def __enter__(self, *args):
+        return self
+
+    def __exit__(self, *args):
+        self.terminate()
 
     def selected_book(self):
         try:

src/calibre/gui2/dialogs/fetch_metadata.ui

@@ -23,7 +23,7 @@
    <item>
     <widget class="QLabel" name="tlabel" >
      <property name="text" >
-      <string>&lt;p>calibre can find metadata for your books from two locations: &lt;b>Google Books&lt;/b> and &lt;b>isbndb.com&lt;/b>. &lt;p>To use isbndb.com you must sign up for a &lt;a href="http://www.isbndb.com">free account&lt;/a> and exter you access key below.</string>
+      <string>&lt;p>calibre can find metadata for your books from two locations: &lt;b>Google Books&lt;/b> and &lt;b>isbndb.com&lt;/b>. &lt;p>To use isbndb.com you must sign up for a &lt;a href="http://www.isbndb.com">free account&lt;/a> and enter your access key below.</string>
      </property>
      <property name="alignment" >
       <set>Qt::AlignCenter</set>
@@ -60,6 +60,16 @@
      </item>
     </layout>
    </item>
+   <item>
+    <widget class="QLabel" name="warning" >
+     <property name="text" >
+      <string/>
+     </property>
+     <property name="wordWrap" >
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
    <item>
     <widget class="QGroupBox" name="groupBox" >
      <property name="title" >

src/calibre/gui2/dialogs/jobs.py

@@ -50,7 +50,10 @@ class JobsDialog(QDialog, Ui_JobsDialog):
         self.running_time_timer.start(1000)
 
     def update_running_time(self, *args):
-        self.model.running_time_updated()
+        try:
+            self.model.running_time_updated()
+        except: # Raises random exceptions on OS X
+            pass
 
     def kill_job(self):
         for index in self.jobs_view.selectedIndexes():

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -70,14 +70,14 @@ class MetadataBulkDialog(QDialog, Ui_MetadataBulkDialog):
             pub = qstring_to_unicode(self.publisher.text())
             if pub:
                 self.db.set_publisher(id, pub, notify=False)
-            tags = qstring_to_unicode(self.tags.text()).strip()
-            if tags:
-                tags = map(lambda x: x.strip(), tags.split(','))
-                self.db.set_tags(id, tags, append=True, notify=False)
             remove_tags = qstring_to_unicode(self.remove_tags.text()).strip()
             if remove_tags:
                 remove_tags = [i.strip() for i in remove_tags.split(',')]
                 self.db.unapply_tags(id, remove_tags, notify=False)
+            tags = qstring_to_unicode(self.tags.text()).strip()
+            if tags:
+                tags = map(lambda x: x.strip(), tags.split(','))
+                self.db.set_tags(id, tags, append=True, notify=False)
             if self.write_series:
                 self.db.set_series(id, qstring_to_unicode(self.series.currentText()), notify=False)
                 

src/calibre/gui2/dialogs/metadata_single.py

@@ -1,12 +1,13 @@
+from __future__ import with_statement
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 '''
 The dialog used to edit meta information for a book as well as
 add/remove formats
 '''
-import os
+import os, time, traceback
 
-from PyQt4.QtCore import SIGNAL, QObject, QCoreApplication, Qt
+from PyQt4.QtCore import SIGNAL, QObject, QCoreApplication, Qt, QTimer, QThread
 from PyQt4.QtGui import QPixmap, QListWidgetItem, QErrorMessage, QDialog, QCompleter
 
 
@@ -16,14 +17,35 @@ from calibre.gui2.dialogs.metadata_single_ui import Ui_MetadataSingleDialog
 from calibre.gui2.dialogs.fetch_metadata import FetchMetadata
 from calibre.gui2.dialogs.tag_editor import TagEditor
 from calibre.gui2.dialogs.password import PasswordDialog
+from calibre.gui2.widgets import ProgressIndicator
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.ebooks.metadata import authors_to_sort_string, string_to_authors, authors_to_string
-from calibre.ebooks.metadata.library_thing import login, cover_from_isbn, LibraryThingError
+from calibre.ebooks.metadata.library_thing import login, cover_from_isbn
 from calibre import islinux
 from calibre.ebooks.metadata.meta import get_metadata
 from calibre.utils.config import prefs
 from calibre.customize.ui import run_plugins_on_import
 
+class CoverFetcher(QThread):
+
+    def __init__(self, username, password, isbn, timeout):
+        self.username = username
+        self.password = password
+        self.timeout = timeout
+        self.isbn = isbn
+        QThread.__init__(self)
+        self.exception = self.traceback = self.cover_data = None
+
+    def run(self):
+        try:
+            login(self.username, self.password, force=False)
+            self.cover_data = cover_from_isbn(self.isbn, timeout=self.timeout)[0]
+        except Exception, e:
+            self.exception = e
+            self.traceback = traceback.format_exc()
+
+
+
 class Format(QListWidgetItem):
     def __init__(self, parent, ext, size, path=None):
         self.path = path
@@ -172,6 +194,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter)
         self.splitter.setStretchFactor(100, 1)
         self.db = db
+        self.pi = ProgressIndicator(self)
         self.accepted_callback = accepted_callback
         self.id = db.id(row)
         self.row = row
@@ -338,30 +361,49 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                     return
             self.fetch_cover_button.setEnabled(False)
             self.setCursor(Qt.WaitCursor)
-            QCoreApplication.instance().processEvents()
+            self.cover_fetcher = CoverFetcher(d.username(), d.password(), isbn,
-            try:
+                                              self.timeout)
-                login(d.username(), d.password(), force=False)
+            self.cover_fetcher.start()
-                cover_data = cover_from_isbn(isbn, timeout=self.timeout)[0]
+            self._hangcheck = QTimer(self)
-            
+            self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck)
-                pix = QPixmap()
+            self.cf_start_time = time.time()
-                pix.loadFromData(cover_data)
+            self.pi.start(_('Downloading cover...'))
-                if pix.isNull():
+            self._hangcheck.start(100)
-                    error_dialog(self.window, _('Bad cover'), 
-                                 _('The cover is not a valid picture')).exec_()
-                else:
-                    self.cover.setPixmap(pix)
-                    self.cover_changed = True
-                    self.cpixmap = pix
-            except LibraryThingError, err:
-                error_dialog(self, _('Cannot fetch cover'), 
-                    _('<b>Could not fetch cover.</b><br/>')+repr(err)).exec_()
-            finally:
-                self.fetch_cover_button.setEnabled(True)
-                self.unsetCursor()
-                
         else:
             error_dialog(self, _('Cannot fetch cover'),
-                _('You must specify the ISBN identifier for this book.')).exec_()
+            _('You must specify the ISBN identifier for this book.')).exec_()
+
+    def hangcheck(self):
+        if not (self.cover_fetcher.isFinished() or time.time()-self.cf_start_time > 150):
+            return
+
+        self._hangcheck.stop()
+        try:
+            if self.cover_fetcher.isRunning():
+                self.cover_fetcher.terminate()
+                error_dialog(self, _('Cannot fetch cover'),
+                    _('<b>Could not fetch cover.</b><br/>')+
+                    _('The download timed out.')).exec_()
+                return
+            if self.cover_fetcher.exception is not None:
+                err = self.cover_fetcher.exception
+                error_dialog(self, _('Cannot fetch cover'),
+                    _('<b>Could not fetch cover.</b><br/>')+repr(err)).exec_()
+                return
+
+            pix = QPixmap()
+            pix.loadFromData(self.cover_fetcher.cover_data)
+            if pix.isNull():
+                error_dialog(self.window, _('Bad cover'),
+                             _('The cover is not a valid picture')).exec_()
+            else:
+                self.cover.setPixmap(pix)
+                self.cover_changed = True
+                self.cpixmap = pix
+        finally:
+            self.fetch_cover_button.setEnabled(True)
+            self.unsetCursor()
+            self.pi.stop()
 
 
     def fetch_metadata(self):
@@ -371,7 +413,8 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         publisher = qstring_to_unicode(self.publisher.currentText())
         if isbn or title or author or publisher:
             d = FetchMetadata(self, isbn, title, author, publisher, self.timeout)
-            d.exec_()
+            with d:
+                d.exec_()
             if d.result() == QDialog.Accepted:
                 book = d.selected_book()
                 if book:
@@ -387,7 +430,9 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
                             prefix += '\n'
                         self.comments.setText(prefix + summ)
         else:
-            error_dialog(self, 'Cannot fetch metadata', 'You must specify at least one of ISBN, Title, Authors or Publisher')
+            error_dialog(self, _('Cannot fetch metadata'),
+                         _('You must specify at least one of ISBN, Title, '
+                           'Authors or Publisher'))
 
     def enable_series_index(self, *args):
         self.series_index.setEnabled(True)

src/calibre/gui2/dialogs/search.py

@@ -20,7 +20,8 @@ class SearchDialog(QDialog, Ui_Dialog):
         return [t.strip() for t in phrases + raw.split()]
 
     def search_string(self):
-        all, any, phrase, none = map(lambda x: unicode(x.text()), (self.all, self.any, self.phrase, self.none))
+        all, any, phrase, none = map(lambda x: unicode(x.text()),
+                (self.all, self.any, self.phrase, self.none))
         all, any, none = map(self.tokens, (all, any, none))
         phrase = phrase.strip()
         all = ' and '.join(all)
@@ -32,7 +33,7 @@ class SearchDialog(QDialog, Ui_Dialog):
         if all:
             ans += (' and ' if ans else '') + all
         if none:
-            ans += (' and not ' if ans else '') + none
+            ans += (' and not ' if ans else 'not ') + none
         if any:
             ans += (' or ' if ans else '') + any
         return ans

src/calibre/gui2/dialogs/test_email.ui

@@ -0,0 +1,103 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>542</width>
+    <height>418</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Test email settings</string>
+  </property>
+  <property name="windowIcon">
+   <iconset resource="../images.qrc">
+    <normaloff>:/images/config.svg</normaloff>:/images/config.svg</iconset>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <widget class="QLabel" name="from_">
+     <property name="text">
+      <string>Send test mail from %s to:</string>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QLineEdit" name="to"/>
+   </item>
+   <item>
+    <widget class="QLabel" name="label">
+     <property name="text">
+      <string/>
+     </property>
+     <property name="wordWrap">
+      <bool>true</bool>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QPushButton" name="test_button">
+     <property name="text">
+      <string>&amp;Test</string>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QPlainTextEdit" name="log"/>
+   </item>
+   <item>
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../images.qrc"/>
+ </resources>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/images/mail.svg

@@ -0,0 +1,270 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<!-- Created with Inkscape (http://www.inkscape.org/) -->
+<svg
+   xmlns:i="http://ns.adobe.com/AdobeIllustrator/10.0/"
+   xmlns:dc="http://purl.org/dc/elements/1.1/"
+   xmlns:cc="http://web.resource.org/cc/"
+   xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+   xmlns:svg="http://www.w3.org/2000/svg"
+   xmlns="http://www.w3.org/2000/svg"
+   xmlns:xlink="http://www.w3.org/1999/xlink"
+   xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
+   xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
+   width="128"
+   height="128"
+   id="svg3007"
+   sodipodi:version="0.32"
+   inkscape:version="0.45.1"
+   version="1.0"
+   sodipodi:docbase="/home/david/Documents/Projects/KDE/Oxygen/kdelibs/scalable/actions"
+   sodipodi:docname="mail.svgz"
+   inkscape:output_extension="org.inkscape.output.svgz.inkscape"
+   inkscape:export-filename="/home/david/Documents/Projects/KDE/Oxygen/kdelibs/scalable/actions/mail.png"
+   inkscape:export-xdpi="90"
+   inkscape:export-ydpi="90">
+  <defs
+     id="defs3009">
+    <linearGradient
+       id="polygon3293_1_"
+       gradientUnits="userSpaceOnUse"
+       x1="615.5"
+       y1="-584.6021"
+       x2="615.5"
+       y2="-595.8521"
+       gradientTransform="matrix(4,0,0,-4,-2402,-2314.406)">
+      <stop
+         offset="0"
+         style="stop-color:#6193CF"
+         id="stop2997" />
+      <stop
+         offset="1"
+         style="stop-color:#EEEEEE"
+         id="stop2999" />
+    </linearGradient>
+    <linearGradient
+       id="polygon3286_1_"
+       gradientUnits="userSpaceOnUse"
+       x1="615.5"
+       y1="-589.8511"
+       x2="615.5"
+       y2="-580.6011"
+       gradientTransform="matrix(4,0,0,-4,-2402,-2314.406)">
+      <stop
+         offset="0"
+         style="stop-color:#6193CF"
+         id="stop2991" />
+      <stop
+         offset="1"
+         style="stop-color:#D1DFF1"
+         id="stop2993" />
+    </linearGradient>
+    <linearGradient
+       id="rect3244_1_"
+       gradientUnits="userSpaceOnUse"
+       x1="59.9995"
+       y1="4"
+       x2="59.9995"
+       y2="72.0005"
+       gradientTransform="matrix(1,0,0,1.0588235,0,-0.2352941)">
+      <stop
+         offset="0"
+         style="stop-color:#A4C0E4"
+         id="stop2983" />
+      <stop
+         offset="0.25"
+         style="stop-color:#D1DFF1"
+         id="stop2985" />
+      <stop
+         offset="0.85"
+         style="stop-color:#FFFFFF"
+         id="stop2987" />
+    </linearGradient>
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#rect3244_1_"
+       id="linearGradient2212"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(1,0,0,1.0588235,0,-0.2352941)"
+       x1="59.9995"
+       y1="4"
+       x2="59.9995"
+       y2="72.0005" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#polygon3286_1_"
+       id="linearGradient2214"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(4,0,0,-4,-2402,-2314.406)"
+       x1="615.5"
+       y1="-589.8511"
+       x2="615.5"
+       y2="-580.6011" />
+    <linearGradient
+       inkscape:collect="always"
+       xlink:href="#polygon3293_1_"
+       id="linearGradient2216"
+       gradientUnits="userSpaceOnUse"
+       gradientTransform="matrix(4,0,0,-4,-2402,-2314.406)"
+       x1="615.5"
+       y1="-584.6021"
+       x2="615.5"
+       y2="-595.8521" />
+  </defs>
+  <sodipodi:namedview
+     id="base"
+     pagecolor="#ffffff"
+     bordercolor="#666666"
+     borderopacity="1.0"
+     gridtolerance="10000"
+     guidetolerance="10"
+     objecttolerance="10"
+     inkscape:pageopacity="0.0"
+     inkscape:pageshadow="2"
+     inkscape:zoom="1"
+     inkscape:cx="64"
+     inkscape:cy="64"
+     inkscape:document-units="px"
+     inkscape:current-layer="g2972"
+     width="128px"
+     height="128px"
+     inkscape:showpageshadow="false"
+     inkscape:window-width="794"
+     inkscape:window-height="731"
+     inkscape:window-x="0"
+     inkscape:window-y="0"
+     showgrid="true"
+     gridspacingx="4px"
+     gridspacingy="4px"
+     gridempspacing="2"
+     showborder="false" />
+  <metadata
+     id="metadata3012">
+    <rdf:RDF>
+      <cc:Work
+         rdf:about="">
+        <dc:format>image/svg+xml</dc:format>
+        <dc:type
+           rdf:resource="http://purl.org/dc/dcmitype/StillImage" />
+      </cc:Work>
+    </rdf:RDF>
+  </metadata>
+  <g
+     inkscape:label="Layer 1"
+     inkscape:groupmode="layer"
+     id="layer1">
+    <switch
+       id="switch2966"
+       transform="translate(4,12)">
+      <foreignObject
+         requiredExtensions="http://ns.adobe.com/AdobeIllustrator/10.0/"
+         x="0"
+         y="0"
+         width="1"
+         height="1"
+         id="foreignObject2968">
+        <i:pgfRef
+           xlink:href="#adobe_illustrator_pgf" />
+      </foreignObject>
+      <g
+         i:extraneous="self"
+         id="g2970">
+        <g
+           id="g2972">
+          <linearGradient
+             id="linearGradient3033"
+             gradientUnits="userSpaceOnUse"
+             x1="59.9995"
+             y1="4"
+             x2="59.9995"
+             y2="72.000504">
+            <stop
+               offset="0"
+               style="stop-color:#A4C0E4"
+               id="stop3035" />
+            <stop
+               offset="0.25"
+               style="stop-color:#D1DFF1"
+               id="stop3037" />
+            <stop
+               offset="0.85"
+               style="stop-color:#FFFFFF"
+               id="stop3039" />
+          </linearGradient>
+          <linearGradient
+             id="linearGradient3042"
+             gradientUnits="userSpaceOnUse"
+             x1="615.5"
+             y1="-589.85107"
+             x2="615.5"
+             y2="-580.60107"
+             gradientTransform="matrix(4,0,0,-4,-2402,-2314.406)">
+            <stop
+               offset="0"
+               style="stop-color:#6193CF"
+               id="stop3044" />
+            <stop
+               offset="1"
+               style="stop-color:#D1DFF1"
+               id="stop3046" />
+          </linearGradient>
+          <linearGradient
+             id="linearGradient3049"
+             gradientUnits="userSpaceOnUse"
+             x1="615.5"
+             y1="-584.60211"
+             x2="615.5"
+             y2="-595.85211"
+             gradientTransform="matrix(4,0,0,-4,-2402,-2314.406)">
+            <stop
+               offset="0"
+               style="stop-color:#6193CF"
+               id="stop3051" />
+            <stop
+               offset="1"
+               style="stop-color:#EEEEEE"
+               id="stop3053" />
+          </linearGradient>
+          <g
+             id="g2202"
+             transform="translate(0,8)">
+            <path
+               style="opacity:0.1"
+               id="path2974"
+               d="M 4,0 C 1.794,0 0,1.8884211 0,4.2105263 L 0,75.789474 C 0,78.111579 1.794,80 4,80 L 116,80 C 118.206,80 120,78.111579 120,75.789474 L 120,4.2105263 C 120,1.8884211 118.206,0 116,0 L 4,0 z " />
+            <path
+               style="opacity:0.15"
+               id="path2976"
+               d="M 4,1 C 2.346,1 1,2.4187568 1,4.1621622 L 1,75.837838 C 1,77.581243 2.346,79 4,79 L 116,79 C 117.654,79 119,77.581243 119,75.837838 L 119,4.1621622 C 119,2.4187568 117.654,1 116,1 L 4,1 z " />
+            <path
+               style="opacity:0.2"
+               id="path2978"
+               d="M 4,2 C 2.897,2 2,2.9468333 2,4.1111111 L 2,75.888889 C 2,77.053167 2.897,78 4,78 L 116,78 C 117.103,78 118,77.053167 118,75.888889 L 118,4.1111111 C 118,2.9468333 117.103,2 116,2 L 4,2 z " />
+            <path
+               style="opacity:0.25"
+               id="path2980"
+               d="M 4,3 C 3.448,3 3,3.4736 3,4.0571428 L 3,75.942857 C 3,76.527457 3.448,77 4,77 L 116,77 C 116.553,77 117,76.527457 117,75.942857 L 117,4.0571428 C 117,3.4736 116.553,3 116,3 L 4,3 z " />
+            <rect
+               style="fill:url(#linearGradient2212)"
+               height="72"
+               width="112"
+               y="4"
+               x="4"
+               id="rect3244_9_" />
+            <polygon
+               style="fill:url(#linearGradient2214)"
+               points="4,8 4,12 60,45 116,12 116,8 60,41 4,8 "
+               id="polygon3286_9_" />
+            <polygon
+               style="fill:url(#linearGradient2216)"
+               points="116,69 116,65 59.997,24 4,65 4,69 59.997,28 116,69 "
+               id="polygon3293_9_" />
+            <polygon
+               style="fill:#ffffff"
+               id="polygon3002"
+               points="4,8 60.004,40.967 116,8 116,4 4,4 4,8 " />
+          </g>
+        </g>
+      </g>
+    </switch>
+  </g>
+</svg>

src/calibre/gui2/jobs2.py

@@ -135,13 +135,13 @@ class JobManager(QAbstractTableModel):
         self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'),
                   self.index(row, 0), self.index(row, 3))
 
-    def running_time_updated(self):
+    def running_time_updated(self, *args):
         for job in self.jobs:
             if not job.is_running:
                 continue
             row = self.jobs.index(job)
             self.emit(SIGNAL('dataChanged(QModelIndex, QModelIndex)'),
-                      self.index(row, 3), self.index(row, 3))
+                        self.index(row, 3), self.index(row, 3))
 
     def has_device_jobs(self):
         for job in self.jobs:

src/calibre/gui2/library.py

@@ -93,8 +93,8 @@ class DateDelegate(QStyledItemDelegate):
 
     def createEditor(self, parent, option, index):
         qde = QStyledItemDelegate.createEditor(self, parent, option, index)
-        qde.setDisplayFormat('MM/dd/yyyy')
+        qde.setDisplayFormat(unicode(qde.displayFormat()).replace('yy', 'yyyy'))
-        qde.setMinimumDate(QDate(-4000,1,1))
+        qde.setMinimumDate(QDate(101,1,1))
         qde.setCalendarPopup(True)
         return qde
 
@@ -637,7 +637,8 @@ class BooksView(TableView):
 
     def columns_sorted(self, rating_col, timestamp_col):
         for i in range(self.model().columnCount(None)):
-            if self.itemDelegateForColumn(i) == self.rating_delegate:
+            if self.itemDelegateForColumn(i) in (self.rating_delegate,
+                    self.timestamp_delegate):
                 self.setItemDelegateForColumn(i, self.itemDelegate())
         if rating_col > -1:
             self.setItemDelegateForColumn(rating_col, self.rating_delegate)
@@ -712,6 +713,9 @@ class BooksView(TableView):
     def set_editable(self, editable):
         self._model.set_editable(editable)
 
+    def set_editable(self, editable):
+        self._model.set_editable(editable)
+
     def connect_to_search_box(self, sb):
         QObject.connect(sb, SIGNAL('search(PyQt_PyObject, PyQt_PyObject)'),
                         self._model.search)
@@ -961,7 +965,7 @@ class DeviceBooksModel(BooksModel):
                 return QVariant('Marked for deletion')
             col = index.column()
             if col in [0, 1] or (col == 4 and self.db.supports_tags()):
-                return QVariant("Double click to <b>edit</b> me<br><br>")
+                return QVariant(_("Double click to <b>edit</b> me<br><br>"))
         return NONE
 
     def headerData(self, section, orientation, role):
@@ -1006,6 +1010,10 @@ class DeviceBooksModel(BooksModel):
         self.editable = editable
         
 
+    def set_editable(self, editable):
+        self.editable = editable
+
+
 class SearchBox(QLineEdit):
 
     INTERVAL = 1000 #: Time to wait before emitting search signal

src/calibre/gui2/main.ui

@@ -1,9 +1,8 @@
-<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0" >
-<ui version="4.0">
  <author>Kovid Goyal</author>
  <class>MainWindow</class>
- <widget class="QMainWindow" name="MainWindow">
+ <widget class="QMainWindow" name="MainWindow" >
-  <property name="geometry">
+  <property name="geometry" >
    <rect>
     <x>0</x>
     <y>0</y>
@@ -11,140 +10,149 @@
     <height>822</height>
    </rect>
   </property>
-  <property name="sizePolicy">
+  <property name="sizePolicy" >
-   <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+   <sizepolicy vsizetype="Preferred" hsizetype="Preferred" >
     <horstretch>0</horstretch>
     <verstretch>0</verstretch>
    </sizepolicy>
   </property>
-  <property name="contextMenuPolicy">
+  <property name="contextMenuPolicy" >
    <enum>Qt::NoContextMenu</enum>
   </property>
-  <property name="windowTitle">
+  <property name="windowTitle" >
    <string>__appname__</string>
   </property>
-  <property name="windowIcon">
+  <property name="windowIcon" >
-   <iconset resource="images.qrc">
+   <iconset resource="images.qrc" >
     <normaloff>:/library</normaloff>:/library</iconset>
   </property>
-  <widget class="QWidget" name="centralwidget">
+  <widget class="QWidget" name="centralwidget" >
-   <layout class="QGridLayout" name="gridLayout">
+   <layout class="QGridLayout" name="gridLayout" >
-    <item row="0" column="0">
+    <item row="0" column="0" >
-     <layout class="QHBoxLayout" name="horizontalLayout_3">
+     <layout class="QHBoxLayout" name="horizontalLayout_3" >
       <item>
-       <widget class="LocationView" name="location_view">
+       <widget class="LocationView" name="location_view" >
-        <property name="sizePolicy">
+        <property name="sizePolicy" >
-         <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+         <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
           <horstretch>0</horstretch>
           <verstretch>0</verstretch>
          </sizepolicy>
         </property>
-        <property name="maximumSize">
+        <property name="maximumSize" >
          <size>
           <width>16777215</width>
           <height>100</height>
          </size>
         </property>
-        <property name="verticalScrollBarPolicy">
+        <property name="verticalScrollBarPolicy" >
          <enum>Qt::ScrollBarAlwaysOff</enum>
         </property>
-        <property name="horizontalScrollBarPolicy">
+        <property name="horizontalScrollBarPolicy" >
          <enum>Qt::ScrollBarAsNeeded</enum>
         </property>
-        <property name="tabKeyNavigation">
+        <property name="editTriggers" >
+         <set>QAbstractItemView::NoEditTriggers</set>
+        </property>
+        <property name="tabKeyNavigation" >
          <bool>true</bool>
         </property>
-        <property name="showDropIndicator" stdset="0">
+        <property name="showDropIndicator" stdset="0" >
          <bool>true</bool>
         </property>
-        <property name="iconSize">
+        <property name="selectionMode" >
+         <enum>QAbstractItemView::NoSelection</enum>
+        </property>
+        <property name="selectionBehavior" >
+         <enum>QAbstractItemView::SelectRows</enum>
+        </property>
+        <property name="iconSize" >
          <size>
           <width>40</width>
           <height>40</height>
          </size>
         </property>
-        <property name="movement">
+        <property name="movement" >
          <enum>QListView::Static</enum>
         </property>
-        <property name="flow">
+        <property name="flow" >
          <enum>QListView::LeftToRight</enum>
         </property>
-        <property name="gridSize">
+        <property name="gridSize" >
          <size>
           <width>175</width>
           <height>90</height>
          </size>
         </property>
-        <property name="viewMode">
+        <property name="viewMode" >
          <enum>QListView::ListMode</enum>
         </property>
-        <property name="wordWrap">
+        <property name="wordWrap" >
          <bool>true</bool>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QToolButton" name="donate_button">
+       <widget class="QToolButton" name="donate_button" >
-        <property name="cursor">
+        <property name="cursor" >
          <cursorShape>PointingHandCursor</cursorShape>
         </property>
-        <property name="text">
+        <property name="text" >
          <string>...</string>
         </property>
-        <property name="icon">
+        <property name="icon" >
-         <iconset resource="images.qrc">
+         <iconset resource="images.qrc" >
           <normaloff>:/images/donate.svg</normaloff>:/images/donate.svg</iconset>
         </property>
-        <property name="iconSize">
+        <property name="iconSize" >
          <size>
           <width>64</width>
           <height>64</height>
          </size>
         </property>
-        <property name="autoRaise">
+        <property name="autoRaise" >
          <bool>true</bool>
         </property>
        </widget>
       </item>
       <item>
-       <layout class="QVBoxLayout" name="verticalLayout_3">
+       <layout class="QVBoxLayout" name="verticalLayout_3" >
         <item>
-         <widget class="QLabel" name="vanity">
+         <widget class="QLabel" name="vanity" >
-          <property name="sizePolicy">
+          <property name="sizePolicy" >
-           <sizepolicy hsizetype="Preferred" vsizetype="Preferred">
+           <sizepolicy vsizetype="Preferred" hsizetype="Preferred" >
             <horstretch>0</horstretch>
             <verstretch>0</verstretch>
            </sizepolicy>
           </property>
-          <property name="maximumSize">
+          <property name="maximumSize" >
            <size>
             <width>16777215</width>
             <height>90</height>
            </size>
           </property>
-          <property name="text">
+          <property name="text" >
            <string/>
           </property>
-          <property name="textFormat">
+          <property name="textFormat" >
            <enum>Qt::RichText</enum>
           </property>
-          <property name="openExternalLinks">
+          <property name="openExternalLinks" >
            <bool>true</bool>
           </property>
          </widget>
         </item>
         <item>
-         <layout class="QHBoxLayout" name="horizontalLayout_2">
+         <layout class="QHBoxLayout" name="horizontalLayout_2" >
           <item>
-           <widget class="QLabel" name="label_2">
+           <widget class="QLabel" name="label_2" >
-            <property name="text">
+            <property name="text" >
              <string>Output:</string>
             </property>
            </widget>
           </item>
           <item>
-           <widget class="QComboBox" name="output_format">
+           <widget class="QComboBox" name="output_format" >
-            <property name="toolTip">
+            <property name="toolTip" >
              <string>Set the output format that is used when converting ebooks and downloading news</string>
             </property>
            </widget>
@@ -155,99 +163,99 @@
       </item>
      </layout>
     </item>
-    <item row="1" column="0">
+    <item row="1" column="0" >
-     <layout class="QHBoxLayout">
+     <layout class="QHBoxLayout" >
-      <property name="spacing">
+      <property name="spacing" >
        <number>6</number>
       </property>
-      <property name="margin">
+      <property name="margin" >
        <number>0</number>
       </property>
       <item>
-       <widget class="QToolButton" name="advanced_search_button">
+       <widget class="QToolButton" name="advanced_search_button" >
-        <property name="toolTip">
+        <property name="toolTip" >
          <string>Advanced search</string>
         </property>
-        <property name="text">
+        <property name="text" >
          <string>...</string>
         </property>
-        <property name="icon">
+        <property name="icon" >
-         <iconset resource="images.qrc">
+         <iconset resource="images.qrc" >
           <normaloff>:/images/search.svg</normaloff>:/images/search.svg</iconset>
         </property>
-        <property name="shortcut">
+        <property name="shortcut" >
          <string>Alt+S</string>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QLabel" name="label">
+       <widget class="QLabel" name="label" >
-        <property name="text">
+        <property name="text" >
          <string>&amp;Search:</string>
         </property>
-        <property name="buddy">
+        <property name="buddy" >
          <cstring>search</cstring>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="SearchBox" name="search">
+       <widget class="SearchBox" name="search" >
-        <property name="enabled">
+        <property name="enabled" >
          <bool>true</bool>
         </property>
-        <property name="sizePolicy">
+        <property name="sizePolicy" >
-         <sizepolicy hsizetype="Expanding" vsizetype="Fixed">
+         <sizepolicy vsizetype="Fixed" hsizetype="Expanding" >
           <horstretch>1</horstretch>
           <verstretch>0</verstretch>
          </sizepolicy>
         </property>
-        <property name="acceptDrops">
+        <property name="acceptDrops" >
          <bool>false</bool>
         </property>
-        <property name="toolTip">
+        <property name="toolTip" >
-         <string>Search the list of books by title or author&lt;br&gt;&lt;br&gt;Words separated by spaces are ANDed</string>
+         <string>Search the list of books by title or author&lt;br>&lt;br>Words separated by spaces are ANDed</string>
         </property>
-        <property name="whatsThis">
+        <property name="whatsThis" >
-         <string>Search the list of books by title, author, publisher, tags and comments&lt;br&gt;&lt;br&gt;Words separated by spaces are ANDed</string>
+         <string>Search the list of books by title, author, publisher, tags and comments&lt;br>&lt;br>Words separated by spaces are ANDed</string>
         </property>
-        <property name="autoFillBackground">
+        <property name="autoFillBackground" >
          <bool>false</bool>
         </property>
-        <property name="text">
+        <property name="text" >
          <string/>
         </property>
-        <property name="frame">
+        <property name="frame" >
          <bool>true</bool>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="QToolButton" name="clear_button">
+       <widget class="QToolButton" name="clear_button" >
-        <property name="toolTip">
+        <property name="toolTip" >
          <string>Reset Quick Search</string>
         </property>
-        <property name="text">
+        <property name="text" >
          <string>...</string>
         </property>
-        <property name="icon">
+        <property name="icon" >
-         <iconset resource="images.qrc">
+         <iconset resource="images.qrc" >
           <normaloff>:/images/clear_left.svg</normaloff>:/images/clear_left.svg</iconset>
         </property>
        </widget>
       </item>
       <item>
-       <widget class="Line" name="line">
+       <widget class="Line" name="line" >
-        <property name="orientation">
+        <property name="orientation" >
          <enum>Qt::Vertical</enum>
         </property>
        </widget>
       </item>
       <item>
        <spacer>
-        <property name="orientation">
+        <property name="orientation" >
          <enum>Qt::Horizontal</enum>
         </property>
-        <property name="sizeHint" stdset="0">
+        <property name="sizeHint" stdset="0" >
          <size>
           <width>20</width>
           <height>20</height>
@@ -256,77 +264,77 @@
        </spacer>
       </item>
       <item>
-       <widget class="QToolButton" name="config_button">
+       <widget class="QToolButton" name="config_button" >
-        <property name="toolTip">
+        <property name="toolTip" >
          <string>Configuration</string>
         </property>
-        <property name="text">
+        <property name="text" >
          <string>...</string>
         </property>
-        <property name="icon">
+        <property name="icon" >
-         <iconset resource="images.qrc">
+         <iconset resource="images.qrc" >
           <normaloff>:/images/config.svg</normaloff>:/images/config.svg</iconset>
         </property>
        </widget>
       </item>
      </layout>
     </item>
-    <item row="2" column="0">
+    <item row="2" column="0" >
-     <widget class="QStackedWidget" name="stack">
+     <widget class="QStackedWidget" name="stack" >
-      <property name="sizePolicy">
+      <property name="sizePolicy" >
-       <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+       <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
         <horstretch>100</horstretch>
         <verstretch>100</verstretch>
        </sizepolicy>
       </property>
-      <property name="currentIndex">
+      <property name="currentIndex" >
        <number>0</number>
       </property>
-      <widget class="QWidget" name="library">
+      <widget class="QWidget" name="library" >
-       <layout class="QVBoxLayout" name="verticalLayout_2">
+       <layout class="QVBoxLayout" name="verticalLayout_2" >
         <item>
-         <layout class="QHBoxLayout" name="horizontalLayout">
+         <layout class="QHBoxLayout" name="horizontalLayout" >
           <item>
-           <layout class="QVBoxLayout" name="verticalLayout">
+           <layout class="QVBoxLayout" name="verticalLayout" >
             <item>
-             <widget class="QRadioButton" name="match_any">
+             <widget class="QRadioButton" name="match_any" >
-              <property name="text">
+              <property name="text" >
                <string>Match any</string>
               </property>
-              <property name="checked">
+              <property name="checked" >
                <bool>false</bool>
               </property>
              </widget>
             </item>
             <item>
-             <widget class="QRadioButton" name="match_all">
+             <widget class="QRadioButton" name="match_all" >
-              <property name="text">
+              <property name="text" >
                <string>Match all</string>
               </property>
-              <property name="checked">
+              <property name="checked" >
                <bool>true</bool>
               </property>
              </widget>
             </item>
             <item>
-             <widget class="QCheckBox" name="popularity">
+             <widget class="QCheckBox" name="popularity" >
-              <property name="text">
+              <property name="text" >
                <string>Sort by &amp;popularity</string>
               </property>
              </widget>
             </item>
             <item>
-             <widget class="TagsView" name="tags_view">
+             <widget class="TagsView" name="tags_view" >
-              <property name="tabKeyNavigation">
+              <property name="tabKeyNavigation" >
                <bool>true</bool>
               </property>
-              <property name="alternatingRowColors">
+              <property name="alternatingRowColors" >
                <bool>true</bool>
               </property>
-              <property name="animated">
+              <property name="animated" >
                <bool>true</bool>
               </property>
-              <property name="headerHidden">
+              <property name="headerHidden" >
                <bool>true</bool>
               </property>
              </widget>
@@ -334,35 +342,35 @@
            </layout>
           </item>
           <item>
-           <widget class="BooksView" name="library_view">
+           <widget class="BooksView" name="library_view" >
-            <property name="sizePolicy">
+            <property name="sizePolicy" >
-             <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+             <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
               <horstretch>100</horstretch>
               <verstretch>10</verstretch>
              </sizepolicy>
             </property>
-            <property name="acceptDrops">
+            <property name="acceptDrops" >
              <bool>true</bool>
             </property>
-            <property name="dragEnabled">
+            <property name="dragEnabled" >
              <bool>true</bool>
             </property>
-            <property name="dragDropOverwriteMode">
+            <property name="dragDropOverwriteMode" >
              <bool>false</bool>
             </property>
-            <property name="dragDropMode">
+            <property name="dragDropMode" >
              <enum>QAbstractItemView::DragDrop</enum>
             </property>
-            <property name="alternatingRowColors">
+            <property name="alternatingRowColors" >
              <bool>true</bool>
             </property>
-            <property name="selectionBehavior">
+            <property name="selectionBehavior" >
              <enum>QAbstractItemView::SelectRows</enum>
             </property>
-            <property name="showGrid">
+            <property name="showGrid" >
              <bool>false</bool>
             </property>
-            <property name="wordWrap">
+            <property name="wordWrap" >
              <bool>false</bool>
             </property>
            </widget>
@@ -371,76 +379,76 @@
         </item>
        </layout>
       </widget>
-      <widget class="QWidget" name="main_memory">
+      <widget class="QWidget" name="main_memory" >
-       <layout class="QGridLayout">
+       <layout class="QGridLayout" >
-        <item row="0" column="0">
+        <item row="0" column="0" >
-         <widget class="DeviceBooksView" name="memory_view">
+         <widget class="DeviceBooksView" name="memory_view" >
-          <property name="sizePolicy">
+          <property name="sizePolicy" >
-           <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+           <sizepolicy vsizetype="Expanding" hsizetype="Expanding" >
             <horstretch>100</horstretch>
             <verstretch>10</verstretch>
            </sizepolicy>
           </property>
-          <property name="acceptDrops">
+          <property name="acceptDrops" >
            <bool>true</bool>
           </property>
-          <property name="dragEnabled">
+          <property name="dragEnabled" >
            <bool>true</bool>
           </property>
-          <property name="dragDropOverwriteMode">
+          <property name="dragDropOverwriteMode" >
            <bool>false</bool>
           </property>
-          <property name="dragDropMode">
+          <property name="dragDropMode" >
            <enum>QAbstractItemView::DragDrop</enum>
           </property>
-          <property name="alternatingRowColors">
+          <property name="alternatingRowColors" >
            <bool>true</bool>
           </property>
-          <property name="selectionBehavior">
+          <property name="selectionBehavior" >
            <enum>QAbstractItemView::SelectRows</enum>
           </property>
-          <property name="showGrid">
+          <property name="showGrid" >
            <bool>false</bool>
           </property>
-          <property name="wordWrap">
+          <property name="wordWrap" >
            <bool>false</bool>
           </property>
          </widget>
         </item>
        </layout>
       </widget>
-      <widget class="QWidget" name="page">
+      <widget class="QWidget" name="page" >
-       <layout class="QGridLayout">
+       <layout class="QGridLayout" >
-        <item row="0" column="0">
+        <item row="0" column="0" >
-         <widget class="DeviceBooksView" name="card_view">
+         <widget class="DeviceBooksView" name="card_view" >
-          <property name="sizePolicy">
+          <property name="sizePolicy" >
-           <sizepolicy hsizetype="Preferred" vsizetype="Expanding">
+           <sizepolicy vsizetype="Expanding" hsizetype="Preferred" >
             <horstretch>10</horstretch>
             <verstretch>10</verstretch>
            </sizepolicy>
           </property>
-          <property name="acceptDrops">
+          <property name="acceptDrops" >
            <bool>true</bool>
           </property>
-          <property name="dragEnabled">
+          <property name="dragEnabled" >
            <bool>true</bool>
           </property>
-          <property name="dragDropOverwriteMode">
+          <property name="dragDropOverwriteMode" >
            <bool>false</bool>
           </property>
-          <property name="dragDropMode">
+          <property name="dragDropMode" >
            <enum>QAbstractItemView::DragDrop</enum>
           </property>
-          <property name="alternatingRowColors">
+          <property name="alternatingRowColors" >
            <bool>true</bool>
           </property>
-          <property name="selectionBehavior">
+          <property name="selectionBehavior" >
            <enum>QAbstractItemView::SelectRows</enum>
           </property>
-          <property name="showGrid">
+          <property name="showGrid" >
            <bool>false</bool>
           </property>
-          <property name="wordWrap">
+          <property name="wordWrap" >
            <bool>false</bool>
           </property>
          </widget>
@@ -451,234 +459,225 @@
     </item>
    </layout>
   </widget>
-  <widget class="QToolBar" name="tool_bar">
+  <widget class="QToolBar" name="tool_bar" >
-   <property name="minimumSize">
+   <property name="minimumSize" >
     <size>
      <width>0</width>
      <height>0</height>
     </size>
    </property>
-   <property name="contextMenuPolicy">
+   <property name="contextMenuPolicy" >
     <enum>Qt::PreventContextMenu</enum>
    </property>
-   <property name="movable">
+   <property name="movable" >
     <bool>false</bool>
    </property>
-   <property name="orientation">
+   <property name="orientation" >
     <enum>Qt::Horizontal</enum>
    </property>
-   <property name="iconSize">
+   <property name="iconSize" >
     <size>
      <width>48</width>
      <height>48</height>
     </size>
    </property>
-   <property name="toolButtonStyle">
+   <property name="toolButtonStyle" >
     <enum>Qt::ToolButtonTextUnderIcon</enum>
    </property>
-   <attribute name="toolBarArea">
+   <attribute name="toolBarArea" >
     <enum>TopToolBarArea</enum>
    </attribute>
-   <attribute name="toolBarBreak">
+   <attribute name="toolBarBreak" >
     <bool>false</bool>
    </attribute>
-   <addaction name="action_add"/>
+   <addaction name="action_add" />
-   <addaction name="action_edit"/>
+   <addaction name="action_edit" />
-   <addaction name="action_convert"/>
+   <addaction name="action_convert" />
-   <addaction name="action_view"/>
+   <addaction name="action_view" />
-   <addaction name="action_news"/>
+   <addaction name="action_news" />
-   <addaction name="separator"/>
+   <addaction name="separator" />
-   <addaction name="action_sync"/>
+   <addaction name="action_sync" />
-   <addaction name="action_save"/>
+   <addaction name="action_save" />
-   <addaction name="action_del"/>
+   <addaction name="action_del" />
-   <addaction name="separator"/>
+   <addaction name="separator" />
-   <addaction name="action_preferences"/>
+   <addaction name="action_preferences" />
   </widget>
-  <widget class="QStatusBar" name="statusBar">
+  <widget class="QStatusBar" name="statusBar" >
-   <property name="mouseTracking">
+   <property name="mouseTracking" >
     <bool>true</bool>
    </property>
   </widget>
-  <action name="action_add">
+  <action name="action_add" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/add_book.svg</normaloff>:/images/add_book.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Add books</string>
    </property>
-   <property name="shortcut">
+   <property name="shortcut" >
     <string>A</string>
    </property>
-   <property name="autoRepeat">
+   <property name="autoRepeat" >
     <bool>false</bool>
    </property>
   </action>
-  <action name="action_del">
+  <action name="action_del" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/trash.svg</normaloff>:/images/trash.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Remove books</string>
    </property>
-   <property name="toolTip">
+   <property name="toolTip" >
     <string>Remove books</string>
    </property>
-   <property name="shortcut">
+   <property name="shortcut" >
     <string>Del</string>
    </property>
   </action>
-  <action name="action_edit">
+  <action name="action_edit" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/edit_input.svg</normaloff>:/images/edit_input.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Edit meta information</string>
    </property>
-   <property name="shortcut">
+   <property name="shortcut" >
     <string>E</string>
    </property>
-   <property name="autoRepeat">
+   <property name="autoRepeat" >
     <bool>false</bool>
    </property>
   </action>
-  <action name="action_sync">
+  <action name="action_sync" >
-   <property name="enabled">
+   <property name="enabled" >
     <bool>false</bool>
    </property>
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/sync.svg</normaloff>:/images/sync.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Send to device</string>
    </property>
   </action>
-  <action name="action_save">
+  <action name="action_save" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/save.svg</normaloff>:/images/save.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Save to disk</string>
    </property>
-   <property name="shortcut">
+   <property name="shortcut" >
     <string>S</string>
    </property>
   </action>
-  <action name="action_news">
+  <action name="action_news" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/news.svg</normaloff>:/images/news.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Fetch news</string>
    </property>
-   <property name="shortcut">
+   <property name="shortcut" >
     <string>F</string>
    </property>
   </action>
-  <action name="action_convert">
+  <action name="action_convert" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/convert.svg</normaloff>:/images/convert.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Convert E-books</string>
    </property>
-   <property name="shortcut">
+   <property name="shortcut" >
     <string>C</string>
    </property>
   </action>
-  <action name="action_view">
+  <action name="action_view" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/view.svg</normaloff>:/images/view.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>View</string>
    </property>
-   <property name="shortcut">
+   <property name="shortcut" >
     <string>V</string>
    </property>
   </action>
-  <action name="action_open_containing_folder">
+  <action name="action_open_containing_folder" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/document_open.svg</normaloff>:/images/document_open.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Open containing folder</string>
    </property>
   </action>
-  <action name="action_show_book_details">
+  <action name="action_show_book_details" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/dialog_information.svg</normaloff>:/images/dialog_information.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Show book details</string>
    </property>
   </action>
-  <action name="action_books_by_same_author">
+  <action name="action_books_by_same_author" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/user_profile.svg</normaloff>:/images/user_profile.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Books by same author</string>
    </property>
   </action>
-  <action name="action_books_in_this_series">
+  <action name="action_books_in_this_series" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/books_in_series.svg</normaloff>:/images/books_in_series.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Books in this series</string>
    </property>
   </action>
-  <action name="action_books_by_this_publisher">
+  <action name="action_books_by_this_publisher" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/publisher.png</normaloff>:/images/publisher.png</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Books by this publisher</string>
    </property>
   </action>
-  <action name="action_books_with_the_same_tags">
+  <action name="action_books_with_the_same_tags" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
      <normaloff>:/images/tags.svg</normaloff>:/images/tags.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Books with the same tags</string>
    </property>
   </action>
-  <action name="action_send_specific_format_to_device">
+  <action name="action_preferences" >
-   <property name="icon">
+   <property name="icon" >
-    <iconset resource="images.qrc">
+    <iconset resource="images.qrc" >
-     <normaloff>:/images/book.svg</normaloff>:/images/book.svg</iconset>
-   </property>
-   <property name="text">
-    <string>Send specific format to device</string>
-   </property>
-  </action>
-  <action name="action_preferences">
-   <property name="icon">
-    <iconset resource="images.qrc">
      <normaloff>:/images/config.svg</normaloff>:/images/config.svg</iconset>
    </property>
-   <property name="text">
+   <property name="text" >
     <string>Preferences</string>
    </property>
-   <property name="toolTip">
+   <property name="toolTip" >
     <string>Configure calibre</string>
    </property>
-   <property name="shortcut">
+   <property name="shortcut" >
     <string>Ctrl+P</string>
    </property>
   </action>
@@ -711,7 +710,7 @@
   </customwidget>
  </customwidgets>
  <resources>
-  <include location="images.qrc"/>
+  <include location="images.qrc" />
  </resources>
  <connections>
   <connection>
@@ -720,11 +719,11 @@
    <receiver>search</receiver>
    <slot>clear()</slot>
    <hints>
-    <hint type="sourcelabel">
+    <hint type="sourcelabel" >
      <x>787</x>
      <y>215</y>
     </hint>
-    <hint type="destinationlabel">
+    <hint type="destinationlabel" >
      <x>755</x>
      <y>213</y>
     </hint>

src/calibre/gui2/widgets.py

@@ -7,9 +7,9 @@ import re, os, traceback
 from PyQt4.QtGui import QListView, QIcon, QFont, QLabel, QListWidget, \
                         QListWidgetItem, QTextCharFormat, QApplication, \
                         QSyntaxHighlighter, QCursor, QColor, QWidget, QDialog, \
-                        QPixmap
+                        QPixmap, QMovie, QPalette
 from PyQt4.QtCore import QAbstractListModel, QVariant, Qt, SIGNAL, \
-                         QObject, QRegExp, QString, QSettings, QSize
+                         QRegExp, QSettings, QSize, QModelIndex
 
 from calibre.gui2.jobs2 import DetailView
 from calibre.gui2 import human_readable, NONE, TableView, \
@@ -21,6 +21,42 @@ from calibre.ebooks.metadata.meta import metadata_from_filename
 from calibre.utils.config import prefs
 from calibre.gui2.dialogs.warning_ui import Ui_Dialog as Ui_WarningDialog
 
+class ProgressIndicator(QWidget):
+
+    def __init__(self, *args):
+        QWidget.__init__(self, *args)
+        self.setGeometry(0, 0, 300, 350)
+        self.movie = QMovie(':/images/jobs-animated.mng')
+        self.ml = QLabel(self)
+        self.ml.setMovie(self.movie)
+        self.movie.start()
+        self.movie.setPaused(True)
+        self.status = QLabel(self)
+        self.status.setWordWrap(True)
+        self.status.setAlignment(Qt.AlignHCenter|Qt.AlignTop)
+        self.status.font().setBold(True)
+        self.status.font().setPointSize(self.font().pointSize()+6)
+        self.setVisible(False)
+
+    def start(self, msg=''):
+        view = self.parent()
+        pwidth, pheight = view.size().width(), view.size().height()
+        self.resize(pwidth, min(pheight, 250))
+        self.move(0, (pheight-self.size().height())/2.)
+        self.ml.resize(self.ml.sizeHint())
+        self.ml.move(int((self.size().width()-self.ml.size().width())/2.), 0)
+        self.status.resize(self.size().width(), self.size().height()-self.ml.size().height()-10)
+        self.status.move(0, self.ml.size().height()+10)
+        self.status.setText(msg)
+        self.setVisible(True)
+        self.movie.setPaused(False)
+
+    def stop(self):
+        if self.movie.state() == self.movie.Running:
+            self.movie.setPaused(True)
+            self.setVisible(False)
+
+
 class WarningDialog(QDialog, Ui_WarningDialog):
 
     def __init__(self, title, msg, details, parent=None):
@@ -168,6 +204,13 @@ class LocationModel(QAbstractListModel):
             font = QFont('monospace')
             font.setBold(row == self.highlight_row)
             data = QVariant(font)
+        elif role == Qt.ForegroundRole and row == self.highlight_row:
+            return QVariant(QApplication.palette().brush(
+                QPalette.HighlightedText))
+        elif role == Qt.BackgroundRole and row == self.highlight_row:
+            return QVariant(QApplication.palette().brush(
+                QPalette.Highlight))
+
         return data
 
     def headerData(self, section, orientation, role):
@@ -182,7 +225,8 @@ class LocationModel(QAbstractListModel):
 
     def location_changed(self, row):
         self.highlight_row = row
-        self.reset()
+        self.emit(SIGNAL('dataChanged(QModelIndex,QModelIndex)'),
+                self.index(0), self.index(self.rowCount(QModelIndex())-1))
 
 class LocationView(QListView):
 
@@ -190,17 +234,19 @@ class LocationView(QListView):
         QListView.__init__(self, parent)
         self.setModel(LocationModel(self))
         self.reset()
-        QObject.connect(self.selectionModel(), SIGNAL('currentChanged(QModelIndex, QModelIndex)'), self.current_changed)
         self.setCursor(Qt.PointingHandCursor)
+        self.currentChanged = self.current_changed
 
     def count_changed(self, new_count):
         self.model().count = new_count
         self.model().reset()
 
     def current_changed(self, current, previous):
-        i = current.row()
+        if current.isValid():
-        location = 'library' if i == 0 else 'main' if i == 1 else 'card'
+            i = current.row()
-        self.emit(SIGNAL('location_selected(PyQt_PyObject)'), location)
+            location = 'library' if i == 0 else 'main' if i == 1 else 'card'
+            self.emit(SIGNAL('location_selected(PyQt_PyObject)'), location)
+            self.model().location_changed(i)
 
     def location_changed(self, row):
         if 0 <= row and row <= 2:

src/calibre/library/database2.py

@@ -226,7 +226,11 @@ class ResultCache(SearchQueryParser):
         Returns a list of affected rows or None if the rows are filtered.
         '''
         for id in ids:
-            self._data[id] = conn.get('SELECT * from meta WHERE id=?', (id,))[0]
+            try:
+                self._data[id] = conn.get('SELECT * from meta WHERE id=?',
+                        (id,))[0]
+            except IndexError:
+                return None
         try:
             return map(self.row, ids)
         except ValueError:
@@ -1568,3 +1572,4 @@ books_series_link      feeds
 
         return duplicates
 
+

src/calibre/libunrar.py

@@ -25,7 +25,8 @@ if iswindows:
 else:
     Structure = _Structure
 if hasattr(sys, 'frozen') and iswindows:
-    _libunrar = cdll.LoadLibrary(os.path.join(os.path.dirname(sys.executable), 'unrar.dll'))
+    _libunrar = cdll.LoadLibrary(os.path.join(os.path.dirname(sys.executable),
+        'unrar.dll'))
 _libunrar = load_library(_librar_name, cdll)
 
 RAR_OM_LIST    = 0

src/calibre/linux.py

@@ -40,6 +40,8 @@ entry_points = {
              'calibre-parallel   = calibre.parallel:main',
              'calibre-customize  = calibre.customize.ui:main',
              'pdfmanipulate      = calibre.ebooks.pdf.manipulate:main',
+             'fetch-ebook-metadata = calibre.ebooks.metadata.fetch:main',
+             'calibre-smtp = calibre.utils.smtp:main',
         ],
         'gui_scripts'    : [
             __appname__+' = calibre.gui2.main:main',
@@ -159,6 +161,7 @@ def setup_completion(fatal_errors):
         from calibre.ebooks.epub.from_comic import option_parser as comic2epub
         from calibre.ebooks.metadata.fetch import option_parser as fem_op
         from calibre.gui2.main import option_parser as guiop
+        from calibre.utils.smtp import option_parser as smtp_op
         any_formats = ['epub', 'htm', 'html', 'xhtml', 'xhtm', 'rar', 'zip',
              'txt', 'lit', 'rtf', 'pdf', 'prc', 'mobi', 'fb2', 'odt']
         f = open_file('/etc/bash_completion.d/libprs500')
@@ -193,6 +196,7 @@ def setup_completion(fatal_errors):
         f.write(opts_and_words('feeds2epub', feeds2epub, feed_titles))
         f.write(opts_and_words('feeds2mobi', feeds2mobi, feed_titles))
         f.write(opts_and_words('fetch-ebook-metadata', fem_op, []))
+        f.write(opts_and_words('calibre-smtp', smtp_op, []))
         f.write('''
 _prs500_ls()
 {
@@ -543,6 +547,3 @@ main = post_install
 if __name__ == '__main__':
     post_install()
 
-
-
-

src/calibre/trac/plugins/download.py

@@ -5,7 +5,7 @@ import re, textwrap
 
 DEPENDENCIES = [
             #(Generic, version, gentoo, ubuntu, fedora)
-            ('python', '2.5', None, None, None),
+            ('python', '2.6', None, None, None),
             ('setuptools', '0.6c5', 'setuptools', 'python-setuptools', 'python-setuptools-devel'),
             ('Python Imaging Library', '1.1.6', 'imaging', 'python-imaging', 'python-imaging'),
             ('libusb', '0.1.12', None, None, None),
@@ -18,6 +18,7 @@ DEPENDENCIES = [
             ('lxml', '2.1.5', 'lxml', 'python-lxml', 'python-lxml'),
             ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil'),
             ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'),
+            ('dnspython', '1.6.0', 'dnspython', 'dnspython', 'dnspython', 'dnspython'),
             ]
 
 

src/calibre/utils/smtp.py

@@ -0,0 +1,240 @@
+from __future__ import with_statement
+__license__ = 'GPL 3'
+__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+'''
+This module implements a simple commandline SMTP client that supports:
+
+  * Delivery via an SMTP relay with SSL or TLS
+  * Background delivery with failures being saved in a maildir mailbox
+'''
+
+import sys, traceback, os
+from email import encoders
+
+def create_mail(from_, to, subject, text=None, attachment_data=None,
+                 attachment_type=None, attachment_name=None):
+    assert text or attachment_data
+
+    from email.mime.multipart import MIMEMultipart
+
+    outer = MIMEMultipart()
+    outer['Subject'] = subject
+    outer['To'] = to
+    outer['From'] = from_
+    outer.preamble = 'You will not see this in a MIME-aware mail reader.\n'
+
+    if text is not None:
+        from email.mime.text import MIMEText
+        msg = MIMEText(text)
+        outer.attach(msg)
+
+    if attachment_data is not None:
+        from email.mime.base import MIMEBase
+        assert attachment_data and attachment_name
+        try:
+            maintype, subtype = attachment_type.split('/', 1)
+        except AttributeError:
+            maintype, subtype = 'application', 'octet-stream'
+        msg = MIMEBase(maintype, subtype)
+        msg.set_payload(attachment_data)
+        encoders.encode_base64(msg)
+        msg.add_header('Content-Disposition', 'attachment',
+                       filename=attachment_name)
+        outer.attach(msg)
+
+    return outer.as_string()
+
+def get_mx(host, verbose=0):
+    import dns.resolver
+    if verbose:
+        print 'Find mail exchanger for', host
+    answers = list(dns.resolver.query(host, 'MX'))
+    answers.sort(cmp=lambda x, y: cmp(int(x.preference), int(y.preference)))
+    return [str(x.exchange) for x in answers]
+
+def sendmail_direct(from_, to, msg, timeout, localhost, verbose):
+    import smtplib
+    hosts = get_mx(to.split('@')[-1].strip(), verbose)
+    timeout=None # Non blocking sockets sometimes don't work
+    s = smtplib.SMTP(timeout=timeout, local_hostname=localhost)
+    s.set_debuglevel(verbose)
+    if not hosts:
+        raise ValueError('No mail server found for address: %s'%to)
+    last_error = last_traceback = None
+    for host in hosts:
+        try:
+            s.connect(host, 25)
+            s.sendmail(from_, [to], msg)
+            return s.quit()
+        except Exception, e:
+            last_error, last_traceback = e, traceback.format_exc()
+    if last_error is not None:
+        print last_traceback
+        raise IOError('Failed to send mail: '+repr(last_error))
+
+
+def sendmail(msg, from_, to, localhost=None, verbose=0, timeout=30,
+             relay=None, username=None, password=None, encryption='TLS',
+             port=-1):
+    if relay is None:
+        for x in to:
+            return sendmail_direct(from_, x, msg, timeout, localhost, verbose)
+    import smtplib
+    cls = smtplib.SMTP if encryption == 'TLS' else smtplib.SMTP_SSL
+    timeout = None # Non-blocking sockets sometimes don't work
+    port = int(port)
+    s = cls(timeout=timeout, local_hostname=localhost)
+    s.set_debuglevel(verbose)
+    if port < 0:
+        port = 25 if encryption == 'TLS' else 465
+    s.connect(relay, port)
+    if encryption == 'TLS':
+        s.starttls()
+        s.ehlo()
+    if username is not None and password is not None:
+        s.login(username, password)
+    s.sendmail(from_, to, msg)
+    return s.quit()
+
+def option_parser():
+    try:
+        from calibre.utils.config import OptionParser
+        OptionParser
+    except ImportError:
+        from optparse import OptionParser
+    import textwrap
+    parser = OptionParser(textwrap.dedent('''\
+    %prog [options] [from to text]
+
+    Send mail using the SMTP protocol. %prog has two modes of operation. In the
+    compose mode you specify from to and text and these are used to build and
+    send an email message. In the filter mode, %prog reads a complete email
+    message from STDIN and sends it.
+
+    text is the body of the email message.
+    If text is not specified, a complete email message is read from STDIN.
+    from is the email address of the sender and to is the email address
+    of the recipient. When a complete email is read from STDIN, from and to
+    are only used in the SMTP negotiation, the message headers are not modified.
+    '''))
+    c=parser.add_option_group('COMPOSE MAIL',
+        'Options to compose an email. Ignored if text is not specified').add_option
+    c('-a', '--attachment', help='File to attach to the email')
+    c('-s', '--subject', help='Subject of the email')
+
+    parser.add_option('-l', '--localhost',
+                      help=('Host name of localhost. Used when connecting '
+                            'to SMTP server.'))
+    r=parser.add_option_group('SMTP RELAY',
+        'Options to use an SMTP relay server to send mail. '
+        '%prog will try to send the email directly unless --relay is '
+        'specified.').add_option
+    r('-r', '--relay', help=('An SMTP relay server to use to send mail.'))
+    r('-p', '--port', default=-1,
+      help='Port to connect to on relay server. Default is to use 465 if '
+      'encryption method is SSL and 25 otherwise.')
+    r('-u', '--username', help='Username for relay')
+    r('-p', '--password', help='Password for relay')
+    r('-e', '--encryption-method', default='TLS',
+      choices=['TLS', 'SSL'],
+      help='Encryption method to use when connecting to relay. Choices are '
+      'TLS and SSL. Default is TLS.')
+    parser.add_option('-o', '--outbox', help='Path to maildir folder to store '
+                      'failed email messages in.')
+    parser.add_option('-f', '--fork', default=False, action='store_true',
+                      help='Fork and deliver message in background. '
+                      'If you use this option, you should also use --outbox '
+                      'to handle delivery failures.')
+    parser.add_option('-t', '--timeout', help='Timeout for connection')
+    parser.add_option('-v', '--verbose', default=0, action='count',
+                      help='Be more verbose')
+    return parser
+
+def extract_email_address(raw):
+    from email.utils import parseaddr
+    return parseaddr(raw)[-1]
+
+def compose_mail(from_, to, text, subject=None, attachment=None,
+        attachment_name=None):
+    attachment_type = attachment_data = None
+    if attachment is not None:
+        try:
+            from calibre import guess_type
+            guess_type
+        except ImportError:
+            from mimetypes import guess_type
+        attachment_data = attachment.read() if hasattr(attachment, 'read') \
+                            else open(attachment, 'rb').read()
+        attachment_type = guess_type(getattr(attachment, 'name', attachment))[0]
+        if attachment_name is None:
+            attachment_name = os.path.basename(getattr(attachment,
+                'name', attachment))
+    subject = subject if subject else 'no subject'
+    return create_mail(from_, to, subject, text=text,
+            attachment_data=attachment_data, attachment_type=attachment_type,
+            attachment_name=attachment_name)
+
+def main(args=sys.argv):
+    parser = option_parser()
+    opts, args = parser.parse_args(args)
+
+
+    if len(args) > 1:
+        msg = compose_mail(args[1], args[2], args[3], subject=opts.subject,
+                           attachment=opts.attachment)
+        from_, to = args[1:3]
+        efrom, eto = map(extract_email_address, (from_, to))
+        eto = [eto]
+    else:
+        msg = sys.stdin.read()
+        from email.parser import Parser
+        from email.utils import getaddresses
+        eml = Parser.parsestr(msg, headersonly=True)
+        tos = eml.get_all('to', [])
+        ccs = eml.get_all('cc', [])
+        eto = getaddresses(tos + ccs)
+        if not eto:
+            raise ValueError('Email from STDIN does not specify any recipients')
+        efrom = getaddresses(eml.get_all('from', []))
+        if not efrom:
+            raise ValueError('Email from STDIN does not specify a sender')
+        efrom = efrom[0]
+
+
+    outbox = None
+    if opts.outbox is not None:
+        outbox = os.path.abspath(os.path.expanduser(opts.outbox))
+        from mailbox import Maildir
+        outbox = Maildir(opts.outbox, factory=None)
+    if opts.fork:
+        if os.fork() != 0:
+            return 0
+    try:
+        sendmail(msg, efrom, eto, localhost=opts.localhost, verbose=opts.verbose,
+             timeout=opts.timeout, relay=opts.relay, username=opts.username,
+             password=opts.password, port=opts.port,
+             encryption=opts.encryption_method)
+    except:
+        if outbox is not None:
+            outbox.add(msg)
+            print 'Delivery failed. Message saved to', opts.outbox
+        raise
+    return 0
+
+def config(defaults=None):
+    from calibre.utils.config import Config, StringConfig
+    desc = _('Control email delivery')
+    c = Config('smtp',desc) if defaults is None else StringConfig(defaults,desc)
+    c.add_opt('from_')
+    c.add_opt('accounts', default={})
+    c.add_opt('relay_host')
+    c.add_opt('relay_port', default=25)
+    c.add_opt('relay_username')
+    c.add_opt('relay_password')
+    c.add_opt('encryption', default='TLS', choices=['TLS', 'SSL'])
+    return c
+
+
+if __name__ == '__main__':
+    sys.exit(main())

src/calibre/web/feeds/news.py

@@ -379,7 +379,7 @@ class BasicNewsRecipe(object):
         if raw:
             return _raw
         if not isinstance(_raw, unicode) and self.encoding:
-            _raw = _raw.decode(self.encoding)
+            _raw = _raw.decode(self.encoding, 'replace')
         massage = list(BeautifulSoup.MARKUP_MASSAGE)
         massage.append((re.compile(r'&(\S+?);'), lambda match: entity_to_unicode(match, encoding=self.encoding)))
         return BeautifulSoup(_raw, markupMassage=massage)

src/calibre/web/feeds/recipes/__init__.py

@@ -36,7 +36,8 @@ recipe_modules = ['recipe_' + r for r in (
            'el_universal', 'mediapart', 'wikinews_en', 'ecogeek', 'daily_mail',
            'new_york_review_of_books_no_sub', 'politico', 'adventuregamers',
            'mondedurable', 'instapaper', 'dnevnik_cro', 'vecernji_list',
-           'nacional_cro', '24sata',
+           'nacional_cro', '24sata', 'dnevni_avaz', 'glas_srpske', '24sata_rs',
+           'krstarica', 'krstarica_en', 'tanjug',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_24sata_rs.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+
+'''
+24sata.rs
+'''
+
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class Ser24Sata(BasicNewsRecipe):
+    title                 = '24 Sata - Sr'
+    __author__            = 'Darko Miletic'
+    description           = '24 sata portal vesti iz Srbije'
+    publisher             = 'Ringier d.o.o.'
+    category              = 'news, politics, entertainment, Serbia'
+    oldest_article        = 1
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    remove_javascript     = True
+    language              = _('Serbian')
+
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        , '--ignore-tables'
+                        ]
+
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True'
+
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
+    feeds = [(u'Vesti Dana', u'http://www.24sata.rs/rss.php')]
+
+    def preprocess_html(self, soup):
+        soup.html['xml:lang'] = 'sr-Latn-RS'
+        soup.html['lang']     = 'sr-Latn-RS'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
+        soup.head.insert(0,mtag)
+        return soup
+
+    def print_version(self, url):
+        article, sep, rest = url.partition('#')
+        return article.replace('/show.php','/_print.php')
+

src/calibre/web/feeds/recipes/recipe_b92.py

@@ -5,7 +5,6 @@ __copyright__ = '2008-2009, Darko Miletic <darko.miletic at gmail.com>'
 '''
 b92.net
 '''
-
 import re
 from calibre.web.feeds.news import BasicNewsRecipe
 
@@ -13,57 +12,53 @@ class B92(BasicNewsRecipe):
     title                 = 'B92'
     __author__            = 'Darko Miletic'
     description           = 'Dnevne vesti iz Srbije i sveta'    
-    oldest_article        = 2
+    publisher             = 'B92'
-    publisher             = 'B92.net'
     category              = 'news, politics, Serbia'
+    oldest_article        = 1
     max_articles_per_feed = 100
-    remove_javascript     = True
     no_stylesheets        = True
     use_embedded_content  = False
-    cover_url = 'http://static.b92.net/images/fp/logo.gif'
+    remove_javascript     = True
+    encoding              = 'cp1250'
     language              = _('Serbian')
-    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: sans1, sans-serif}'
+    extra_css             = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}'
     
     html2lrf_options = [
-                          '--comment'  , description
+                          '--comment', description
-                        , '--category' , category
+                        , '--category', category
                         , '--publisher', publisher
                         , '--ignore-tables'
                         ]
     
-    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True' 
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\nlinearize_tables=True\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em}"' 
-
-    keep_only_tags = [ dict(name='div', attrs={'class':'sama_vest'}) ]
     
     preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
     
+    keep_only_tags     = [dict(name='table', attrs={'class':'maindocument'})]
+
+    remove_tags = [
+                     dict(name='ul', attrs={'class':'comment-nav'})
+                    ,dict(name=['embed','link','base']            )
+                  ]
+
     feeds          = [
                         (u'Vesti', u'http://www.b92.net/info/rss/vesti.xml')
                        ,(u'Biz'  , u'http://www.b92.net/info/rss/biz.xml'  )
-                       ,(u'Zivot', u'http://www.b92.net/info/rss/zivot.xml')
-                       ,(u'Sport', u'http://www.b92.net/info/rss/sport.xml')
                      ]
 
     def print_version(self, url):
-        main, sep, article_id = url.partition('nav_id=')
+        return url + '&version=print'
-        rmain, rsep, rrest = main.partition('.php?')
-        mrmain , rsepp, nnt = rmain.rpartition('/')
-        mprmain, rrsep, news_type = mrmain.rpartition('/')
-        nurl = 'http://www.b92.net/mobilni/' + news_type + '/index.php?nav_id=' + article_id
-        brbiz, biz, bizrest = rmain.partition('/biz/')
-        if biz:
-            nurl = 'http://www.b92.net/mobilni/biz/index.php?nav_id=' + article_id
-        return nurl
 
     def preprocess_html(self, soup):
-        lng = 'sr-Latn-RS'
+        del soup.body['onload']
-        soup.html['xml:lang'] = lng
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>'
-        soup.html['lang']     = lng
-        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-RS"/>'
         soup.head.insert(0,mtag)
         for item in soup.findAll(style=True):
             del item['style']
-        for item in soup.findAll(name='img',align=True):
+        for item in soup.findAll(align=True):
             del item['align']
-            item.insert(0,'<br /><br />')
+        for item in soup.findAll('font'):
+            item.name='p'
+            if item.has_key('size'):
+               del item['size']
         return soup

src/calibre/web/feeds/recipes/recipe_dnevni_avaz.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+
+'''
+dnevniavaz.ba
+'''
+
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class DnevniAvaz(BasicNewsRecipe):
+    title                 = 'Dnevni Avaz'
+    __author__            = 'Darko Miletic'
+    description           = 'Latest news from Bosnia'
+    publisher             = 'Dnevni Avaz'
+    category              = 'news, politics, Bosnia and Herzegovina'
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    remove_javascript     = True
+    cover_url             = 'http://www.dnevniavaz.ba/img/logo.gif'
+    lang                  = 'bs-BA'
+    language              = _('Bosnian')
+
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"'
+
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
+    keep_only_tags = [dict(name='div', attrs={'id':['fullarticle-title','fullarticle-leading','fullarticle-date','fullarticle-text','articleauthor']})]
+
+    remove_tags = [dict(name=['object','link','base'])]
+
+    feeds = [
+               (u'Najnovije'     , u'http://www.dnevniavaz.ba/rss/novo'     )
+              ,(u'Najpopularnije', u'http://www.dnevniavaz.ba/rss/popularno')
+            ]
+
+    def preprocess_html(self, soup):
+        soup.html['xml:lang'] = self.lang
+        soup.html['lang']     = self.lang
+        mtag = '<meta http-equiv="Content-Language" content="bs-BA"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
+        soup.head.insert(0,mtag)
+        return soup

src/calibre/web/feeds/recipes/recipe_faznet.py

@@ -1,29 +1,50 @@
 __license__   = 'GPL v3'
-__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
+__copyright__ = '2008-2009, Kovid Goyal <kovid at kovidgoyal.net>, Darko Miletic <darko at gmail.com>'
 '''
 Profile to download FAZ.net
 '''
-import re
+
 from calibre.web.feeds.news import BasicNewsRecipe
  
-
 class FazNet(BasicNewsRecipe): 
-
+    title                 = 'FAZ NET'
-    title = 'FAZ NET'
+    __author__            = 'Kovid Goyal, Darko Miletic'
-    __author__ = 'Kovid Goyal'
+    description           = 'Frankfurter Allgemeine Zeitung'
-    description = 'Frankfurter Allgemeine Zeitung'
+    publisher             = 'FAZ Electronic Media GmbH'
-    use_embedded_content   = False
+    category              = 'news, politics, Germany'
-    language = _('German') 
+    use_embedded_content  = False
+    language              = _('German') 
     max_articles_per_feed = 30 
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    remove_javascript     = True
 
-    preprocess_regexps = [
+    html2lrf_options = [
-       (re.compile(r'Zum Thema</span>.*?</BODY>', re.IGNORECASE | re.DOTALL), 
+                          '--comment', description
-        lambda match : ''),
+                        , '--category', category
-    ]    
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+    
+    keep_only_tags = [dict(name='div', attrs={'class':'Article'})]
+
+    remove_tags = [
+                     dict(name=['object','link','embed','base'])
+                    ,dict(name='div', attrs={'class':['LinkBoxModulSmall','ModulVerlagsInfo']})
+                  ]
     
 
     feeds = [ ('FAZ.NET', 'http://www.faz.net/s/Rub/Tpl~Epartner~SRss_.xml') ] 
 
     def print_version(self, url):
-        return url.replace('.html?rss_aktuell', '~Afor~Eprint.html') 
+        article, sep, rest = url.partition('?')    
+        return article.replace('.html', '~Afor~Eprint.html') 
 
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>'
+        soup.head.insert(0,mtag)        
+        del soup.body['onload']
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup

src/calibre/web/feeds/recipes/recipe_glas_srpske.py

@@ -0,0 +1,96 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+
+'''
+glassrpske.com
+'''
+
+import re
+from calibre.web.feeds.recipes import BasicNewsRecipe
+
+class GlasSrpske(BasicNewsRecipe):
+    title                 = 'Glas Srpske'
+    __author__            = 'Darko Miletic'
+    description           = 'Latest news from republika srpska'
+    publisher             = 'GLAS SRPSKE'
+    category              = 'Novine, Dnevne novine, Vijesti, Novosti, Ekonomija, Sport, Crna Hronika, Banja Luka,, Republika Srpska, Bosna i Hercegovina'
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    encoding              = 'utf-8'
+    use_embedded_content  = False
+    remove_javascript     = True    
+    cover_url             = 'http://www.glassrpske.com/var/slike/glassrpske-logo.png'
+    lang                  = 'sr-BA'
+    language              = _('Serbian')
+    INDEX                 = 'http://www.glassrpske.com'
+
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em} img {margin-top: 0em; margin-bottom: 0.4em}"' 
+     
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+    
+    keep_only_tags = [dict(name='div', attrs={'class':'gl_cv paragraf'})]
+
+    remove_tags = [dict(name=['object','link','base'])]
+    
+    feeds = [ 
+               (u'Novosti'   , u'http://www.glassrpske.com/vijest/2/novosti/lat/'   )
+              ,(u'Drustvo'   , u'http://www.glassrpske.com/vijest/3/drustvo/lat/'   )
+              ,(u'Biznis'    , u'http://www.glassrpske.com/vijest/4/ekonomija/lat/' )
+              ,(u'Kroz RS'   , u'http://www.glassrpske.com/vijest/5/krozrs/lat/'    )
+              ,(u'Hronika'   , u'http://www.glassrpske.com/vijest/6/hronika/lat/'   )
+              ,(u'Srbija'    , u'http://www.glassrpske.com/vijest/8/srbija/lat/'    )
+              ,(u'Region'    , u'http://www.glassrpske.com/vijest/18/region/lat/'   )
+              ,(u'Svijet'    , u'http://www.glassrpske.com/vijest/12/svijet/lat/'   )
+              ,(u'Kultura'   , u'http://www.glassrpske.com/vijest/9/kultura/lat/'   )
+              ,(u'Banja Luka', u'http://www.glassrpske.com/vijest/10/banjaluka/lat/')
+              ,(u'Jet Set'   , u'http://www.glassrpske.com/vijest/11/jetset/lat/'   )
+              ,(u'Muzika'    , u'http://www.glassrpske.com/vijest/19/muzika/lat/'   )
+              ,(u'Sport'     , u'http://www.glassrpske.com/vijest/13/sport/lat/'    )
+              ,(u'Kolumne'   , u'http://www.glassrpske.com/vijest/16/kolumne/lat/'  )
+              ,(u'Plus'      , u'http://www.glassrpske.com/vijest/7/plus/lat/'      )              
+            ]
+
+    def preprocess_html(self, soup):
+        soup.html['xml:lang'] = self.lang
+        soup.html['lang']     = self.lang
+        mtag = '<meta http-equiv="Content-Language" content="sr-BA"/>\n<meta http-equiv="Content-Type" content="text/html; charset=utf-8">'
+        soup.head.insert(0,mtag)
+        return soup        
+        
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            soup = self.index_to_soup(feedurl)
+            for item in soup.findAll('div', attrs={'class':'gl_rub'}):
+                atag    = item.find('a')
+                ptag    = item.find('p')
+                datetag = item.find('span')
+                url           = self.INDEX + atag['href']
+                title         = self.tag_to_string(atag)
+                description   = self.tag_to_string(ptag)
+                date,sep,rest = self.tag_to_string(ptag).partition('|')
+                articles.append({
+                                      'title'      :title
+                                     ,'date'       :date
+                                     ,'url'        :url
+                                     ,'description':description
+                                    })
+            totalfeeds.append((feedtitle, articles))
+        return totalfeeds
+        
+        

src/calibre/web/feeds/recipes/recipe_joelonsoftware.py

@@ -15,6 +15,7 @@ class Joelonsoftware(BasicNewsRecipe):
     language = _('English')
     no_stylesheets = True
     use_embedded_content  = True
+    oldest_article = 60
 
     cover_url = 'http://www.joelonsoftware.com/RssJoelOnSoftware.jpg'
 

src/calibre/web/feeds/recipes/recipe_krstarica.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+'''
+vesti.krstarica.com
+'''
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Krstarica(BasicNewsRecipe):
+    title                 = 'Krstarica - Vesti'
+    __author__            = 'Darko Miletic'
+    description           = 'Dnevne vesti iz Srbije i sveta'    
+    publisher             = 'Krstarica'
+    category              = 'news, politics, Serbia'
+    oldest_article        = 1
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    use_embedded_content  = False
+    remove_javascript     = True
+    encoding              = 'utf-8'
+    language              = _('Serbian')
+    extra_css             = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"\noverride_css=" p {text-indent: 0em; margin-top: 0em; margin-bottom: 0.5em}"' 
+    
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+    
+    feeds          = [
+                        (u'Vesti dana'         , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=aktuelno&lang=0'     )
+                       ,(u'Srbija'             , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=scg&lang=0'          )
+                       ,(u'Svet'               , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=svet&lang=0'         )
+                       ,(u'Politika'           , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=politika&lang=0'     )
+                       ,(u'Ekonomija'          , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=ekonomija&lang=0'    )
+                       ,(u'Drustvo'            , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=drustvo&lang=0'      )
+                       ,(u'Kultura'            , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=kultura&lang=0'      )
+                       ,(u'Nauka i Tehnologija', u'http://vesti.krstarica.com/index.php?rss=1&rubrika=nauka&lang=0'        )
+                       ,(u'Medicina'           , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=medicina&lang=0'     )
+                       ,(u'Sport'              , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=sport&lang=0'        )
+                       ,(u'Zanimljivosti'      , u'http://vesti.krstarica.com/index.php?rss=1&rubrika=zanimljivosti&lang=0')
+                     ]
+
+    def preprocess_html(self, soup):
+        mtag = '<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>'
+        soup.head.insert(0,mtag)
+        titletag = soup.find('h4')
+        if titletag:
+           realtag = titletag.parent.parent
+           realtag.extract()
+           for item in soup.findAll(['table','center']):
+               item.extract()
+           soup.body.insert(1,realtag)            
+           realtag.name = 'div'
+        for item in soup.findAll(style=True):
+            del item['style']
+        for item in soup.findAll(align=True):
+            del item['align']
+        return soup