Pull from trunk

installer/linux/freeze.py

@@ -81,7 +81,8 @@ def freeze():
                 'PyQt4.QtScript.so', 'PyQt4.QtSql.so', 'PyQt4.QtTest.so', 'qt',
                 'glib', 'gobject']
     
-    packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg']
+    packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg', 
+                'dateutil']
     
     includes += ['calibre.web.feeds.recipes.'+r for r in recipe_modules]
     

installer/osx/freeze.py

@@ -342,6 +342,7 @@ def main():
                                        'calibre.ebooks.lrf.any.*', 'calibre.ebooks.lrf.feeds.*',
                                        'keyword', 'codeop', 'pydoc', 'readline',
                                        'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*',
+                                       'dateutil',
                                        ],
                          'packages' : ['PIL', 'Authorization', 'lxml'],
                          'excludes' : ['IPython'],

installer/windows/freeze.py

@@ -179,7 +179,8 @@ def main(args=sys.argv):
                                              'calibre.ebooks.lrf.fonts.prs500.*',
                                              'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
                                              ],
-                                  'packages'  : ['PIL', 'lxml', 'cherrypy'],
+                                  'packages'  : ['PIL', 'lxml', 'cherrypy',
+                                                 'dateutil'],
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",
                                                  "_imagingtk", "ImageTk", "FixTk"
                                                 ],

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.134'
+__version__   = '0.4.136'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/customize/ui.py

@@ -153,6 +153,7 @@ def set_file_type_metadata(stream, mi, ftype):
                         plugin.set_metadata(stream, mi, ftype.lower().strip())
                         break
                     except:
+                        print 'Failed to set metadata for', repr(getattr(mi, 'title', ''))  
                         traceback.print_exc()
     
                 

src/calibre/devices/cybookg3/driver.py

@@ -17,8 +17,8 @@ class CYBOOKG3(USBMS):
     # Be sure these have an entry in calibre.devices.mime
     FORMATS     = ['mobi', 'prc', 'html', 'pdf', 'rtf', 'txt']
     
-    VENDOR_ID   = 0x0bda
-    PRODUCT_ID  = 0x0703
+    VENDOR_ID   = [0x0bda, 0x3034]
+    PRODUCT_ID  = [0x0703, 0x1795]
     BCD         = [0x110, 0x132]
     
     VENDOR_NAME = 'BOOKEEN'

src/calibre/devices/kindle/driver.py

@@ -12,8 +12,8 @@ class KINDLE(USBMS):
     # Ordered list of supported formats
     FORMATS     = ['azw', 'mobi', 'prc', 'txt']
     
-    VENDOR_ID   = 0x1949
-    PRODUCT_ID  = 0x0001
+    VENDOR_ID   = [0x1949]
+    PRODUCT_ID  = [0x0001]
     BCD         = [0x399]
     
     VENDOR_NAME = 'KINDLE'

src/calibre/devices/prs505/driver.py

@@ -248,6 +248,7 @@ class PRS505(Device):
                 time.sleep(3)
                 self.open_osx()
         if self._card_prefix is not None:
+            try:
                 cachep = os.path.join(self._card_prefix, self.CACHE_XML)
                 if not os.path.exists(cachep):
                     os.makedirs(os.path.dirname(cachep), mode=0777)
@@ -257,6 +258,10 @@ class PRS505(Device):
 </cache>
 '''.encode('utf8'))
                     f.close()
+            except:
+                self._card_prefix = None
+                import traceback
+                traceback.print_exc()
             
     def set_progress_reporter(self, pr):
         self.report_progress = pr

src/calibre/devices/scanner.py

@@ -58,9 +58,12 @@ class DeviceScanner(object):
         return False
     
     def is_device_connected(self, device):
+        vendor_ids  = device.VENDOR_ID if hasattr(device.VENDOR_ID, '__len__') else [device.VENDOR_ID]
+        product_ids =  device.PRODUCT_ID if hasattr(device.PRODUCT_ID, '__len__') else [device.PRODUCT_ID]
         if iswindows:
-            vid, pid = 'vid_%4.4x'%device.VENDOR_ID, 'pid_%4.4x'%device.PRODUCT_ID
-            vidd, pidd = 'vid_%i'%device.VENDOR_ID, 'pid_%i'%device.PRODUCT_ID
+            for vendor_id, product_id in zip(vendor_ids, product_ids):
+                vid, pid = 'vid_%4.4x'%vendor_id, 'pid_%4.4x'%product_id
+                vidd, pidd = 'vid_%i'%vendor_id, 'pid_%i'%product_id
                 for device_id in self.devices:
                     if (vid in device_id or vidd in device_id) and (pid in device_id or pidd in device_id):
                         if self.test_bcd_windows(device_id, getattr(device, 'BCD', None)):
@@ -68,7 +71,7 @@ class DeviceScanner(object):
                                 return True
         else:
             for vendor, product, bcdDevice in self.devices:
-                if device.VENDOR_ID == vendor and device.PRODUCT_ID == product:
+                if vendor in vendor_ids and product in product_ids:
                     if self.test_bcd(bcdDevice, getattr(device, 'BCD', None)):
                         if device.can_handle((vendor, product, bcdDevice)):
                             return True

src/calibre/devices/usbms/device.py

@@ -74,18 +74,21 @@ class Device(_Device):
     def get_fdi(cls):
         fdi = ''
 
+        for vid in cls.VENDOR_ID:
+            for pid in cls.PRODUCT_ID:
                 fdi_base_values = dict(
                                        app=__appname__,
                                        deviceclass=cls.__name__,
-                               vendor_id=hex(cls.VENDOR_ID),
-                               product_id=hex(cls.PRODUCT_ID),
+                                       vendor_id=hex(vid),
+                                       product_id=hex(pid),
                                        main_memory=cls.MAIN_MEMORY_VOLUME_LABEL,
                                        storage_card=cls.STORAGE_CARD_VOLUME_LABEL,
                                   )
+
                 if cls.BCD is None:
                     fdi_base_values['BCD_start'] = ''
                     fdi_base_values['BCD_end'] = ''
-            fdi = cls.FDI_TEMPLATE % fdi_base_values
+                    fdi += cls.FDI_TEMPLATE % fdi_base_values
                 else:
                     for bcd in cls.BCD:
                         fdi_bcd_values = fdi_base_values

src/calibre/ebooks/epub/from_html.py

@@ -74,7 +74,9 @@ def check_links(opf_path, pretty_print):
         html_files = []
         for item in opf.itermanifest():
             if 'html' in item.get('media-type', '').lower():
-                f = item.get('href').split('/')[-1].decode('utf-8')
+                f = item.get('href').split('/')[-1]
+                if isinstance(f, str):
+                    f = f.decode('utf-8')
                 html_files.append(os.path.abspath(content(f)))
         
         for path in html_files:

src/calibre/ebooks/html.py

@@ -330,7 +330,8 @@ class PreProcessor(object):
                    sanitize_head),
                   # Convert all entities, since lxml doesn't handle them well
                   (re.compile(r'&(\S+?);'), convert_entities),
-                  
+                  # Remove the <![if/endif tags inserted by everybody's darling, MS Word
+                  (re.compile(r'(?i)<{0,1}!\[(end){0,1}if[^>]*>'), lambda match: ''),
                   ]
                      
     # Fix pdftohtml markup

src/calibre/ebooks/lrf/html/convert_from.py

@@ -917,7 +917,8 @@ class HTMLConverter(object, LoggingInterface):
                                                          blockStyle=self.current_block.blockStyle)
         
     
-    def process_image(self, path, tag_css, width=None, height=None, dropcaps=False):
+    def process_image(self, path, tag_css, width=None, height=None, 
+                      dropcaps=False, rescale=False):
         def detect_encoding(im):
             fmt = im.format
             if fmt == 'JPG':
@@ -936,10 +937,6 @@ class HTMLConverter(object, LoggingInterface):
             return
         encoding = detect_encoding(im)
 
-        if width == None or height == None:            
-            width, height = im.size
-        
-        factor = 720./self.profile.dpi
         
         def scale_image(width, height):
             if width <= 0:
@@ -955,8 +952,15 @@ class HTMLConverter(object, LoggingInterface):
                 return pt.name
             except (IOError, SystemError), err: # PIL chokes on interlaced PNG images as well a some GIF images
                 self.log_warning(_('Unable to process image %s. Error: %s')%(path, err))
-                return None
         
+        if width == None or height == None:            
+            width, height = im.size
+        elif rescale and (width < im.size[0] or height < im.size[1]):
+            path = scale_image(width, height)
+            if not path:
+                return
+            
+        factor = 720./self.profile.dpi
         pheight = int(self.current_page.pageStyle.attrs['textheight'])
         pwidth  = int(self.current_page.pageStyle.attrs['textwidth'])
         
@@ -1518,7 +1522,8 @@ class HTMLConverter(object, LoggingInterface):
                         except:
                             pass
                         dropcaps = tag.has_key('class') and tag['class'] == 'libprs500_dropcaps'
-                        self.process_image(path, tag_css, width, height, dropcaps=dropcaps)
+                        self.process_image(path, tag_css, width, height, 
+                                           dropcaps=dropcaps, rescale=True)
                     elif not urlparse(tag['src'])[0]:
                         self.log_warn('Could not find image: '+tag['src'])
                 else:

src/calibre/ebooks/metadata/__init__.py

@@ -188,7 +188,8 @@ class MetaInformation(object):
         for attr in ('author_sort', 'title_sort', 'comments', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'tags', 'cover_data', 'application_id', 'guide',
-                     'manifest', 'spine', 'toc', 'cover', 'language', 'book_producer'):
+                     'manifest', 'spine', 'toc', 'cover', 'language',
+                     'book_producer', 'timestamp'):
             if hasattr(mi, attr):
                 setattr(ans, attr, getattr(mi, attr))
 
@@ -213,7 +214,7 @@ class MetaInformation(object):
         for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
                   'series', 'series_index', 'rating', 'isbn', 'language',
                   'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer',
+                  'book_producer', 'timestamp'
                   ):
             setattr(self, x, getattr(mi, x, None))
 
@@ -231,7 +232,8 @@ class MetaInformation(object):
         for attr in ('author_sort', 'title_sort', 'comments', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'application_id', 'manifest', 'spine', 'toc',
-                     'cover', 'language', 'guide', 'book_producer'):
+                     'cover', 'language', 'guide', 'book_producer',
+                     'timestamp'):
             val = getattr(mi, attr, None)
             if val is not None:
                 setattr(self, attr, val)
@@ -279,6 +281,8 @@ class MetaInformation(object):
             fmt('Language', self.language)
         if self.rating is not None:
             fmt('Rating', self.rating)
+        if self.timestamp is not None:
+            fmt('Timestamp', self.timestamp.isoformat(' '))
         return u'\n'.join(ans)
 
     def to_html(self):
@@ -290,14 +294,14 @@ class MetaInformation(object):
         ans += [('ISBN', unicode(self.isbn))]
         ans += [(_('Tags'), u', '.join([unicode(t) for t in self.tags]))]
         if self.series:
-            ans += [(_('Series'), unicode(self.series))+ ' #%s'%self.format_series_index()]
+            ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
         ans += [(_('Language'), unicode(self.language))]
+        if self.timestamp is not None:
+            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
         for i, x in enumerate(ans):
             ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
         return u'<table>%s</table>'%u'\n'.join(ans)
 
-
-
     def __str__(self):
         return self.__unicode__().encode('utf-8')
 

src/calibre/ebooks/metadata/library_thing.py

@@ -38,8 +38,14 @@ def cover_from_isbn(isbn, timeout=5.):
         browser = _browser()
     _timeout = socket.getdefaulttimeout()
     socket.setdefaulttimeout(timeout)
+    src = None                
     try:
         src = browser.open('http://www.librarything.com/isbn/'+isbn).read().decode('utf-8', 'replace')
+    except Exception, err:
+        if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
+            err = LibraryThingError(_('LibraryThing.com timed out. Try again later.'))
+        raise err
+    else:
         s = BeautifulSoup(src)
         url = s.find('td', attrs={'class':'left'})
         if url is None:

src/calibre/ebooks/metadata/meta.py

@@ -31,8 +31,14 @@ def metadata_from_formats(formats):
     mi = MetaInformation(None, None)
     formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)],
                                      METADATA_PRIORITIES[path_to_ext(y)]))
-    for path in formats:
-        ext = path_to_ext(path)
+    extensions = list(map(path_to_ext, formats))
+    if 'opf' in extensions:
+        opf = formats[extensions.index('opf')]
+        mi2 = opf_metadata(opf)
+        if mi2 is not None and mi2.title:
+            return mi2
+    
+    for path, ext in zip(formats, extensions):
         stream = open(path, 'rb')
         try:
             mi.smart_update(get_metadata(stream, stream_type=ext, use_libprs_metadata=True))

src/calibre/ebooks/metadata/mobi.py

@@ -85,9 +85,9 @@ class MetadataUpdater(object):
         image_base, = unpack('>I', record0[108:112])
         flags, = unpack('>I', record0[128:132])
         have_exth = self.have_exth = (flags & 0x40) != 0
+        self.cover_record = self.thumbnail_record = None
         if not have_exth:
             return
-        self.cover_record = self.thumbnail_record = None
         exth_off = unpack('>I', record0[20:24])[0] + 16 + record0.start
         exth = self.exth = StreamSlicer(stream, exth_off, record0.stop)
         nitems, = unpack('>I', exth[8:12])
@@ -142,6 +142,8 @@ class MetadataUpdater(object):
         exth = ['EXTH', pack('>II', len(exth) + 12, len(recs)), exth, pad]
         exth = ''.join(exth)
         title = (mi.title or _('Unknown')).encode(self.codec, 'replace')
+        if getattr(self, 'exth', None) is None:
+            raise MobiError('No existing EXTH record. Cannot update metadata.')
         title_off = (self.exth.start - self.record0.start) + len(exth)
         title_len = len(title)
         trail = len(self.exth) - len(exth) - len(title)
@@ -150,8 +152,12 @@ class MetadataUpdater(object):
         self.exth[:] = ''.join([exth, title, '\0' * trail])
         self.record0[84:92] = pack('>II', title_off, title_len)
         self.record0[92:96] = iana2mobi(mi.language)
-        if mi.cover_data[1]:
-            data =  mi.cover_data[1]
+        if mi.cover_data[1] or mi.cover:
+            try:
+                data =  mi.cover_data[1] if mi.cover_data[1] else open(mi.cover, 'rb').read()
+            except:
+                pass
+            else:
                 if self.cover_record is not None:
                     size = len(self.cover_record)
                     cover = rescale_image(data, size)

src/calibre/ebooks/metadata/opf.xml

@@ -10,7 +10,7 @@
         <dc:creator opf:role="aut" py:for="i, author in enumerate(mi.authors)" py:attrs="{'opf:file-as':mi.author_sort} if mi.author_sort and i == 0 else {}">${author}</dc:creator>
         <dc:contributor opf:role="bkp" py:with="attrs={'opf:file-as':__appname__}" py:attrs="attrs">${'%s (%s)'%(__appname__, __version__)} [http://${__appname__}.kovidgoyal.net]</dc:contributor> 
         <dc:identifier opf:scheme="${__appname__}" id="${__appname__}_id">${mi.application_id}</dc:identifier>
-    
+        <dc:date py:if="getattr(mi, 'timestamp', None) is not None">${mi.timestamp.isoformat()}</dc:date>    
         <dc:language>${mi.language if mi.language else 'UND'}</dc:language>
         <dc:type py:if="getattr(mi, 'category', False)">${mi.category}</dc:type>
         <dc:description py:if="mi.comments">${mi.comments}</dc:description>

src/calibre/ebooks/metadata/opf2.py

@@ -12,6 +12,7 @@ from urllib import unquote
 from urlparse import urlparse
 
 from lxml import etree
+from dateutil import parser
 
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre import relpath
@@ -436,6 +437,7 @@ class OPF(object):
     series          = MetadataField('series', is_dc=False)
     series_index    = MetadataField('series_index', is_dc=False, formatter=int, none_is=1)
     rating          = MetadataField('rating', is_dc=False, formatter=int)
+    timestamp       = MetadataField('date', formatter=parser.parse)
 
 
     def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True):
@@ -618,7 +620,7 @@ class OPF(object):
         def fset(self, val):
             remove = list(self.authors_path(self.metadata))
             for elem in remove:
-                self.metadata.remove(elem)
+                elem.getparent().remove(elem)
             for author in val:
                 attrib = {'{%s}role'%self.NAMESPACES['opf']: 'aut'}
                 elem = self.create_metadata_element('creator', attrib=attrib)

src/calibre/ebooks/mobi/langcodes.py

@@ -306,13 +306,15 @@ IANA_MOBI = \
      'zu': {None: (53, 0)}}
 
 def iana2mobi(icode):
+    langdict, subtags = IANA_MOBI[None], []
+    if icode:
         subtags = list(icode.split('-'))
-    langdict = IANA_MOBI[None]
         while len(subtags) > 0:
             lang = subtags.pop(0).lower()
             if lang in IANA_MOBI:
                 langdict = IANA_MOBI[lang]
                 break
+            
     mcode = langdict[None]
     while len(subtags) > 0:
         subtag = subtags.pop(0)

src/calibre/ebooks/mobi/reader.py

@@ -308,8 +308,11 @@ class MobiReader(object):
             if 'filepos-id' in attrib:
                 attrib['id'] = attrib.pop('filepos-id')
             if 'filepos' in attrib:
-                filepos = int(attrib.pop('filepos'))
-                attrib['href'] = "#filepos%d" % filepos
+                filepos = attrib.pop('filepos')
+                try:
+                    attrib['href'] = "#filepos%d" % int(filepos)
+                except:
+                    attrib['href'] = filepos
             if tag.tag == 'img':
                 recindex = None
                 for attr in self.IMAGE_ATTRS:

src/calibre/ebooks/oeb/stylizer.py

@@ -17,6 +17,7 @@ import types
 import re
 import copy
 from itertools import izip
+from weakref import WeakKeyDictionary
 from xml.dom import SyntaxErr as CSSSyntaxError
 import cssutils
 from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \
@@ -107,7 +108,7 @@ class CSSSelector(etree.XPath):
 
 
 class Stylizer(object):    
-    STYLESHEETS = {}
+    STYLESHEETS = WeakKeyDictionary()
     
     def __init__(self, tree, path, oeb, profile=PROFILES['PRS505']):
         self.oeb = oeb
@@ -132,18 +133,19 @@ class Stylizer(object):
                  and elem.get('type', CSS_MIME) in OEB_STYLES:
                 href = urlnormalize(elem.attrib['href'])
                 path = item.abshref(href)
-                if path not in oeb.manifest.hrefs:
+                sitem = oeb.manifest.hrefs.get(path, None)
+                if sitem is None:
                     self.logger.warn(
                         'Stylesheet %r referenced by file %r not in manifest' %
                         (path, item.href))
                     continue
-                if path in self.STYLESHEETS:
-                    stylesheet = self.STYLESHEETS[path]
+                if sitem in self.STYLESHEETS:
+                    stylesheet = self.STYLESHEETS[sitem]
                 else:
                     data = self._fetch_css_file(path)[1]
                     stylesheet = parser.parseString(data, href=path)
                     stylesheet.namespaces['h'] = XHTML_NS
-                    self.STYLESHEETS[path] = stylesheet
+                    self.STYLESHEETS[sitem] = stylesheet
                 stylesheets.append(stylesheet)
         rules = []
         index = 0

src/calibre/gui2/__init__.py

@@ -1,7 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 """ The GUI """
-import sys, os, re, StringIO, traceback
+import sys, os, re, StringIO, traceback, time
 from PyQt4.QtCore import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \
                          QByteArray, QLocale, QUrl, QTranslator, QCoreApplication, \
                          QModelIndex
@@ -14,6 +14,9 @@ from calibre import __author__, islinux, iswindows, isosx
 from calibre.startup import get_lang
 from calibre.utils.config import Config, ConfigProxy, dynamic
 import calibre.resources as resources
+from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
+from calibre.ebooks.metadata import MetaInformation
+
 
 NONE = QVariant() #: Null value to return from the data function of item models
 
@@ -149,6 +152,40 @@ class Dispatcher(QObject):
     def dispatch(self, args, kwargs):
         self.func(*args, **kwargs)
 
+class GetMetadata(QObject):
+    '''
+    Convenience class to ensure that metadata readers are used only in the
+    GUI thread. Must be instantiated in the GUI thread.
+    '''
+    
+    def __init__(self):
+        QObject.__init__(self)
+        self.connect(self, SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                     self._get_metadata, Qt.QueuedConnection)
+        self.connect(self, SIGNAL('idispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                     self._from_formats, Qt.QueuedConnection)
+        
+    def __call__(self, id, *args, **kwargs):
+        self.emit(SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                  id, args, kwargs)
+    
+    def from_formats(self, id, *args, **kwargs):
+        self.emit(SIGNAL('idispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                  id, args, kwargs)
+    
+    def _from_formats(self, id, args, kwargs):
+        try:
+            mi = metadata_from_formats(*args, **kwargs)
+        except:
+            mi = MetaInformation('', [_('Unknown')])
+        self.emit(SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), id, mi)
+    
+    def _get_metadata(self, id, args, kwargs):
+        try:
+            mi = get_metadata(*args, **kwargs)
+        except:
+            mi = MetaInformation('', [_('Unknown')])
+        self.emit(SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'), id, mi)
 
 class TableView(QTableView):
     def __init__(self, parent):

src/calibre/gui2/add.py

@@ -0,0 +1,224 @@
+'''
+UI for adding books to the database
+'''
+import os
+
+from PyQt4.Qt import QThread, SIGNAL, QMutex, QWaitCondition, Qt
+
+from calibre.gui2.dialogs.progress import ProgressDialog
+from calibre.constants import preferred_encoding
+from calibre.gui2.widgets import WarningDialog
+
+class Add(QThread):
+    
+    def __init__(self):
+        QThread.__init__(self)
+        self._lock = QMutex()
+        self._waiting = QWaitCondition()
+    
+    def is_canceled(self):
+        if self.pd.canceled:
+            self.canceled = True
+        return self.canceled
+    
+    def wait_for_condition(self):
+        self._lock.lock()
+        self._waiting.wait(self._lock)
+        self._lock.unlock()
+        
+    def wake_up(self):
+        self._waiting.wakeAll()
+ 
+class AddFiles(Add):
+    
+    def __init__(self, paths, default_thumbnail, get_metadata, db=None):
+        Add.__init__(self)
+        self.paths = paths
+        self.get_metadata = get_metadata
+        self.default_thumbnail = default_thumbnail
+        self.db = db
+        self.formats, self.metadata, self.names, self.infos = [], [], [], []
+        self.duplicates = []
+        self.number_of_books_added = 0
+        self.connect(self.get_metadata, 
+                     SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'),
+                     self.metadata_delivered)
+    
+    def metadata_delivered(self, id, mi):
+        if self.is_canceled():
+            self.reading.wakeAll()
+            return
+        if not mi.title:
+            mi.title = os.path.splitext(self.names[id])[0]
+        mi.title = mi.title if isinstance(mi.title, unicode) else \
+                   mi.title.decode(preferred_encoding, 'replace')
+        self.metadata.append(mi)
+        self.infos.append({'title':mi.title, 
+                           'authors':', '.join(mi.authors),
+                           'cover':self.default_thumbnail, 'tags':[]})
+        if self.db is not None:
+            duplicates, num = self.db.add_books(self.paths[id:id+1], 
+                                self.formats[id:id+1], [mi], 
+                                add_duplicates=False)
+            self.number_of_books_added += num
+            if duplicates:
+                if not self.duplicates:
+                    self.duplicates = [[], [], [], []]
+                for i in range(4):
+                    self.duplicates[i] += duplicates[i]
+        self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), 
+                      mi.title, id)
+        self.wake_up()
+    
+    def create_progress_dialog(self, title, msg, parent):
+        self._parent = parent
+        self.pd = ProgressDialog(title, msg, -1, len(self.paths)-1, parent)
+        self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                     self.update_progress_dialog)
+        self.pd.setModal(True)
+        self.pd.show()
+        self.connect(self, SIGNAL('finished()'), self.pd.hide)
+        return self.pd
+        
+       
+    def update_progress_dialog(self, title, count):
+        self.pd.set_value(count)
+        if self.db is not None:
+            self.pd.set_msg(_('Added %s to library')%title)
+        else:
+            self.pd.set_msg(_('Read metadata from ')+title)
+        
+        
+    def run(self):
+        self.canceled = False
+        for c, book in enumerate(self.paths):
+            if self.pd.canceled:
+                self.canceled = True
+                break
+            format = os.path.splitext(book)[1]
+            format = format[1:] if format else None
+            stream = open(book, 'rb')
+            self.formats.append(format)
+            self.names.append(os.path.basename(book))
+            self.get_metadata(c, stream, stream_type=format, 
+                                  use_libprs_metadata=True)
+            self.wait_for_condition()
+        
+                    
+    def process_duplicates(self):
+        if self.duplicates:
+            files = _('<p>Books with the same title as the following already '
+                      'exist in the database. Add them anyway?<ul>')
+            for mi in self.duplicates[2]:
+                files += '<li>'+mi.title+'</li>\n'
+            d = WarningDialog (_('Duplicates found!'), 
+                              _('Duplicates found!'), 
+                              files+'</ul></p>', parent=self._parent)
+            if d.exec_() == d.Accepted:
+                num = self.db.add_books(*self.duplicates, 
+                                  **dict(add_duplicates=True))[1]
+                self.number_of_books_added += num
+                
+            
+class AddRecursive(Add):
+    
+    def __init__(self, path, db, get_metadata, single_book_per_directory, parent):
+        self.path = path
+        self.db = db
+        self.get_metadata = get_metadata
+        self.single_book_per_directory = single_book_per_directory
+        self.duplicates, self.books, self.metadata = [], [], []
+        self.number_of_books_added = 0
+        self.canceled = False
+        Add.__init__(self)
+        self.connect(self.get_metadata, 
+                     SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'),
+                     self.metadata_delivered, Qt.QueuedConnection)
+        self.connect(self, SIGNAL('searching_done()'), self.searching_done,
+                     Qt.QueuedConnection)    
+        self._parent = parent
+        self.pd = ProgressDialog(_('Adding books recursively...'),
+                                 _('Searching for books in all sub-directories...'), 
+                                 0, 0, parent)
+        self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                     self.update_progress_dialog)
+        self.connect(self, SIGNAL('update(PyQt_PyObject)'), self.pd.set_msg,
+                     Qt.QueuedConnection)
+        self.connect(self, SIGNAL('pupdate(PyQt_PyObject)'), self.pd.set_value,
+                     Qt.QueuedConnection)
+        self.pd.setModal(True)
+        self.pd.show()
+        self.connect(self, SIGNAL('finished()'), self.pd.hide)
+        
+    def update_progress_dialog(self, title, count):
+        self.pd.set_value(count)
+        if title:
+            self.pd.set_msg(_('Read metadata from ')+title)
+    
+    def metadata_delivered(self, id, mi):
+        if self.is_canceled():
+            self.reading.wakeAll()
+            return
+        self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                          mi.title, id)
+        self.metadata.append((mi if mi.title else None, self.books[id]))
+        if len(self.metadata) >= len(self.books):
+            self.metadata = [x for x in self.metadata if x[0] is not None]
+            self.pd.set_min(-1)
+            self.pd.set_max(len(self.metadata)-1)
+            self.pd.set_value(-1)
+            self.pd.set_msg(_('Adding books to database...'))
+        self.wake_up()
+    
+    def searching_done(self):
+        self.pd.set_min(-1)
+        self.pd.set_max(len(self.books)-1)
+        self.pd.set_value(-1)
+        self.pd.set_msg(_('Reading metadata...'))
+    
+   
+    def run(self):
+        root = os.path.abspath(self.path)
+        for dirpath in os.walk(root):
+            if self.is_canceled():
+                return
+            self.emit(SIGNAL('update(PyQt_PyObject)'), 
+                      _('Searching in')+' '+dirpath[0])
+            self.books += list(self.db.find_books_in_directory(dirpath[0], 
+                                            self.single_book_per_directory))
+        self.books = [formats for formats in self.books if formats]
+        # Reset progress bar
+        self.emit(SIGNAL('searching_done()'))
+        
+        for c, formats in enumerate(self.books):
+            self.get_metadata.from_formats(c, formats)
+            self.wait_for_condition()
+            
+        # Add books to database
+        for c, x in enumerate(self.metadata):
+            mi, formats = x
+            if self.is_canceled():
+                break
+            if self.db.has_book(mi):
+                self.duplicates.append((mi, formats))
+            else:
+                self.db.import_book(mi, formats, notify=False)
+                self.number_of_books_added += 1
+            self.emit(SIGNAL('pupdate(PyQt_PyObject)'), c)
+ 
+        
+    def process_duplicates(self):
+        if self.duplicates:
+            files = _('<p>Books with the same title as the following already '
+                      'exist in the database. Add them anyway?<ul>')
+            for mi in self.duplicates:
+                files += '<li>'+mi[0].title+'</li>\n'
+            d = WarningDialog (_('Duplicates found!'), 
+                              _('Duplicates found!'), 
+                              files+'</ul></p>', parent=self._parent)
+            if d.exec_() == d.Accepted:
+                for mi, formats in self.duplicates:
+                    self.db.import_book(mi, formats, notify=False)
+                    self.number_of_books_added += 1
+        
+        

src/calibre/gui2/dialogs/confirm_delete.py

@@ -5,7 +5,7 @@ __docformat__ = 'restructuredtext en'
 
 from calibre.gui2 import dynamic
 from calibre.gui2.dialogs.confirm_delete_ui import Ui_Dialog
-from PyQt4.Qt import QDialog, SIGNAL
+from PyQt4.Qt import QDialog, SIGNAL, Qt
 
 def _config_name(name):
     return name + '_again'
@@ -19,6 +19,7 @@ class Dialog(QDialog, Ui_Dialog):
         self.msg.setText(msg)
         self.name = name
         self.connect(self.again, SIGNAL('stateChanged(int)'), self.toggle)
+        self.buttonBox.setFocus(Qt.OtherFocusReason)
         
 
     def toggle(self, x):

src/calibre/gui2/dialogs/epub.ui

@@ -105,36 +105,6 @@
                 <string>Book Cover</string>
                </property>
                <layout class="QGridLayout" name="_2" >
-                <item row="0" column="0" >
-                 <layout class="QHBoxLayout" name="_3" >
-                  <item>
-                   <widget class="ImageView" name="cover" >
-                    <property name="text" >
-                     <string/>
-                    </property>
-                    <property name="pixmap" >
-                     <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
-                    </property>
-                    <property name="scaledContents" >
-                     <bool>true</bool>
-                    </property>
-                    <property name="alignment" >
-                     <set>Qt::AlignCenter</set>
-                    </property>
-                   </widget>
-                  </item>
-                 </layout>
-                </item>
-                <item row="2" column="0" >
-                 <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
-                  <property name="text" >
-                   <string>Use cover from &amp;source file</string>
-                  </property>
-                  <property name="checked" >
-                   <bool>true</bool>
-                  </property>
-                 </widget>
-                </item>
                 <item row="1" column="0" >
                  <layout class="QVBoxLayout" name="_4" >
                   <property name="spacing" >
@@ -186,6 +156,36 @@
                   </item>
                  </layout>
                 </item>
+                <item row="2" column="0" >
+                 <widget class="QCheckBox" name="opt_prefer_metadata_cover" >
+                  <property name="text" >
+                   <string>Use cover from &amp;source file</string>
+                  </property>
+                  <property name="checked" >
+                   <bool>true</bool>
+                  </property>
+                 </widget>
+                </item>
+                <item row="0" column="0" >
+                 <layout class="QHBoxLayout" name="_3" >
+                  <item>
+                   <widget class="ImageView" name="cover" >
+                    <property name="text" >
+                     <string/>
+                    </property>
+                    <property name="pixmap" >
+                     <pixmap resource="../images.qrc" >:/images/book.svg</pixmap>
+                    </property>
+                    <property name="scaledContents" >
+                     <bool>true</bool>
+                    </property>
+                    <property name="alignment" >
+                     <set>Qt::AlignCenter</set>
+                    </property>
+                   </widget>
+                  </item>
+                 </layout>
+                </item>
                </layout>
                <zorder>opt_prefer_metadata_cover</zorder>
                <zorder></zorder>
@@ -507,6 +507,13 @@
                </property>
               </widget>
              </item>
+             <item row="9" column="0" >
+              <widget class="QCheckBox" name="opt_remove_first_image" >
+               <property name="text" >
+                <string>Remove &amp;first image from source file</string>
+               </property>
+              </widget>
+             </item>
             </layout>
            </item>
            <item>

src/calibre/gui2/dialogs/progress.py

@@ -20,6 +20,7 @@ class ProgressDialog(QDialog, Ui_Dialog):
         self.setWindowModality(Qt.ApplicationModal)
         self.set_min(min)
         self.set_max(max)
+        self.bar.setValue(min)
         self.canceled = False
         
         self.connect(self.button_box, SIGNAL('rejected()'), self._canceled)

src/calibre/gui2/library.py

@@ -1,3 +1,4 @@
+from calibre.ebooks.metadata import authors_to_string
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import os, textwrap, traceback, time, re
@@ -18,6 +19,7 @@ from calibre.library.database2 import FIELD_MAP
 from calibre.gui2 import NONE, TableView, qstring_to_unicode, config, \
                          error_dialog
 from calibre.utils.search_query_parser import SearchQueryParser
+from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 
 class LibraryDelegate(QItemDelegate):
     COLOR    = QColor("blue")
@@ -370,35 +372,24 @@ class BooksModel(QAbstractTableModel):
         if not rows_are_ids:
             rows = [self.db.id(row.row()) for row in rows]
         for id in rows:
-            au = self.db.authors(id, index_is_id=True)
-            tags = self.db.tags(id, index_is_id=True)
-            if not au:
-                au = _('Unknown')
-            au = au.split(',')
-            if len(au) > 1:
-                t = ', '.join(au[:-1])
-                t += ' & ' + au[-1]
-                au = t
-            else:
-                au = ' & '.join(au)
-            if not tags:
-                tags = []
-            else:
-                tags = tags.split(',')
-            series = self.db.series(id, index_is_id=True)
-            if series is not None:
-                tags.append(series)
-            mi = {
-                  'title'   : self.db.title(id, index_is_id=True),
+            mi = self.db.get_metadata(id, index_is_id=True)
+            au = authors_to_string(mi.authors if mi.authors else [_('Unknown')])
+            tags = mi.tags if mi.tags else []
+            if mi.series is not None:
+                tags.append(mi.series)
+            info = {
+                  'title'   : mi.title,
                   'authors' : au,
                   'cover'   : self.db.cover(id, index_is_id=True),
                   'tags'    : tags,
-                  'comments': self.db.comments(id, index_is_id=True),
+                  'comments': mi.comments,
+                  }
+            if mi.series is not None:
+                info['tag order'] = {
+                    mi.series:self.db.books_in_series_of(id, index_is_id=True)
                 }
-            if series is not None:
-                mi['tag order'] = {series:self.db.books_in_series_of(id, index_is_id=True)}
 
-            metadata.append(mi)
+            metadata.append(info)
         return metadata
 
     def get_preferred_formats_from_ids(self, ids, all_formats, mode='r+b'):
@@ -423,7 +414,7 @@ class BooksModel(QAbstractTableModel):
                      
             
     
-    def get_preferred_formats(self, rows, formats, paths=False):
+    def get_preferred_formats(self, rows, formats, paths=False, set_metadata=False):
         ans = []
         for row in (row.row() for row in rows):
             format = None
@@ -441,6 +432,9 @@ class BooksModel(QAbstractTableModel):
                 pt = PersistentTemporaryFile(suffix='.'+format)
                 pt.write(self.db.format(row, format))
                 pt.flush()
+                if set_metadata:
+                    _set_metadata(pt, self.db.get_metadata(row, get_cover=True),
+                                  format)
                 pt.close() if paths else pt.seek(0)
                 ans.append(pt)
             else:

src/calibre/gui2/main.py

@@ -13,7 +13,6 @@ from PyQt4.QtSvg import QSvgRenderer
 from calibre import __version__, __appname__, islinux, sanitize_file_name, \
                     iswindows, isosx, preferred_encoding
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.ebooks.metadata.meta import get_metadata
 from calibre.devices.errors import FreeSpaceError
 from calibre.devices.interface import Device
 from calibre.utils.config import prefs, dynamic
@@ -23,7 +22,7 @@ from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            set_sidebar_directories, Dispatcher, \
                            SingleApplication, Application, available_height, \
                            max_available_height, config, info_dialog, \
-                           available_width
+                           available_width, GetMetadata
 from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
 from calibre.gui2.dialogs.scheduler import Scheduler
 from calibre.gui2.update import CheckForUpdates
@@ -49,7 +48,6 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.library.database2 import LibraryDatabase2, CoverCache
 from calibre.parallel import JobKilled
 from calibre.utils.filenames import ascii_filename
-from calibre.gui2.widgets import WarningDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 
 class Main(MainWindow, Ui_MainWindow):
@@ -78,6 +76,7 @@ class Main(MainWindow, Ui_MainWindow):
         self.setupUi(self)
         self.setWindowTitle(__appname__)
         self.verbose = opts.verbose
+        self.get_metadata = GetMetadata()
         self.read_settings()
         self.job_manager = JobManager()
         self.jobs_dialog = JobsDialog(self, self.job_manager)
@@ -391,7 +390,7 @@ class Main(MainWindow, Ui_MainWindow):
     def change_output_format(self, x):
         of = unicode(x).strip()
         if of != prefs['output_format']:
-            if of not in ('LRF',):
+            if of not in ('LRF', 'EPUB'):
                 warning_dialog(self, 'Warning', 
                                '<p>%s support is still in beta. If you find bugs, please report them by opening a <a href="http://calibre.kovidgoyal.net">ticket</a>.'%of).exec_()
             prefs.set('output_format', of)
@@ -608,35 +607,25 @@ class Main(MainWindow, Ui_MainWindow):
     ################################# Add books ################################
 
     def add_recursive(self, single):
-        root = choose_dir(self, 'recursive book import root dir dialog', 'Select root folder')
+        root = choose_dir(self, 'recursive book import root dir dialog', 
+                          'Select root folder')
         if not root:
             return
-        progress = ProgressDialog(_('Adding books recursively...'),
-                                   min=0, max=0, parent=self)
-        progress.show()
-        def callback(msg):
-            if msg != '.':
-                progress.set_msg((_('Added ')+msg) if msg else _('Searching...'))
-            QApplication.processEvents()
-            QApplication.sendPostedEvents()
-            QApplication.flush()
-            return progress.canceled
-        try:
-            duplicates = self.library_view.model().db.recursive_import(root, single, callback=callback)
-        finally:
-            progress.hide()            
-        if duplicates:
-            files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
-            for mi, formats in duplicates:
-                files += '<li>'+mi.title+'</li>\n'
-            d = WarningDialog(_('Duplicates found!'), _('Duplicates found!'), 
-                              files+'</ul></p>', self)
-            if d.exec_() == QDialog.Accepted:
-                for mi, formats in duplicates:
-                    self.library_view.model().db.import_book(mi, formats )
+        from calibre.gui2.add import AddRecursive
+        self._add_recursive_thread = AddRecursive(root, 
+                                self.library_view.model().db, self.get_metadata,
+                                single, self)
+        self.connect(self._add_recursive_thread, SIGNAL('finished()'),
+                     self._recursive_files_added)
+        self._add_recursive_thread.start()
     
-        self.library_view.model().resort()
+    def _recursive_files_added(self):
+        self._add_recursive_thread.process_duplicates()
+        if self._add_recursive_thread.number_of_books_added > 0:
+            self.library_view.model().resort(reset=False)
             self.library_view.model().research()
+            self.library_view.model().count_changed()
+        self._add_recursive_thread = None
         
     def add_recursive_single(self, checked):
         '''
@@ -686,62 +675,40 @@ class Main(MainWindow, Ui_MainWindow):
             return
         to_device = self.stack.currentIndex() != 0
         self._add_books(books, to_device)
-        if to_device:
-            self.status_bar.showMessage(_('Uploading books to device.'), 2000)
+        
 
     def _add_books(self, paths, to_device, on_card=None):
         if on_card is None:
             on_card = self.stack.currentIndex() == 2
         if not paths:
             return
-        # Get format and metadata information
-        formats, metadata, names, infos = [], [], [], []
-        progress = ProgressDialog(_('Adding books...'), _('Reading metadata...'),
-                                  min=0, max=len(paths), parent=self)
-        progress.show()
-        try:
-            for c, book in enumerate(paths):
-                progress.set_value(c)
-                if progress.canceled:
-                    return
-                format = os.path.splitext(book)[1]
-                format = format[1:] if format else None
-                stream = open(book, 'rb')
-                try:
-                    mi = get_metadata(stream, stream_type=format, use_libprs_metadata=True)
-                except:
-                    mi = MetaInformation(None, None)
-                if not mi.title:
-                    mi.title = os.path.splitext(os.path.basename(book))[0]
-                if not mi.authors:
-                    mi.authors = [_('Unknown')]
-                formats.append(format)
-                metadata.append(mi)
-                names.append(os.path.basename(book))
-                infos.append({'title':mi.title, 'authors':', '.join(mi.authors),
-                              'cover':self.default_thumbnail, 'tags':[]})
-                title = mi.title if isinstance(mi.title, unicode) else mi.title.decode(preferred_encoding, 'replace')
-                progress.set_msg(_('Read metadata from ')+title)
+        from calibre.gui2.add import AddFiles
+        self._add_files_thread = AddFiles(paths, self.default_thumbnail, 
+                                          self.get_metadata,
+                                          None if to_device else \
+                                          self.library_view.model().db
+                                          )
+        self._add_files_thread.send_to_device = to_device
+        self._add_files_thread.on_card = on_card
+        self._add_files_thread.create_progress_dialog(_('Adding books...'),
+                                                _('Reading metadata...'), self)
+        self.connect(self._add_files_thread, SIGNAL('finished()'),
+                     self._files_added)
+        self._add_files_thread.start()
     
-            if not to_device:
-                progress.set_msg(_('Adding books to database...'))
-                model = self.library_view.model()
-                
-                paths = list(paths)
-                duplicates, number_added = model.add_books(paths, formats, metadata)
-                if duplicates:
-                    files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
-                    for mi in duplicates[2]:
-                        files += '<li>'+mi.title+'</li>\n'
-                    d = WarningDialog(_('Duplicates found!'), _('Duplicates found!'), files+'</ul></p>', parent=self)
-                    if d.exec_() == QDialog.Accepted:
-                        num = model.add_books(*duplicates, **dict(add_duplicates=True))[1]
-                        number_added += num
-                model.books_added(number_added)
+    def _files_added(self):
+        t = self._add_files_thread
+        self._add_files_thread = None
+        if not t.canceled:
+            if t.send_to_device:
+                self.upload_books(t.paths, 
+                                  list(map(sanitize_file_name, t.names)), 
+                                  t.infos, on_card=t.on_card)
+                self.status_bar.showMessage(_('Uploading books to device.'), 2000)
             else:
-                self.upload_books(paths, list(map(sanitize_file_name, names)), infos, on_card=on_card)
-        finally:
-            progress.hide()
+                t.process_duplicates()
+        if t.number_of_books_added > 0:
+            self.library_view.model().books_added(t.number_of_books_added)
         
     def upload_books(self, files, names, metadata, on_card=False, memory=None):
         '''
@@ -798,7 +765,10 @@ class Main(MainWindow, Ui_MainWindow):
         if not rows or len(rows) == 0:
             return
         if self.stack.currentIndex() == 0:
-            if not confirm('<p>'+_('The selected books will be <b>permanently deleted</b> and the files removed from your computer. Are you sure?')+'</p>', 'library_delete_books', self):
+            if not confirm('<p>'+_('The selected books will be '
+                                   '<b>permanently deleted</b> and the files '
+                                   'removed from your computer. Are you sure?')
+                                +'</p>', 'library_delete_books', self):
                 return
             view.model().delete_books(rows)
         else:
@@ -929,7 +899,8 @@ class Main(MainWindow, Ui_MainWindow):
                 mi['cover'] = self.cover_to_thumbnail(cdata)
         metadata = iter(metadata)
         _files   = self.library_view.model().get_preferred_formats(rows,
-                                    self.device_manager.device_class.FORMATS, paths=True)
+                                    self.device_manager.device_class.FORMATS, 
+                                    paths=True, set_metadata=True)
         files = [getattr(f, 'name', None) for f in _files]
         bad, good, gf, names, remove_ids = [], [], [], [], []
         for f in files:
@@ -1223,6 +1194,8 @@ class Main(MainWindow, Ui_MainWindow):
                 format = 'LRF'
             if 'EPUB' in formats:
                 format = 'EPUB'
+            if 'MOBI' in formats:
+                format = 'MOBI'
             if not formats:
                 d = error_dialog(self, _('Cannot view'),
                         _('%s has no available formats.')%(title,))
@@ -1404,8 +1377,15 @@ class Main(MainWindow, Ui_MainWindow):
     def initialize_database(self):
         self.library_path = prefs['library_path']
         if self.library_path is None: # Need to migrate to new database layout
+            base = os.path.expanduser('~')
+            if iswindows:
+                from calibre import plugins
+                from PyQt4.Qt import QDir
+                base = plugins['winutil'][0].special_folder_path(plugins['winutil'][0].CSIDL_PERSONAL)
+                if not base or not os.path.exists(base):
+                    base = unicode(QDir.homePath()).replace('/', os.sep)
             dir = unicode(QFileDialog.getExistingDirectory(self, 
-                            _('Choose a location for your ebook library.'), os.getcwd()))
+                            _('Choose a location for your ebook library.'), base))
             if not dir:
                 dir = os.path.expanduser('~/Library')
             self.library_path = os.path.abspath(dir)
@@ -1591,6 +1571,11 @@ def main(args=sys.argv):
             print 'Restarting with:', e, sys.argv
             os.execvp(e, sys.argv)
         else:
+            if iswindows:
+                try:
+                    main.system_tray_icon.hide()
+                except:
+                    pass
             return ret
     return 0
 

src/calibre/gui2/viewer/documentview.py

@@ -15,6 +15,7 @@ from calibre.utils.config import Config, StringConfig
 from calibre.gui2.viewer.config_ui import Ui_Dialog
 from calibre.gui2.viewer.js import bookmarks, referencing
 from calibre.ptempfile import PersistentTemporaryFile
+from calibre.constants import iswindows
 
 def load_builtin_fonts():
     from calibre.ebooks.lrf.fonts.liberation import LiberationMono_BoldItalic
@@ -56,9 +57,12 @@ def config(defaults=None):
               help=_('Set the user CSS stylesheet. This can be used to customize the look of all books.'))
     
     fonts = c.add_group('FONTS', _('Font options'))
-    fonts('serif_family', default='Liberation Serif', help=_('The serif font family'))
-    fonts('sans_family', default='Liberation Sans', help=_('The sans-serif font family'))
-    fonts('mono_family', default='Liberation Mono', help=_('The monospaced font family'))
+    fonts('serif_family', default='Times New Roman' if iswindows else 'Liberation Serif', 
+          help=_('The serif font family'))
+    fonts('sans_family', default='Verdana' if iswindows else 'Liberation Sans', 
+          help=_('The sans-serif font family'))
+    fonts('mono_family', default='Courier New' if iswindows else 'Liberation Mono', 
+          help=_('The monospaced font family'))
     fonts('default_font_size', default=20, help=_('The standard font size in px'))
     fonts('mono_font_size', default=16, help=_('The monospaced font size in px'))
     fonts('standard_font', default='serif', help=_('The standard font type'))

src/calibre/library/database.py

@@ -4,14 +4,10 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Backend that implements storage of ebooks in an sqlite database.
 '''
 import sqlite3 as sqlite
-import datetime, re, os, cPickle, traceback, sre_constants
+import datetime, re, cPickle, sre_constants
 from zlib import compress, decompress
 
-from calibre import sanitize_file_name
-from calibre.ebooks.metadata.meta import set_metadata, metadata_from_formats
-from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.web.feeds.recipes import migrate_automatic_profile_to_automatic_recipe
 
 class Concatenate(object):
@@ -1389,225 +1385,14 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
     def all_ids(self):
         return [i[0] for i in self.conn.get('SELECT id FROM books')]
 
-    def export_to_dir(self, dir, indices, byauthor=False, single_dir=False,
-                      index_is_id=False, callback=None):
-        if not os.path.exists(dir):
-            raise IOError('Target directory does not exist: '+dir)
-        by_author = {}
-        count = 0
-        for index in indices:
-            id = index if index_is_id else self.id(index)
-            au = self.conn.get('SELECT author_sort FROM books WHERE id=?',
-                                   (id,), all=False)
-            if not au:
-                au = self.authors(index, index_is_id=index_is_id)
-                if not au:
-                    au = _('Unknown')
-                au = au.split(',')[0]
-            if not by_author.has_key(au):
-                by_author[au] = []
-            by_author[au].append(index)
-        for au in by_author.keys():
-            apath = os.path.join(dir, sanitize_file_name(au))
-            if not single_dir and not os.path.exists(apath):
-                os.mkdir(apath)
-            for idx in by_author[au]:
-                title = re.sub(r'\s', ' ', self.title(idx, index_is_id=index_is_id))
-                tpath = os.path.join(apath, sanitize_file_name(title))
-                id = idx if index_is_id else self.id(idx)
-                id = str(id)
-                if not single_dir and not os.path.exists(tpath):
-                    os.mkdir(tpath)
-
-                name = au + ' - ' + title if byauthor else title + ' - ' + au
-                name += '_'+id
-                base  = dir if single_dir else tpath
-                mi = self.get_metadata(idx, index_is_id=index_is_id)
-                cover = self.cover(idx, index_is_id=index_is_id)
-                if cover is not None:
-                    cname = sanitize_file_name(name) + '.jpg'
-                    cpath = os.path.join(base, cname)
-                    open(cpath, 'wb').write(cover)
-                    mi.cover = cname
-                f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb')
-                if not mi.authors:
-                    mi.authors = [_('Unknown')]
-                opf = OPFCreator(base, mi)
-                opf.render(f)
-                f.close()
-                
-                fmts = self.formats(idx, index_is_id=index_is_id)
-                if not fmts:
-                    fmts = ''
-                for fmt in fmts.split(','):
-                    data = self.format(idx, fmt, index_is_id=index_is_id)
-                    if not data:
-                        continue
-                    fname = name +'.'+fmt.lower()
-                    fname = sanitize_file_name(fname)
-                    f = open(os.path.join(base, fname), 'w+b')
-                    f.write(data)
-                    f.flush()
-                    f.seek(0)
-                    try:
-                        set_metadata(f, mi, fmt.lower())
-                    except:
-                        print 'Error setting metadata for book:', mi.title
-                        traceback.print_exc()
-                    f.close()
-                count += 1
-                if callable(callback):
-                    if not callback(count, mi.title):
-                        return
                          
 
 
-    def import_book(self, mi, formats):
-        series_index = 1 if mi.series_index is None else mi.series_index
-        if not mi.authors:
-            mi.authors = [_('Unknown')]
-        aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
-        obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)',
-                          (mi.title, None, series_index, aus))
-        id = obj.lastrowid
-        self.conn.commit()
-        self.set_metadata(id, mi)
-        for path in formats:
-            ext = os.path.splitext(path)[1][1:].lower()
-            stream = open(path, 'rb')
-            stream.seek(0, 2)
-            usize = stream.tell()
-            stream.seek(0)
-            data = sqlite.Binary(compress(stream.read()))
-            try:
-                self.conn.execute('INSERT INTO data(book, format, uncompressed_size, data) VALUES (?,?,?,?)',
-                              (id, ext, usize, data))
-            except sqlite.IntegrityError:
-                self.conn.execute('UPDATE data SET uncompressed_size=?, data=? WHERE book=? AND format=?',
-                                  (usize, data, id, ext))
-        self.conn.commit()
-
-    def import_book_directory_multiple(self, dirpath, callback=None):
-        dirpath = os.path.abspath(dirpath)
-        duplicates = []
-        books = {}
-        for path in os.listdir(dirpath):
-            if callable(callback):
-                callback('.')
-            path = os.path.abspath(os.path.join(dirpath, path))
-            if os.path.isdir(path) or not os.access(path, os.R_OK):
-                continue
-            ext = os.path.splitext(path)[1]
-            if not ext:
-                continue
-            ext = ext[1:].lower()
-            if ext not in BOOK_EXTENSIONS:
-                continue
-
-            key = os.path.splitext(path)[0]
-            if not books.has_key(key):
-                books[key] = []
-
-            books[key].append(path)
-
-        for formats in books.values():
-            mi = metadata_from_formats(formats)
-            if mi.title is None:
-                continue
-            if self.has_book(mi):
-                duplicates.append((mi, formats))
-                continue
-            self.import_book(mi, formats)
-            if callable(callback):
-                if callback(mi.title):
-                    break
-        return duplicates
-
-
-    def import_book_directory(self, dirpath, callback=None):
-        dirpath = os.path.abspath(dirpath)
-        formats = []
-        for path in os.listdir(dirpath):
-            if callable(callback):
-                callback('.')
-            path = os.path.abspath(os.path.join(dirpath, path))
-            if os.path.isdir(path) or not os.access(path, os.R_OK):
-                continue
-            ext = os.path.splitext(path)[1]
-            if not ext:
-                continue
-            ext = ext[1:].lower()
-            if ext not in BOOK_EXTENSIONS:
-                continue
-            formats.append(path)
-
-        if not formats:
-            return
-        
-        mi = metadata_from_formats(formats)
-        if mi.title is None:
-            return
-        if self.has_book(mi):
-            return [(mi, formats)]
-        self.import_book(mi, formats)
-        if callable(callback):
-            callback(mi.title)
-            
     
 
     def has_id(self, id):
         return self.conn.get('SELECT id FROM books where id=?', (id,), all=False) is not None
 
-    def recursive_import(self, root, single_book_per_directory=True, callback=None):
-        root = os.path.abspath(root)
-        duplicates  = []
-        for dirpath in os.walk(root):
-            res = self.import_book_directory(dirpath[0], callback=callback) if \
-                single_book_per_directory else \
-                  self.import_book_directory_multiple(dirpath[0], callback=callback)
-            if res is not None:
-                duplicates.extend(res)
-            if callable(callback):
-                if callback(''):
-                    break
-            
-        return duplicates
-
-    def export_single_format_to_dir(self, dir, indices, format, 
-                                    index_is_id=False, callback=None):
-        dir = os.path.abspath(dir)
-        if not index_is_id:
-            indices = map(self.id, indices)
-        failures = []
-        for count, id in enumerate(indices):
-            try:
-                data = self.format(id, format, index_is_id=True)
-                if not data:
-                    failures.append((id, self.title(id, index_is_id=True)))
-                    continue
-            except:
-                failures.append((id, self.title(id, index_is_id=True)))
-                continue
-            title = self.title(id, index_is_id=True)
-            au = self.authors(id, index_is_id=True)
-            if not au:
-                au = _('Unknown')
-            fname = '%s - %s.%s'%(title, au, format.lower())
-            fname = sanitize_file_name(fname)
-            if not os.path.exists(dir):
-                os.makedirs(dir)
-            f = open(os.path.join(dir, fname), 'w+b')
-            f.write(data)
-            f.seek(0)
-            try:
-                set_metadata(f, self.get_metadata(id, index_is_id=True), stream_type=format.lower())
-            except:
-                pass
-            f.close()
-            if callable(callback):
-                if not callback(count, title):
-                    break
-        return failures
     
     
 

src/calibre/library/database2.py

@@ -12,19 +12,23 @@ from itertools import repeat
 from datetime import datetime
 
 from PyQt4.QtCore import QCoreApplication, QThread, QReadWriteLock
-from PyQt4.QtGui import QApplication, QPixmap, QImage
+from PyQt4.QtGui import QApplication, QImage
 __app = None
 
 from calibre.library import title_sort
 from calibre.library.database import LibraryDatabase
 from calibre.library.sqlite import connect, IntegrityError
 from calibre.utils.search_query_parser import SearchQueryParser
-from calibre.ebooks.metadata import string_to_authors, authors_to_string
-from calibre.ebooks.metadata.meta import get_metadata
+from calibre.ebooks.metadata import string_to_authors, authors_to_string, MetaInformation
+from calibre.ebooks.metadata.meta import get_metadata, set_metadata, \
+    metadata_from_formats
+from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import run_plugins_on_import
+
 from calibre import sanitize_file_name
+from calibre.ebooks import BOOK_EXTENSIONS
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 
@@ -376,8 +380,10 @@ class LibraryDatabase2(LibraryDatabase):
             return row[loc]
 
         for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn',
-                     'publisher', 'rating', 'series', 'series_index', 'tags', 'title'):
-            setattr(self, prop, functools.partial(get_property, loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
+                     'publisher', 'rating', 'series', 'series_index', 'tags',
+                     'title', 'timestamp'):
+            setattr(self, prop, functools.partial(get_property,
+                    loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
 
     def initialize_database(self):
         from calibre.resources import metadata_sqlite
@@ -556,7 +562,8 @@ class LibraryDatabase2(LibraryDatabase):
                 traceback.print_exc()
                 continue
 
-    def cover(self, index, index_is_id=False, as_file=False, as_image=False):
+    def cover(self, index, index_is_id=False, as_file=False, as_image=False,
+              as_path=False):
         '''
         Return the cover image as a bytestring (in JPEG format) or None.
 
@@ -566,6 +573,8 @@ class LibraryDatabase2(LibraryDatabase):
         id = index if  index_is_id else self.id(index)
         path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
         if os.access(path, os.R_OK):
+            if as_path:
+                return path
             f = open(path, 'rb')
             if as_image:
                 img = QImage()
@@ -573,6 +582,31 @@ class LibraryDatabase2(LibraryDatabase):
                 return img
             return f if as_file else f.read()
 
+    def get_metadata(self, idx, index_is_id=False, get_cover=False):
+        '''
+        Convenience method to return metadata as a L{MetaInformation} object.
+        '''
+        aum = self.authors(idx, index_is_id=index_is_id)
+        if aum: aum = [a.strip().replace('|', ',') for a in aum.split(',')]
+        mi = MetaInformation(self.title(idx, index_is_id=index_is_id), aum)
+        mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
+        mi.comments    = self.comments(idx, index_is_id=index_is_id)
+        mi.publisher   = self.publisher(idx, index_is_id=index_is_id)
+        mi.timestamp   = self.timestamp(idx, index_is_id=index_is_id)
+        tags = self.tags(idx, index_is_id=index_is_id)
+        if tags:
+            mi.tags = [i.strip() for i in tags.split(',')]
+        mi.series = self.series(idx, index_is_id=index_is_id)
+        if mi.series:
+            mi.series_index = self.series_index(idx, index_is_id=index_is_id)
+        mi.rating = self.rating(idx, index_is_id=index_is_id)
+        mi.isbn = self.isbn(idx, index_is_id=index_is_id)
+        id = idx if index_is_id else self.id(idx)
+        mi.application_id = id
+        if get_cover:
+            mi.cover = self.cover(id, index_is_id=True, as_path=True)
+        return mi
+
     def has_book(self, mi):
         title = mi.title
         if title:
@@ -599,7 +633,7 @@ class LibraryDatabase2(LibraryDatabase):
             if not QCoreApplication.instance():
                 global __app
                 __app = QApplication([])
-            p = QPixmap()
+            p = QImage()
             if callable(getattr(data, 'read', None)):
                 data = data.read()
             p.loadFromData(data)
@@ -853,6 +887,8 @@ class LibraryDatabase2(LibraryDatabase):
             self.set_isbn(id, mi.isbn, notify=False)
         if mi.series_index and mi.series_index > 0:
             self.set_series_index(id, mi.series_index, notify=False)
+        if getattr(mi, 'timestamp', None) is not None:
+            self.set_timestamp(id, mi.timestamp, notify=False)
         self.set_path(id, True)
         self.notify('metadata', [id])
 
@@ -1114,7 +1150,7 @@ class LibraryDatabase2(LibraryDatabase):
     def add_books(self, paths, formats, metadata, uris=[], add_duplicates=True):
         '''
         Add a book to the database. The result cache is not updated.
-        @param paths: List of paths to book files or file-like objects
+        :param:`paths` List of paths to book files or file-like objects
         '''
         formats, metadata, uris = iter(formats), iter(metadata), iter(uris)
         duplicates = []
@@ -1152,17 +1188,17 @@ class LibraryDatabase2(LibraryDatabase):
         self.conn.commit()
         self.data.refresh_ids(self.conn, ids) # Needed to update format list and size
         if duplicates:
-            paths    = tuple(duplicate[0] for duplicate in duplicates)
-            formats  = tuple(duplicate[1] for duplicate in duplicates)
-            metadata = tuple(duplicate[2] for duplicate in duplicates)
-            uris     = tuple(duplicate[3] for duplicate in duplicates)
+            paths    = list(duplicate[0] for duplicate in duplicates)
+            formats  = list(duplicate[1] for duplicate in duplicates)
+            metadata = list(duplicate[2] for duplicate in duplicates)
+            uris     = list(duplicate[3] for duplicate in duplicates)
             return (paths, formats, metadata, uris), len(ids)
         return None, len(ids)
 
-    def import_book(self, mi, formats):
+    def import_book(self, mi, formats, notify=True):
         series_index = 1 if mi.series_index is None else mi.series_index
         if not mi.authors:
-            mi.authors = ['Unknown']
+            mi.authors = [_('Unknown')]
         aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
         obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)',
                           (mi.title, None, series_index, aus))
@@ -1172,10 +1208,13 @@ class LibraryDatabase2(LibraryDatabase):
         self.set_metadata(id, mi)
         for path in formats:
             ext = os.path.splitext(path)[1][1:].lower()
+            if ext == 'opf':
+                continue
             stream = open(path, 'rb')
             self.add_format(id, ext, stream, index_is_id=True)
         self.conn.commit()
         self.data.refresh_ids(self.conn, [id]) # Needed to update format list and size
+        if notify:
             self.notify('add', [id])
 
     def move_library_to(self, newloc, progress=None):
@@ -1323,4 +1362,190 @@ books_series_link      feeds
         progress.reset()
         return len(books)
 
+    def export_to_dir(self, dir, indices, byauthor=False, single_dir=False,
+                      index_is_id=False, callback=None):
+        if not os.path.exists(dir):
+            raise IOError('Target directory does not exist: '+dir)
+        by_author = {}
+        count = 0
+        for index in indices:
+            id = index if index_is_id else self.id(index)
+            au = self.conn.get('SELECT author_sort FROM books WHERE id=?',
+                                   (id,), all=False)
+            if not au:
+                au = self.authors(index, index_is_id=index_is_id)
+                if not au:
+                    au = _('Unknown')
+                au = au.split(',')[0]
+            if not by_author.has_key(au):
+                by_author[au] = []
+            by_author[au].append(index)
+        for au in by_author.keys():
+            apath = os.path.join(dir, sanitize_file_name(au))
+            if not single_dir and not os.path.exists(apath):
+                os.mkdir(apath)
+            for idx in by_author[au]:
+                title = re.sub(r'\s', ' ', self.title(idx, index_is_id=index_is_id))
+                tpath = os.path.join(apath, sanitize_file_name(title))
+                id = idx if index_is_id else self.id(idx)
+                id = str(id)
+                if not single_dir and not os.path.exists(tpath):
+                    os.mkdir(tpath)
 
+                name = au + ' - ' + title if byauthor else title + ' - ' + au
+                name += '_'+id
+                base  = dir if single_dir else tpath
+                mi = self.get_metadata(idx, index_is_id=index_is_id, get_cover=True)
+                f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb')
+                if not mi.authors:
+                    mi.authors = [_('Unknown')]
+                cdata = self.cover(int(id), index_is_id=True)
+                if cdata is not None:
+                    cname = sanitize_file_name(name)+'.jpg'
+                    open(os.path.join(base, cname), 'wb').write(cdata)
+                    mi.cover = cname
+                opf = OPFCreator(base, mi)
+                opf.render(f)
+                f.close()
+
+                fmts = self.formats(idx, index_is_id=index_is_id)
+                if not fmts:
+                    fmts = ''
+                for fmt in fmts.split(','):
+                    data = self.format(idx, fmt, index_is_id=index_is_id)
+                    if not data:
+                        continue
+                    fname = name +'.'+fmt.lower()
+                    fname = sanitize_file_name(fname)
+                    f = open(os.path.join(base, fname), 'w+b')
+                    f.write(data)
+                    f.flush()
+                    f.seek(0)
+                    try:
+                        set_metadata(f, mi, fmt.lower())
+                    except:
+                        pass
+                    f.close()
+                count += 1
+                if callable(callback):
+                    if not callback(count, mi.title):
+                        return
+
+    def export_single_format_to_dir(self, dir, indices, format,
+                                    index_is_id=False, callback=None):
+        dir = os.path.abspath(dir)
+        if not index_is_id:
+            indices = map(self.id, indices)
+        failures = []
+        for count, id in enumerate(indices):
+            try:
+                data = self.format(id, format, index_is_id=True)
+                if not data:
+                    failures.append((id, self.title(id, index_is_id=True)))
+                    continue
+            except:
+                failures.append((id, self.title(id, index_is_id=True)))
+                continue
+            title = self.title(id, index_is_id=True)
+            au = self.authors(id, index_is_id=True)
+            if not au:
+                au = _('Unknown')
+            fname = '%s - %s.%s'%(title, au, format.lower())
+            fname = sanitize_file_name(fname)
+            if not os.path.exists(dir):
+                os.makedirs(dir)
+            f = open(os.path.join(dir, fname), 'w+b')
+            f.write(data)
+            f.seek(0)
+            try:
+                set_metadata(f, self.get_metadata(id, index_is_id=True, get_cover=True),
+                             stream_type=format.lower())
+            except:
+                pass
+            f.close()
+            if callable(callback):
+                if not callback(count, title):
+                    break
+        return failures
+
+    def find_books_in_directory(self, dirpath, single_book_per_directory):
+        dirpath = os.path.abspath(dirpath)
+        if single_book_per_directory:
+            formats = []
+            for path in os.listdir(dirpath):
+                path = os.path.abspath(os.path.join(dirpath, path))
+                if os.path.isdir(path) or not os.access(path, os.R_OK):
+                    continue
+                ext = os.path.splitext(path)[1]
+                if not ext:
+                    continue
+                ext = ext[1:].lower()
+                if ext not in BOOK_EXTENSIONS and ext != 'opf':
+                    continue
+                formats.append(path)
+            yield formats
+        else:
+            books = {}
+            for path in os.listdir(dirpath):
+                path = os.path.abspath(os.path.join(dirpath, path))
+                if os.path.isdir(path) or not os.access(path, os.R_OK):
+                    continue
+                ext = os.path.splitext(path)[1]
+                if not ext:
+                    continue
+                ext = ext[1:].lower()
+                if ext not in BOOK_EXTENSIONS:
+                    continue
+
+                key = os.path.splitext(path)[0]
+                if not books.has_key(key):
+                    books[key] = []
+                books[key].append(path)
+
+            for formats in books.values():
+                yield formats
+
+    def import_book_directory_multiple(self, dirpath, callback=None):
+        duplicates = []
+        for formats in self.find_books_in_directory(dirpath, False):
+            mi = metadata_from_formats(formats)
+            if mi.title is None:
+                continue
+            if self.has_book(mi):
+                duplicates.append((mi, formats))
+                continue
+            self.import_book(mi, formats)
+            if callable(callback):
+                if callback(mi.title):
+                    break
+        return duplicates
+
+    def import_book_directory(self, dirpath, callback=None):
+        dirpath = os.path.abspath(dirpath)
+        formats = self.find_books_in_directory(dirpath, True)
+        if not formats:
+            return
+
+        mi = metadata_from_formats(formats)
+        if mi.title is None:
+            return
+        if self.has_book(mi):
+            return [(mi, formats)]
+        self.import_book(mi, formats)
+        if callable(callback):
+            callback(mi.title)
+
+    def recursive_import(self, root, single_book_per_directory=True, callback=None):
+        root = os.path.abspath(root)
+        duplicates  = []
+        for dirpath in os.walk(root):
+            res = self.import_book_directory(dirpath[0], callback=callback) if \
+                single_book_per_directory else \
+                  self.import_book_directory_multiple(dirpath[0], callback=callback)
+            if res is not None:
+                duplicates.extend(res)
+            if callable(callback):
+                if callback(''):
+                    break
+
+        return duplicates

src/calibre/trac/donations/server.py

@@ -236,7 +236,9 @@ Donors per day: %(dpd).2f
         ml   = mdates.MonthLocator() # every month
         fig = plt.figure(1, (8, 4), 96)#, facecolor, edgecolor, frameon, FigureClass)
         ax = fig.add_subplot(111)
+        average = sum(y)/len(y)
         ax.bar(x, y, align='center', width=20, color='g')
+        ax.hlines([average], x[0], x[-1])
         ax.xaxis.set_major_locator(ml)
         ax.xaxis.set_major_formatter(mdates.DateFormatter('%b %y'))
         ax.set_xlim(_months[0].min-timedelta(days=15), _months[-1].min+timedelta(days=15))

src/calibre/trac/plugins/download.py

@@ -30,11 +30,12 @@ class Distribution(object):
         ('libusb', '0.1.12', None, None, None),
         ('Qt', '4.4.0', 'qt', 'libqt4-core libqt4-gui', 'qt4'),
         ('PyQt', '4.4.2', 'PyQt4', 'python-qt4', 'PyQt4'),
-        ('mechanize for python', '0.1.8', 'dev-python/mechanize', 'python-mechanize', 'python-mechanize'),
+        ('mechanize for python', '0.1.11', 'dev-python/mechanize', 'python-mechanize', 'python-mechanize'),
         ('ImageMagick', '6.3.5', 'imagemagick', 'imagemagick', 'ImageMagick'),
         ('xdg-utils', '1.0.2', 'xdg-utils', 'xdg-utils', 'xdg-utils'),
         ('dbus-python', '0.82.2', 'dbus-python', 'python-dbus', 'dbus-python'),
         ('lxml', '2.0.5', 'lxml', 'python-lxml', 'python-lxml'),
+        ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil'),
         ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'),
         ('help2man', '1.36.4', 'help2man', 'help2man', 'help2man'),
         ]

src/calibre/web/feeds/news.py

@@ -91,7 +91,7 @@ class BasicNewsRecipe(object, LoggingInterface):
     #: If True stylesheets are not downloaded and processed
     no_stylesheets         = False
     
-    #: Convenient flag to strip all javascripts tags from the downloaded HTML
+    #: Convenient flag to strip all javascript tags from the downloaded HTML
     remove_javascript      = True
     
     #: If True the GUI will ask the user for a username and password 

src/calibre/web/feeds/recipes/__init__.py

@@ -27,7 +27,8 @@ recipe_modules = ['recipe_' + r for r in (
            'shacknews', 'teleread', 'granma', 'juventudrebelde', 'juventudrebelde_english',
            'la_tercera', 'el_mercurio_chile', 'la_cuarta', 'lanacion_chile', 'la_segunda',
            'jb_online', 'estadao', 'o_globo', 'vijesti', 'elmundo', 'the_oz',
-           'honoluluadvertiser', 'starbulletin', 'exiled',
+           'honoluluadvertiser', 'starbulletin', 'exiled', 'indy_star', 'dna',
+           'pobjeda',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_dna.py

@@ -0,0 +1,41 @@
+'''
+dnaindia.com
+'''
+import re
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class DNAIndia(BasicNewsRecipe):
+    
+    title       = 'DNA India'
+    description = 'Mumbai news, India news, World news, breaking news'
+    __author__  = 'Kovid Goyal'
+    language    = _('English')
+    
+    feeds       = [
+                   ('Top News', 'http://www.dnaindia.com/syndication/rss_topnews.xml'),
+                   ('Popular News', 'http://www.dnaindia.com/syndication/rss_popular.xml'),
+                   ('Recent Columns', 'http://www.dnaindia.com/syndication/rss_column.xml'),
+                   ('Mumbai', 'http://www.dnaindia.com/syndication/rss,catid-1.xml'),
+                   ('India', 'http://www.dnaindia.com/syndication/rss,catid-2.xml'),
+                   ('World', 'http://www.dnaindia.com/syndication/rss,catid-9.xml'),
+                   ('Money', 'http://www.dnaindia.com/syndication/rss,catid-4.xml'),
+                   ('Sports', 'http://www.dnaindia.com/syndication/rss,catid-6.xml'),
+                   ('After Hours', 'http://www.dnaindia.com/syndication/rss,catid-7.xml'),
+                   ('Digital Life', 'http://www.dnaindia.com/syndication/rss,catid-1089741.xml'),                   
+                   ]
+    remove_tags = [{'id':'footer'}, {'class':['bottom', 'categoryHead']}]
+    
+    def print_version(self, url):
+        match = re.search(r'newsid=(\d+)', url)
+        if not match:
+            return url
+        return 'http://www.dnaindia.com/dnaprint.asp?newsid='+match.group(1)
+    
+    def postprocess_html(self, soup, first_fetch):
+        for t in soup.findAll(['table', 'tr', 'td']):
+            t.name = 'div'
+            
+        a = soup.find(href='http://www.3dsyndication.com/')
+        if a is not None:
+            a.parent.extract()
+        return soup

src/calibre/web/feeds/recipes/recipe_honoluluadvertiser.py

@@ -15,6 +15,7 @@ class Honoluluadvertiser(BasicNewsRecipe):
     publisher             = 'Honolulu Advertiser'
     category              = 'news, Honolulu, Hawaii'
     oldest_article        = 2
+    language              = _('English')
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False

src/calibre/web/feeds/recipes/recipe_indy_star.py

@@ -0,0 +1,15 @@
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class AdvancedUserRecipe1234144423(BasicNewsRecipe):
+    title          = u'Indianapolis Star'
+    oldest_article = 5
+    language       = _('English')
+    __author__     = 'Owen Kelly'
+    max_articles_per_feed = 100
+
+    cover_url  = u'http://www2.indystar.com/frontpage/images/today.jpg'
+    
+    feeds          = [(u'Community Headlines', u'http://www.indystar.com/apps/pbcs.dll/section?Category=LOCAL&template=rss&mime=XML'), (u'News Headlines', u'http://www.indystar.com/apps/pbcs.dll/section?Category=NEWS&template=rss&mime=XML'), (u'Business Headlines', u'http://www..indystar.com/apps/pbcs.dll/section?Category=BUSINESS&template=rss&mime=XML'), (u'Sports Headlines', u'http://www.indystar.com/apps/pbcs.dll/section?Category=SPORTS&template=rss&mime=XML'), (u'Lifestyle Headlines', u'http://www.indystar.com/apps/pbcs.dll/section?Category=LIVING&template=rss&mime=XML'), (u'Opinion Headlines', u'http://www.indystar.com/apps/pbcs.dll/section?Category=OPINION&template=rss&mime=XML')]
+
+    def print_version(self, url):
+        return url + '&template=printart'

src/calibre/web/feeds/recipes/recipe_jutarnji.py

@@ -20,6 +20,7 @@ class Jutarnji(BasicNewsRecipe):
     max_articles_per_feed = 100
     simultaneous_downloads = 1
     delay = 1    
+    language              = _('Croatian')
     no_stylesheets        = True
     use_embedded_content  = False
     remove_javascript     = True

src/calibre/web/feeds/recipes/recipe_outlook_india.py

@@ -13,11 +13,10 @@ class OutlookIndia(BasicNewsRecipe):
     
     title = 'Outlook India'
     __author__  = 'Kovid Goyal'
-    description = 'Weekly news magazine focussed on India.'
+    description = 'Weekly news magazine focused on India.'
     language = _('English')
     recursions = 1
     match_regexp = r'full.asp.*&pn=\d+'
-    html2lrf_options = ['--ignore-tables']
     
     remove_tags = [
                    dict(name='img', src="images/space.gif"),
@@ -81,5 +80,8 @@ class OutlookIndia(BasicNewsRecipe):
                 bad.append(table)
         for b in bad:
             b.extract()
+        soup = soup.findAll('html')[0]
+        for t in soup.findAll(['table', 'tr', 'td']):
+            t.name = 'div' 
         return soup
     

src/calibre/web/feeds/recipes/recipe_pobjeda.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+
+'''
+pobjeda.co.me
+'''
+
+import re
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Pobjeda(BasicNewsRecipe):
+    title                 = 'Pobjeda Online'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Montenegro'
+    publisher             = 'Pobjeda a.d.'
+    category              = 'news, politics, Montenegro'    
+    language              = _('Serbian')
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    encoding              = 'utf8'
+    remove_javascript     = True
+    use_embedded_content  = False
+    INDEX                 = u'http://www.pobjeda.co.me'
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+     
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
+    keep_only_tags = [dict(name='div', attrs={'class':'vijest'})]
+
+    remove_tags = [dict(name=['object','link'])]
+
+    feeds = [
+               (u'Politika'          , u'http://www.pobjeda.co.me/rubrika.php?rubrika=1'  )
+              ,(u'Ekonomija'         , u'http://www.pobjeda.co.me/rubrika.php?rubrika=2'  )
+              ,(u'Drustvo'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=3'  )
+              ,(u'Crna Hronika'      , u'http://www.pobjeda.co.me/rubrika.php?rubrika=4'  )
+              ,(u'Kultura'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=5'  )
+              ,(u'Hronika Podgorice' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=7'  )
+              ,(u'Feljton'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=8'  )
+              ,(u'Crna Gora'         , u'http://www.pobjeda.co.me/rubrika.php?rubrika=9'  )
+              ,(u'Svijet'            , u'http://www.pobjeda.co.me/rubrika.php?rubrika=202')
+              ,(u'Ekonomija i Biznis', u'http://www.pobjeda.co.me/dodatak.php?rubrika=11' )
+              ,(u'Djeciji Svijet'    , u'http://www.pobjeda.co.me/dodatak.php?rubrika=12' )
+              ,(u'Kultura i Drustvo' , u'http://www.pobjeda.co.me/dodatak.php?rubrika=13' )
+              ,(u'Agora'             , u'http://www.pobjeda.co.me/dodatak.php?rubrika=133')
+              ,(u'Ekologija'         , u'http://www.pobjeda.co.me/dodatak.php?rubrika=252')
+            ]
+
+    def preprocess_html(self, soup):
+        soup.html['xml:lang'] = 'sr-Latn-ME'
+        soup.html['lang']     = 'sr-Latn-ME'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-ME"/>'
+        soup.head.insert(0,mtag)
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup
+
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('img',attrs={'alt':'Naslovna strana'})
+        if cover_item:
+           cover_url = self.INDEX + cover_item.parent['href']
+        return cover_url
+    
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            soup = self.index_to_soup(feedurl)
+            for item in soup.findAll('div', attrs={'class':'vijest'}):
+                description = self.tag_to_string(item.h2)
+                atag = item.h1.find('a')
+                if atag:
+                    url         = self.INDEX + '/' + atag['href']
+                    title       = self.tag_to_string(atag)
+                    date        = strftime(self.timefmt)
+                    articles.append({
+                                      'title'      :title
+                                     ,'date'       :date
+                                     ,'url'        :url
+                                     ,'description':description
+                                    })
+            totalfeeds.append((feedtitle, articles))
+        return totalfeeds
+        

src/calibre/web/feeds/recipes/recipe_starbulletin.py

@@ -16,6 +16,7 @@ class Starbulletin(BasicNewsRecipe):
     category              = 'news, Honolulu, Hawaii'
     oldest_article        = 2
     max_articles_per_feed = 100
+    language              = _('English')
     no_stylesheets        = True
     use_embedded_content  = False
     encoding              = 'utf8'

src/calibre/web/feeds/recipes/recipe_wash_post.py

@@ -12,19 +12,7 @@ class WashingtonPost(BasicNewsRecipe):
     language = _('English')
 
     
-    preprocess_regexps = [ (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in 
-[
-    	(r'<HEAD>.*?</HEAD>' , lambda match : '<HEAD></HEAD>'),
-    	(r'<div id="apple-rss-sidebar-background">.*?<!-- start Entries -->', lambda match : ''),
-    	(r'<!-- end apple-rss-content-area -->.*?</body>', lambda match : '</body>'),
-    	(r'<script.*?>.*?</script>', lambda match : ''),
-    	(r'<body.*?>.*?.correction {', lambda match : '<body><style>.correction {'),
-    	(r'<span class="display:none;" name="pubDate".*?>.*?</body>', lambda match : '<body>'),
-    	
-    	
-    ]
-    ]   
-     
+    remove_javascript = True   
 
   
     feeds = [ ('Today\'s Highlights', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/03/24/LI2005032400102.xml'),
@@ -38,8 +26,17 @@ class WashingtonPost(BasicNewsRecipe):
      	         ('Editorials', 'http://www.washingtonpost.com/wp-dyn/rss/linkset/2005/05/30/LI2005053000331.xml'),
      	]
     
+    remove_tags = [{'id':['pfmnav', 'ArticleCommentsWrapper']}]
 
 
+    def get_article_url(self, article):
+        return article.get('feedburner_origlink', article.get('link', None))
+
     def print_version(self, url):
-        return (url.rpartition('.')[0] + '_pf.html')
+        return url.rpartition('.')[0] + '_pf.html'
+    
+    def postprocess_html(self, soup, first):
+        for div in soup.findAll(name='div', style=re.compile('margin')):
+            div['style'] = ''
+        return soup