Sync to trunk

installer/linux/freeze.py

@@ -81,7 +81,8 @@ def freeze():
                 'PyQt4.QtScript.so', 'PyQt4.QtSql.so', 'PyQt4.QtTest.so', 'qt',
                 'glib', 'gobject']
     
-    packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg']
+    packages = ['calibre', 'encodings', 'cherrypy', 'cssutils', 'xdg', 
+                'dateutil']
     
     includes += ['calibre.web.feeds.recipes.'+r for r in recipe_modules]
     

installer/osx/freeze.py

@@ -342,6 +342,7 @@ def main():
                                        'calibre.ebooks.lrf.any.*', 'calibre.ebooks.lrf.feeds.*',
                                        'keyword', 'codeop', 'pydoc', 'readline',
                                        'BeautifulSoup', 'calibre.ebooks.lrf.fonts.prs500.*',
+                                       'dateutil',
                                        ],
                          'packages' : ['PIL', 'Authorization', 'lxml'],
                          'excludes' : ['IPython'],

installer/windows/freeze.py

@@ -179,7 +179,8 @@ def main(args=sys.argv):
                                              'calibre.ebooks.lrf.fonts.prs500.*',
                                              'PyQt4.QtWebKit', 'PyQt4.QtNetwork',
                                              ],
-                                  'packages'  : ['PIL', 'lxml', 'cherrypy'],
+                                  'packages'  : ['PIL', 'lxml', 'cherrypy',
+                                                 'dateutil'],
                                   'excludes'  : ["Tkconstants", "Tkinter", "tcl",
                                                  "_imagingtk", "ImageTk", "FixTk"
                                                 ],

src/calibre/constants.py

@@ -2,7 +2,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
 __docformat__ = 'restructuredtext en'
 __appname__   = 'calibre'
-__version__   = '0.4.135'
+__version__   = '0.4.136'
 __author__    = "Kovid Goyal <kovid@kovidgoyal.net>"
 '''
 Various run time constants.

src/calibre/devices/prs505/books.py

@@ -186,7 +186,10 @@ class BookList(_BookList):
         node = self.document.createElement(self.prefix + "text")
         mime = MIME_MAP[name.rpartition('.')[-1].lower()]
         cid = self.max_id()+1
-        sourceid = str(self[0].sourceid) if len(self) else "1"
+        try:
+            sourceid = str(self[0].sourceid) if len(self) else '1'
+        except:
+            sourceid = '1'
         attrs = {
                  "title"  : info["title"], 
                  'titleSorter' : sortable_title(info['title']),

src/calibre/devices/prs505/driver.py

@@ -248,15 +248,20 @@ class PRS505(Device):
                 time.sleep(3)
                 self.open_osx()
         if self._card_prefix is not None:
-            cachep = os.path.join(self._card_prefix, self.CACHE_XML)
+            try:
-            if not os.path.exists(cachep):
+                cachep = os.path.join(self._card_prefix, self.CACHE_XML)
-                os.makedirs(os.path.dirname(cachep), mode=0777)
+                if not os.path.exists(cachep):
-                f = open(cachep, 'wb')
+                    os.makedirs(os.path.dirname(cachep), mode=0777)
-                f.write(u'''<?xml version="1.0" encoding="UTF-8"?>
+                    f = open(cachep, 'wb')
+                    f.write(u'''<?xml version="1.0" encoding="UTF-8"?>
 <cache xmlns="http://www.kinoma.com/FskCache/1">
 </cache>
 '''.encode('utf8'))
-                f.close()
+                    f.close()
+            except:
+                self._card_prefix = None
+                import traceback
+                traceback.print_exc()
             
     def set_progress_reporter(self, pr):
         self.report_progress = pr

src/calibre/ebooks/epub/from_html.py

@@ -74,7 +74,9 @@ def check_links(opf_path, pretty_print):
         html_files = []
         for item in opf.itermanifest():
             if 'html' in item.get('media-type', '').lower():
-                f = item.get('href').split('/')[-1].decode('utf-8')
+                f = item.get('href').split('/')[-1]
+                if isinstance(f, str):
+                    f = f.decode('utf-8')
                 html_files.append(os.path.abspath(content(f)))
         
         for path in html_files:

src/calibre/ebooks/html.py

@@ -330,7 +330,8 @@ class PreProcessor(object):
                    sanitize_head),
                   # Convert all entities, since lxml doesn't handle them well
                   (re.compile(r'&(\S+?);'), convert_entities),
-                  
+                  # Remove the <![if/endif tags inserted by everybody's darling, MS Word
+                  (re.compile(r'(?i)<{0,1}!\[(end){0,1}if[^>]*>'), lambda match: ''),
                   ]
                      
     # Fix pdftohtml markup

src/calibre/ebooks/metadata/__init__.py

@@ -192,7 +192,8 @@ class MetaInformation(object):
         for attr in ('author_sort', 'title_sort', 'comments', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'tags', 'cover_data', 'application_id', 'guide',
-                     'manifest', 'spine', 'toc', 'cover', 'language', 'book_producer'):
+                     'manifest', 'spine', 'toc', 'cover', 'language', 
+                     'book_producer', 'timestamp'):
             if hasattr(mi, attr):
                 setattr(ans, attr, getattr(mi, attr))
         
@@ -217,7 +218,7 @@ class MetaInformation(object):
         for x in ('author_sort', 'title_sort', 'comments', 'category', 'publisher',
                   'series', 'series_index', 'rating', 'isbn', 'language',
                   'application_id', 'manifest', 'toc', 'spine', 'guide', 'cover',
-                  'book_producer',
+                  'book_producer', 'timestamp'
                   ):
             setattr(self, x, getattr(mi, x, None))
     
@@ -235,7 +236,8 @@ class MetaInformation(object):
         for attr in ('author_sort', 'title_sort', 'comments', 'category',
                      'publisher', 'series', 'series_index', 'rating',
                      'isbn', 'application_id', 'manifest', 'spine', 'toc', 
-                     'cover', 'language', 'guide', 'book_producer'):
+                     'cover', 'language', 'guide', 'book_producer', 
+                     'timestamp'):
             if hasattr(mi, attr):
                 val = getattr(mi, attr)
                 if val is not None:
@@ -276,6 +278,8 @@ class MetaInformation(object):
             ans += u'Series   : '+unicode(self.series) + ' #%s\n'%self.format_series_index()  
         if self.language:
             ans += u'Language : '     + unicode(self.language) + u'\n'
+        if self.timestamp is not None:
+            ans += u'Timestamp : ' + self.timestamp.isoformat(' ')
         return ans.strip()
     
     def to_html(self):
@@ -289,12 +293,12 @@ class MetaInformation(object):
         if self.series:
             ans += [(_('Series'), unicode(self.series)+ ' #%s'%self.format_series_index())]
         ans += [(_('Language'), unicode(self.language))]
+        if self.timestamp is not None:
+            ans += [(_('Timestamp'), unicode(self.timestamp.isoformat(' ')))]
         for i, x in enumerate(ans):
             ans[i] = u'<tr><td><b>%s</b></td><td>%s</td></tr>'%x
         return u'<table>%s</table>'%u'\n'.join(ans)
         
-        
-    
     def __str__(self):
         return self.__unicode__().encode('utf-8')
     

src/calibre/ebooks/metadata/library_thing.py

@@ -43,8 +43,9 @@ def cover_from_isbn(isbn, timeout=5.):
         src = browser.open('http://www.librarything.com/isbn/'+isbn).read().decode('utf-8', 'replace')
     except Exception, err:
         if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
-            raise LibraryThingError(_('LibraryThing.com timed out. Try again later.'))
+            err = LibraryThingError(_('LibraryThing.com timed out. Try again later.'))
-        raise
+        raise err
+    else:
         s = BeautifulSoup(src)
         url = s.find('td', attrs={'class':'left'})
         if url is None:

src/calibre/ebooks/metadata/meta.py

@@ -31,8 +31,14 @@ def metadata_from_formats(formats):
     mi = MetaInformation(None, None)
     formats.sort(cmp=lambda x,y: cmp(METADATA_PRIORITIES[path_to_ext(x)],
                                      METADATA_PRIORITIES[path_to_ext(y)]))
-    for path in formats:
+    extensions = list(map(path_to_ext, formats))
-        ext = path_to_ext(path)
+    if 'opf' in extensions:
+        opf = formats[extensions.index('opf')]
+        mi2 = opf_metadata(opf)
+        if mi2 is not None and mi2.title:
+            return mi2
+    
+    for path, ext in zip(formats, extensions):
         stream = open(path, 'rb')
         try:
             mi.smart_update(get_metadata(stream, stream_type=ext, use_libprs_metadata=True))

src/calibre/ebooks/metadata/opf.xml

@@ -10,7 +10,7 @@
         <dc:creator opf:role="aut" py:for="i, author in enumerate(mi.authors)" py:attrs="{'opf:file-as':mi.author_sort} if mi.author_sort and i == 0 else {}">${author}</dc:creator>
         <dc:contributor opf:role="bkp" py:with="attrs={'opf:file-as':__appname__}" py:attrs="attrs">${'%s (%s)'%(__appname__, __version__)} [http://${__appname__}.kovidgoyal.net]</dc:contributor> 
         <dc:identifier opf:scheme="${__appname__}" id="${__appname__}_id">${mi.application_id}</dc:identifier>
-    
+        <dc:date py:if="getattr(mi, 'timestamp', None) is not None">${mi.timestamp.isoformat()}</dc:date>    
         <dc:language>${mi.language if mi.language else 'UND'}</dc:language>
         <dc:type py:if="mi.category">${mi.category}</dc:type>
         <dc:description py:if="mi.comments">${mi.comments}</dc:description>

src/calibre/ebooks/metadata/opf2.py

@@ -12,6 +12,7 @@ from urllib import unquote
 from urlparse import urlparse
 
 from lxml import etree
+from dateutil import parser
 
 from calibre.ebooks.chardet import xml_to_unicode
 from calibre import relpath
@@ -436,6 +437,7 @@ class OPF(object):
     series          = MetadataField('series', is_dc=False)
     series_index    = MetadataField('series_index', is_dc=False, formatter=int, none_is=1)
     rating          = MetadataField('rating', is_dc=False, formatter=int)
+    timestamp       = MetadataField('date', formatter=parser.parse)
 
 
     def __init__(self, stream, basedir=os.getcwdu(), unquote_urls=True):

src/calibre/ebooks/mobi/reader.py

@@ -308,8 +308,11 @@ class MobiReader(object):
             if 'filepos-id' in attrib:
                 attrib['id'] = attrib.pop('filepos-id')
             if 'filepos' in attrib:
-                filepos = int(attrib.pop('filepos'))
+                filepos = attrib.pop('filepos')
-                attrib['href'] = "#filepos%d" % filepos
+                try:
+                    attrib['href'] = "#filepos%d" % int(filepos)
+                except:
+                    attrib['href'] = filepos
             if tag.tag == 'img':
                 recindex = None
                 for attr in self.IMAGE_ATTRS:

src/calibre/gui2/__init__.py

@@ -1,7 +1,7 @@
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 """ The GUI """
-import sys, os, re, StringIO, traceback
+import sys, os, re, StringIO, traceback, time
 from PyQt4.QtCore import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, QSize, \
                          QByteArray, QLocale, QUrl, QTranslator, QCoreApplication, \
                          QModelIndex
@@ -14,6 +14,9 @@ from calibre import __author__, islinux, iswindows, isosx
 from calibre.startup import get_lang
 from calibre.utils.config import Config, ConfigProxy, dynamic
 import calibre.resources as resources
+from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats
+from calibre.ebooks.metadata import MetaInformation
+
 
 NONE = QVariant() #: Null value to return from the data function of item models
 
@@ -148,7 +151,41 @@ class Dispatcher(QObject):
         
     def dispatch(self, args, kwargs):
         self.func(*args, **kwargs)
+
+class GetMetadata(QObject):
+    '''
+    Convenience class to ensure that metadata readers are used only in the
+    GUI thread. Must be instantiated in the GUI thread.
+    '''
+    
+    def __init__(self):
+        QObject.__init__(self)
+        self.connect(self, SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                     self._get_metadata, Qt.QueuedConnection)
+        self.connect(self, SIGNAL('idispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                     self._from_formats, Qt.QueuedConnection)
         
+    def __call__(self, id, *args, **kwargs):
+        self.emit(SIGNAL('edispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                  id, args, kwargs)
+    
+    def from_formats(self, id, *args, **kwargs):
+        self.emit(SIGNAL('idispatch(PyQt_PyObject, PyQt_PyObject, PyQt_PyObject)'),
+                  id, args, kwargs)
+    
+    def _from_formats(self, id, args, kwargs):
+        try:
+            mi = metadata_from_formats(*args, **kwargs)
+        except:
+            mi = MetaInformation('', [_('Unknown')])
+        self.emit(SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), id, mi)
+    
+    def _get_metadata(self, id, args, kwargs):
+        try:
+            mi = get_metadata(*args, **kwargs)
+        except:
+            mi = MetaInformation('', [_('Unknown')])
+        self.emit(SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'), id, mi)
 
 class TableView(QTableView):
     def __init__(self, parent):

src/calibre/gui2/add.py

@@ -0,0 +1,224 @@
+'''
+UI for adding books to the database
+'''
+import os
+
+from PyQt4.Qt import QThread, SIGNAL, QMutex, QWaitCondition, Qt
+
+from calibre.gui2.dialogs.progress import ProgressDialog
+from calibre.constants import preferred_encoding
+from calibre.gui2.widgets import WarningDialog
+
+class Add(QThread):
+    
+    def __init__(self):
+        QThread.__init__(self)
+        self._lock = QMutex()
+        self._waiting = QWaitCondition()
+    
+    def is_canceled(self):
+        if self.pd.canceled:
+            self.canceled = True
+        return self.canceled
+    
+    def wait_for_condition(self):
+        self._lock.lock()
+        self._waiting.wait(self._lock)
+        self._lock.unlock()
+        
+    def wake_up(self):
+        self._waiting.wakeAll()
+ 
+class AddFiles(Add):
+    
+    def __init__(self, paths, default_thumbnail, get_metadata, db=None):
+        Add.__init__(self)
+        self.paths = paths
+        self.get_metadata = get_metadata
+        self.default_thumbnail = default_thumbnail
+        self.db = db
+        self.formats, self.metadata, self.names, self.infos = [], [], [], []
+        self.duplicates = []
+        self.number_of_books_added = 0
+        self.connect(self.get_metadata, 
+                     SIGNAL('metadata(PyQt_PyObject, PyQt_PyObject)'),
+                     self.metadata_delivered)
+    
+    def metadata_delivered(self, id, mi):
+        if self.is_canceled():
+            self.reading.wakeAll()
+            return
+        if not mi.title:
+            mi.title = os.path.splitext(self.names[id])[0]
+        mi.title = mi.title if isinstance(mi.title, unicode) else \
+                   mi.title.decode(preferred_encoding, 'replace')
+        self.metadata.append(mi)
+        self.infos.append({'title':mi.title, 
+                           'authors':', '.join(mi.authors),
+                           'cover':self.default_thumbnail, 'tags':[]})
+        if self.db is not None:
+            duplicates, num = self.db.add_books(self.paths[id:id+1], 
+                                self.formats[id:id+1], [mi], 
+                                add_duplicates=False)
+            self.number_of_books_added += num
+            if duplicates:
+                if not self.duplicates:
+                    self.duplicates = [[], [], [], []]
+                for i in range(4):
+                    self.duplicates[i] += duplicates[i]
+        self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'), 
+                      mi.title, id)
+        self.wake_up()
+    
+    def create_progress_dialog(self, title, msg, parent):
+        self._parent = parent
+        self.pd = ProgressDialog(title, msg, -1, len(self.paths)-1, parent)
+        self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                     self.update_progress_dialog)
+        self.pd.setModal(True)
+        self.pd.show()
+        self.connect(self, SIGNAL('finished()'), self.pd.hide)
+        return self.pd
+        
+       
+    def update_progress_dialog(self, title, count):
+        self.pd.set_value(count)
+        if self.db is not None:
+            self.pd.set_msg(_('Added %s to library')%title)
+        else:
+            self.pd.set_msg(_('Read metadata from ')+title)
+        
+        
+    def run(self):
+        self.canceled = False
+        for c, book in enumerate(self.paths):
+            if self.pd.canceled:
+                self.canceled = True
+                break
+            format = os.path.splitext(book)[1]
+            format = format[1:] if format else None
+            stream = open(book, 'rb')
+            self.formats.append(format)
+            self.names.append(os.path.basename(book))
+            self.get_metadata(c, stream, stream_type=format, 
+                                  use_libprs_metadata=True)
+            self.wait_for_condition()
+        
+                    
+    def process_duplicates(self):
+        if self.duplicates:
+            files = _('<p>Books with the same title as the following already '
+                      'exist in the database. Add them anyway?<ul>')
+            for mi in self.duplicates[2]:
+                files += '<li>'+mi.title+'</li>\n'
+            d = WarningDialog (_('Duplicates found!'), 
+                              _('Duplicates found!'), 
+                              files+'</ul></p>', parent=self._parent)
+            if d.exec_() == d.Accepted:
+                num = self.db.add_books(*self.duplicates, 
+                                  **dict(add_duplicates=True))[1]
+                self.number_of_books_added += num
+                
+            
+class AddRecursive(Add):
+    
+    def __init__(self, path, db, get_metadata, single_book_per_directory, parent):
+        self.path = path
+        self.db = db
+        self.get_metadata = get_metadata
+        self.single_book_per_directory = single_book_per_directory
+        self.duplicates, self.books, self.metadata = [], [], []
+        self.number_of_books_added = 0
+        self.canceled = False
+        Add.__init__(self)
+        self.connect(self.get_metadata, 
+                     SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'),
+                     self.metadata_delivered, Qt.QueuedConnection)
+        self.connect(self, SIGNAL('searching_done()'), self.searching_done,
+                     Qt.QueuedConnection)    
+        self._parent = parent
+        self.pd = ProgressDialog(_('Adding books recursively...'),
+                                 _('Searching for books in all sub-directories...'), 
+                                 0, 0, parent)
+        self.connect(self, SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                     self.update_progress_dialog)
+        self.connect(self, SIGNAL('update(PyQt_PyObject)'), self.pd.set_msg,
+                     Qt.QueuedConnection)
+        self.connect(self, SIGNAL('pupdate(PyQt_PyObject)'), self.pd.set_value,
+                     Qt.QueuedConnection)
+        self.pd.setModal(True)
+        self.pd.show()
+        self.connect(self, SIGNAL('finished()'), self.pd.hide)
+        
+    def update_progress_dialog(self, title, count):
+        self.pd.set_value(count)
+        if title:
+            self.pd.set_msg(_('Read metadata from ')+title)
+    
+    def metadata_delivered(self, id, mi):
+        if self.is_canceled():
+            self.reading.wakeAll()
+            return
+        self.emit(SIGNAL('processed(PyQt_PyObject,PyQt_PyObject)'),
+                          mi.title, id)
+        self.metadata.append((mi if mi.title else None, self.books[id]))
+        if len(self.metadata) >= len(self.books):
+            self.metadata = [x for x in self.metadata if x[0] is not None]
+            self.pd.set_min(-1)
+            self.pd.set_max(len(self.metadata)-1)
+            self.pd.set_value(-1)
+            self.pd.set_msg(_('Adding books to database...'))
+        self.wake_up()
+    
+    def searching_done(self):
+        self.pd.set_min(-1)
+        self.pd.set_max(len(self.books)-1)
+        self.pd.set_value(-1)
+        self.pd.set_msg(_('Reading metadata...'))
+    
+   
+    def run(self):
+        root = os.path.abspath(self.path)
+        for dirpath in os.walk(root):
+            if self.is_canceled():
+                return
+            self.emit(SIGNAL('update(PyQt_PyObject)'), 
+                      _('Searching in')+' '+dirpath[0])
+            self.books += list(self.db.find_books_in_directory(dirpath[0], 
+                                            self.single_book_per_directory))
+        self.books = [formats for formats in self.books if formats]
+        # Reset progress bar
+        self.emit(SIGNAL('searching_done()'))
+        
+        for c, formats in enumerate(self.books):
+            self.get_metadata.from_formats(c, formats)
+            self.wait_for_condition()
+            
+        # Add books to database
+        for c, x in enumerate(self.metadata):
+            mi, formats = x
+            if self.is_canceled():
+                break
+            if self.db.has_book(mi):
+                self.duplicates.append((mi, formats))
+            else:
+                self.db.import_book(mi, formats, notify=False)
+                self.number_of_books_added += 1
+            self.emit(SIGNAL('pupdate(PyQt_PyObject)'), c)
+ 
+        
+    def process_duplicates(self):
+        if self.duplicates:
+            files = _('<p>Books with the same title as the following already '
+                      'exist in the database. Add them anyway?<ul>')
+            for mi in self.duplicates:
+                files += '<li>'+mi[0].title+'</li>\n'
+            d = WarningDialog (_('Duplicates found!'), 
+                              _('Duplicates found!'), 
+                              files+'</ul></p>', parent=self._parent)
+            if d.exec_() == d.Accepted:
+                for mi, formats in self.duplicates:
+                    self.db.import_book(mi, formats, notify=False)
+                    self.number_of_books_added += 1
+        
+        

src/calibre/gui2/dialogs/confirm_delete.py

@@ -5,7 +5,7 @@ __docformat__ = 'restructuredtext en'
 
 from calibre.gui2 import dynamic
 from calibre.gui2.dialogs.confirm_delete_ui import Ui_Dialog
-from PyQt4.Qt import QDialog, SIGNAL
+from PyQt4.Qt import QDialog, SIGNAL, Qt
 
 def _config_name(name):
     return name + '_again'
@@ -19,6 +19,7 @@ class Dialog(QDialog, Ui_Dialog):
         self.msg.setText(msg)
         self.name = name
         self.connect(self.again, SIGNAL('stateChanged(int)'), self.toggle)
+        self.buttonBox.setFocus(Qt.OtherFocusReason)
         
 
     def toggle(self, x):

src/calibre/gui2/dialogs/metadata_single.py

@@ -113,6 +113,10 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
     def set_cover(self):
         row = self.formats.currentRow()
         fmt = self.formats.item(row)
+        if fmt is None:
+            error_dialog(self, _('No format selected'),
+                    _('No format selected')).exec_()
+            return
         ext = fmt.ext.lower()
         if fmt.path is None:
             stream = self.db.format(self.row, ext, as_file=True)
@@ -121,7 +125,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog):
         try:
             mi = get_metadata(stream, ext)
         except:
-            error_dialog(self, _('Could not read metadata'), 
+            error_dialog(self, _('Could not read metadata'),
                          _('Could not read metadata from %s format')%ext).exec_()
             return
         cdata = None

src/calibre/gui2/dialogs/progress.py

@@ -20,6 +20,7 @@ class ProgressDialog(QDialog, Ui_Dialog):
         self.setWindowModality(Qt.ApplicationModal)
         self.set_min(min)
         self.set_max(max)
+        self.bar.setValue(min)
         self.canceled = False
         
         self.connect(self.button_box, SIGNAL('rejected()'), self._canceled)

src/calibre/gui2/library.py

@@ -1,3 +1,4 @@
+from calibre.ebooks.metadata import authors_to_string
 __license__   = 'GPL v3'
 __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import os, textwrap, traceback, time, re
@@ -371,35 +372,24 @@ class BooksModel(QAbstractTableModel):
         if not rows_are_ids:
             rows = [self.db.id(row.row()) for row in rows]
         for id in rows:
-            au = self.db.authors(id, index_is_id=True)
+            mi = self.db.get_metadata(id, index_is_id=True)
-            tags = self.db.tags(id, index_is_id=True)
+            au = authors_to_string(mi.authors if mi.authors else [_('Unknown')])
-            if not au:
+            tags = mi.tags if mi.tags else []
-                au = _('Unknown')
+            if mi.series is not None:
-            au = au.split(',')
+                tags.append(mi.series)
-            if len(au) > 1:
+            info = {
-                t = ', '.join(au[:-1])
+                  'title'   : mi.title,
-                t += ' & ' + au[-1]
-                au = t
-            else:
-                au = ' & '.join(au)
-            if not tags:
-                tags = []
-            else:
-                tags = tags.split(',')
-            series = self.db.series(id, index_is_id=True)
-            if series is not None:
-                tags.append(series)
-            mi = {
-                  'title'   : self.db.title(id, index_is_id=True),
                   'authors' : au,
                   'cover'   : self.db.cover(id, index_is_id=True),
                   'tags'    : tags,
-                  'comments': self.db.comments(id, index_is_id=True),
+                  'comments': mi.comments,
                   }
-            if series is not None:
+            if mi.series is not None:
-                mi['tag order'] = {series:self.db.books_in_series_of(id, index_is_id=True)}
+                info['tag order'] = {
+                    mi.series:self.db.books_in_series_of(id, index_is_id=True)
+                }
 
-            metadata.append(mi)
+            metadata.append(info)
         return metadata
 
     def get_preferred_formats_from_ids(self, ids, all_formats, mode='r+b'):

src/calibre/gui2/main.py

@@ -13,7 +13,6 @@ from PyQt4.QtSvg import QSvgRenderer
 from calibre import __version__, __appname__, islinux, sanitize_file_name, \
                     iswindows, isosx, preferred_encoding
 from calibre.ptempfile import PersistentTemporaryFile
-from calibre.ebooks.metadata.meta import get_metadata
 from calibre.devices.errors import FreeSpaceError
 from calibre.devices.interface import Device
 from calibre.utils.config import prefs, dynamic
@@ -23,7 +22,7 @@ from calibre.gui2 import APP_UID, warning_dialog, choose_files, error_dialog, \
                            set_sidebar_directories, Dispatcher, \
                            SingleApplication, Application, available_height, \
                            max_available_height, config, info_dialog, \
-                           available_width
+                           available_width, GetMetadata
 from calibre.gui2.cover_flow import CoverFlow, DatabaseImages, pictureflowerror
 from calibre.gui2.dialogs.scheduler import Scheduler
 from calibre.gui2.update import CheckForUpdates
@@ -49,7 +48,6 @@ from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.library.database2 import LibraryDatabase2, CoverCache
 from calibre.parallel import JobKilled
 from calibre.utils.filenames import ascii_filename
-from calibre.gui2.widgets import WarningDialog
 from calibre.gui2.dialogs.confirm_delete import confirm
 
 class Main(MainWindow, Ui_MainWindow):
@@ -78,6 +76,7 @@ class Main(MainWindow, Ui_MainWindow):
         self.setupUi(self)
         self.setWindowTitle(__appname__)
         self.verbose = opts.verbose
+        self.get_metadata = GetMetadata()
         self.read_settings()
         self.job_manager = JobManager()
         self.jobs_dialog = JobsDialog(self, self.job_manager)
@@ -391,7 +390,7 @@ class Main(MainWindow, Ui_MainWindow):
     def change_output_format(self, x):
         of = unicode(x).strip()
         if of != prefs['output_format']:
-            if of not in ('LRF',):
+            if of not in ('LRF', 'EPUB'):
                 warning_dialog(self, 'Warning', 
                                '<p>%s support is still in beta. If you find bugs, please report them by opening a <a href="http://calibre.kovidgoyal.net">ticket</a>.'%of).exec_()
             prefs.set('output_format', of)
@@ -608,36 +607,26 @@ class Main(MainWindow, Ui_MainWindow):
     ################################# Add books ################################
 
     def add_recursive(self, single):
-        root = choose_dir(self, 'recursive book import root dir dialog', 'Select root folder')
+        root = choose_dir(self, 'recursive book import root dir dialog', 
+                          'Select root folder')
         if not root:
             return
-        progress = ProgressDialog(_('Adding books recursively...'),
+        from calibre.gui2.add import AddRecursive
-                                   min=0, max=0, parent=self)
+        self._add_recursive_thread = AddRecursive(root, 
-        progress.show()
+                                self.library_view.model().db, self.get_metadata,
-        def callback(msg):
+                                single, self)
-            if msg != '.':
+        self.connect(self._add_recursive_thread, SIGNAL('finished()'),
-                progress.set_msg((_('Added ')+msg) if msg else _('Searching...'))
+                     self._recursive_files_added)
-            QApplication.processEvents()
+        self._add_recursive_thread.start()
-            QApplication.sendPostedEvents()
+    
-            QApplication.flush()
+    def _recursive_files_added(self):
-            return progress.canceled
+        self._add_recursive_thread.process_duplicates()
-        try:
+        if self._add_recursive_thread.number_of_books_added > 0:
-            duplicates = self.library_view.model().db.recursive_import(root, single, callback=callback)
+            self.library_view.model().resort(reset=False)
-        finally:
+            self.library_view.model().research()
-            progress.hide()            
+            self.library_view.model().count_changed()
-        if duplicates:
+        self._add_recursive_thread = None
-            files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
+        
-            for mi, formats in duplicates:
-                files += '<li>'+mi.title+'</li>\n'
-            d = WarningDialog(_('Duplicates found!'), _('Duplicates found!'), 
-                              files+'</ul></p>', self)
-            if d.exec_() == QDialog.Accepted:
-                for mi, formats in duplicates:
-                    self.library_view.model().db.import_book(mi, formats )
-
-        self.library_view.model().resort()
-        self.library_view.model().research()
-
     def add_recursive_single(self, checked):
         '''
         Add books from the local filesystem to either the library or the device
@@ -686,66 +675,41 @@ class Main(MainWindow, Ui_MainWindow):
             return
         to_device = self.stack.currentIndex() != 0
         self._add_books(books, to_device)
-        if to_device:
+        
-            self.status_bar.showMessage(_('Uploading books to device.'), 2000)
 
     def _add_books(self, paths, to_device, on_card=None):
         if on_card is None:
             on_card = self.stack.currentIndex() == 2
         if not paths:
             return
-        # Get format and metadata information
+        from calibre.gui2.add import AddFiles
-        formats, metadata, names, infos = [], [], [], []
+        self._add_files_thread = AddFiles(paths, self.default_thumbnail, 
-        progress = ProgressDialog(_('Adding books...'), _('Reading metadata...'),
+                                          self.get_metadata,
-                                  min=0, max=len(paths), parent=self)
+                                          None if to_device else \
-        progress.show()
+                                          self.library_view.model().db
-        try:
+                                          )
-            for c, book in enumerate(paths):
+        self._add_files_thread.send_to_device = to_device
-                progress.set_value(c+1)
+        self._add_files_thread.on_card = on_card
-                if progress.canceled:
+        self._add_files_thread.create_progress_dialog(_('Adding books...'),
-                    return
+                                                _('Reading metadata...'), self)
-                format = os.path.splitext(book)[1]
+        self.connect(self._add_files_thread, SIGNAL('finished()'),
-                format = format[1:] if format else None
+                     self._files_added)
-                stream = open(book, 'rb')
+        self._add_files_thread.start()
-                try:
+    
-                    mi = get_metadata(stream, stream_type=format, use_libprs_metadata=True)
+    def _files_added(self):
-                except:
+        t = self._add_files_thread
-                    mi = MetaInformation(None, None)
+        self._add_files_thread = None
-                if not mi.title:
+        if not t.canceled:
-                    mi.title = os.path.splitext(os.path.basename(book))[0]
+            if t.send_to_device:
-                if not mi.authors:
+                self.upload_books(t.paths, 
-                    mi.authors = [_('Unknown')]
+                                  list(map(sanitize_file_name, t.names)), 
-                formats.append(format)
+                                  t.infos, on_card=t.on_card)
-                metadata.append(mi)
+                self.status_bar.showMessage(_('Uploading books to device.'), 2000)
-                names.append(os.path.basename(book))
-                infos.append({'title':mi.title, 'authors':', '.join(mi.authors),
-                              'cover':self.default_thumbnail, 'tags':[]})
-                title = mi.title if isinstance(mi.title, unicode) else mi.title.decode(preferred_encoding, 'replace')
-                progress.set_msg(_('Read metadata from ')+title)
-                QApplication.processEvents()
-            
-            if not to_device:
-                progress.set_msg(_('Adding books to database...'))
-                QApplication.processEvents()
-                model = self.library_view.model()
-                
-                paths = list(paths)
-                duplicates, number_added = model.add_books(paths, formats, metadata)
-                if duplicates:
-                    files = _('<p>Books with the same title as the following already exist in the database. Add them anyway?<ul>')
-                    for mi in duplicates[2]:
-                        files += '<li>'+mi.title+'</li>\n'
-                    d = WarningDialog(_('Duplicates found!'), _('Duplicates found!'), files+'</ul></p>', parent=self)
-                    if d.exec_() == QDialog.Accepted:
-                        num = model.add_books(*duplicates, **dict(add_duplicates=True))[1]
-                        number_added += num
-                model.books_added(number_added)
             else:
-                self.upload_books(paths, list(map(sanitize_file_name, names)), infos, on_card=on_card)
+                t.process_duplicates()
-        finally:
+        if t.number_of_books_added > 0:
-            QApplication.processEvents()
+            self.library_view.model().books_added(t.number_of_books_added)
-            progress.hide()
+        
-
     def upload_books(self, files, names, metadata, on_card=False, memory=None):
         '''
         Upload books to device.
@@ -801,7 +765,10 @@ class Main(MainWindow, Ui_MainWindow):
         if not rows or len(rows) == 0:
             return
         if self.stack.currentIndex() == 0:
-            if not confirm('<p>'+_('The selected books will be <b>permanently deleted</b> and the files removed from your computer. Are you sure?')+'</p>', 'library_delete_books', self):
+            if not confirm('<p>'+_('The selected books will be '
+                                   '<b>permanently deleted</b> and the files '
+                                   'removed from your computer. Are you sure?')
+                                +'</p>', 'library_delete_books', self):
                 return
             view.model().delete_books(rows)
         else:
@@ -1410,8 +1377,15 @@ class Main(MainWindow, Ui_MainWindow):
     def initialize_database(self):
         self.library_path = prefs['library_path']
         if self.library_path is None: # Need to migrate to new database layout
+            base = os.path.expanduser('~')
+            if iswindows:
+                from calibre import plugins
+                from PyQt4.Qt import QDir
+                base = plugins['winutil'][0].special_folder_path(plugins['winutil'][0].CSIDL_PERSONAL)
+                if not base or not os.path.exists(base):
+                    base = unicode(QDir.homePath()).replace('/', os.sep)
             dir = unicode(QFileDialog.getExistingDirectory(self, 
-                            _('Choose a location for your ebook library.'), os.getcwd()))
+                            _('Choose a location for your ebook library.'), base))
             if not dir:
                 dir = os.path.expanduser('~/Library')
             self.library_path = os.path.abspath(dir)
@@ -1597,6 +1571,11 @@ def main(args=sys.argv):
             print 'Restarting with:', e, sys.argv
             os.execvp(e, sys.argv)
         else:
+            if iswindows:
+                try:
+                    main.system_tray_icon.hide()
+                except:
+                    pass
             return ret
     return 0
 

src/calibre/gui2/viewer/documentview.py

@@ -8,8 +8,7 @@ __docformat__ = 'restructuredtext en'
 import os, math, re
 from PyQt4.Qt import QWidget, QSize, QSizePolicy, QUrl, SIGNAL, Qt, QTimer, \
                      QPainter, QPalette, QBrush, QFontDatabase, QDialog, \
-                     QByteArray, QColor, QWheelEvent, QPoint, QImage, QRegion, \
+                     QByteArray, QColor, QWheelEvent, QPoint, QImage, QRegion, QFont
-                     QFont, QPrinter, QPrintPreviewDialog, QPrintDialog 
 from PyQt4.QtWebKit import QWebPage, QWebView, QWebSettings
 
 from calibre.utils.config import Config, StringConfig
@@ -310,61 +309,6 @@ class DocumentView(QWebView):
         
     def goto_bookmark(self, bm):
         self.document.goto_bookmark(bm)
-
-    def all_content(self):
-        book_content = ''
-        
-        if self.manager is not None:
-            for path in self.manager.iterator.spine:
-                html = open(path, 'rb').read().decode(path.encoding)
-                book_content += EntityDeclarationProcessor(html).processed_html
-            base_url = QUrl.fromLocalFile(self.manager.iterator.spine[0])
-        else:
-            book_content = self.page().mainFrame().toHtml()
-            base_url = QUrl.fromLocalFile(self.path())
-            
-        return (book_content, base_url)
-
-    def print_preview(self):        
-        print_view = QWebView()
-        book_content, base_url = self.all_content()
-        print_view.setHtml(book_content, base_url)
-        print_view.setTextSizeMultiplier(self.textSizeMultiplier())
-
-        def finished(ok):
-            printer = QPrinter(QPrinter.HighResolution)
-            printer.setPageMargins(1, 1, 1, 1, QPrinter.Inch)
-            
-            previewDialog = QPrintPreviewDialog(printer, self)
-            
-            self.connect(previewDialog, SIGNAL('paintRequested(QPrinter *)'), print_view.print_)
-            previewDialog.exec_()
-            self.disconnect(previewDialog, SIGNAL('paintRequested(QPrinter *)'), print_view.print_)
-
-            self.disconnect(print_view, SIGNAL('loadFinished(bool)'), finished)
-
-        self.connect(print_view, SIGNAL('loadFinished(bool)'), finished)
-        
-    def print_book(self):        
-        print_view = QWebView()
-        book_content, base_url = self.all_content()
-        print_view.setHtml(book_content, base_url)
-        print_view.setTextSizeMultiplier(self.textSizeMultiplier())
-
-        def finished(ok):
-            printer = QPrinter(QPrinter.HighResolution)
-            printer.setPageMargins(1, 1, 1, 1, QPrinter.Inch)
-            
-            printDialog = QPrintDialog(printer, self)
-            printDialog.setWindowTitle(_("Print eBook"))
-            
-            printDialog.exec_()
-            if printDialog.result() == QDialog.Accepted:
-                print_view.print_(printer)
-                
-            self.disconnect(print_view, SIGNAL('loadFinished(bool)'), finished)
-
-        self.connect(print_view, SIGNAL('loadFinished(bool)'), finished)
     
     def config(self, parent=None):
         self.document.do_config(parent)
@@ -612,4 +556,4 @@ class DocumentView(QWebView):
             self.manager.scrolled(self.scroll_fraction)
         return ret
 
-    
+    

src/calibre/gui2/viewer/main.py

@@ -248,8 +248,6 @@ class EbookViewer(MainWindow, Ui_EbookViewer):
         self.connect(self.action_back, SIGNAL('triggered(bool)'), self.back)
         self.connect(self.action_bookmark, SIGNAL('triggered(bool)'), self.bookmark)
         self.connect(self.action_forward, SIGNAL('triggered(bool)'), self.forward)
-        self.connect(self.action_print_preview, SIGNAL('triggered()'), self.view.print_preview)
-        self.connect(self.action_print, SIGNAL('triggered()'), self.view.print_book)
         self.connect(self.action_preferences, SIGNAL('triggered(bool)'), lambda x: self.view.config(self))
         self.connect(self.pos, SIGNAL('valueChanged(double)'), self.goto_page)
         self.connect(self.vertical_scrollbar, SIGNAL('valueChanged(int)'), 

src/calibre/gui2/viewer/main.ui

@@ -27,8 +27,8 @@
       <widget class="QWidget" name="layoutWidget" >
        <layout class="QGridLayout" name="gridLayout" >
         <item row="0" column="0" >
-         <widget class="QWebView" name="view" >
+         <widget class="QWebView" native="1" name="view" >
-          <property name="url" >
+          <property name="url" stdset="0" >
            <url>
             <string>about:blank</string>
            </url>
@@ -87,9 +87,6 @@
    <addaction name="action_bookmark" />
    <addaction name="action_reference_mode" />
    <addaction name="separator" />
-   <addaction name="action_print_preview" />
-   <addaction name="action_print" />
-   <addaction name="separator" />
    <addaction name="action_preferences" />
    <addaction name="action_full_screen" />
   </widget>
@@ -237,24 +234,6 @@
     <string>Toggle full screen</string>
    </property>
   </action>
-  <action name="action_print" >
-   <property name="icon" >
-    <iconset resource="../images.qrc" >
-     <normaloff>:/images/document-print.svg</normaloff>:/images/document-print.svg</iconset>
-   </property>
-   <property name="text" >
-    <string>Print</string>
-   </property>
-  </action>
-  <action name="action_print_preview" >
-   <property name="icon" >
-    <iconset resource="../images.qrc" >
-     <normaloff>:/images/document-print-preview.svg</normaloff>:/images/document-print-preview.svg</iconset>
-   </property>
-   <property name="text" >
-    <string>Print Preview</string>
-   </property>
-  </action>
  </widget>
  <customwidgets>
   <customwidget>

src/calibre/library/database.py

@@ -4,13 +4,10 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Backend that implements storage of ebooks in an sqlite database.
 '''
 import sqlite3 as sqlite
-import datetime, re, os, cPickle, sre_constants
+import datetime, re, cPickle, sre_constants
 from zlib import compress, decompress
 
-from calibre import sanitize_file_name
-from calibre.ebooks.metadata.meta import set_metadata, metadata_from_formats
 from calibre.ebooks.metadata import MetaInformation
-from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.web.feeds.recipes import migrate_automatic_profile_to_automatic_recipe
 
 class Concatenate(object):
@@ -1391,117 +1388,12 @@ ALTER TABLE books ADD COLUMN isbn TEXT DEFAULT "" COLLATE NOCASE;
                          
 
 
-    def import_book(self, mi, formats):
+    
-        series_index = 1 if mi.series_index is None else mi.series_index
-        if not mi.authors:
-            mi.authors = [_('Unknown')]
-        aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
-        obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)',
-                          (mi.title, None, series_index, aus))
-        id = obj.lastrowid
-        self.conn.commit()
-        self.set_metadata(id, mi)
-        for path in formats:
-            ext = os.path.splitext(path)[1][1:].lower()
-            stream = open(path, 'rb')
-            stream.seek(0, 2)
-            usize = stream.tell()
-            stream.seek(0)
-            data = sqlite.Binary(compress(stream.read()))
-            try:
-                self.conn.execute('INSERT INTO data(book, format, uncompressed_size, data) VALUES (?,?,?,?)',
-                              (id, ext, usize, data))
-            except sqlite.IntegrityError:
-                self.conn.execute('UPDATE data SET uncompressed_size=?, data=? WHERE book=? AND format=?',
-                                  (usize, data, id, ext))
-        self.conn.commit()
-
-    def import_book_directory_multiple(self, dirpath, callback=None):
-        dirpath = os.path.abspath(dirpath)
-        duplicates = []
-        books = {}
-        for path in os.listdir(dirpath):
-            if callable(callback):
-                callback('.')
-            path = os.path.abspath(os.path.join(dirpath, path))
-            if os.path.isdir(path) or not os.access(path, os.R_OK):
-                continue
-            ext = os.path.splitext(path)[1]
-            if not ext:
-                continue
-            ext = ext[1:].lower()
-            if ext not in BOOK_EXTENSIONS:
-                continue
-
-            key = os.path.splitext(path)[0]
-            if not books.has_key(key):
-                books[key] = []
-
-            books[key].append(path)
-
-        for formats in books.values():
-            mi = metadata_from_formats(formats)
-            if mi.title is None:
-                continue
-            if self.has_book(mi):
-                duplicates.append((mi, formats))
-                continue
-            self.import_book(mi, formats)
-            if callable(callback):
-                if callback(mi.title):
-                    break
-        return duplicates
-
-
-    def import_book_directory(self, dirpath, callback=None):
-        dirpath = os.path.abspath(dirpath)
-        formats = []
-        for path in os.listdir(dirpath):
-            if callable(callback):
-                callback('.')
-            path = os.path.abspath(os.path.join(dirpath, path))
-            if os.path.isdir(path) or not os.access(path, os.R_OK):
-                continue
-            ext = os.path.splitext(path)[1]
-            if not ext:
-                continue
-            ext = ext[1:].lower()
-            if ext not in BOOK_EXTENSIONS:
-                continue
-            formats.append(path)
-
-        if not formats:
-            return
-        
-        mi = metadata_from_formats(formats)
-        if mi.title is None:
-            return
-        if self.has_book(mi):
-            return [(mi, formats)]
-        self.import_book(mi, formats)
-        if callable(callback):
-            callback(mi.title)
-            
-
 
     def has_id(self, id):
         return self.conn.get('SELECT id FROM books where id=?', (id,), all=False) is not None
 
-    def recursive_import(self, root, single_book_per_directory=True, callback=None):
+    
-        root = os.path.abspath(root)
-        duplicates  = []
-        for dirpath in os.walk(root):
-            res = self.import_book_directory(dirpath[0], callback=callback) if \
-                single_book_per_directory else \
-                  self.import_book_directory_multiple(dirpath[0], callback=callback)
-            if res is not None:
-                duplicates.extend(res)
-            if callable(callback):
-                if callback(''):
-                    break
-            
-        return duplicates
-
     
 
 class SearchToken(object):

src/calibre/library/database2.py

@@ -12,20 +12,24 @@ from itertools import repeat
 from datetime import datetime
 
 from PyQt4.QtCore import QCoreApplication, QThread, QReadWriteLock
-from PyQt4.QtGui import QApplication, QPixmap, QImage
+from PyQt4.QtGui import QApplication, QImage
 __app = None
 
 from calibre.library import title_sort
 from calibre.library.database import LibraryDatabase
 from calibre.library.sqlite import connect, IntegrityError
 from calibre.utils.search_query_parser import SearchQueryParser
-from calibre.ebooks.metadata import string_to_authors, authors_to_string, MetaInformation
+from calibre.ebooks.metadata import string_to_authors, authors_to_string, \
-from calibre.ebooks.metadata.meta import get_metadata, set_metadata
+                                    MetaInformation, authors_to_sort_string
+from calibre.ebooks.metadata.meta import get_metadata, set_metadata, \
+    metadata_from_formats
 from calibre.ebooks.metadata.opf2 import OPFCreator
 from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.customize.ui import run_plugins_on_import
+
 from calibre import sanitize_file_name
+from calibre.ebooks import BOOK_EXTENSIONS
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
 
@@ -377,8 +381,10 @@ class LibraryDatabase2(LibraryDatabase):
             return row[loc]
         
         for prop in ('author_sort', 'authors', 'comment', 'comments', 'isbn', 
-                     'publisher', 'rating', 'series', 'series_index', 'tags', 'title'):
+                     'publisher', 'rating', 'series', 'series_index', 'tags', 
-            setattr(self, prop, functools.partial(get_property, loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
+                     'title', 'timestamp'):
+            setattr(self, prop, functools.partial(get_property, 
+                    loc=FIELD_MAP['comments' if prop == 'comment' else prop]))
         
     def initialize_database(self):
         from calibre.resources import metadata_sqlite
@@ -587,6 +593,7 @@ class LibraryDatabase2(LibraryDatabase):
         mi.author_sort = self.author_sort(idx, index_is_id=index_is_id)
         mi.comments    = self.comments(idx, index_is_id=index_is_id)
         mi.publisher   = self.publisher(idx, index_is_id=index_is_id)
+        mi.timestamp   = self.timestamp(idx, index_is_id=index_is_id)
         tags = self.tags(idx, index_is_id=index_is_id)
         if tags:
             mi.tags = [i.strip() for i in tags.split(',')]
@@ -627,7 +634,7 @@ class LibraryDatabase2(LibraryDatabase):
             if not QCoreApplication.instance():
                 global __app
                 __app = QApplication([])
-            p = QPixmap()
+            p = QImage()
             if callable(getattr(data, 'read', None)):
                 data = data.read()
             p.loadFromData(data)
@@ -881,6 +888,8 @@ class LibraryDatabase2(LibraryDatabase):
             self.set_isbn(id, mi.isbn, notify=False)
         if mi.series_index and mi.series_index > 0:
             self.set_series_index(id, mi.series_index, notify=False)
+        if getattr(mi, 'timestamp', None) is not None:
+            self.set_timestamp(id, mi.timestamp, notify=False)
         self.set_path(id, True)
         self.notify('metadata', [id])
         
@@ -1142,7 +1151,7 @@ class LibraryDatabase2(LibraryDatabase):
     def add_books(self, paths, formats, metadata, uris=[], add_duplicates=True):
         '''
         Add a book to the database. The result cache is not updated.
-        @param paths: List of paths to book files or file-like objects
+        :param:`paths` List of paths to book files or file-like objects
         '''
         formats, metadata, uris = iter(formats), iter(metadata), iter(uris)
         duplicates = []
@@ -1180,31 +1189,40 @@ class LibraryDatabase2(LibraryDatabase):
         self.conn.commit()
         self.data.refresh_ids(self.conn, ids) # Needed to update format list and size
         if duplicates:
-            paths    = tuple(duplicate[0] for duplicate in duplicates)
+            paths    = list(duplicate[0] for duplicate in duplicates)
-            formats  = tuple(duplicate[1] for duplicate in duplicates)
+            formats  = list(duplicate[1] for duplicate in duplicates)
-            metadata = tuple(duplicate[2] for duplicate in duplicates)
+            metadata = list(duplicate[2] for duplicate in duplicates)
-            uris     = tuple(duplicate[3] for duplicate in duplicates)
+            uris     = list(duplicate[3] for duplicate in duplicates)
             return (paths, formats, metadata, uris), len(ids)
         return None, len(ids)
      
-    def import_book(self, mi, formats):
+    def import_book(self, mi, formats, notify=True):
         series_index = 1 if mi.series_index is None else mi.series_index
+        if not mi.title:
+            mi.title = _('Unknown')
         if not mi.authors:
-            mi.authors = ['Unknown']
+            mi.authors = [_('Unknown')]
-        aus = mi.author_sort if mi.author_sort else ', '.join(mi.authors)
+        aus = mi.author_sort if mi.author_sort else authors_to_sort_string(mi.authors)
+        if isinstance(aus, str):
+            aus = aus.decode(preferred_encoding, 'replace')
+        title = mi.title if isinstance(mi.title, unicode) else \
+                mi.title.decode(preferred_encoding, 'replace')
         obj = self.conn.execute('INSERT INTO books(title, uri, series_index, author_sort) VALUES (?, ?, ?, ?)', 
-                          (mi.title, None, series_index, aus))
+                          (title, None, series_index, aus))
         id = obj.lastrowid
         self.data.books_added([id], self.conn)
         self.set_path(id, True)
         self.set_metadata(id, mi)
         for path in formats:
             ext = os.path.splitext(path)[1][1:].lower()
+            if ext == 'opf':
+                continue
             stream = open(path, 'rb')
             self.add_format(id, ext, stream, index_is_id=True)
         self.conn.commit()
         self.data.refresh_ids(self.conn, [id]) # Needed to update format list and size
-        self.notify('add', [id])
+        if notify:
+            self.notify('add', [id])
         
     def move_library_to(self, newloc, progress=None):
         header = _(u'<p>Copying books to %s<br><center>')%newloc
@@ -1388,6 +1406,11 @@ books_series_link      feeds
                 f = open(os.path.join(base, sanitize_file_name(name)+'.opf'), 'wb')
                 if not mi.authors:
                     mi.authors = [_('Unknown')]
+                cdata = self.cover(int(id), index_is_id=True)
+                if cdata is not None:
+                    cname = sanitize_file_name(name)+'.jpg'
+                    open(os.path.join(base, cname), 'wb').write(cdata)
+                    mi.cover = cname
                 opf = OPFCreator(base, mi)
                 opf.render(f)
                 f.close()
@@ -1451,6 +1474,88 @@ books_series_link      feeds
                 if not callback(count, title):
                     break
         return failures
+    
+    def find_books_in_directory(self, dirpath, single_book_per_directory):
+        dirpath = os.path.abspath(dirpath)
+        if single_book_per_directory:
+            formats = []
+            for path in os.listdir(dirpath):
+                path = os.path.abspath(os.path.join(dirpath, path))
+                if os.path.isdir(path) or not os.access(path, os.R_OK):
+                    continue
+                ext = os.path.splitext(path)[1]
+                if not ext:
+                    continue
+                ext = ext[1:].lower()
+                if ext not in BOOK_EXTENSIONS and ext != 'opf':
+                    continue
+                formats.append(path)
+            yield formats
+        else:
+            books = {}
+            for path in os.listdir(dirpath):
+                path = os.path.abspath(os.path.join(dirpath, path))
+                if os.path.isdir(path) or not os.access(path, os.R_OK):
+                    continue
+                ext = os.path.splitext(path)[1]
+                if not ext:
+                    continue
+                ext = ext[1:].lower()
+                if ext not in BOOK_EXTENSIONS:
+                    continue
+    
+                key = os.path.splitext(path)[0]
+                if not books.has_key(key):
+                    books[key] = []
+                books[key].append(path)
+            
+            for formats in books.values():
+                yield formats
+
+    def import_book_directory_multiple(self, dirpath, callback=None):
+        duplicates = []
+        for formats in self.find_books_in_directory(dirpath, False):
+            mi = metadata_from_formats(formats)
+            if mi.title is None:
+                continue
+            if self.has_book(mi):
+                duplicates.append((mi, formats))
+                continue
+            self.import_book(mi, formats)
+            if callable(callback):
+                if callback(mi.title):
+                    break
+        return duplicates
+
+    def import_book_directory(self, dirpath, callback=None):
+        dirpath = os.path.abspath(dirpath)
+        formats = self.find_books_in_directory(dirpath, True)
+        if not formats:
+            return
+        
+        mi = metadata_from_formats(formats)
+        if mi.title is None:
+            return
+        if self.has_book(mi):
+            return [(mi, formats)]
+        self.import_book(mi, formats)
+        if callable(callback):
+            callback(mi.title)
+            
+    def recursive_import(self, root, single_book_per_directory=True, callback=None):
+        root = os.path.abspath(root)
+        duplicates  = []
+        for dirpath in os.walk(root):
+            res = self.import_book_directory(dirpath[0], callback=callback) if \
+                single_book_per_directory else \
+                  self.import_book_directory_multiple(dirpath[0], callback=callback)
+            if res is not None:
+                duplicates.extend(res)
+            if callable(callback):
+                if callback(''):
+                    break
+            
+        return duplicates
 
 
         

src/calibre/library/sqlite.py

@@ -12,11 +12,50 @@ from sqlite3 import IntegrityError
 from threading import Thread
 from Queue import Queue
 from threading import RLock
+from datetime import tzinfo, datetime, timedelta
 
 from calibre.library import title_sort
 
 global_lock = RLock()
 
+def convert_timestamp(val):
+    datepart, timepart = val.split(' ')
+    tz, mult = None, 1
+    x = timepart.split('+')
+    if len(x) > 1:
+        timepart, tz = x
+    else:
+        x = timepart.split('-')
+        if len(x) > 1:
+            timepart, tz = x
+            mult = -1
+
+    year, month, day = map(int, datepart.split("-"))
+    timepart_full = timepart.split(".")
+    hours, minutes, seconds = map(int, timepart_full[0].split(":"))
+    if len(timepart_full) == 2:
+        microseconds = int(timepart_full[1])
+    else:
+        microseconds = 0
+    if tz is not None:
+        h, m = map(int, tz.split(':'))
+        delta = timedelta(minutes=mult*(60*h + m))
+        tz = type('CustomTZ', (tzinfo,), {'utcoffset':lambda self, dt:delta, 
+                                          'dst':lambda self,dt:timedelta(0)})()
+
+    val = datetime(year, month, day, hours, minutes, seconds, microseconds, 
+                   tzinfo=tz)
+    if tz is not None:
+        val = datetime(*(val.utctimetuple()[:6]))
+    return val
+
+def adapt_datetime(dt):
+    dt = datetime(*(dt.utctimetuple()[:6]))
+    return dt.isoformat(' ')
+
+sqlite.register_adapter(datetime, adapt_datetime)
+sqlite.register_converter('timestamp', convert_timestamp)
+
 class Concatenate(object):
     '''String concatenation aggregator for sqlite'''
     def __init__(self, sep=','):

src/calibre/manual/templates/layout.html

@@ -0,0 +1,12 @@
+{% extends "!layout.html" %}
+{% block sidebarlogo %}
+{{ super() }}
+         <form action="https://www.paypal.com/cgi-bin/webscr" method="post">
+               <input type="hidden" name="cmd" value="_s-xclick" />
+               <input type="hidden" name="hosted_button_id" value="3028915" />
+               <input type="image" src="https://www.paypal.com/en_US/i/btn/btn_donate_LG.gif" border="0" name="submit" alt="Donate to support calibre development" style="border:0pt" />
+               <img alt="" border="0" src="https://www.paypal.com/en_US/i/scr/pixel.gif" width="1" height="1"/>
+           </form>
+           <hr/>
+{% endblock %}
+

src/calibre/trac/plugins/download.py

@@ -35,6 +35,7 @@ class Distribution(object):
         ('xdg-utils', '1.0.2', 'xdg-utils', 'xdg-utils', 'xdg-utils'),
         ('dbus-python', '0.82.2', 'dbus-python', 'python-dbus', 'dbus-python'),
         ('lxml', '2.0.5', 'lxml', 'python-lxml', 'python-lxml'),
+        ('python-dateutil', '1.4.1', 'python-dateutil', 'python-dateutil', 'python-dateutil'),
         ('BeautifulSoup', '3.0.5', 'beautifulsoup', 'python-beautifulsoup', 'python-BeautifulSoup'),
         ('help2man', '1.36.4', 'help2man', 'help2man', 'help2man'),
         ]

src/calibre/web/feeds/recipes/__init__.py

@@ -28,6 +28,7 @@ recipe_modules = ['recipe_' + r for r in (
            'la_tercera', 'el_mercurio_chile', 'la_cuarta', 'lanacion_chile', 'la_segunda',
            'jb_online', 'estadao', 'o_globo', 'vijesti', 'elmundo', 'the_oz',
            'honoluluadvertiser', 'starbulletin', 'exiled', 'indy_star', 'dna',
+           'pobjeda',
           )]
 
 import re, imp, inspect, time, os

src/calibre/web/feeds/recipes/recipe_pobjeda.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env  python
+
+__license__   = 'GPL v3'
+__copyright__ = '2009, Darko Miletic <darko.miletic at gmail.com>'
+
+'''
+pobjeda.co.me
+'''
+
+import re
+from calibre import strftime
+from calibre.web.feeds.news import BasicNewsRecipe
+
+class Pobjeda(BasicNewsRecipe):
+    title                 = 'Pobjeda Online'
+    __author__            = 'Darko Miletic'
+    description           = 'News from Montenegro'
+    publisher             = 'Pobjeda a.d.'
+    category              = 'news, politics, Montenegro'    
+    language              = _('Serbian')
+    oldest_article        = 2
+    max_articles_per_feed = 100
+    no_stylesheets        = True
+    remove_javascript     = True
+    encoding              = 'utf8'
+    remove_javascript     = True
+    use_embedded_content  = False
+    INDEX                 = u'http://www.pobjeda.co.me'
+    extra_css = '@font-face {font-family: "serif1";src:url(res:///opt/sony/ebook/FONT/tt0011m_.ttf)} body{text-align: justify; font-family: serif1, serif} .article_description{font-family: serif1, serif}'
+    
+    html2lrf_options = [
+                          '--comment', description
+                        , '--category', category
+                        , '--publisher', publisher
+                        ]
+    
+    html2epub_options = 'publisher="' + publisher + '"\ncomments="' + description + '"\ntags="' + category + '"' 
+     
+    preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')]
+
+    keep_only_tags = [dict(name='div', attrs={'class':'vijest'})]
+
+    remove_tags = [dict(name=['object','link'])]
+
+    feeds = [
+               (u'Politika'          , u'http://www.pobjeda.co.me/rubrika.php?rubrika=1'  )
+              ,(u'Ekonomija'         , u'http://www.pobjeda.co.me/rubrika.php?rubrika=2'  )
+              ,(u'Drustvo'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=3'  )
+              ,(u'Crna Hronika'      , u'http://www.pobjeda.co.me/rubrika.php?rubrika=4'  )
+              ,(u'Kultura'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=5'  )
+              ,(u'Hronika Podgorice' , u'http://www.pobjeda.co.me/rubrika.php?rubrika=7'  )
+              ,(u'Feljton'           , u'http://www.pobjeda.co.me/rubrika.php?rubrika=8'  )
+              ,(u'Crna Gora'         , u'http://www.pobjeda.co.me/rubrika.php?rubrika=9'  )
+              ,(u'Svijet'            , u'http://www.pobjeda.co.me/rubrika.php?rubrika=202')
+              ,(u'Ekonomija i Biznis', u'http://www.pobjeda.co.me/dodatak.php?rubrika=11' )
+              ,(u'Djeciji Svijet'    , u'http://www.pobjeda.co.me/dodatak.php?rubrika=12' )
+              ,(u'Kultura i Drustvo' , u'http://www.pobjeda.co.me/dodatak.php?rubrika=13' )
+              ,(u'Agora'             , u'http://www.pobjeda.co.me/dodatak.php?rubrika=133')
+              ,(u'Ekologija'         , u'http://www.pobjeda.co.me/dodatak.php?rubrika=252')
+            ]
+
+    def preprocess_html(self, soup):
+        soup.html['xml:lang'] = 'sr-Latn-ME'
+        soup.html['lang']     = 'sr-Latn-ME'
+        mtag = '<meta http-equiv="Content-Language" content="sr-Latn-ME"/>'
+        soup.head.insert(0,mtag)
+        for item in soup.findAll(style=True):
+            del item['style']
+        return soup
+
+    def get_cover_url(self):
+        cover_url = None
+        soup = self.index_to_soup(self.INDEX)
+        cover_item = soup.find('img',attrs={'alt':'Naslovna strana'})
+        if cover_item:
+           cover_url = self.INDEX + cover_item.parent['href']
+        return cover_url
+    
+    def parse_index(self):
+        totalfeeds = []
+        lfeeds = self.get_feeds()
+        for feedobj in lfeeds:
+            feedtitle, feedurl = feedobj
+            self.report_progress(0, _('Fetching feed')+' %s...'%(feedtitle if feedtitle else feedurl))
+            articles = []
+            soup = self.index_to_soup(feedurl)
+            for item in soup.findAll('div', attrs={'class':'vijest'}):
+                description = self.tag_to_string(item.h2)
+                atag = item.h1.find('a')
+                if atag:
+                    url         = self.INDEX + '/' + atag['href']
+                    title       = self.tag_to_string(atag)
+                    date        = strftime(self.timefmt)
+                    articles.append({
+                                      'title'      :title
+                                     ,'date'       :date
+                                     ,'url'        :url
+                                     ,'description':description
+                                    })
+            totalfeeds.append((feedtitle, articles))
+        return totalfeeds
+        

src/calibre/web/feeds/recipes/recipe_politika.py

@@ -16,6 +16,7 @@ class Politika(BasicNewsRecipe):
     publisher             = 'Politika novine i Magazini d.o.o'
     category              = 'news, politics, Serbia'            
     oldest_article        = 2
+    language              = _('Serbian')
     max_articles_per_feed = 100
     no_stylesheets        = True
     use_embedded_content  = False