Try to ensure that all file I/O on library files happes in the LibraryDatabase2 class. This is in preparation for the new multi-threaded db backend. Also: Content server now sends the Content-Disposition header when sending ebook files. After uploading books to the device, delete the associated temp files. If the user tries to drag and drop more than 25 books out of calibre, only the first 25 will be sent.

src/calibre/db/backend.py

@@ -222,7 +222,9 @@ class DB(object, SchemaUpgrade):
         if self.user_version == 0:
             self.initialize_database()
 
-        SchemaUpgrade.__init__(self)
+        with self.conn:
+            SchemaUpgrade.__init__(self)
+
         # Guarantee that the library_id is set
         self.library_id
 

src/calibre/db/errors.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
+from __future__ import (unicode_literals, division, absolute_import,
+                        print_function)
+
+__license__   = 'GPL v3'
+__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
+__docformat__ = 'restructuredtext en'
+
+
+class NoSuchFormat(ValueError):
+    pass
+

src/calibre/devices/apple/driver.py

@@ -107,6 +107,7 @@ class DriverBase(DeviceConfig, DevicePlugin):
     # Needed for config_widget to work
     FORMATS = ['epub', 'pdf']
     USER_CAN_ADD_NEW_FORMATS = False
+    KEEP_TEMP_FILES_AFTER_UPLOAD = True
 
     # Hide the standard customization widgets
     SUPPORTS_SUB_DIRS = False

src/calibre/devices/interface.py

@@ -327,12 +327,7 @@ class DevicePlugin(Plugin):
         free space on the device. The text of the FreeSpaceError must contain the
         word "card" if ``on_card`` is not None otherwise it must contain the word "memory".
 
-        :param files: A list of paths and/or file-like objects. If they are paths and
+        :param files: A list of paths
-                      the paths point to temporary files, they may have an additional
-                      attribute, original_file_path pointing to the originals. They may have
-                      another optional attribute, deleted_after_upload which if True means
-                      that the file pointed to by original_file_path will be deleted after
-                      being uploaded to the device.
         :param names: A list of file names that the books should have
                       once uploaded to the device. len(names) == len(files)
         :param metadata: If not None, it is a list of :class:`Metadata` objects.

src/calibre/ebooks/metadata/worker.py

@@ -15,7 +15,7 @@ from calibre.utils.ipc.server import Server
 from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory
 from calibre import prints, isbytestring
 from calibre.constants import filesystem_encoding
-
+from calibre.db.errors import NoSuchFormat
 
 def debug(*args):
     prints(*args)
@@ -201,27 +201,35 @@ class SaveWorker(Thread):
         self.spare_server = spare_server
         self.start()
 
-    def collect_data(self, ids):
+    def collect_data(self, ids, tdir):
         from calibre.ebooks.metadata.opf2 import metadata_to_opf
         data = {}
         for i in set(ids):
-            mi = self.db.get_metadata(i, index_is_id=True, get_cover=True)
+            mi = self.db.get_metadata(i, index_is_id=True, get_cover=True,
+                    cover_as_data=True)
             opf = metadata_to_opf(mi)
             if isbytestring(opf):
                 opf = opf.decode('utf-8')
             cpath = None
-            if mi.cover:
+            if mi.cover_data and mi.cover_data[1]:
-                cpath = mi.cover
+                cpath = os.path.join(tdir, 'cover_%s.jpg'%i)
+                with lopen(cpath, 'wb') as f:
+                    f.write(mi.cover_data[1])
                 if isbytestring(cpath):
                     cpath = cpath.decode(filesystem_encoding)
             formats = {}
             if mi.formats:
                 for fmt in mi.formats:
-                    fpath = self.db.format_abspath(i, fmt, index_is_id=True)
+                    fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower()))
-                    if fpath is not None:
+                    with lopen(fpath, 'wb') as f:
-                        if isbytestring(fpath):
+                        try:
-                            fpath = fpath.decode(filesystem_encoding)
+                            self.db.copy_format_to(i, fmt, f, index_is_id=True)
-                        formats[fmt.lower()] = fpath
+                        except NoSuchFormat:
+                            continue
+                        else:
+                            if isbytestring(fpath):
+                                fpath = fpath.decode(filesystem_encoding)
+                            formats[fmt.lower()] = fpath
             data[i] = [opf, cpath, formats, mi.last_modified.isoformat()]
         return data
 
@@ -244,7 +252,7 @@ class SaveWorker(Thread):
 
         for i, task in enumerate(tasks):
             tids = [x[-1] for x in task]
-            data = self.collect_data(tids)
+            data = self.collect_data(tids, tdir)
             dpath = os.path.join(tdir, '%d.json'%i)
             with open(dpath, 'wb') as f:
                 f.write(json.dumps(data, ensure_ascii=False).encode('utf-8'))

src/calibre/gui2/actions/copy_to_library.py

@@ -53,13 +53,18 @@ class Worker(Thread): # {{{
         from calibre.library.database2 import LibraryDatabase2
         newdb = LibraryDatabase2(self.loc)
         for i, x in enumerate(self.ids):
-            mi = self.db.get_metadata(x, index_is_id=True, get_cover=True)
+            mi = self.db.get_metadata(x, index_is_id=True, get_cover=True,
+                    cover_as_data=True)
             self.progress(i, mi.title)
             fmts = self.db.formats(x, index_is_id=True)
             if not fmts: fmts = []
             else: fmts = fmts.split(',')
-            paths = [self.db.format_abspath(x, fmt, index_is_id=True) for fmt in
+            paths = []
-                    fmts]
+            for fmt in fmts:
+                p = self.db.format(x, fmt, index_is_id=True,
+                    as_path=True)
+                if p:
+                    paths.append(p)
             added = False
             if prefs['add_formats_to_existing']:
                 identical_book_list = newdb.find_identical_books(mi)
@@ -75,6 +80,11 @@ class Worker(Thread): # {{{
                 if co is not None:
                     newdb.set_conversion_options(x, 'PIPE', co)
             self.processed.add(x)
+            for path in paths:
+                try:
+                    os.remove(path)
+                except:
+                    pass
 # }}}
 
 class CopyToLibraryAction(InterfaceAction):

src/calibre/gui2/actions/edit_metadata.py

@@ -17,6 +17,7 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor
 from calibre.gui2.actions import InterfaceAction
 from calibre.ebooks.metadata import authors_to_string
 from calibre.utils.icu import sort_key
+from calibre.db.errors import NoSuchFormat
 
 class EditMetadataAction(InterfaceAction):
 
@@ -265,7 +266,7 @@ class EditMetadataAction(InterfaceAction):
                                 +'</p>', 'merge_too_many_books', self.gui):
                 return
 
-        dest_id, src_books, src_ids = self.books_to_merge(rows)
+        dest_id, src_ids = self.books_to_merge(rows)
         title = self.gui.library_view.model().db.title(dest_id, index_is_id=True)
         if safe_merge:
             if not confirm('<p>'+_(
@@ -277,7 +278,7 @@ class EditMetadataAction(InterfaceAction):
                 'Please confirm you want to proceed.')%title
             +'</p>', 'merge_books_safe', self.gui):
                 return
-            self.add_formats(dest_id, src_books)
+            self.add_formats(dest_id, self.formats_for_books(rows))
             self.merge_metadata(dest_id, src_ids)
         elif merge_only_formats:
             if not confirm('<p>'+_(
@@ -293,7 +294,7 @@ class EditMetadataAction(InterfaceAction):
                 'Are you <b>sure</b> you want to proceed?')%title
             +'</p>', 'merge_only_formats', self.gui):
                 return
-            self.add_formats(dest_id, src_books)
+            self.add_formats(dest_id, self.formats_for_books(rows))
             self.delete_books_after_merge(src_ids)
         else:
             if not confirm('<p>'+_(
@@ -308,7 +309,7 @@ class EditMetadataAction(InterfaceAction):
                 'Are you <b>sure</b> you want to proceed?')%title
             +'</p>', 'merge_books', self.gui):
                 return
-            self.add_formats(dest_id, src_books)
+            self.add_formats(dest_id, self.formats_for_books(rows))
             self.merge_metadata(dest_id, src_ids)
             self.delete_books_after_merge(src_ids)
             # leave the selection highlight on first selected book
@@ -329,8 +330,22 @@ class EditMetadataAction(InterfaceAction):
                     self.gui.library_view.model().db.add_format(dest_id, fmt, f, index_is_id=True,
                             notify=False, replace=replace)
 
+    def formats_for_books(self, rows):
+        m = self.gui.library_view.model()
+        ans = []
+        for id_ in map(m.id, rows):
+            dbfmts = m.db.formats(id_, index_is_id=True)
+            if dbfmts:
+                for fmt in dbfmts.split(','):
+                    try:
+                        path = m.db.format(id_, fmt, index_is_id=True,
+                                as_path=True)
+                        ans.append(path)
+                    except NoSuchFormat:
+                        continue
+        return ans
+
     def books_to_merge(self, rows):
-        src_books = []
         src_ids = []
         m = self.gui.library_view.model()
         for i, row in enumerate(rows):
@@ -339,22 +354,19 @@ class EditMetadataAction(InterfaceAction):
                 dest_id = id_
             else:
                 src_ids.append(id_)
-                dbfmts = m.db.formats(id_, index_is_id=True)
+        return [dest_id, src_ids]
-                if dbfmts:
-                    for fmt in dbfmts.split(','):
-                        src_books.append(m.db.format_abspath(id_, fmt,
-                            index_is_id=True))
-        return [dest_id, src_books, src_ids]
 
     def delete_books_after_merge(self, ids_to_delete):
         self.gui.library_view.model().delete_books_by_id(ids_to_delete)
 
     def merge_metadata(self, dest_id, src_ids):
         db = self.gui.library_view.model().db
-        dest_mi = db.get_metadata(dest_id, index_is_id=True, get_cover=True)
+        dest_mi = db.get_metadata(dest_id, index_is_id=True)
         orig_dest_comments = dest_mi.comments
+        dest_cover = db.cover(dest_id, index_is_id=True)
+        had_orig_cover = bool(dest_cover)
         for src_id in src_ids:
-            src_mi = db.get_metadata(src_id, index_is_id=True, get_cover=True)
+            src_mi = db.get_metadata(src_id, index_is_id=True)
             if src_mi.comments and orig_dest_comments != src_mi.comments:
                 if not dest_mi.comments:
                     dest_mi.comments = src_mi.comments
@@ -372,8 +384,10 @@ class EditMetadataAction(InterfaceAction):
                     dest_mi.tags = src_mi.tags
                 else:
                     dest_mi.tags.extend(src_mi.tags)
-            if src_mi.cover and not dest_mi.cover:
+            if not dest_cover:
-                dest_mi.cover = src_mi.cover
+                src_cover = db.cover(src_id, index_is_id=True)
+                if src_cover:
+                    dest_cover = src_cover
             if not dest_mi.publisher:
                 dest_mi.publisher = src_mi.publisher
             if not dest_mi.rating:
@@ -382,6 +396,8 @@ class EditMetadataAction(InterfaceAction):
                 dest_mi.series = src_mi.series
                 dest_mi.series_index = src_mi.series_index
         db.set_metadata(dest_id, dest_mi, ignore_errors=False)
+        if not had_orig_cover and dest_cover:
+            db.set_cover(dest_id, dest_cover)
 
         for key in db.field_metadata: #loop thru all defined fields
           if db.field_metadata[key]['is_custom']:

src/calibre/gui2/actions/tweak_epub.py

@@ -5,6 +5,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import os
+
 from calibre.gui2 import error_dialog
 from calibre.gui2.actions import InterfaceAction
 from calibre.gui2.dialogs.tweak_epub import TweakEpub
@@ -30,8 +32,8 @@ class TweakEpubAction(InterfaceAction):
         # Confirm 'EPUB' in formats
         book_id = self.gui.library_view.model().id(row)
         try:
-            path_to_epub = self.gui.library_view.model().db.format_abspath(
+            path_to_epub = self.gui.library_view.model().db.format(
-                    book_id, 'EPUB', index_is_id=True)
+                    book_id, 'EPUB', index_is_id=True, as_path=True)
         except:
             path_to_epub = None
 
@@ -45,6 +47,7 @@ class TweakEpubAction(InterfaceAction):
         if dlg.exec_() == dlg.Accepted:
             self.update_db(book_id, dlg._output)
         dlg.cleanup()
+        os.remove(path_to_epub)
 
     def update_db(self, book_id, rebuilt):
         '''

src/calibre/gui2/add.py

@@ -445,6 +445,7 @@ class Saver(QObject): # {{{
         self.pd.setModal(True)
         self.pd.show()
         self.pd.set_min(0)
+        self.pd.set_msg(_('Collecting data, please wait...'))
         self._parent = parent
         self.callback = callback
         self.callback_called = False

src/calibre/gui2/convert/regex_builder.py

@@ -4,7 +4,7 @@ __license__ = 'GPL 3'
 __copyright__ = '2009, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import re
+import re, os
 
 from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal
 from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \
@@ -134,7 +134,12 @@ class RegexBuilder(QDialog, Ui_RegexBuilder):
                          _('Cannot build regex using the GUI builder without a book.'),
                          show=True)
             return False
-        self.open_book(db.format_abspath(book_id, format, index_is_id=True))
+        fpath = db.format(book_id, format, index_is_id=True,
+            as_path=True)
+        try:
+            self.open_book(fpath)
+        finally:
+            os.remove(fpath)
         return True
 
     def open_book(self, pathtoebook):

src/calibre/gui2/convert/single.py

@@ -106,7 +106,6 @@ class Config(ResizableDialog, Ui_Dialog):
     Configuration dialog for single book conversion. If accepted, has the
     following important attributes
 
-    input_path - Path to input file
     output_format - Output format (without a leading .)
     input_format  - Input format (without a leading .)
     opf_path - Path to OPF file with user specified metadata
@@ -156,13 +155,10 @@ class Config(ResizableDialog, Ui_Dialog):
         oidx = self.groups.currentIndex().row()
         input_format = self.input_format
         output_format = self.output_format
-        input_path = self.db.format_abspath(self.book_id, input_format,
-                index_is_id=True)
-        self.input_path = input_path
         output_path = 'dummy.'+output_format
         log = Log()
         log.outputs = []
-        self.plumber = Plumber(input_path, output_path, log)
+        self.plumber = Plumber('dummy.'+input_format, output_path, log)
 
         def widget_factory(cls):
             return cls(self.stack, self.plumber.get_option_by_name,

src/calibre/gui2/device.py

@@ -396,8 +396,17 @@ class DeviceManager(Thread): # {{{
                             if DEBUG:
                                 prints(traceback.format_exc(), file=sys.__stdout__)
 
-        return self.device.upload_books(files, names, on_card,
+        try:
-                                        metadata=metadata, end_session=False)
+            return self.device.upload_books(files, names, on_card,
+                    metadata=metadata, end_session=False)
+        finally:
+            if metadata:
+                for mi in metadata:
+                    try:
+                        if mi.cover:
+                            os.remove(mi.cover)
+                    except:
+                        pass
 
     def upload_books(self, done, files, names, on_card=None, titles=None,
                      metadata=None, plugboards=None, add_as_step_to_job=None):
@@ -1072,8 +1081,6 @@ class DeviceMixin(object): # {{{
                             'the device?'), autos):
                         self.iactions['Convert Books'].auto_convert_news(auto, format)
             files = [f for f in files if f is not None]
-            for f in files:
-                f.deleted_after_upload = del_on_upload
             if not files:
                 self.news_to_be_synced = set([])
                 return
@@ -1315,8 +1322,17 @@ class DeviceMixin(object): # {{{
             self.card_b_view if on_card == 'cardb' else self.memory_view
         view.model().resort(reset=False)
         view.model().research()
-        for f in files:
+        if files:
-            getattr(f, 'close', lambda : True)()
+            for f in files:
+                # Remove temporary files
+                try:
+                    rem = not getattr(
+                            self.device_manager.device,
+                            'KEEP_TEMP_FILES_AFTER_UPLOAD', False)
+                    if rem and 'caltmpfmt.' in f:
+                        os.remove(f)
+                except:
+                    pass
 
     def book_on_device(self, id, reset=False):
         '''

src/calibre/gui2/dialogs/metadata_bulk.py

@@ -24,7 +24,7 @@ from calibre.utils.config import prefs, tweaks
 from calibre.utils.magick.draw import identify_data
 from calibre.utils.date import qt_to_dt
 
-def get_cover_data(path): # {{{
+def get_cover_data(stream, ext): # {{{
     from calibre.ebooks.metadata.meta import get_metadata
     old = prefs['read_file_metadata']
     if not old:
@@ -32,8 +32,8 @@ def get_cover_data(path): # {{{
     cdata = area = None
 
     try:
-        mi = get_metadata(open(path, 'rb'),
+        with stream:
-                os.path.splitext(path)[1][1:].lower())
+            mi = get_metadata(stream, ext)
         if mi.cover and os.access(mi.cover, os.R_OK):
             cdata = open(mi.cover).read()
         elif mi.cover_data[1] is not None:
@@ -186,9 +186,10 @@ class MyBlockingBusy(QDialog): # {{{
                 if fmts:
                     covers = []
                     for fmt in fmts.split(','):
-                        fmt = self.db.format_abspath(id, fmt, index_is_id=True)
+                        fmtf = self.db.format(id, fmt, index_is_id=True,
-                        if not fmt: continue
+                                as_file=True)
-                        cdata, area = get_cover_data(fmt)
+                        if fmtf is None: continue
+                        cdata, area = get_cover_data(fmtf, fmt)
                         if cdata:
                             covers.append((cdata, area))
                     covers.sort(key=lambda x: x[1])

src/calibre/gui2/email.py

@@ -174,7 +174,8 @@ class EmailMixin(object): # {{{
         else:
             _auto_ids = []
 
-        full_metadata = self.library_view.model().metadata_for(ids)
+        full_metadata = self.library_view.model().metadata_for(ids,
+                get_cover=False)
 
         bad, remove_ids, jobnames = [], [], []
         texts, subjects, attachments, attachment_names = [], [], [], []

src/calibre/gui2/library/models.py

@@ -5,8 +5,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import shutil, functools, re, os, traceback
+import functools, re, os, traceback
-from contextlib import closing
 from collections import defaultdict
 
 from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage,
@@ -36,14 +35,6 @@ TIME_FMT = '%d %b %Y'
 ALIGNMENT_MAP = {'left': Qt.AlignLeft, 'right': Qt.AlignRight, 'center':
         Qt.AlignHCenter}
 
-class FormatPath(unicode):
-
-    def __new__(cls, path, orig_file_path):
-        ans = unicode.__new__(cls, path)
-        ans.orig_file_path = orig_file_path
-        ans.deleted_after_upload = False
-        return ans
-
 _default_image = None
 
 def default_image():
@@ -391,10 +382,14 @@ class BooksModel(QAbstractTableModel): # {{{
         data = self.current_changed(index, None, False)
         return data
 
-    def metadata_for(self, ids):
+    def metadata_for(self, ids, get_cover=True):
+        '''
+        WARNING: if get_cover=True temp files are created for mi.cover.
+        Remember to delete them once you are done with them.
+        '''
         ans = []
         for id in ids:
-            mi = self.db.get_metadata(id, index_is_id=True, get_cover=True)
+            mi = self.db.get_metadata(id, index_is_id=True, get_cover=get_cover)
             ans.append(mi)
         return ans
 
@@ -449,18 +444,14 @@ class BooksModel(QAbstractTableModel): # {{{
                     format = f
                     break
             if format is not None:
-                pt = PersistentTemporaryFile(suffix='.'+format)
+                pt = PersistentTemporaryFile(suffix='caltmpfmt.'+format)
-                with closing(self.db.format(id, format, index_is_id=True,
+                self.db.copy_format_to(id, format, pt, index_is_id=True)
-                    as_file=True)) as src:
-                    shutil.copyfileobj(src, pt)
-                    pt.flush()
-                    if getattr(src, 'name', None):
-                        pt.orig_file_path = os.path.abspath(src.name)
                 pt.seek(0)
                 if set_metadata:
                     try:
-                        _set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True),
+                        _set_metadata(pt, self.db.get_metadata(
-                                  format)
+                            id, get_cover=True, index_is_id=True,
+                            cover_as_data=True), format)
                     except:
                         traceback.print_exc()
                 pt.close()
@@ -468,9 +459,7 @@ class BooksModel(QAbstractTableModel): # {{{
                     if isbytestring(x):
                         x = x.decode(filesystem_encoding)
                     return x
-                name, op = map(to_uni, map(os.path.abspath, (pt.name,
+                ans.append(to_uni(os.path.abspath(pt.name)))
-                    pt.orig_file_path)))
-                ans.append(FormatPath(name, op))
             else:
                 need_auto.append(id)
                 if not exclude_auto:
@@ -499,13 +488,11 @@ class BooksModel(QAbstractTableModel): # {{{
                     break
             if format is not None:
                 pt = PersistentTemporaryFile(suffix='.'+format)
-                with closing(self.db.format(row, format, as_file=True)) as src:
+                self.db.copy_format_to(id, format, pt, index_is_id=True)
-                    shutil.copyfileobj(src, pt)
-                    pt.flush()
                 pt.seek(0)
                 if set_metadata:
-                    _set_metadata(pt, self.db.get_metadata(row, get_cover=True),
+                    _set_metadata(pt, self.db.get_metadata(row, get_cover=True,
-                                  format)
+                        cover_as_data=True), format)
                 pt.close() if paths else pt.seek(0)
                 ans.append(pt)
             else:

src/calibre/gui2/library/views.py

@@ -584,14 +584,15 @@ class BooksView(QTableView): # {{{
         m = self.model()
         db = m.db
         rows = self.selectionModel().selectedRows()
-        selected = map(m.id, rows)
+        selected = list(map(m.id, rows))
         ids = ' '.join(map(str, selected))
         md = QMimeData()
         md.setData('application/calibre+from_library', ids)
         fmt = prefs['output_format']
 
         def url_for_id(i):
-            ans = db.format_abspath(i, fmt, index_is_id=True)
+            ans = db.format(i, fmt, index_is_id=True, as_path=True,
+                    preserve_filename=True)
             if ans is None:
                 fmts = db.formats(i, index_is_id=True)
                 if fmts:
@@ -599,14 +600,13 @@ class BooksView(QTableView): # {{{
                 else:
                     fmts = []
                 for f in fmts:
-                    ans = db.format_abspath(i, f, index_is_id=True)
+                    ans = db.format(i, f, index_is_id=True, as_path=True,
-                    if ans is not None:
+                            preserve_filename=True)
-                        break
             if ans is None:
                 ans = db.abspath(i, index_is_id=True)
             return QUrl.fromLocalFile(ans)
 
-        md.setUrls([url_for_id(i) for i in selected])
+        md.setUrls([url_for_id(i) for i in selected[:25]])
         drag = QDrag(self)
         col = self.selectionModel().currentIndex().column()
         md.column_name = self.column_map[col]

src/calibre/gui2/metadata/basic_widgets.py

@@ -688,7 +688,8 @@ class FormatsManager(QWidget): # {{{
             else:
                 stream = open(fmt.path, 'r+b')
             try:
-                mi = get_metadata(stream, ext)
+                with stream:
+                    mi = get_metadata(stream, ext)
                 return mi, ext
             except:
                 error_dialog(self, _('Could not read metadata'),

src/calibre/gui2/tools.py

@@ -51,12 +51,15 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
                 #    continue
 
                 mi = db.get_metadata(book_id, True)
-                in_file = db.format_abspath(book_id, d.input_format, True)
+                in_file = PersistentTemporaryFile('.'+d.input_format)
+                with in_file:
+                    db.copy_format_to(book_id, d.input_format, in_file,
+                            index_is_id=True)
 
                 out_file = PersistentTemporaryFile('.' + d.output_format)
                 out_file.write(d.output_format)
                 out_file.close()
-                temp_files = []
+                temp_files = [in_file]
 
                 try:
                     dtitle = unicode(mi.title)
@@ -74,7 +77,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{
                     recs.append(('cover', d.cover_file.name,
                         OptionRecommendation.HIGH))
                     temp_files.append(d.cover_file)
-                args = [in_file, out_file.name, recs]
+                args = [in_file.name, out_file.name, recs]
                 temp_files.append(out_file)
                 jobs.append(('gui_convert_override', args, desc, d.output_format.upper(), book_id, temp_files))
 
@@ -142,12 +145,15 @@ class QueueBulk(QProgressDialog):
         try:
             input_format = get_input_format_for_book(self.db, book_id, None)[0]
             mi, opf_file = create_opf_file(self.db, book_id)
-            in_file = self.db.format_abspath(book_id, input_format, True)
+            in_file = PersistentTemporaryFile('.'+input_format)
+            with in_file:
+                self.db.copy_format_to(book_id, input_format, in_file,
+                        index_is_id=True)
 
             out_file = PersistentTemporaryFile('.' + self.output_format)
             out_file.write(self.output_format)
             out_file.close()
-            temp_files = []
+            temp_files = [in_file]
 
             combined_recs = GuiRecommendations()
             default_recs = bulk_defaults_for_input_format(input_format)
@@ -183,7 +189,7 @@ class QueueBulk(QProgressDialog):
             self.setLabelText(_('Queueing ')+dtitle)
             desc = _('Convert book %d of %d (%s)') % (self.i, len(self.book_ids), dtitle)
 
-            args = [in_file, out_file.name, lrecs]
+            args = [in_file.name, out_file.name, lrecs]
             temp_files.append(out_file)
             self.jobs.append(('gui_convert_override', args, desc, self.output_format.upper(), book_id, temp_files))
 

src/calibre/library/__init__.py

@@ -61,22 +61,4 @@ def generate_test_db(library_path, # {{{
     print 'Time per record:', t/float(num_of_records)
 # }}}
 
-def cover_load_timing(path=None):
-    from PyQt4.Qt import QApplication, QImage
-    import os, time
-    app = QApplication([])
-    app
-    d = db(path)
-    paths = [d.cover(i, index_is_id=True, as_path=True) for i in
-            d.data.iterallids()]
-    paths = [p for p in paths if (p and os.path.exists(p) and os.path.isfile(p))]
-
-    start = time.time()
-
-    for p in paths:
-        with open(p, 'rb') as f:
-            img = QImage()
-            img.loadFromData(f.read())
-
-    print 'Average load time:', (time.time() - start)/len(paths), 'seconds'
 

src/calibre/library/catalog.py

@@ -5137,6 +5137,7 @@ Author '{0}':
                     OptionRecommendation.HIGH))
 
             # If cover exists, use it
+            cpath = None
             try:
                 search_text = 'title:"%s" author:%s' % (
                         opts.catalog_title.replace('"', '\\"'), 'calibre')
@@ -5157,5 +5158,10 @@ Author '{0}':
             plumber.merge_ui_recommendations(recommendations)
             plumber.run()
 
+            try:
+                os.remove(cpath)
+            except:
+                pass
+
         # returns to gui2.actions.catalog:catalog_generated()
         return catalog.error

src/calibre/library/database2.py

@@ -12,8 +12,6 @@ import threading, random
 from itertools import repeat
 from math import ceil
 
-from PyQt4.QtGui import QImage
-
 from calibre import prints
 from calibre.ebooks.metadata import (title_sort, author_to_author_sort,
         string_to_authors, authors_to_string)
@@ -27,7 +25,7 @@ from calibre.library.sqlite import connect, IntegrityError
 from calibre.library.prefs import DBPrefs
 from calibre.ebooks.metadata.book.base import Metadata
 from calibre.constants import preferred_encoding, iswindows, filesystem_encoding
-from calibre.ptempfile import PersistentTemporaryFile
+from calibre.ptempfile import PersistentTemporaryFile, base_dir
 from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import ascii_filename
@@ -39,8 +37,10 @@ from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format
 from calibre.utils.magick.draw import save_cover_data_to
 from calibre.utils.recycle_bin import delete_file, delete_tree
 from calibre.utils.formatter_functions import load_user_template_functions
+from calibre.db.errors import NoSuchFormat
 
 copyfile = os.link if hasattr(os, 'link') else shutil.copyfile
+SPOOL_SIZE = 30*1024*1024
 
 class Tag(object):
 
@@ -601,14 +601,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f:
                     f.write(cdata)
             for format in formats:
-                # Get data as string (can't use file as source and target files may be the same)
+                with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream:
-                f = self.format(id, format, index_is_id=True, as_file=True)
+                    try:
-                if f is None:
+                        self.copy_format_to(id, format, stream, index_is_id=True)
-                    continue
+                        stream.seek(0)
-                with tempfile.SpooledTemporaryFile(max_size=30*(1024**2)) as stream:
+                    except NoSuchFormat:
-                    with f:
+                        continue
-                        shutil.copyfileobj(f, stream)
-                    stream.seek(0)
                     self.add_format(id, format, stream, index_is_id=True,
                         path=tpath, notify=False)
         self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
@@ -661,32 +659,53 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 continue
 
     def cover(self, index, index_is_id=False, as_file=False, as_image=False,
-              as_path=False):
+            as_path=False):
         '''
         Return the cover image as a bytestring (in JPEG format) or None.
 
-        `as_file` : If True return the image as an open file object
+        WARNING: Using as_path will copy the cover to a temp file and return
-        `as_image`: If True return the image as a QImage object
+        the path to the temp file. You should delete the temp file when you are
+        done with it.
+
+        :param as_file: If True return the image as an open file object (a SpooledTemporaryFile)
+        :param as_image: If True return the image as a QImage object
         '''
-        id = index if  index_is_id else self.id(index)
+        id = index if index_is_id else self.id(index)
         path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
         if os.access(path, os.R_OK):
-            if as_path:
-                return path
             try:
                 f = lopen(path, 'rb')
             except (IOError, OSError):
                 time.sleep(0.2)
                 f = lopen(path, 'rb')
-            if as_image:
+            with f:
-                img = QImage()
+                if as_path:
-                img.loadFromData(f.read())
+                    pt = PersistentTemporaryFile('_dbcover.jpg')
-                f.close()
+                    with pt:
-                return img
+                        shutil.copyfileobj(f, pt)
-            ans = f if as_file else f.read()
+                    return pt.name
-            if ans is not f:
+                if as_file:
-                f.close()
+                    ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE)
-            return ans
+                    shutil.copyfileobj(f, ret)
+                    ret.seek(0)
+                else:
+                    ret = f.read()
+                    if as_image:
+                        from PyQt4.Qt import QImage
+                        i = QImage()
+                        i.loadFromData(ret)
+                        ret = i
+            return ret
+
+    def cover_last_modified(self, index, index_is_id=False):
+        id = index if  index_is_id else self.id(index)
+        path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
+        try:
+            return utcfromtimestamp(os.stat(path).st_mtime)
+        except:
+            # Cover doesn't exist
+            pass
+        return self.last_modified()
 
     ### The field-style interface. These use field keys.
 
@@ -859,7 +878,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         return (path, mi, sequence)
 
     def get_metadata(self, idx, index_is_id=False, get_cover=False,
-                     get_user_categories=True):
+                     get_user_categories=True, cover_as_data=False):
         '''
         Convenience method to return metadata as a :class:`Metadata` object.
         Note that the list of formats is not verified.
@@ -934,7 +953,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         mi.user_categories = user_cat_vals
 
         if get_cover:
-            mi.cover = self.cover(id, index_is_id=True, as_path=True)
+            if cover_as_data:
+                cdata = self.cover(id, index_is_id=True)
+                if cdata:
+                    mi.cover_data = ('jpeg', cdata)
+            else:
+                mi.cover = self.cover(id, index_is_id=True, as_path=True)
         return mi
 
     def has_book(self, mi):
@@ -1099,7 +1123,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             return utcfromtimestamp(os.stat(path).st_mtime)
 
     def format_abspath(self, index, format, index_is_id=False):
-        'Return absolute path to the ebook file of format `format`'
+        '''
+        Return absolute path to the ebook file of format `format`
+
+        WARNING: This method will return a dummy path for a network backend DB,
+        so do not rely on it, use format(..., as_path=True) instead.
+
+        Currently used only in calibredb list, the viewer and the catalogs (via
+        get_data_as_dict()).
+
+        Apart from the viewer, I don't believe any of the others do any file
+        I/O with the results of this call.
+        '''
         id = index if index_is_id else self.id(index)
         try:
             name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False)
@@ -1119,25 +1154,63 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 shutil.copyfile(candidates[0], fmt_path)
                 return fmt_path
 
-    def format(self, index, format, index_is_id=False, as_file=False, mode='r+b'):
+    def copy_format_to(self, index, fmt, dest, index_is_id=False):
+        '''
+        Copy the format ``fmt`` to the file like object ``dest``. If the
+        specified format does not exist, raises :class:`NoSuchFormat` error.
+        '''
+        path = self.format_abspath(index, fmt, index_is_id=index_is_id)
+        if path is None:
+            id_ = index if index_is_id else self.id(index)
+            raise NoSuchFormat('Record %d has no %s file'%(id_, fmt))
+        with lopen(path, 'rb') as f:
+            shutil.copyfileobj(f, dest)
+        if hasattr(dest, 'flush'):
+            dest.flush()
+
+    def format(self, index, format, index_is_id=False, as_file=False,
+            mode='r+b', as_path=False, preserve_filename=False):
         '''
         Return the ebook format as a bytestring or `None` if the format doesn't exist,
         or we don't have permission to write to the ebook file.
 
-        `as_file`: If True the ebook format is returned as a file object opened in `mode`
+        :param as_file: If True the ebook format is returned as a file object. Note
+                        that the file object is a SpooledTemporaryFile, so if what you want to
+                        do is copy the format to another file, use :method:`copy_format_to`
+                        instead for performance.
+        :param as_path: Copies the format file to a temp file and returns the
+                        path to the temp file
+        :param preserve_filename: If True and returning a path the filename is
+                                  the same as that used in the library. Note that using
+                                  this means that repeated calls yield the same
+                                  temp file (which is re-created each time)
+        :param mode: This is ignored (present for legacy compatibility)
         '''
         path = self.format_abspath(index, format, index_is_id=index_is_id)
         if path is not None:
-            f = lopen(path, mode)
+            with lopen(path, mode) as f:
-            try:
+                if as_path:
-                ret = f if as_file else f.read()
+                    if preserve_filename:
-            except IOError:
+                        bd = base_dir()
-                f.seek(0)
+                        d = os.path.join(bd, 'format_abspath')
-                out = cStringIO.StringIO()
+                        try:
-                shutil.copyfileobj(f, out)
+                            os.makedirs(d)
-                ret = out.getvalue()
+                        except:
-            if not as_file:
+                            pass
-                f.close()
+                        fname = os.path.basename(path)
+                        ret = os.path.join(d, fname)
+                        with lopen(ret, 'wb') as f2:
+                            shutil.copyfileobj(f, f2)
+                    else:
+                        with PersistentTemporaryFile('.'+format.lower()) as pt:
+                            shutil.copyfileobj(f, pt)
+                            ret = pt.name
+                elif as_file:
+                    ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE)
+                    shutil.copyfileobj(f, ret)
+                    ret.seek(0)
+                else:
+                    ret = f.read()
             return ret
 
     def add_format_with_hooks(self, index, format, fpath, index_is_id=False,

src/calibre/library/save_to_disk.py

@@ -6,7 +6,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, traceback, cStringIO, re, shutil
+import os, traceback, cStringIO, re
 
 from calibre.constants import DEBUG
 from calibre.utils.config import Config, StringConfig, tweaks
@@ -238,7 +238,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250,
 
 def save_book_to_disk(id_, db, root, opts, length):
     mi = db.get_metadata(id_, index_is_id=True)
-    cover = db.cover(id_, index_is_id=True, as_path=True)
+    cover = db.cover(id_, index_is_id=True)
     plugboards = db.prefs.get('plugboards', {})
 
     available_formats = db.formats(id_, index_is_id=True)
@@ -252,12 +252,20 @@ def save_book_to_disk(id_, db, root, opts, length):
     if fmts:
         fmts = fmts.split(',')
         for fmt in fmts:
-            fpath = db.format_abspath(id_, fmt, index_is_id=True)
+            fpath = db.format(id_, fmt, index_is_id=True, as_path=True)
             if fpath is not None:
                 formats[fmt.lower()] = fpath
 
-    return do_save_book_to_disk(id_, mi, cover, plugboards,
+    try:
+        return do_save_book_to_disk(id_, mi, cover, plugboards,
             formats, root, opts, length)
+    finally:
+        for temp in formats.itervalues():
+            try:
+                os.remove(temp)
+            except:
+                pass
+
 
 
 def do_save_book_to_disk(id_, mi, cover, plugboards,
@@ -289,10 +297,9 @@ def do_save_book_to_disk(id_, mi, cover, plugboards,
             raise
 
     ocover = mi.cover
-    if opts.save_cover and cover and os.access(cover, os.R_OK):
+    if opts.save_cover and cover:
         with open(base_path+'.jpg', 'wb') as f:
-            with open(cover, 'rb') as s:
+            f.write(cover)
-                shutil.copyfileobj(s, f)
         mi.cover = base_name+'.jpg'
     else:
         mi.cover = None
@@ -395,8 +402,13 @@ def save_serialized_to_disk(ids, data, plugboards, root, opts, callback):
             pass
         tb = ''
         try:
-            failed, id, title = do_save_book_to_disk(x, mi, cover, plugboards,
+            with open(cover, 'rb') as f:
-                    format_map, root, opts, length)
+                cover = f.read()
+        except:
+            cover = None
+        try:
+            failed, id, title = do_save_book_to_disk(x, mi, cover,
+                plugboards, format_map, root, opts, length)
             tb = _('Requested formats not available')
         except:
             failed, id, title = True, x, mi.title

src/calibre/library/server/content.py

@@ -10,12 +10,13 @@ import re, os, posixpath
 import cherrypy
 
 from calibre import fit_image, guess_type
-from calibre.utils.date import fromtimestamp
+from calibre.utils.date import fromtimestamp, utcnow
 from calibre.library.caches import SortKeyGenerator
 from calibre.library.save_to_disk import find_plugboard
-
+from calibre.ebooks.metadata import authors_to_string
-from calibre.utils.magick.draw import save_cover_data_to, Image, \
+from calibre.utils.magick.draw import (save_cover_data_to, Image,
-        thumbnail as generate_thumbnail
+        thumbnail as generate_thumbnail)
+from calibre.utils.filenames import ascii_filename
 
 plugboard_content_server_value = 'content_server'
 plugboard_content_server_formats = ['epub']
@@ -46,7 +47,7 @@ class ContentServer(object):
     # Utility methods {{{
     def last_modified(self, updated):
         '''
-        Generates a local independent, english timestamp from a datetime
+        Generates a locale independent, english timestamp from a datetime
         object
         '''
         lm = updated.strftime('day, %d month %Y %H:%M:%S GMT')
@@ -151,14 +152,12 @@ class ContentServer(object):
         try:
             cherrypy.response.headers['Content-Type'] = 'image/jpeg'
             cherrypy.response.timeout = 3600
-            cover = self.db.cover(id, index_is_id=True, as_file=True)
+            cover = self.db.cover(id, index_is_id=True)
             if cover is None:
                 cover = self.default_cover
                 updated = self.build_time
             else:
-                with cover as f:
+                updated = self.db.cover_last_modified(id, index_is_id=True)
-                    updated = fromtimestamp(os.fstat(f.fileno()).st_mtime)
-                    cover = f.read()
             cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
 
             if thumbnail:
@@ -187,9 +186,9 @@ class ContentServer(object):
                 mode='rb')
         if fmt is None:
             raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format))
+        mi = self.db.get_metadata(id, index_is_id=True)
         if format == 'EPUB':
             # Get the original metadata
-            mi = self.db.get_metadata(id, index_is_id=True)
 
             # Get any EPUB plugboards for the content server
             plugboards = self.db.prefs.get('plugboards', {})
@@ -203,24 +202,22 @@ class ContentServer(object):
                 newmi = mi
 
             # Write the updated file
-            from tempfile import TemporaryFile
             from calibre.ebooks.metadata.meta import set_metadata
-            raw = fmt.read()
-            fmt = TemporaryFile()
-            fmt.write(raw)
-            fmt.seek(0)
             set_metadata(fmt, newmi, 'epub')
             fmt.seek(0)
 
         mt = guess_type('dummy.'+format.lower())[0]
         if mt is None:
             mt = 'application/octet-stream'
+        au = authors_to_string(mi.authors if mi.authors else [_('Unknown')])
+        title = mi.title if mi.title else _('Unknown')
+        fname = u'%s - %s_%s.%s'%(title[:30], au[:30], id, format.lower())
+        fname = ascii_filename(fname).replace('"', '_')
         cherrypy.response.headers['Content-Type'] = mt
+        cherrypy.response.headers['Content-Disposition'] = \
+                b'attachment; filename="%s"'%fname
         cherrypy.response.timeout = 3600
-        path = getattr(fmt, 'name', None)
+        cherrypy.response.headers['Last-Modified'] = self.last_modified(utcnow())
-        if path and os.path.exists(path):
-            updated = fromtimestamp(os.stat(path).st_mtime)
-            cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
         return fmt
     # }}}