From f1a4d06e512a1ab63a727a148005fdacf4a52e36 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 21 Jun 2011 20:43:48 -0600 Subject: [PATCH] Try to ensure that all file I/O on library files happes in the LibraryDatabase2 class. This is in preparation for the new multi-threaded db backend. Also: Content server now sends the Content-Disposition header when sending ebook files. After uploading books to the device, delete the associated temp files. If the user tries to drag and drop more than 25 books out of calibre, only the first 25 will be sent. --- src/calibre/db/backend.py | 4 +- src/calibre/db/errors.py | 13 ++ src/calibre/devices/apple/driver.py | 1 + src/calibre/devices/interface.py | 7 +- src/calibre/ebooks/metadata/worker.py | 30 ++-- src/calibre/gui2/actions/copy_to_library.py | 16 +- src/calibre/gui2/actions/edit_metadata.py | 46 ++++-- src/calibre/gui2/actions/tweak_epub.py | 7 +- src/calibre/gui2/add.py | 1 + src/calibre/gui2/convert/regex_builder.py | 9 +- src/calibre/gui2/convert/single.py | 6 +- src/calibre/gui2/device.py | 28 +++- src/calibre/gui2/dialogs/metadata_bulk.py | 13 +- src/calibre/gui2/email.py | 3 +- src/calibre/gui2/library/models.py | 45 ++---- src/calibre/gui2/library/views.py | 12 +- src/calibre/gui2/metadata/basic_widgets.py | 3 +- src/calibre/gui2/tools.py | 18 ++- src/calibre/library/__init__.py | 18 --- src/calibre/library/catalog.py | 6 + src/calibre/library/database2.py | 155 ++++++++++++++------ src/calibre/library/save_to_disk.py | 30 ++-- src/calibre/library/server/content.py | 35 ++--- 23 files changed, 319 insertions(+), 187 deletions(-) create mode 100644 src/calibre/db/errors.py diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 159612e52d..ba683dde50 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -222,7 +222,9 @@ class DB(object, SchemaUpgrade): if self.user_version == 0: self.initialize_database() - SchemaUpgrade.__init__(self) + with self.conn: + SchemaUpgrade.__init__(self) + # Guarantee that the library_id is set self.library_id diff --git a/src/calibre/db/errors.py b/src/calibre/db/errors.py new file mode 100644 index 0000000000..d2657f9904 --- /dev/null +++ b/src/calibre/db/errors.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +class NoSuchFormat(ValueError): + pass + diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index dea5844028..cc531b7476 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -107,6 +107,7 @@ class DriverBase(DeviceConfig, DevicePlugin): # Needed for config_widget to work FORMATS = ['epub', 'pdf'] USER_CAN_ADD_NEW_FORMATS = False + KEEP_TEMP_FILES_AFTER_UPLOAD = True # Hide the standard customization widgets SUPPORTS_SUB_DIRS = False diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index b265331ace..925f9eafd0 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -327,12 +327,7 @@ class DevicePlugin(Plugin): free space on the device. The text of the FreeSpaceError must contain the word "card" if ``on_card`` is not None otherwise it must contain the word "memory". - :param files: A list of paths and/or file-like objects. If they are paths and - the paths point to temporary files, they may have an additional - attribute, original_file_path pointing to the originals. They may have - another optional attribute, deleted_after_upload which if True means - that the file pointed to by original_file_path will be deleted after - being uploaded to the device. + :param files: A list of paths :param names: A list of file names that the books should have once uploaded to the device. len(names) == len(files) :param metadata: If not None, it is a list of :class:`Metadata` objects. diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index c335cc4c13..ca8707258b 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -15,7 +15,7 @@ from calibre.utils.ipc.server import Server from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory from calibre import prints, isbytestring from calibre.constants import filesystem_encoding - +from calibre.db.errors import NoSuchFormat def debug(*args): prints(*args) @@ -201,27 +201,35 @@ class SaveWorker(Thread): self.spare_server = spare_server self.start() - def collect_data(self, ids): + def collect_data(self, ids, tdir): from calibre.ebooks.metadata.opf2 import metadata_to_opf data = {} for i in set(ids): - mi = self.db.get_metadata(i, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(i, index_is_id=True, get_cover=True, + cover_as_data=True) opf = metadata_to_opf(mi) if isbytestring(opf): opf = opf.decode('utf-8') cpath = None - if mi.cover: - cpath = mi.cover + if mi.cover_data and mi.cover_data[1]: + cpath = os.path.join(tdir, 'cover_%s.jpg'%i) + with lopen(cpath, 'wb') as f: + f.write(mi.cover_data[1]) if isbytestring(cpath): cpath = cpath.decode(filesystem_encoding) formats = {} if mi.formats: for fmt in mi.formats: - fpath = self.db.format_abspath(i, fmt, index_is_id=True) - if fpath is not None: - if isbytestring(fpath): - fpath = fpath.decode(filesystem_encoding) - formats[fmt.lower()] = fpath + fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower())) + with lopen(fpath, 'wb') as f: + try: + self.db.copy_format_to(i, fmt, f, index_is_id=True) + except NoSuchFormat: + continue + else: + if isbytestring(fpath): + fpath = fpath.decode(filesystem_encoding) + formats[fmt.lower()] = fpath data[i] = [opf, cpath, formats, mi.last_modified.isoformat()] return data @@ -244,7 +252,7 @@ class SaveWorker(Thread): for i, task in enumerate(tasks): tids = [x[-1] for x in task] - data = self.collect_data(tids) + data = self.collect_data(tids, tdir) dpath = os.path.join(tdir, '%d.json'%i) with open(dpath, 'wb') as f: f.write(json.dumps(data, ensure_ascii=False).encode('utf-8')) diff --git a/src/calibre/gui2/actions/copy_to_library.py b/src/calibre/gui2/actions/copy_to_library.py index 7190d1486f..97880faaa1 100644 --- a/src/calibre/gui2/actions/copy_to_library.py +++ b/src/calibre/gui2/actions/copy_to_library.py @@ -53,13 +53,18 @@ class Worker(Thread): # {{{ from calibre.library.database2 import LibraryDatabase2 newdb = LibraryDatabase2(self.loc) for i, x in enumerate(self.ids): - mi = self.db.get_metadata(x, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(x, index_is_id=True, get_cover=True, + cover_as_data=True) self.progress(i, mi.title) fmts = self.db.formats(x, index_is_id=True) if not fmts: fmts = [] else: fmts = fmts.split(',') - paths = [self.db.format_abspath(x, fmt, index_is_id=True) for fmt in - fmts] + paths = [] + for fmt in fmts: + p = self.db.format(x, fmt, index_is_id=True, + as_path=True) + if p: + paths.append(p) added = False if prefs['add_formats_to_existing']: identical_book_list = newdb.find_identical_books(mi) @@ -75,6 +80,11 @@ class Worker(Thread): # {{{ if co is not None: newdb.set_conversion_options(x, 'PIPE', co) self.processed.add(x) + for path in paths: + try: + os.remove(path) + except: + pass # }}} class CopyToLibraryAction(InterfaceAction): diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 650947100e..718ece46d2 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -17,6 +17,7 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor from calibre.gui2.actions import InterfaceAction from calibre.ebooks.metadata import authors_to_string from calibre.utils.icu import sort_key +from calibre.db.errors import NoSuchFormat class EditMetadataAction(InterfaceAction): @@ -265,7 +266,7 @@ class EditMetadataAction(InterfaceAction): +'

', 'merge_too_many_books', self.gui): return - dest_id, src_books, src_ids = self.books_to_merge(rows) + dest_id, src_ids = self.books_to_merge(rows) title = self.gui.library_view.model().db.title(dest_id, index_is_id=True) if safe_merge: if not confirm('

'+_( @@ -277,7 +278,7 @@ class EditMetadataAction(InterfaceAction): 'Please confirm you want to proceed.')%title +'

', 'merge_books_safe', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.merge_metadata(dest_id, src_ids) elif merge_only_formats: if not confirm('

'+_( @@ -293,7 +294,7 @@ class EditMetadataAction(InterfaceAction): 'Are you sure you want to proceed?')%title +'

', 'merge_only_formats', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.delete_books_after_merge(src_ids) else: if not confirm('

'+_( @@ -308,7 +309,7 @@ class EditMetadataAction(InterfaceAction): 'Are you sure you want to proceed?')%title +'

', 'merge_books', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.merge_metadata(dest_id, src_ids) self.delete_books_after_merge(src_ids) # leave the selection highlight on first selected book @@ -329,8 +330,22 @@ class EditMetadataAction(InterfaceAction): self.gui.library_view.model().db.add_format(dest_id, fmt, f, index_is_id=True, notify=False, replace=replace) + def formats_for_books(self, rows): + m = self.gui.library_view.model() + ans = [] + for id_ in map(m.id, rows): + dbfmts = m.db.formats(id_, index_is_id=True) + if dbfmts: + for fmt in dbfmts.split(','): + try: + path = m.db.format(id_, fmt, index_is_id=True, + as_path=True) + ans.append(path) + except NoSuchFormat: + continue + return ans + def books_to_merge(self, rows): - src_books = [] src_ids = [] m = self.gui.library_view.model() for i, row in enumerate(rows): @@ -339,22 +354,19 @@ class EditMetadataAction(InterfaceAction): dest_id = id_ else: src_ids.append(id_) - dbfmts = m.db.formats(id_, index_is_id=True) - if dbfmts: - for fmt in dbfmts.split(','): - src_books.append(m.db.format_abspath(id_, fmt, - index_is_id=True)) - return [dest_id, src_books, src_ids] + return [dest_id, src_ids] def delete_books_after_merge(self, ids_to_delete): self.gui.library_view.model().delete_books_by_id(ids_to_delete) def merge_metadata(self, dest_id, src_ids): db = self.gui.library_view.model().db - dest_mi = db.get_metadata(dest_id, index_is_id=True, get_cover=True) + dest_mi = db.get_metadata(dest_id, index_is_id=True) orig_dest_comments = dest_mi.comments + dest_cover = db.cover(dest_id, index_is_id=True) + had_orig_cover = bool(dest_cover) for src_id in src_ids: - src_mi = db.get_metadata(src_id, index_is_id=True, get_cover=True) + src_mi = db.get_metadata(src_id, index_is_id=True) if src_mi.comments and orig_dest_comments != src_mi.comments: if not dest_mi.comments: dest_mi.comments = src_mi.comments @@ -372,8 +384,10 @@ class EditMetadataAction(InterfaceAction): dest_mi.tags = src_mi.tags else: dest_mi.tags.extend(src_mi.tags) - if src_mi.cover and not dest_mi.cover: - dest_mi.cover = src_mi.cover + if not dest_cover: + src_cover = db.cover(src_id, index_is_id=True) + if src_cover: + dest_cover = src_cover if not dest_mi.publisher: dest_mi.publisher = src_mi.publisher if not dest_mi.rating: @@ -382,6 +396,8 @@ class EditMetadataAction(InterfaceAction): dest_mi.series = src_mi.series dest_mi.series_index = src_mi.series_index db.set_metadata(dest_id, dest_mi, ignore_errors=False) + if not had_orig_cover and dest_cover: + db.set_cover(dest_id, dest_cover) for key in db.field_metadata: #loop thru all defined fields if db.field_metadata[key]['is_custom']: diff --git a/src/calibre/gui2/actions/tweak_epub.py b/src/calibre/gui2/actions/tweak_epub.py index c9f2d7a8c6..d3924e7cd3 100755 --- a/src/calibre/gui2/actions/tweak_epub.py +++ b/src/calibre/gui2/actions/tweak_epub.py @@ -5,6 +5,8 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os + from calibre.gui2 import error_dialog from calibre.gui2.actions import InterfaceAction from calibre.gui2.dialogs.tweak_epub import TweakEpub @@ -30,8 +32,8 @@ class TweakEpubAction(InterfaceAction): # Confirm 'EPUB' in formats book_id = self.gui.library_view.model().id(row) try: - path_to_epub = self.gui.library_view.model().db.format_abspath( - book_id, 'EPUB', index_is_id=True) + path_to_epub = self.gui.library_view.model().db.format( + book_id, 'EPUB', index_is_id=True, as_path=True) except: path_to_epub = None @@ -45,6 +47,7 @@ class TweakEpubAction(InterfaceAction): if dlg.exec_() == dlg.Accepted: self.update_db(book_id, dlg._output) dlg.cleanup() + os.remove(path_to_epub) def update_db(self, book_id, rebuilt): ''' diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 44b5bb446b..3dbf4b94df 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -445,6 +445,7 @@ class Saver(QObject): # {{{ self.pd.setModal(True) self.pd.show() self.pd.set_min(0) + self.pd.set_msg(_('Collecting data, please wait...')) self._parent = parent self.callback = callback self.callback_called = False diff --git a/src/calibre/gui2/convert/regex_builder.py b/src/calibre/gui2/convert/regex_builder.py index bf32bf472a..f79e6df3fe 100644 --- a/src/calibre/gui2/convert/regex_builder.py +++ b/src/calibre/gui2/convert/regex_builder.py @@ -4,7 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -import re +import re, os from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \ @@ -134,7 +134,12 @@ class RegexBuilder(QDialog, Ui_RegexBuilder): _('Cannot build regex using the GUI builder without a book.'), show=True) return False - self.open_book(db.format_abspath(book_id, format, index_is_id=True)) + fpath = db.format(book_id, format, index_is_id=True, + as_path=True) + try: + self.open_book(fpath) + finally: + os.remove(fpath) return True def open_book(self, pathtoebook): diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index 3575fb5ffb..15e670ee32 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -106,7 +106,6 @@ class Config(ResizableDialog, Ui_Dialog): Configuration dialog for single book conversion. If accepted, has the following important attributes - input_path - Path to input file output_format - Output format (without a leading .) input_format - Input format (without a leading .) opf_path - Path to OPF file with user specified metadata @@ -156,13 +155,10 @@ class Config(ResizableDialog, Ui_Dialog): oidx = self.groups.currentIndex().row() input_format = self.input_format output_format = self.output_format - input_path = self.db.format_abspath(self.book_id, input_format, - index_is_id=True) - self.input_path = input_path output_path = 'dummy.'+output_format log = Log() log.outputs = [] - self.plumber = Plumber(input_path, output_path, log) + self.plumber = Plumber('dummy.'+input_format, output_path, log) def widget_factory(cls): return cls(self.stack, self.plumber.get_option_by_name, diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index a527cc2e27..d5805a8e09 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -396,8 +396,17 @@ class DeviceManager(Thread): # {{{ if DEBUG: prints(traceback.format_exc(), file=sys.__stdout__) - return self.device.upload_books(files, names, on_card, - metadata=metadata, end_session=False) + try: + return self.device.upload_books(files, names, on_card, + metadata=metadata, end_session=False) + finally: + if metadata: + for mi in metadata: + try: + if mi.cover: + os.remove(mi.cover) + except: + pass def upload_books(self, done, files, names, on_card=None, titles=None, metadata=None, plugboards=None, add_as_step_to_job=None): @@ -1072,8 +1081,6 @@ class DeviceMixin(object): # {{{ 'the device?'), autos): self.iactions['Convert Books'].auto_convert_news(auto, format) files = [f for f in files if f is not None] - for f in files: - f.deleted_after_upload = del_on_upload if not files: self.news_to_be_synced = set([]) return @@ -1315,8 +1322,17 @@ class DeviceMixin(object): # {{{ self.card_b_view if on_card == 'cardb' else self.memory_view view.model().resort(reset=False) view.model().research() - for f in files: - getattr(f, 'close', lambda : True)() + if files: + for f in files: + # Remove temporary files + try: + rem = not getattr( + self.device_manager.device, + 'KEEP_TEMP_FILES_AFTER_UPLOAD', False) + if rem and 'caltmpfmt.' in f: + os.remove(f) + except: + pass def book_on_device(self, id, reset=False): ''' diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index ce21eba00e..7c7c78629c 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -24,7 +24,7 @@ from calibre.utils.config import prefs, tweaks from calibre.utils.magick.draw import identify_data from calibre.utils.date import qt_to_dt -def get_cover_data(path): # {{{ +def get_cover_data(stream, ext): # {{{ from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: @@ -32,8 +32,8 @@ def get_cover_data(path): # {{{ cdata = area = None try: - mi = get_metadata(open(path, 'rb'), - os.path.splitext(path)[1][1:].lower()) + with stream: + mi = get_metadata(stream, ext) if mi.cover and os.access(mi.cover, os.R_OK): cdata = open(mi.cover).read() elif mi.cover_data[1] is not None: @@ -186,9 +186,10 @@ class MyBlockingBusy(QDialog): # {{{ if fmts: covers = [] for fmt in fmts.split(','): - fmt = self.db.format_abspath(id, fmt, index_is_id=True) - if not fmt: continue - cdata, area = get_cover_data(fmt) + fmtf = self.db.format(id, fmt, index_is_id=True, + as_file=True) + if fmtf is None: continue + cdata, area = get_cover_data(fmtf, fmt) if cdata: covers.append((cdata, area)) covers.sort(key=lambda x: x[1]) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index b82f421e1e..b9c760abff 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -174,7 +174,8 @@ class EmailMixin(object): # {{{ else: _auto_ids = [] - full_metadata = self.library_view.model().metadata_for(ids) + full_metadata = self.library_view.model().metadata_for(ids, + get_cover=False) bad, remove_ids, jobnames = [], [], [] texts, subjects, attachments, attachment_names = [], [], [], [] diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index f49c6db59a..40d6e2b6cf 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -5,8 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import shutil, functools, re, os, traceback -from contextlib import closing +import functools, re, os, traceback from collections import defaultdict from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage, @@ -36,14 +35,6 @@ TIME_FMT = '%d %b %Y' ALIGNMENT_MAP = {'left': Qt.AlignLeft, 'right': Qt.AlignRight, 'center': Qt.AlignHCenter} -class FormatPath(unicode): - - def __new__(cls, path, orig_file_path): - ans = unicode.__new__(cls, path) - ans.orig_file_path = orig_file_path - ans.deleted_after_upload = False - return ans - _default_image = None def default_image(): @@ -391,10 +382,14 @@ class BooksModel(QAbstractTableModel): # {{{ data = self.current_changed(index, None, False) return data - def metadata_for(self, ids): + def metadata_for(self, ids, get_cover=True): + ''' + WARNING: if get_cover=True temp files are created for mi.cover. + Remember to delete them once you are done with them. + ''' ans = [] for id in ids: - mi = self.db.get_metadata(id, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(id, index_is_id=True, get_cover=get_cover) ans.append(mi) return ans @@ -449,18 +444,14 @@ class BooksModel(QAbstractTableModel): # {{{ format = f break if format is not None: - pt = PersistentTemporaryFile(suffix='.'+format) - with closing(self.db.format(id, format, index_is_id=True, - as_file=True)) as src: - shutil.copyfileobj(src, pt) - pt.flush() - if getattr(src, 'name', None): - pt.orig_file_path = os.path.abspath(src.name) + pt = PersistentTemporaryFile(suffix='caltmpfmt.'+format) + self.db.copy_format_to(id, format, pt, index_is_id=True) pt.seek(0) if set_metadata: try: - _set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True), - format) + _set_metadata(pt, self.db.get_metadata( + id, get_cover=True, index_is_id=True, + cover_as_data=True), format) except: traceback.print_exc() pt.close() @@ -468,9 +459,7 @@ class BooksModel(QAbstractTableModel): # {{{ if isbytestring(x): x = x.decode(filesystem_encoding) return x - name, op = map(to_uni, map(os.path.abspath, (pt.name, - pt.orig_file_path))) - ans.append(FormatPath(name, op)) + ans.append(to_uni(os.path.abspath(pt.name))) else: need_auto.append(id) if not exclude_auto: @@ -499,13 +488,11 @@ class BooksModel(QAbstractTableModel): # {{{ break if format is not None: pt = PersistentTemporaryFile(suffix='.'+format) - with closing(self.db.format(row, format, as_file=True)) as src: - shutil.copyfileobj(src, pt) - pt.flush() + self.db.copy_format_to(id, format, pt, index_is_id=True) pt.seek(0) if set_metadata: - _set_metadata(pt, self.db.get_metadata(row, get_cover=True), - format) + _set_metadata(pt, self.db.get_metadata(row, get_cover=True, + cover_as_data=True), format) pt.close() if paths else pt.seek(0) ans.append(pt) else: diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 3ca898d15a..9aa926f5c5 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -584,14 +584,15 @@ class BooksView(QTableView): # {{{ m = self.model() db = m.db rows = self.selectionModel().selectedRows() - selected = map(m.id, rows) + selected = list(map(m.id, rows)) ids = ' '.join(map(str, selected)) md = QMimeData() md.setData('application/calibre+from_library', ids) fmt = prefs['output_format'] def url_for_id(i): - ans = db.format_abspath(i, fmt, index_is_id=True) + ans = db.format(i, fmt, index_is_id=True, as_path=True, + preserve_filename=True) if ans is None: fmts = db.formats(i, index_is_id=True) if fmts: @@ -599,14 +600,13 @@ class BooksView(QTableView): # {{{ else: fmts = [] for f in fmts: - ans = db.format_abspath(i, f, index_is_id=True) - if ans is not None: - break + ans = db.format(i, f, index_is_id=True, as_path=True, + preserve_filename=True) if ans is None: ans = db.abspath(i, index_is_id=True) return QUrl.fromLocalFile(ans) - md.setUrls([url_for_id(i) for i in selected]) + md.setUrls([url_for_id(i) for i in selected[:25]]) drag = QDrag(self) col = self.selectionModel().currentIndex().column() md.column_name = self.column_map[col] diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index f865a5c62c..303cc51c74 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -688,7 +688,8 @@ class FormatsManager(QWidget): # {{{ else: stream = open(fmt.path, 'r+b') try: - mi = get_metadata(stream, ext) + with stream: + mi = get_metadata(stream, ext) return mi, ext except: error_dialog(self, _('Could not read metadata'), diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 39224c8b35..03b32ca2fb 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -51,12 +51,15 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ # continue mi = db.get_metadata(book_id, True) - in_file = db.format_abspath(book_id, d.input_format, True) + in_file = PersistentTemporaryFile('.'+d.input_format) + with in_file: + db.copy_format_to(book_id, d.input_format, in_file, + index_is_id=True) out_file = PersistentTemporaryFile('.' + d.output_format) out_file.write(d.output_format) out_file.close() - temp_files = [] + temp_files = [in_file] try: dtitle = unicode(mi.title) @@ -74,7 +77,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ recs.append(('cover', d.cover_file.name, OptionRecommendation.HIGH)) temp_files.append(d.cover_file) - args = [in_file, out_file.name, recs] + args = [in_file.name, out_file.name, recs] temp_files.append(out_file) jobs.append(('gui_convert_override', args, desc, d.output_format.upper(), book_id, temp_files)) @@ -142,12 +145,15 @@ class QueueBulk(QProgressDialog): try: input_format = get_input_format_for_book(self.db, book_id, None)[0] mi, opf_file = create_opf_file(self.db, book_id) - in_file = self.db.format_abspath(book_id, input_format, True) + in_file = PersistentTemporaryFile('.'+input_format) + with in_file: + self.db.copy_format_to(book_id, input_format, in_file, + index_is_id=True) out_file = PersistentTemporaryFile('.' + self.output_format) out_file.write(self.output_format) out_file.close() - temp_files = [] + temp_files = [in_file] combined_recs = GuiRecommendations() default_recs = bulk_defaults_for_input_format(input_format) @@ -183,7 +189,7 @@ class QueueBulk(QProgressDialog): self.setLabelText(_('Queueing ')+dtitle) desc = _('Convert book %d of %d (%s)') % (self.i, len(self.book_ids), dtitle) - args = [in_file, out_file.name, lrecs] + args = [in_file.name, out_file.name, lrecs] temp_files.append(out_file) self.jobs.append(('gui_convert_override', args, desc, self.output_format.upper(), book_id, temp_files)) diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py index 2e00db32c4..84a7acbc73 100644 --- a/src/calibre/library/__init__.py +++ b/src/calibre/library/__init__.py @@ -61,22 +61,4 @@ def generate_test_db(library_path, # {{{ print 'Time per record:', t/float(num_of_records) # }}} -def cover_load_timing(path=None): - from PyQt4.Qt import QApplication, QImage - import os, time - app = QApplication([]) - app - d = db(path) - paths = [d.cover(i, index_is_id=True, as_path=True) for i in - d.data.iterallids()] - paths = [p for p in paths if (p and os.path.exists(p) and os.path.isfile(p))] - - start = time.time() - - for p in paths: - with open(p, 'rb') as f: - img = QImage() - img.loadFromData(f.read()) - - print 'Average load time:', (time.time() - start)/len(paths), 'seconds' diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 8508fb266f..1b8bb365ab 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -5137,6 +5137,7 @@ Author '{0}': OptionRecommendation.HIGH)) # If cover exists, use it + cpath = None try: search_text = 'title:"%s" author:%s' % ( opts.catalog_title.replace('"', '\\"'), 'calibre') @@ -5157,5 +5158,10 @@ Author '{0}': plumber.merge_ui_recommendations(recommendations) plumber.run() + try: + os.remove(cpath) + except: + pass + # returns to gui2.actions.catalog:catalog_generated() return catalog.error diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index efb130676a..4c61438e35 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -12,8 +12,6 @@ import threading, random from itertools import repeat from math import ceil -from PyQt4.QtGui import QImage - from calibre import prints from calibre.ebooks.metadata import (title_sort, author_to_author_sort, string_to_authors, authors_to_string) @@ -27,7 +25,7 @@ from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs from calibre.ebooks.metadata.book.base import Metadata from calibre.constants import preferred_encoding, iswindows, filesystem_encoding -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, base_dir from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring from calibre.utils.filenames import ascii_filename @@ -39,8 +37,10 @@ from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions +from calibre.db.errors import NoSuchFormat copyfile = os.link if hasattr(os, 'link') else shutil.copyfile +SPOOL_SIZE = 30*1024*1024 class Tag(object): @@ -601,14 +601,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f: f.write(cdata) for format in formats: - # Get data as string (can't use file as source and target files may be the same) - f = self.format(id, format, index_is_id=True, as_file=True) - if f is None: - continue - with tempfile.SpooledTemporaryFile(max_size=30*(1024**2)) as stream: - with f: - shutil.copyfileobj(f, stream) - stream.seek(0) + with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream: + try: + self.copy_format_to(id, format, stream, index_is_id=True) + stream.seek(0) + except NoSuchFormat: + continue self.add_format(id, format, stream, index_is_id=True, path=tpath, notify=False) self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id)) @@ -661,32 +659,53 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): continue def cover(self, index, index_is_id=False, as_file=False, as_image=False, - as_path=False): + as_path=False): ''' Return the cover image as a bytestring (in JPEG format) or None. - `as_file` : If True return the image as an open file object - `as_image`: If True return the image as a QImage object + WARNING: Using as_path will copy the cover to a temp file and return + the path to the temp file. You should delete the temp file when you are + done with it. + + :param as_file: If True return the image as an open file object (a SpooledTemporaryFile) + :param as_image: If True return the image as a QImage object ''' - id = index if index_is_id else self.id(index) + id = index if index_is_id else self.id(index) path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') if os.access(path, os.R_OK): - if as_path: - return path try: f = lopen(path, 'rb') except (IOError, OSError): time.sleep(0.2) f = lopen(path, 'rb') - if as_image: - img = QImage() - img.loadFromData(f.read()) - f.close() - return img - ans = f if as_file else f.read() - if ans is not f: - f.close() - return ans + with f: + if as_path: + pt = PersistentTemporaryFile('_dbcover.jpg') + with pt: + shutil.copyfileobj(f, pt) + return pt.name + if as_file: + ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE) + shutil.copyfileobj(f, ret) + ret.seek(0) + else: + ret = f.read() + if as_image: + from PyQt4.Qt import QImage + i = QImage() + i.loadFromData(ret) + ret = i + return ret + + def cover_last_modified(self, index, index_is_id=False): + id = index if index_is_id else self.id(index) + path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') + try: + return utcfromtimestamp(os.stat(path).st_mtime) + except: + # Cover doesn't exist + pass + return self.last_modified() ### The field-style interface. These use field keys. @@ -859,7 +878,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return (path, mi, sequence) def get_metadata(self, idx, index_is_id=False, get_cover=False, - get_user_categories=True): + get_user_categories=True, cover_as_data=False): ''' Convenience method to return metadata as a :class:`Metadata` object. Note that the list of formats is not verified. @@ -934,7 +953,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.user_categories = user_cat_vals if get_cover: - mi.cover = self.cover(id, index_is_id=True, as_path=True) + if cover_as_data: + cdata = self.cover(id, index_is_id=True) + if cdata: + mi.cover_data = ('jpeg', cdata) + else: + mi.cover = self.cover(id, index_is_id=True, as_path=True) return mi def has_book(self, mi): @@ -1099,7 +1123,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return utcfromtimestamp(os.stat(path).st_mtime) def format_abspath(self, index, format, index_is_id=False): - 'Return absolute path to the ebook file of format `format`' + ''' + Return absolute path to the ebook file of format `format` + + WARNING: This method will return a dummy path for a network backend DB, + so do not rely on it, use format(..., as_path=True) instead. + + Currently used only in calibredb list, the viewer and the catalogs (via + get_data_as_dict()). + + Apart from the viewer, I don't believe any of the others do any file + I/O with the results of this call. + ''' id = index if index_is_id else self.id(index) try: name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) @@ -1119,25 +1154,63 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): shutil.copyfile(candidates[0], fmt_path) return fmt_path - def format(self, index, format, index_is_id=False, as_file=False, mode='r+b'): + def copy_format_to(self, index, fmt, dest, index_is_id=False): + ''' + Copy the format ``fmt`` to the file like object ``dest``. If the + specified format does not exist, raises :class:`NoSuchFormat` error. + ''' + path = self.format_abspath(index, fmt, index_is_id=index_is_id) + if path is None: + id_ = index if index_is_id else self.id(index) + raise NoSuchFormat('Record %d has no %s file'%(id_, fmt)) + with lopen(path, 'rb') as f: + shutil.copyfileobj(f, dest) + if hasattr(dest, 'flush'): + dest.flush() + + def format(self, index, format, index_is_id=False, as_file=False, + mode='r+b', as_path=False, preserve_filename=False): ''' Return the ebook format as a bytestring or `None` if the format doesn't exist, or we don't have permission to write to the ebook file. - `as_file`: If True the ebook format is returned as a file object opened in `mode` + :param as_file: If True the ebook format is returned as a file object. Note + that the file object is a SpooledTemporaryFile, so if what you want to + do is copy the format to another file, use :method:`copy_format_to` + instead for performance. + :param as_path: Copies the format file to a temp file and returns the + path to the temp file + :param preserve_filename: If True and returning a path the filename is + the same as that used in the library. Note that using + this means that repeated calls yield the same + temp file (which is re-created each time) + :param mode: This is ignored (present for legacy compatibility) ''' path = self.format_abspath(index, format, index_is_id=index_is_id) if path is not None: - f = lopen(path, mode) - try: - ret = f if as_file else f.read() - except IOError: - f.seek(0) - out = cStringIO.StringIO() - shutil.copyfileobj(f, out) - ret = out.getvalue() - if not as_file: - f.close() + with lopen(path, mode) as f: + if as_path: + if preserve_filename: + bd = base_dir() + d = os.path.join(bd, 'format_abspath') + try: + os.makedirs(d) + except: + pass + fname = os.path.basename(path) + ret = os.path.join(d, fname) + with lopen(ret, 'wb') as f2: + shutil.copyfileobj(f, f2) + else: + with PersistentTemporaryFile('.'+format.lower()) as pt: + shutil.copyfileobj(f, pt) + ret = pt.name + elif as_file: + ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) + shutil.copyfileobj(f, ret) + ret.seek(0) + else: + ret = f.read() return ret def add_format_with_hooks(self, index, format, fpath, index_is_id=False, diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 5f49833564..b5c4e2faf3 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, cStringIO, re, shutil +import os, traceback, cStringIO, re from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks @@ -238,7 +238,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, def save_book_to_disk(id_, db, root, opts, length): mi = db.get_metadata(id_, index_is_id=True) - cover = db.cover(id_, index_is_id=True, as_path=True) + cover = db.cover(id_, index_is_id=True) plugboards = db.prefs.get('plugboards', {}) available_formats = db.formats(id_, index_is_id=True) @@ -252,12 +252,20 @@ def save_book_to_disk(id_, db, root, opts, length): if fmts: fmts = fmts.split(',') for fmt in fmts: - fpath = db.format_abspath(id_, fmt, index_is_id=True) + fpath = db.format(id_, fmt, index_is_id=True, as_path=True) if fpath is not None: formats[fmt.lower()] = fpath - return do_save_book_to_disk(id_, mi, cover, plugboards, + try: + return do_save_book_to_disk(id_, mi, cover, plugboards, formats, root, opts, length) + finally: + for temp in formats.itervalues(): + try: + os.remove(temp) + except: + pass + def do_save_book_to_disk(id_, mi, cover, plugboards, @@ -289,10 +297,9 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, raise ocover = mi.cover - if opts.save_cover and cover and os.access(cover, os.R_OK): + if opts.save_cover and cover: with open(base_path+'.jpg', 'wb') as f: - with open(cover, 'rb') as s: - shutil.copyfileobj(s, f) + f.write(cover) mi.cover = base_name+'.jpg' else: mi.cover = None @@ -395,8 +402,13 @@ def save_serialized_to_disk(ids, data, plugboards, root, opts, callback): pass tb = '' try: - failed, id, title = do_save_book_to_disk(x, mi, cover, plugboards, - format_map, root, opts, length) + with open(cover, 'rb') as f: + cover = f.read() + except: + cover = None + try: + failed, id, title = do_save_book_to_disk(x, mi, cover, + plugboards, format_map, root, opts, length) tb = _('Requested formats not available') except: failed, id, title = True, x, mi.title diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index 08de4faecd..853ec7829c 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -10,12 +10,13 @@ import re, os, posixpath import cherrypy from calibre import fit_image, guess_type -from calibre.utils.date import fromtimestamp +from calibre.utils.date import fromtimestamp, utcnow from calibre.library.caches import SortKeyGenerator from calibre.library.save_to_disk import find_plugboard - -from calibre.utils.magick.draw import save_cover_data_to, Image, \ - thumbnail as generate_thumbnail +from calibre.ebooks.metadata import authors_to_string +from calibre.utils.magick.draw import (save_cover_data_to, Image, + thumbnail as generate_thumbnail) +from calibre.utils.filenames import ascii_filename plugboard_content_server_value = 'content_server' plugboard_content_server_formats = ['epub'] @@ -46,7 +47,7 @@ class ContentServer(object): # Utility methods {{{ def last_modified(self, updated): ''' - Generates a local independent, english timestamp from a datetime + Generates a locale independent, english timestamp from a datetime object ''' lm = updated.strftime('day, %d month %Y %H:%M:%S GMT') @@ -151,14 +152,12 @@ class ContentServer(object): try: cherrypy.response.headers['Content-Type'] = 'image/jpeg' cherrypy.response.timeout = 3600 - cover = self.db.cover(id, index_is_id=True, as_file=True) + cover = self.db.cover(id, index_is_id=True) if cover is None: cover = self.default_cover updated = self.build_time else: - with cover as f: - updated = fromtimestamp(os.fstat(f.fileno()).st_mtime) - cover = f.read() + updated = self.db.cover_last_modified(id, index_is_id=True) cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) if thumbnail: @@ -187,9 +186,9 @@ class ContentServer(object): mode='rb') if fmt is None: raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format)) + mi = self.db.get_metadata(id, index_is_id=True) if format == 'EPUB': # Get the original metadata - mi = self.db.get_metadata(id, index_is_id=True) # Get any EPUB plugboards for the content server plugboards = self.db.prefs.get('plugboards', {}) @@ -203,24 +202,22 @@ class ContentServer(object): newmi = mi # Write the updated file - from tempfile import TemporaryFile from calibre.ebooks.metadata.meta import set_metadata - raw = fmt.read() - fmt = TemporaryFile() - fmt.write(raw) - fmt.seek(0) set_metadata(fmt, newmi, 'epub') fmt.seek(0) mt = guess_type('dummy.'+format.lower())[0] if mt is None: mt = 'application/octet-stream' + au = authors_to_string(mi.authors if mi.authors else [_('Unknown')]) + title = mi.title if mi.title else _('Unknown') + fname = u'%s - %s_%s.%s'%(title[:30], au[:30], id, format.lower()) + fname = ascii_filename(fname).replace('"', '_') cherrypy.response.headers['Content-Type'] = mt + cherrypy.response.headers['Content-Disposition'] = \ + b'attachment; filename="%s"'%fname cherrypy.response.timeout = 3600 - path = getattr(fmt, 'name', None) - if path and os.path.exists(path): - updated = fromtimestamp(os.stat(path).st_mtime) - cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) + cherrypy.response.headers['Last-Modified'] = self.last_modified(utcnow()) return fmt # }}}