diff --git a/src/calibre/db/backend.py b/src/calibre/db/backend.py index 159612e52d..ba683dde50 100644 --- a/src/calibre/db/backend.py +++ b/src/calibre/db/backend.py @@ -222,7 +222,9 @@ class DB(object, SchemaUpgrade): if self.user_version == 0: self.initialize_database() - SchemaUpgrade.__init__(self) + with self.conn: + SchemaUpgrade.__init__(self) + # Guarantee that the library_id is set self.library_id diff --git a/src/calibre/db/errors.py b/src/calibre/db/errors.py new file mode 100644 index 0000000000..d2657f9904 --- /dev/null +++ b/src/calibre/db/errors.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + + +class NoSuchFormat(ValueError): + pass + diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index dea5844028..cc531b7476 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -107,6 +107,7 @@ class DriverBase(DeviceConfig, DevicePlugin): # Needed for config_widget to work FORMATS = ['epub', 'pdf'] USER_CAN_ADD_NEW_FORMATS = False + KEEP_TEMP_FILES_AFTER_UPLOAD = True # Hide the standard customization widgets SUPPORTS_SUB_DIRS = False diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index b265331ace..925f9eafd0 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -327,12 +327,7 @@ class DevicePlugin(Plugin): free space on the device. The text of the FreeSpaceError must contain the word "card" if ``on_card`` is not None otherwise it must contain the word "memory". - :param files: A list of paths and/or file-like objects. If they are paths and - the paths point to temporary files, they may have an additional - attribute, original_file_path pointing to the originals. They may have - another optional attribute, deleted_after_upload which if True means - that the file pointed to by original_file_path will be deleted after - being uploaded to the device. + :param files: A list of paths :param names: A list of file names that the books should have once uploaded to the device. len(names) == len(files) :param metadata: If not None, it is a list of :class:`Metadata` objects. diff --git a/src/calibre/ebooks/metadata/worker.py b/src/calibre/ebooks/metadata/worker.py index c335cc4c13..ca8707258b 100644 --- a/src/calibre/ebooks/metadata/worker.py +++ b/src/calibre/ebooks/metadata/worker.py @@ -15,7 +15,7 @@ from calibre.utils.ipc.server import Server from calibre.ptempfile import PersistentTemporaryDirectory, TemporaryDirectory from calibre import prints, isbytestring from calibre.constants import filesystem_encoding - +from calibre.db.errors import NoSuchFormat def debug(*args): prints(*args) @@ -201,27 +201,35 @@ class SaveWorker(Thread): self.spare_server = spare_server self.start() - def collect_data(self, ids): + def collect_data(self, ids, tdir): from calibre.ebooks.metadata.opf2 import metadata_to_opf data = {} for i in set(ids): - mi = self.db.get_metadata(i, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(i, index_is_id=True, get_cover=True, + cover_as_data=True) opf = metadata_to_opf(mi) if isbytestring(opf): opf = opf.decode('utf-8') cpath = None - if mi.cover: - cpath = mi.cover + if mi.cover_data and mi.cover_data[1]: + cpath = os.path.join(tdir, 'cover_%s.jpg'%i) + with lopen(cpath, 'wb') as f: + f.write(mi.cover_data[1]) if isbytestring(cpath): cpath = cpath.decode(filesystem_encoding) formats = {} if mi.formats: for fmt in mi.formats: - fpath = self.db.format_abspath(i, fmt, index_is_id=True) - if fpath is not None: - if isbytestring(fpath): - fpath = fpath.decode(filesystem_encoding) - formats[fmt.lower()] = fpath + fpath = os.path.join(tdir, 'fmt_%s.%s'%(i, fmt.lower())) + with lopen(fpath, 'wb') as f: + try: + self.db.copy_format_to(i, fmt, f, index_is_id=True) + except NoSuchFormat: + continue + else: + if isbytestring(fpath): + fpath = fpath.decode(filesystem_encoding) + formats[fmt.lower()] = fpath data[i] = [opf, cpath, formats, mi.last_modified.isoformat()] return data @@ -244,7 +252,7 @@ class SaveWorker(Thread): for i, task in enumerate(tasks): tids = [x[-1] for x in task] - data = self.collect_data(tids) + data = self.collect_data(tids, tdir) dpath = os.path.join(tdir, '%d.json'%i) with open(dpath, 'wb') as f: f.write(json.dumps(data, ensure_ascii=False).encode('utf-8')) diff --git a/src/calibre/gui2/actions/copy_to_library.py b/src/calibre/gui2/actions/copy_to_library.py index 7190d1486f..97880faaa1 100644 --- a/src/calibre/gui2/actions/copy_to_library.py +++ b/src/calibre/gui2/actions/copy_to_library.py @@ -53,13 +53,18 @@ class Worker(Thread): # {{{ from calibre.library.database2 import LibraryDatabase2 newdb = LibraryDatabase2(self.loc) for i, x in enumerate(self.ids): - mi = self.db.get_metadata(x, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(x, index_is_id=True, get_cover=True, + cover_as_data=True) self.progress(i, mi.title) fmts = self.db.formats(x, index_is_id=True) if not fmts: fmts = [] else: fmts = fmts.split(',') - paths = [self.db.format_abspath(x, fmt, index_is_id=True) for fmt in - fmts] + paths = [] + for fmt in fmts: + p = self.db.format(x, fmt, index_is_id=True, + as_path=True) + if p: + paths.append(p) added = False if prefs['add_formats_to_existing']: identical_book_list = newdb.find_identical_books(mi) @@ -75,6 +80,11 @@ class Worker(Thread): # {{{ if co is not None: newdb.set_conversion_options(x, 'PIPE', co) self.processed.add(x) + for path in paths: + try: + os.remove(path) + except: + pass # }}} class CopyToLibraryAction(InterfaceAction): diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 650947100e..718ece46d2 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -17,6 +17,7 @@ from calibre.gui2.dialogs.tag_list_editor import TagListEditor from calibre.gui2.actions import InterfaceAction from calibre.ebooks.metadata import authors_to_string from calibre.utils.icu import sort_key +from calibre.db.errors import NoSuchFormat class EditMetadataAction(InterfaceAction): @@ -265,7 +266,7 @@ class EditMetadataAction(InterfaceAction): +'

', 'merge_too_many_books', self.gui): return - dest_id, src_books, src_ids = self.books_to_merge(rows) + dest_id, src_ids = self.books_to_merge(rows) title = self.gui.library_view.model().db.title(dest_id, index_is_id=True) if safe_merge: if not confirm('

'+_( @@ -277,7 +278,7 @@ class EditMetadataAction(InterfaceAction): 'Please confirm you want to proceed.')%title +'

', 'merge_books_safe', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.merge_metadata(dest_id, src_ids) elif merge_only_formats: if not confirm('

'+_( @@ -293,7 +294,7 @@ class EditMetadataAction(InterfaceAction): 'Are you sure you want to proceed?')%title +'

', 'merge_only_formats', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.delete_books_after_merge(src_ids) else: if not confirm('

'+_( @@ -308,7 +309,7 @@ class EditMetadataAction(InterfaceAction): 'Are you sure you want to proceed?')%title +'

', 'merge_books', self.gui): return - self.add_formats(dest_id, src_books) + self.add_formats(dest_id, self.formats_for_books(rows)) self.merge_metadata(dest_id, src_ids) self.delete_books_after_merge(src_ids) # leave the selection highlight on first selected book @@ -329,8 +330,22 @@ class EditMetadataAction(InterfaceAction): self.gui.library_view.model().db.add_format(dest_id, fmt, f, index_is_id=True, notify=False, replace=replace) + def formats_for_books(self, rows): + m = self.gui.library_view.model() + ans = [] + for id_ in map(m.id, rows): + dbfmts = m.db.formats(id_, index_is_id=True) + if dbfmts: + for fmt in dbfmts.split(','): + try: + path = m.db.format(id_, fmt, index_is_id=True, + as_path=True) + ans.append(path) + except NoSuchFormat: + continue + return ans + def books_to_merge(self, rows): - src_books = [] src_ids = [] m = self.gui.library_view.model() for i, row in enumerate(rows): @@ -339,22 +354,19 @@ class EditMetadataAction(InterfaceAction): dest_id = id_ else: src_ids.append(id_) - dbfmts = m.db.formats(id_, index_is_id=True) - if dbfmts: - for fmt in dbfmts.split(','): - src_books.append(m.db.format_abspath(id_, fmt, - index_is_id=True)) - return [dest_id, src_books, src_ids] + return [dest_id, src_ids] def delete_books_after_merge(self, ids_to_delete): self.gui.library_view.model().delete_books_by_id(ids_to_delete) def merge_metadata(self, dest_id, src_ids): db = self.gui.library_view.model().db - dest_mi = db.get_metadata(dest_id, index_is_id=True, get_cover=True) + dest_mi = db.get_metadata(dest_id, index_is_id=True) orig_dest_comments = dest_mi.comments + dest_cover = db.cover(dest_id, index_is_id=True) + had_orig_cover = bool(dest_cover) for src_id in src_ids: - src_mi = db.get_metadata(src_id, index_is_id=True, get_cover=True) + src_mi = db.get_metadata(src_id, index_is_id=True) if src_mi.comments and orig_dest_comments != src_mi.comments: if not dest_mi.comments: dest_mi.comments = src_mi.comments @@ -372,8 +384,10 @@ class EditMetadataAction(InterfaceAction): dest_mi.tags = src_mi.tags else: dest_mi.tags.extend(src_mi.tags) - if src_mi.cover and not dest_mi.cover: - dest_mi.cover = src_mi.cover + if not dest_cover: + src_cover = db.cover(src_id, index_is_id=True) + if src_cover: + dest_cover = src_cover if not dest_mi.publisher: dest_mi.publisher = src_mi.publisher if not dest_mi.rating: @@ -382,6 +396,8 @@ class EditMetadataAction(InterfaceAction): dest_mi.series = src_mi.series dest_mi.series_index = src_mi.series_index db.set_metadata(dest_id, dest_mi, ignore_errors=False) + if not had_orig_cover and dest_cover: + db.set_cover(dest_id, dest_cover) for key in db.field_metadata: #loop thru all defined fields if db.field_metadata[key]['is_custom']: diff --git a/src/calibre/gui2/actions/tweak_epub.py b/src/calibre/gui2/actions/tweak_epub.py index c9f2d7a8c6..d3924e7cd3 100755 --- a/src/calibre/gui2/actions/tweak_epub.py +++ b/src/calibre/gui2/actions/tweak_epub.py @@ -5,6 +5,8 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' +import os + from calibre.gui2 import error_dialog from calibre.gui2.actions import InterfaceAction from calibre.gui2.dialogs.tweak_epub import TweakEpub @@ -30,8 +32,8 @@ class TweakEpubAction(InterfaceAction): # Confirm 'EPUB' in formats book_id = self.gui.library_view.model().id(row) try: - path_to_epub = self.gui.library_view.model().db.format_abspath( - book_id, 'EPUB', index_is_id=True) + path_to_epub = self.gui.library_view.model().db.format( + book_id, 'EPUB', index_is_id=True, as_path=True) except: path_to_epub = None @@ -45,6 +47,7 @@ class TweakEpubAction(InterfaceAction): if dlg.exec_() == dlg.Accepted: self.update_db(book_id, dlg._output) dlg.cleanup() + os.remove(path_to_epub) def update_db(self, book_id, rebuilt): ''' diff --git a/src/calibre/gui2/add.py b/src/calibre/gui2/add.py index 44b5bb446b..3dbf4b94df 100644 --- a/src/calibre/gui2/add.py +++ b/src/calibre/gui2/add.py @@ -445,6 +445,7 @@ class Saver(QObject): # {{{ self.pd.setModal(True) self.pd.show() self.pd.set_min(0) + self.pd.set_msg(_('Collecting data, please wait...')) self._parent = parent self.callback = callback self.callback_called = False diff --git a/src/calibre/gui2/convert/regex_builder.py b/src/calibre/gui2/convert/regex_builder.py index bf32bf472a..f79e6df3fe 100644 --- a/src/calibre/gui2/convert/regex_builder.py +++ b/src/calibre/gui2/convert/regex_builder.py @@ -4,7 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2009, John Schember ' __docformat__ = 'restructuredtext en' -import re +import re, os from PyQt4.QtCore import SIGNAL, Qt, pyqtSignal from PyQt4.QtGui import QDialog, QWidget, QDialogButtonBox, \ @@ -134,7 +134,12 @@ class RegexBuilder(QDialog, Ui_RegexBuilder): _('Cannot build regex using the GUI builder without a book.'), show=True) return False - self.open_book(db.format_abspath(book_id, format, index_is_id=True)) + fpath = db.format(book_id, format, index_is_id=True, + as_path=True) + try: + self.open_book(fpath) + finally: + os.remove(fpath) return True def open_book(self, pathtoebook): diff --git a/src/calibre/gui2/convert/single.py b/src/calibre/gui2/convert/single.py index 3575fb5ffb..15e670ee32 100644 --- a/src/calibre/gui2/convert/single.py +++ b/src/calibre/gui2/convert/single.py @@ -106,7 +106,6 @@ class Config(ResizableDialog, Ui_Dialog): Configuration dialog for single book conversion. If accepted, has the following important attributes - input_path - Path to input file output_format - Output format (without a leading .) input_format - Input format (without a leading .) opf_path - Path to OPF file with user specified metadata @@ -156,13 +155,10 @@ class Config(ResizableDialog, Ui_Dialog): oidx = self.groups.currentIndex().row() input_format = self.input_format output_format = self.output_format - input_path = self.db.format_abspath(self.book_id, input_format, - index_is_id=True) - self.input_path = input_path output_path = 'dummy.'+output_format log = Log() log.outputs = [] - self.plumber = Plumber(input_path, output_path, log) + self.plumber = Plumber('dummy.'+input_format, output_path, log) def widget_factory(cls): return cls(self.stack, self.plumber.get_option_by_name, diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index a527cc2e27..d5805a8e09 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -396,8 +396,17 @@ class DeviceManager(Thread): # {{{ if DEBUG: prints(traceback.format_exc(), file=sys.__stdout__) - return self.device.upload_books(files, names, on_card, - metadata=metadata, end_session=False) + try: + return self.device.upload_books(files, names, on_card, + metadata=metadata, end_session=False) + finally: + if metadata: + for mi in metadata: + try: + if mi.cover: + os.remove(mi.cover) + except: + pass def upload_books(self, done, files, names, on_card=None, titles=None, metadata=None, plugboards=None, add_as_step_to_job=None): @@ -1072,8 +1081,6 @@ class DeviceMixin(object): # {{{ 'the device?'), autos): self.iactions['Convert Books'].auto_convert_news(auto, format) files = [f for f in files if f is not None] - for f in files: - f.deleted_after_upload = del_on_upload if not files: self.news_to_be_synced = set([]) return @@ -1315,8 +1322,17 @@ class DeviceMixin(object): # {{{ self.card_b_view if on_card == 'cardb' else self.memory_view view.model().resort(reset=False) view.model().research() - for f in files: - getattr(f, 'close', lambda : True)() + if files: + for f in files: + # Remove temporary files + try: + rem = not getattr( + self.device_manager.device, + 'KEEP_TEMP_FILES_AFTER_UPLOAD', False) + if rem and 'caltmpfmt.' in f: + os.remove(f) + except: + pass def book_on_device(self, id, reset=False): ''' diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index ce21eba00e..7c7c78629c 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -24,7 +24,7 @@ from calibre.utils.config import prefs, tweaks from calibre.utils.magick.draw import identify_data from calibre.utils.date import qt_to_dt -def get_cover_data(path): # {{{ +def get_cover_data(stream, ext): # {{{ from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: @@ -32,8 +32,8 @@ def get_cover_data(path): # {{{ cdata = area = None try: - mi = get_metadata(open(path, 'rb'), - os.path.splitext(path)[1][1:].lower()) + with stream: + mi = get_metadata(stream, ext) if mi.cover and os.access(mi.cover, os.R_OK): cdata = open(mi.cover).read() elif mi.cover_data[1] is not None: @@ -186,9 +186,10 @@ class MyBlockingBusy(QDialog): # {{{ if fmts: covers = [] for fmt in fmts.split(','): - fmt = self.db.format_abspath(id, fmt, index_is_id=True) - if not fmt: continue - cdata, area = get_cover_data(fmt) + fmtf = self.db.format(id, fmt, index_is_id=True, + as_file=True) + if fmtf is None: continue + cdata, area = get_cover_data(fmtf, fmt) if cdata: covers.append((cdata, area)) covers.sort(key=lambda x: x[1]) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index b82f421e1e..b9c760abff 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -174,7 +174,8 @@ class EmailMixin(object): # {{{ else: _auto_ids = [] - full_metadata = self.library_view.model().metadata_for(ids) + full_metadata = self.library_view.model().metadata_for(ids, + get_cover=False) bad, remove_ids, jobnames = [], [], [] texts, subjects, attachments, attachment_names = [], [], [], [] diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index f49c6db59a..40d6e2b6cf 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -5,8 +5,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import shutil, functools, re, os, traceback -from contextlib import closing +import functools, re, os, traceback from collections import defaultdict from PyQt4.Qt import (QAbstractTableModel, Qt, pyqtSignal, QIcon, QImage, @@ -36,14 +35,6 @@ TIME_FMT = '%d %b %Y' ALIGNMENT_MAP = {'left': Qt.AlignLeft, 'right': Qt.AlignRight, 'center': Qt.AlignHCenter} -class FormatPath(unicode): - - def __new__(cls, path, orig_file_path): - ans = unicode.__new__(cls, path) - ans.orig_file_path = orig_file_path - ans.deleted_after_upload = False - return ans - _default_image = None def default_image(): @@ -391,10 +382,14 @@ class BooksModel(QAbstractTableModel): # {{{ data = self.current_changed(index, None, False) return data - def metadata_for(self, ids): + def metadata_for(self, ids, get_cover=True): + ''' + WARNING: if get_cover=True temp files are created for mi.cover. + Remember to delete them once you are done with them. + ''' ans = [] for id in ids: - mi = self.db.get_metadata(id, index_is_id=True, get_cover=True) + mi = self.db.get_metadata(id, index_is_id=True, get_cover=get_cover) ans.append(mi) return ans @@ -449,18 +444,14 @@ class BooksModel(QAbstractTableModel): # {{{ format = f break if format is not None: - pt = PersistentTemporaryFile(suffix='.'+format) - with closing(self.db.format(id, format, index_is_id=True, - as_file=True)) as src: - shutil.copyfileobj(src, pt) - pt.flush() - if getattr(src, 'name', None): - pt.orig_file_path = os.path.abspath(src.name) + pt = PersistentTemporaryFile(suffix='caltmpfmt.'+format) + self.db.copy_format_to(id, format, pt, index_is_id=True) pt.seek(0) if set_metadata: try: - _set_metadata(pt, self.db.get_metadata(id, get_cover=True, index_is_id=True), - format) + _set_metadata(pt, self.db.get_metadata( + id, get_cover=True, index_is_id=True, + cover_as_data=True), format) except: traceback.print_exc() pt.close() @@ -468,9 +459,7 @@ class BooksModel(QAbstractTableModel): # {{{ if isbytestring(x): x = x.decode(filesystem_encoding) return x - name, op = map(to_uni, map(os.path.abspath, (pt.name, - pt.orig_file_path))) - ans.append(FormatPath(name, op)) + ans.append(to_uni(os.path.abspath(pt.name))) else: need_auto.append(id) if not exclude_auto: @@ -499,13 +488,11 @@ class BooksModel(QAbstractTableModel): # {{{ break if format is not None: pt = PersistentTemporaryFile(suffix='.'+format) - with closing(self.db.format(row, format, as_file=True)) as src: - shutil.copyfileobj(src, pt) - pt.flush() + self.db.copy_format_to(id, format, pt, index_is_id=True) pt.seek(0) if set_metadata: - _set_metadata(pt, self.db.get_metadata(row, get_cover=True), - format) + _set_metadata(pt, self.db.get_metadata(row, get_cover=True, + cover_as_data=True), format) pt.close() if paths else pt.seek(0) ans.append(pt) else: diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 3ca898d15a..9aa926f5c5 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -584,14 +584,15 @@ class BooksView(QTableView): # {{{ m = self.model() db = m.db rows = self.selectionModel().selectedRows() - selected = map(m.id, rows) + selected = list(map(m.id, rows)) ids = ' '.join(map(str, selected)) md = QMimeData() md.setData('application/calibre+from_library', ids) fmt = prefs['output_format'] def url_for_id(i): - ans = db.format_abspath(i, fmt, index_is_id=True) + ans = db.format(i, fmt, index_is_id=True, as_path=True, + preserve_filename=True) if ans is None: fmts = db.formats(i, index_is_id=True) if fmts: @@ -599,14 +600,13 @@ class BooksView(QTableView): # {{{ else: fmts = [] for f in fmts: - ans = db.format_abspath(i, f, index_is_id=True) - if ans is not None: - break + ans = db.format(i, f, index_is_id=True, as_path=True, + preserve_filename=True) if ans is None: ans = db.abspath(i, index_is_id=True) return QUrl.fromLocalFile(ans) - md.setUrls([url_for_id(i) for i in selected]) + md.setUrls([url_for_id(i) for i in selected[:25]]) drag = QDrag(self) col = self.selectionModel().currentIndex().column() md.column_name = self.column_map[col] diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index f865a5c62c..303cc51c74 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -688,7 +688,8 @@ class FormatsManager(QWidget): # {{{ else: stream = open(fmt.path, 'r+b') try: - mi = get_metadata(stream, ext) + with stream: + mi = get_metadata(stream, ext) return mi, ext except: error_dialog(self, _('Could not read metadata'), diff --git a/src/calibre/gui2/tools.py b/src/calibre/gui2/tools.py index 39224c8b35..03b32ca2fb 100644 --- a/src/calibre/gui2/tools.py +++ b/src/calibre/gui2/tools.py @@ -51,12 +51,15 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ # continue mi = db.get_metadata(book_id, True) - in_file = db.format_abspath(book_id, d.input_format, True) + in_file = PersistentTemporaryFile('.'+d.input_format) + with in_file: + db.copy_format_to(book_id, d.input_format, in_file, + index_is_id=True) out_file = PersistentTemporaryFile('.' + d.output_format) out_file.write(d.output_format) out_file.close() - temp_files = [] + temp_files = [in_file] try: dtitle = unicode(mi.title) @@ -74,7 +77,7 @@ def convert_single_ebook(parent, db, book_ids, auto_conversion=False, # {{{ recs.append(('cover', d.cover_file.name, OptionRecommendation.HIGH)) temp_files.append(d.cover_file) - args = [in_file, out_file.name, recs] + args = [in_file.name, out_file.name, recs] temp_files.append(out_file) jobs.append(('gui_convert_override', args, desc, d.output_format.upper(), book_id, temp_files)) @@ -142,12 +145,15 @@ class QueueBulk(QProgressDialog): try: input_format = get_input_format_for_book(self.db, book_id, None)[0] mi, opf_file = create_opf_file(self.db, book_id) - in_file = self.db.format_abspath(book_id, input_format, True) + in_file = PersistentTemporaryFile('.'+input_format) + with in_file: + self.db.copy_format_to(book_id, input_format, in_file, + index_is_id=True) out_file = PersistentTemporaryFile('.' + self.output_format) out_file.write(self.output_format) out_file.close() - temp_files = [] + temp_files = [in_file] combined_recs = GuiRecommendations() default_recs = bulk_defaults_for_input_format(input_format) @@ -183,7 +189,7 @@ class QueueBulk(QProgressDialog): self.setLabelText(_('Queueing ')+dtitle) desc = _('Convert book %d of %d (%s)') % (self.i, len(self.book_ids), dtitle) - args = [in_file, out_file.name, lrecs] + args = [in_file.name, out_file.name, lrecs] temp_files.append(out_file) self.jobs.append(('gui_convert_override', args, desc, self.output_format.upper(), book_id, temp_files)) diff --git a/src/calibre/library/__init__.py b/src/calibre/library/__init__.py index 2e00db32c4..84a7acbc73 100644 --- a/src/calibre/library/__init__.py +++ b/src/calibre/library/__init__.py @@ -61,22 +61,4 @@ def generate_test_db(library_path, # {{{ print 'Time per record:', t/float(num_of_records) # }}} -def cover_load_timing(path=None): - from PyQt4.Qt import QApplication, QImage - import os, time - app = QApplication([]) - app - d = db(path) - paths = [d.cover(i, index_is_id=True, as_path=True) for i in - d.data.iterallids()] - paths = [p for p in paths if (p and os.path.exists(p) and os.path.isfile(p))] - - start = time.time() - - for p in paths: - with open(p, 'rb') as f: - img = QImage() - img.loadFromData(f.read()) - - print 'Average load time:', (time.time() - start)/len(paths), 'seconds' diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 8508fb266f..1b8bb365ab 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -5137,6 +5137,7 @@ Author '{0}': OptionRecommendation.HIGH)) # If cover exists, use it + cpath = None try: search_text = 'title:"%s" author:%s' % ( opts.catalog_title.replace('"', '\\"'), 'calibre') @@ -5157,5 +5158,10 @@ Author '{0}': plumber.merge_ui_recommendations(recommendations) plumber.run() + try: + os.remove(cpath) + except: + pass + # returns to gui2.actions.catalog:catalog_generated() return catalog.error diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index efb130676a..4c61438e35 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -12,8 +12,6 @@ import threading, random from itertools import repeat from math import ceil -from PyQt4.QtGui import QImage - from calibre import prints from calibre.ebooks.metadata import (title_sort, author_to_author_sort, string_to_authors, authors_to_string) @@ -27,7 +25,7 @@ from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs from calibre.ebooks.metadata.book.base import Metadata from calibre.constants import preferred_encoding, iswindows, filesystem_encoding -from calibre.ptempfile import PersistentTemporaryFile +from calibre.ptempfile import PersistentTemporaryFile, base_dir from calibre.customize.ui import run_plugins_on_import from calibre import isbytestring from calibre.utils.filenames import ascii_filename @@ -39,8 +37,10 @@ from calibre.ebooks import BOOK_EXTENSIONS, check_ebook_format from calibre.utils.magick.draw import save_cover_data_to from calibre.utils.recycle_bin import delete_file, delete_tree from calibre.utils.formatter_functions import load_user_template_functions +from calibre.db.errors import NoSuchFormat copyfile = os.link if hasattr(os, 'link') else shutil.copyfile +SPOOL_SIZE = 30*1024*1024 class Tag(object): @@ -601,14 +601,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f: f.write(cdata) for format in formats: - # Get data as string (can't use file as source and target files may be the same) - f = self.format(id, format, index_is_id=True, as_file=True) - if f is None: - continue - with tempfile.SpooledTemporaryFile(max_size=30*(1024**2)) as stream: - with f: - shutil.copyfileobj(f, stream) - stream.seek(0) + with tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) as stream: + try: + self.copy_format_to(id, format, stream, index_is_id=True) + stream.seek(0) + except NoSuchFormat: + continue self.add_format(id, format, stream, index_is_id=True, path=tpath, notify=False) self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id)) @@ -661,32 +659,53 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): continue def cover(self, index, index_is_id=False, as_file=False, as_image=False, - as_path=False): + as_path=False): ''' Return the cover image as a bytestring (in JPEG format) or None. - `as_file` : If True return the image as an open file object - `as_image`: If True return the image as a QImage object + WARNING: Using as_path will copy the cover to a temp file and return + the path to the temp file. You should delete the temp file when you are + done with it. + + :param as_file: If True return the image as an open file object (a SpooledTemporaryFile) + :param as_image: If True return the image as a QImage object ''' - id = index if index_is_id else self.id(index) + id = index if index_is_id else self.id(index) path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') if os.access(path, os.R_OK): - if as_path: - return path try: f = lopen(path, 'rb') except (IOError, OSError): time.sleep(0.2) f = lopen(path, 'rb') - if as_image: - img = QImage() - img.loadFromData(f.read()) - f.close() - return img - ans = f if as_file else f.read() - if ans is not f: - f.close() - return ans + with f: + if as_path: + pt = PersistentTemporaryFile('_dbcover.jpg') + with pt: + shutil.copyfileobj(f, pt) + return pt.name + if as_file: + ret = tempfile.SpooledTemporaryFile(SPOOL_SIZE) + shutil.copyfileobj(f, ret) + ret.seek(0) + else: + ret = f.read() + if as_image: + from PyQt4.Qt import QImage + i = QImage() + i.loadFromData(ret) + ret = i + return ret + + def cover_last_modified(self, index, index_is_id=False): + id = index if index_is_id else self.id(index) + path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg') + try: + return utcfromtimestamp(os.stat(path).st_mtime) + except: + # Cover doesn't exist + pass + return self.last_modified() ### The field-style interface. These use field keys. @@ -859,7 +878,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return (path, mi, sequence) def get_metadata(self, idx, index_is_id=False, get_cover=False, - get_user_categories=True): + get_user_categories=True, cover_as_data=False): ''' Convenience method to return metadata as a :class:`Metadata` object. Note that the list of formats is not verified. @@ -934,7 +953,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.user_categories = user_cat_vals if get_cover: - mi.cover = self.cover(id, index_is_id=True, as_path=True) + if cover_as_data: + cdata = self.cover(id, index_is_id=True) + if cdata: + mi.cover_data = ('jpeg', cdata) + else: + mi.cover = self.cover(id, index_is_id=True, as_path=True) return mi def has_book(self, mi): @@ -1099,7 +1123,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): return utcfromtimestamp(os.stat(path).st_mtime) def format_abspath(self, index, format, index_is_id=False): - 'Return absolute path to the ebook file of format `format`' + ''' + Return absolute path to the ebook file of format `format` + + WARNING: This method will return a dummy path for a network backend DB, + so do not rely on it, use format(..., as_path=True) instead. + + Currently used only in calibredb list, the viewer and the catalogs (via + get_data_as_dict()). + + Apart from the viewer, I don't believe any of the others do any file + I/O with the results of this call. + ''' id = index if index_is_id else self.id(index) try: name = self.conn.get('SELECT name FROM data WHERE book=? AND format=?', (id, format), all=False) @@ -1119,25 +1154,63 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): shutil.copyfile(candidates[0], fmt_path) return fmt_path - def format(self, index, format, index_is_id=False, as_file=False, mode='r+b'): + def copy_format_to(self, index, fmt, dest, index_is_id=False): + ''' + Copy the format ``fmt`` to the file like object ``dest``. If the + specified format does not exist, raises :class:`NoSuchFormat` error. + ''' + path = self.format_abspath(index, fmt, index_is_id=index_is_id) + if path is None: + id_ = index if index_is_id else self.id(index) + raise NoSuchFormat('Record %d has no %s file'%(id_, fmt)) + with lopen(path, 'rb') as f: + shutil.copyfileobj(f, dest) + if hasattr(dest, 'flush'): + dest.flush() + + def format(self, index, format, index_is_id=False, as_file=False, + mode='r+b', as_path=False, preserve_filename=False): ''' Return the ebook format as a bytestring or `None` if the format doesn't exist, or we don't have permission to write to the ebook file. - `as_file`: If True the ebook format is returned as a file object opened in `mode` + :param as_file: If True the ebook format is returned as a file object. Note + that the file object is a SpooledTemporaryFile, so if what you want to + do is copy the format to another file, use :method:`copy_format_to` + instead for performance. + :param as_path: Copies the format file to a temp file and returns the + path to the temp file + :param preserve_filename: If True and returning a path the filename is + the same as that used in the library. Note that using + this means that repeated calls yield the same + temp file (which is re-created each time) + :param mode: This is ignored (present for legacy compatibility) ''' path = self.format_abspath(index, format, index_is_id=index_is_id) if path is not None: - f = lopen(path, mode) - try: - ret = f if as_file else f.read() - except IOError: - f.seek(0) - out = cStringIO.StringIO() - shutil.copyfileobj(f, out) - ret = out.getvalue() - if not as_file: - f.close() + with lopen(path, mode) as f: + if as_path: + if preserve_filename: + bd = base_dir() + d = os.path.join(bd, 'format_abspath') + try: + os.makedirs(d) + except: + pass + fname = os.path.basename(path) + ret = os.path.join(d, fname) + with lopen(ret, 'wb') as f2: + shutil.copyfileobj(f, f2) + else: + with PersistentTemporaryFile('.'+format.lower()) as pt: + shutil.copyfileobj(f, pt) + ret = pt.name + elif as_file: + ret = tempfile.SpooledTemporaryFile(max_size=SPOOL_SIZE) + shutil.copyfileobj(f, ret) + ret.seek(0) + else: + ret = f.read() return ret def add_format_with_hooks(self, index, format, fpath, index_is_id=False, diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 5f49833564..b5c4e2faf3 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -6,7 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, traceback, cStringIO, re, shutil +import os, traceback, cStringIO, re from calibre.constants import DEBUG from calibre.utils.config import Config, StringConfig, tweaks @@ -238,7 +238,7 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, def save_book_to_disk(id_, db, root, opts, length): mi = db.get_metadata(id_, index_is_id=True) - cover = db.cover(id_, index_is_id=True, as_path=True) + cover = db.cover(id_, index_is_id=True) plugboards = db.prefs.get('plugboards', {}) available_formats = db.formats(id_, index_is_id=True) @@ -252,12 +252,20 @@ def save_book_to_disk(id_, db, root, opts, length): if fmts: fmts = fmts.split(',') for fmt in fmts: - fpath = db.format_abspath(id_, fmt, index_is_id=True) + fpath = db.format(id_, fmt, index_is_id=True, as_path=True) if fpath is not None: formats[fmt.lower()] = fpath - return do_save_book_to_disk(id_, mi, cover, plugboards, + try: + return do_save_book_to_disk(id_, mi, cover, plugboards, formats, root, opts, length) + finally: + for temp in formats.itervalues(): + try: + os.remove(temp) + except: + pass + def do_save_book_to_disk(id_, mi, cover, plugboards, @@ -289,10 +297,9 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, raise ocover = mi.cover - if opts.save_cover and cover and os.access(cover, os.R_OK): + if opts.save_cover and cover: with open(base_path+'.jpg', 'wb') as f: - with open(cover, 'rb') as s: - shutil.copyfileobj(s, f) + f.write(cover) mi.cover = base_name+'.jpg' else: mi.cover = None @@ -395,8 +402,13 @@ def save_serialized_to_disk(ids, data, plugboards, root, opts, callback): pass tb = '' try: - failed, id, title = do_save_book_to_disk(x, mi, cover, plugboards, - format_map, root, opts, length) + with open(cover, 'rb') as f: + cover = f.read() + except: + cover = None + try: + failed, id, title = do_save_book_to_disk(x, mi, cover, + plugboards, format_map, root, opts, length) tb = _('Requested formats not available') except: failed, id, title = True, x, mi.title diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index 08de4faecd..853ec7829c 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -10,12 +10,13 @@ import re, os, posixpath import cherrypy from calibre import fit_image, guess_type -from calibre.utils.date import fromtimestamp +from calibre.utils.date import fromtimestamp, utcnow from calibre.library.caches import SortKeyGenerator from calibre.library.save_to_disk import find_plugboard - -from calibre.utils.magick.draw import save_cover_data_to, Image, \ - thumbnail as generate_thumbnail +from calibre.ebooks.metadata import authors_to_string +from calibre.utils.magick.draw import (save_cover_data_to, Image, + thumbnail as generate_thumbnail) +from calibre.utils.filenames import ascii_filename plugboard_content_server_value = 'content_server' plugboard_content_server_formats = ['epub'] @@ -46,7 +47,7 @@ class ContentServer(object): # Utility methods {{{ def last_modified(self, updated): ''' - Generates a local independent, english timestamp from a datetime + Generates a locale independent, english timestamp from a datetime object ''' lm = updated.strftime('day, %d month %Y %H:%M:%S GMT') @@ -151,14 +152,12 @@ class ContentServer(object): try: cherrypy.response.headers['Content-Type'] = 'image/jpeg' cherrypy.response.timeout = 3600 - cover = self.db.cover(id, index_is_id=True, as_file=True) + cover = self.db.cover(id, index_is_id=True) if cover is None: cover = self.default_cover updated = self.build_time else: - with cover as f: - updated = fromtimestamp(os.fstat(f.fileno()).st_mtime) - cover = f.read() + updated = self.db.cover_last_modified(id, index_is_id=True) cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) if thumbnail: @@ -187,9 +186,9 @@ class ContentServer(object): mode='rb') if fmt is None: raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format)) + mi = self.db.get_metadata(id, index_is_id=True) if format == 'EPUB': # Get the original metadata - mi = self.db.get_metadata(id, index_is_id=True) # Get any EPUB plugboards for the content server plugboards = self.db.prefs.get('plugboards', {}) @@ -203,24 +202,22 @@ class ContentServer(object): newmi = mi # Write the updated file - from tempfile import TemporaryFile from calibre.ebooks.metadata.meta import set_metadata - raw = fmt.read() - fmt = TemporaryFile() - fmt.write(raw) - fmt.seek(0) set_metadata(fmt, newmi, 'epub') fmt.seek(0) mt = guess_type('dummy.'+format.lower())[0] if mt is None: mt = 'application/octet-stream' + au = authors_to_string(mi.authors if mi.authors else [_('Unknown')]) + title = mi.title if mi.title else _('Unknown') + fname = u'%s - %s_%s.%s'%(title[:30], au[:30], id, format.lower()) + fname = ascii_filename(fname).replace('"', '_') cherrypy.response.headers['Content-Type'] = mt + cherrypy.response.headers['Content-Disposition'] = \ + b'attachment; filename="%s"'%fname cherrypy.response.timeout = 3600 - path = getattr(fmt, 'name', None) - if path and os.path.exists(path): - updated = fromtimestamp(os.stat(path).st_mtime) - cherrypy.response.headers['Last-Modified'] = self.last_modified(updated) + cherrypy.response.headers['Last-Modified'] = self.last_modified(utcnow()) return fmt # }}}