diff --git a/src/calibre/ebooks/metadata/meta.py b/src/calibre/ebooks/metadata/meta.py index 26b0b6c59b..73ba7e77f4 100644 --- a/src/calibre/ebooks/metadata/meta.py +++ b/src/calibre/ebooks/metadata/meta.py @@ -222,6 +222,11 @@ def forked_read_metadata(path, tdir): from calibre.ebooks.metadata.opf2 import metadata_to_opf with open(path, 'rb') as f: fmt = os.path.splitext(path)[1][1:].lower() + f.seek(0, 2) + sz = f.tell() + with open(os.path.join(tdir, 'size.txt'), 'wb') as s: + s.write(str(sz).encode('ascii')) + f.seek(0) mi = get_metadata(f, fmt) if mi.cover_data and mi.cover_data[1]: with open(os.path.join(tdir, 'cover.jpg'), 'wb') as f: diff --git a/src/calibre/gui2/auto_add.py b/src/calibre/gui2/auto_add.py index 71d2b8ecd0..0319a67210 100644 --- a/src/calibre/gui2/auto_add.py +++ b/src/calibre/gui2/auto_add.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2012, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, tempfile, shutil +import os, tempfile, shutil, time from threading import Thread, Event from PyQt4.Qt import (QFileSystemWatcher, QObject, Qt, pyqtSignal, QTimer) @@ -41,25 +41,58 @@ class Worker(Thread): traceback.print_exc() def auto_add(self): - from calibre.utils.ipc.simple_worker import fork_job + from calibre.utils.ipc.simple_worker import fork_job, WorkerError from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.metadata.meta import metadata_from_filename - files = [x for x in os.listdir(self.path) if x not in self.staging - and os.path.isfile(os.path.join(self.path, x)) and - os.access(os.path.join(self.path, x), os.R_OK|os.W_OK) and - os.path.splitext(x)[1][1:].lower() in self.be] + files = [x for x in os.listdir(self.path) if + # Must not be in the process of being added to the db + x not in self.staging + # Firefox creates 0 byte placeholder files when downloading + and os.stat(os.path.join(self.path, x)).st_size > 0 + # Must be a file + and os.path.isfile(os.path.join(self.path, x)) + # Must have read and write permissions + and os.access(os.path.join(self.path, x), os.R_OK|os.W_OK) + # Must be a known ebook file type + and os.path.splitext(x)[1][1:].lower() in self.be + ] data = {} + # Give any in progress copies time to complete + time.sleep(2) + for fname in files: f = os.path.join(self.path, fname) + + # Try opening the file for reading, if the OS prevents us, then at + # least on windows, it means the file is open in another + # application for writing. We will get notified by + # QFileSystemWatcher when writing is completed, so ignore for now. + try: + open(f, 'rb').close() + except: + continue tdir = tempfile.mkdtemp(dir=self.tdir) try: fork_job('calibre.ebooks.metadata.meta', 'forked_read_metadata', (f, tdir), no_output=True) + except WorkerError as e: + prints('Failed to read metadata from:', fname) + prints(e.orig_tb) except: import traceback traceback.print_exc() + # Ensure that the pre-metadata file size is present. If it isn't, + # write 0 so that the file is rescanned + szpath = os.path.join(tdir, 'size.txt') + try: + with open(szpath, 'rb') as f: + int(f.read()) + except: + with open(szpath, 'wb') as f: + f.write(b'0') + opfpath = os.path.join(tdir, 'metadata.opf') try: if os.stat(opfpath).st_size < 30: @@ -125,8 +158,23 @@ class AutoAdder(QObject): m = gui.library_view.model() count = 0 + needs_rescan = False + for fname, tdir in data.iteritems(): paths = [os.path.join(self.worker.path, fname)] + sz = os.path.join(tdir, 'size.txt') + if not os.access(sz, os.R_OK): + continue + try: + with open(sz, 'rb') as f: + sz = int(f.read()) + if sz != os.stat(paths[0]).st_size: + raise Exception('Looks like the file was written to after' + ' we tried to read metadata') + except: + needs_rescan = True + continue + mi = os.path.join(tdir, 'metadata.opf') if not os.access(mi, os.R_OK): continue @@ -135,7 +183,7 @@ class AutoAdder(QObject): m.add_books(paths, [os.path.splitext(fname)[1][1:].upper()], mi, add_duplicates=True) try: - os.remove(os.path.join(self.worker.path, fname)) + os.remove(paths[0]) try: self.worker.staging.remove(fname) except KeyError: @@ -153,4 +201,7 @@ class AutoAdder(QObject): if hasattr(gui, 'db_images'): gui.db_images.reset() + if needs_rescan: + QTimer.singleShot(2000, self.dir_changed) + diff --git a/src/calibre/utils/ipc/simple_worker.py b/src/calibre/utils/ipc/simple_worker.py index a7a30200a1..8454986360 100644 --- a/src/calibre/utils/ipc/simple_worker.py +++ b/src/calibre/utils/ipc/simple_worker.py @@ -19,7 +19,7 @@ from calibre.utils.ipc.launch import Worker class WorkerError(Exception): def __init__(self, msg, orig_tb=''): Exception.__init__(self, msg) - self.org_tb = orig_tb + self.orig_tb = orig_tb class ConnectedWorker(Thread):