Auto-adding: Do not add incomplete files when files are downloaded directly into the auto add folder. Fixes #926578 (Automatic adding produces duplicate entries for downloaded files)

src/calibre/ebooks/metadata/meta.py

@@ -222,6 +222,11 @@ def forked_read_metadata(path, tdir):
     from calibre.ebooks.metadata.opf2 import metadata_to_opf
     with open(path, 'rb') as f:
         fmt = os.path.splitext(path)[1][1:].lower()
+        f.seek(0, 2)
+        sz = f.tell()
+        with open(os.path.join(tdir, 'size.txt'), 'wb') as s:
+            s.write(str(sz).encode('ascii'))
+        f.seek(0)
         mi = get_metadata(f, fmt)
     if mi.cover_data and mi.cover_data[1]:
         with open(os.path.join(tdir, 'cover.jpg'), 'wb') as f:

src/calibre/gui2/auto_add.py

@@ -7,7 +7,7 @@ __license__   = 'GPL v3'
 __copyright__ = '2012, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import os, tempfile, shutil
+import os, tempfile, shutil, time
 from threading import Thread, Event
 
 from PyQt4.Qt import (QFileSystemWatcher, QObject, Qt, pyqtSignal, QTimer)
@@ -41,25 +41,58 @@ class Worker(Thread):
                 traceback.print_exc()
 
     def auto_add(self):
-        from calibre.utils.ipc.simple_worker import fork_job
+        from calibre.utils.ipc.simple_worker import fork_job, WorkerError
         from calibre.ebooks.metadata.opf2 import metadata_to_opf
         from calibre.ebooks.metadata.meta import metadata_from_filename
 
-        files = [x for x in os.listdir(self.path) if x not in self.staging
+        files = [x for x in os.listdir(self.path) if
-                and os.path.isfile(os.path.join(self.path, x)) and
+                    # Must not be in the process of being added to the db
-                os.access(os.path.join(self.path, x), os.R_OK|os.W_OK) and
+                    x not in self.staging
-                os.path.splitext(x)[1][1:].lower() in self.be]
+                    # Firefox creates 0 byte placeholder files when downloading
+                    and os.stat(os.path.join(self.path, x)).st_size > 0
+                    # Must be a file
+                    and os.path.isfile(os.path.join(self.path, x))
+                    # Must have read and write permissions
+                    and os.access(os.path.join(self.path, x), os.R_OK|os.W_OK)
+                    # Must be a known ebook file type
+                    and os.path.splitext(x)[1][1:].lower() in self.be
+                ]
         data = {}
+        # Give any in progress copies time to complete
+        time.sleep(2)
+
         for fname in files:
             f = os.path.join(self.path, fname)
+
+            # Try opening the file for reading, if the OS prevents us, then at
+            # least on windows, it means the file is open in another
+            # application for writing. We will get notified by
+            # QFileSystemWatcher when writing is completed, so ignore for now.
+            try:
+                open(f, 'rb').close()
+            except:
+                continue
             tdir = tempfile.mkdtemp(dir=self.tdir)
             try:
                 fork_job('calibre.ebooks.metadata.meta',
                         'forked_read_metadata', (f, tdir), no_output=True)
+            except WorkerError as e:
+                prints('Failed to read metadata from:', fname)
+                prints(e.orig_tb)
             except:
                 import traceback
                 traceback.print_exc()
 
+            # Ensure that the pre-metadata file size is present. If it isn't,
+            # write 0 so that the file is rescanned
+            szpath = os.path.join(tdir, 'size.txt')
+            try:
+                with open(szpath, 'rb') as f:
+                    int(f.read())
+            except:
+                with open(szpath, 'wb') as f:
+                    f.write(b'0')
+
             opfpath = os.path.join(tdir, 'metadata.opf')
             try:
                 if os.stat(opfpath).st_size < 30:
@@ -125,8 +158,23 @@ class AutoAdder(QObject):
         m = gui.library_view.model()
         count = 0
 
+        needs_rescan = False
+
         for fname, tdir in data.iteritems():
             paths = [os.path.join(self.worker.path, fname)]
+            sz = os.path.join(tdir, 'size.txt')
+            if not os.access(sz, os.R_OK):
+                continue
+            try:
+                with open(sz, 'rb') as f:
+                    sz = int(f.read())
+                if sz != os.stat(paths[0]).st_size:
+                    raise Exception('Looks like the file was written to after'
+                            ' we tried to read metadata')
+            except:
+                needs_rescan = True
+                continue
+
             mi = os.path.join(tdir, 'metadata.opf')
             if not os.access(mi, os.R_OK):
                 continue
@@ -135,7 +183,7 @@ class AutoAdder(QObject):
             m.add_books(paths, [os.path.splitext(fname)[1][1:].upper()], mi,
                     add_duplicates=True)
             try:
-                os.remove(os.path.join(self.worker.path, fname))
+                os.remove(paths[0])
                 try:
                     self.worker.staging.remove(fname)
                 except KeyError:
@@ -153,4 +201,7 @@ class AutoAdder(QObject):
             if hasattr(gui, 'db_images'):
                 gui.db_images.reset()
 
+        if needs_rescan:
+            QTimer.singleShot(2000, self.dir_changed)
+
 

src/calibre/utils/ipc/simple_worker.py

@@ -19,7 +19,7 @@ from calibre.utils.ipc.launch import Worker
 class WorkerError(Exception):
     def __init__(self, msg, orig_tb=''):
         Exception.__init__(self, msg)
-        self.org_tb = orig_tb
+        self.orig_tb = orig_tb
 
 class ConnectedWorker(Thread):