Make title/author renames completely atomic on windows

src/calibre/library/database2.py

@@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en'
 The database used to store ebook metadata
 '''
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re, \
-        json, uuid, hashlib, copy, errno
+        json, uuid, hashlib, copy
 from collections import defaultdict
 import threading, random
 from itertools import repeat
@@ -31,7 +31,7 @@ from calibre.ptempfile import (PersistentTemporaryFile,
 from calibre.customize.ui import run_plugins_on_import
 from calibre import isbytestring
 from calibre.utils.filenames import (ascii_filename, samefile,
-            windows_is_folder_in_use)
+        WindowsAtomicFolderMove)
 from calibre.utils.date import (utcnow, now as nowf, utcfromtimestamp,
         parse_only_date, UNDEFINED_DATE)
 from calibre.utils.config import prefs, tweaks, from_json, to_json
@@ -641,45 +641,43 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if name and name != fname:
                 changed = True
                 break
-        tpath = os.path.join(self.library_path, *path.split('/'))
-        if not os.path.exists(tpath):
-            os.makedirs(tpath)
         if path == current_path and not changed:
             return
-
         spath = os.path.join(self.library_path, *current_path.split('/'))
+        tpath = os.path.join(self.library_path, *path.split('/'))
 
-        if current_path and os.path.exists(spath): # Migrate existing files
+        wam = WindowsAtomicFolderMove(spath) if iswindows else None
-            if iswindows:
+        try:
-                uf = windows_is_folder_in_use(spath)
+            if not os.path.exists(tpath):
-                if uf is not None:
+                os.makedirs(tpath)
-                    err = IOError(errno.EACCES,
+
-                            _('File is open in another process'))
+            if current_path and os.path.exists(spath): # Migrate existing files
-                    err.filename = uf
+                self.copy_cover_to(id, os.path.join(tpath, 'cover.jpg'),
-                    raise err
+                        index_is_id=True, windows_atomic_move=wam)
-            cdata = self.cover(id, index_is_id=True)
+                for format in formats:
-            if cdata is not None:
+                    copy_function = functools.partial(self.copy_format_to, id,
-                with lopen(os.path.join(tpath, 'cover.jpg'), 'wb') as f:
+                            format, index_is_id=True, windows_atomic_move=wam)
-                    f.write(cdata)
+                    try:
-            for format in formats:
+                        self.add_format(id, format, None, index_is_id=True,
-                copy_function = functools.partial(self.copy_format_to, id,
+                            path=tpath, notify=False, copy_function=copy_function)
-                        format, index_is_id=True)
+                    except NoSuchFormat:
-                try:
+                        continue
-                    self.add_format(id, format, None, index_is_id=True,
+            self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
-                        path=tpath, notify=False, copy_function=copy_function)
+            self.dirtied([id], commit=False)
-                except NoSuchFormat:
+            self.conn.commit()
-                    continue
+            self.data.set(id, self.FIELD_MAP['path'], path, row_is_id=True)
-        self.conn.execute('UPDATE books SET path=? WHERE id=?', (path, id))
+            # Delete not needed directories
-        self.dirtied([id], commit=False)
+            if current_path and os.path.exists(spath):
-        self.conn.commit()
+                if not samefile(spath, tpath):
-        self.data.set(id, self.FIELD_MAP['path'], path, row_is_id=True)
+                    if wam is not None:
-        # Delete not needed directories
+                        wam.delete_originals()
-        if current_path and os.path.exists(spath):
+                    self.rmtree(spath, permanent=True)
-            if not samefile(spath, tpath):
+                    parent = os.path.dirname(spath)
-                self.rmtree(spath, permanent=True)
+                    if len(os.listdir(parent)) == 0:
-                parent = os.path.dirname(spath)
+                        self.rmtree(parent, permanent=True)
-                if len(os.listdir(parent)) == 0:
+        finally:
-                    self.rmtree(parent, permanent=True)
+            if wam is not None:
+                wam.close_handles()
 
         curpath = self.library_path
         c1, c2 = current_path.split('/'), path.split('/')
@@ -1348,26 +1346,77 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     return None
                 return fmt_path
 
-    def copy_format_to(self, index, fmt, dest, index_is_id=False):
+    def copy_format_to(self, index, fmt, dest, index_is_id=False,
+            windows_atomic_move=None):
         '''
         Copy the format ``fmt`` to the file like object ``dest``. If the
         specified format does not exist, raises :class:`NoSuchFormat` error.
         dest can also be a path, in which case the format is copied to it, iff
         the path is different from the current path (taking case sensitivity
         into account).
+
+        windows_atomic_move is an internally used parameter. You should not use
+        it in any code outside this module.
         '''
         path = self.format_abspath(index, fmt, index_is_id=index_is_id)
         if path is None:
             id_ = index if index_is_id else self.id(index)
             raise NoSuchFormat('Record %d has no %s file'%(id_, fmt))
-        if hasattr(dest, 'write'):
+        if windows_atomic_move is not None:
-            with lopen(path, 'rb') as f:
+            if not isinstance(dest, basestring):
-                shutil.copyfileobj(f, dest)
+                raise Exception("Error, you must pass the dest as a path when"
-            if hasattr(dest, 'flush'):
+                        " using windows_atomic_move")
-                dest.flush()
+            if dest and not samefile(dest, path):
-        elif dest and not samefile(dest, path):
+                windows_atomic_move.copy_path_to(path, dest)
-            with lopen(path, 'rb') as f, lopen(dest, 'wb') as d:
+        else:
-                shutil.copyfileobj(f, d)
+            if hasattr(dest, 'write'):
+                with lopen(path, 'rb') as f:
+                    shutil.copyfileobj(f, dest)
+                if hasattr(dest, 'flush'):
+                    dest.flush()
+            elif dest and not samefile(dest, path):
+                with lopen(path, 'rb') as f, lopen(dest, 'wb') as d:
+                    shutil.copyfileobj(f, d)
+
+    def copy_cover_to(self, index, dest, index_is_id=False,
+            windows_atomic_move=None):
+        '''
+        Copy the format cover to the file like object ``dest``. Returns False
+        if no cover exists or dest is the same file as the current cover.
+        dest can also be a path in which case the cover is
+        copied to it iff the path is different from the current path (taking
+        case sensitivity into account).
+
+        windows_atomic_move is an internally used parameter. You should not use
+        it in any code outside this module.
+        '''
+        id = index if index_is_id else self.id(index)
+        path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
+        if windows_atomic_move is not None:
+            if not isinstance(dest, basestring):
+                raise Exception("Error, you must pass the dest as a path when"
+                        " using windows_atomic_move")
+            if os.access(path, os.R_OK) and dest and not samefile(dest, path):
+                windows_atomic_move.copy_path_to(path, dest)
+                return True
+        else:
+            if os.access(path, os.R_OK):
+                try:
+                    f = lopen(path, 'rb')
+                except (IOError, OSError):
+                    time.sleep(0.2)
+                f = lopen(path, 'rb')
+                with f:
+                    if hasattr(dest, 'write'):
+                        shutil.copyfileobj(f, dest)
+                        if hasattr(dest, 'flush'):
+                            dest.flush()
+                        return True
+                    elif dest and not samefile(dest, path):
+                        with lopen(dest, 'wb') as d:
+                            shutil.copyfileobj(f, d)
+                        return True
+        return False
 
     def format(self, index, format, index_is_id=False, as_file=False,
             mode='r+b', as_path=False, preserve_filename=False):

src/calibre/utils/filenames.py

@@ -3,7 +3,7 @@ Make strings safe for use as ASCII filenames, while trying to preserve as much
 meaning as possible.
 '''
 
-import os
+import os, errno
 from math import ceil
 
 from calibre import sanitize_file_name, isbytestring, force_unicode
@@ -249,32 +249,75 @@ def samefile(src, dst):
             os.path.normcase(os.path.abspath(dst)))
     return samestring
 
-def windows_is_file_opened(path):
+class WindowsAtomicFolderMove(object):
-    import win32file, winerror
-    from pywintypes import error
-    if isbytestring(path): path = path.decode(filesystem_encoding)
-    try:
-        h = win32file.CreateFile(path, win32file.GENERIC_READ, 0, None,
-            win32file.OPEN_EXISTING, 0, 0)
-    except error as e:
-        if getattr(e, 'winerror', 0) == winerror.ERROR_SHARING_VIOLATION:
-            return True
-    else:
-        win32file.CloseHandle(h)
-    return False
 
-def windows_is_folder_in_use(path):
     '''
-    Returns the path to a file that is used in another process in the specified
+    Move all the files inside a specified folder in an atomic fashion,
-    folder, or None if no such file exists. Note
+    preventing any other process from locking a file while the operation is
-    that this function is not a guarantee. A file may well be opened in the
+    incomplete. Raises an IOError if another process has locked a file before
-    folder after this function returns. However, it is useful to handle the
+    the operation starts. Note that this only operates on the files in the
-    common case of a sharing violation gracefully most of the time.
+    folder, not any sub-folders.
     '''
-    if isbytestring(path): path = path.decode(filesystem_encoding)
+
-    for x in os.listdir(path):
+    def __init__(self, path):
-        f = os.path.join(path, x)
+        self.handle_map = {}
-        if windows_is_file_opened(f):
+
-            return f
+        import win32file, winerror
-    return None
+        from pywintypes import error
+
+        if isbytestring(path): path = path.decode(filesystem_encoding)
+
+        if not os.path.exists(path):
+            return
+
+        for x in os.listdir(path):
+            f = os.path.normcase(os.path.abspath(os.path.join(path, x)))
+            if not os.path.isfile(f): continue
+            try:
+                h = win32file.CreateFile(f, win32file.GENERIC_READ,
+                        win32file.FILE_SHARE_DELETE, None,
+                        win32file.OPEN_EXISTING, win32file.FILE_FLAG_SEQUENTIAL_SCAN, 0)
+            except error as e:
+                self.close_handles()
+                if getattr(e, 'winerror', 0) == winerror.ERROR_SHARING_VIOLATION:
+                    err = IOError(errno.EACCES,
+                            _('File is open in another process'))
+                    err.filename = f
+                    raise err
+                raise
+            except:
+                self.close_handles()
+                raise
+            self.handle_map[f] = h
+
+    def copy_path_to(self, path, dest):
+        import win32file
+        handle = None
+        for p, h in self.handle_map.iteritems():
+            if samefile_windows(path, p):
+                handle = h
+                break
+        if handle is None:
+            raise ValueError(u'The file %r did not exist when this move'
+                    ' operation was started'%path)
+        with lopen(dest, 'wb') as f:
+            while True:
+                hr, raw = win32file.ReadFile(handle, 1024*1024)
+                if hr != 0:
+                    raise IOError(hr, u'Error while reading from %r'%path)
+                if not raw:
+                    break
+                f.write(raw)
+
+    def close_handles(self):
+        import win32file
+        for h in self.handle_map.itervalues():
+            win32file.CloseHandle(h)
+        self.handle_map = {}
+
+    def delete_originals(self):
+        import win32file
+        for path in self.handle_map.iterkeys():
+            win32file.DeleteFile(path)
+        self.close_handles()