Code to manage temp dirs inside the cache directory

The temp dirs are cleaned up on subsequent application starts even if
left behind by a previous crash. And they persist as long as the process
that created them runs.

src/calibre/constants.py

@@ -95,9 +95,6 @@ def debug():
     DEBUG = True
 
 
-_cache_dir = None
-
-
 def _get_cache_dir():
     confcache = os.path.join(config_dir, u'caches')
     if isportable:
@@ -131,10 +128,10 @@ def _get_cache_dir():
 
 
 def cache_dir():
-    global _cache_dir
+    ans = getattr(cache_dir, 'ans', None)
-    if _cache_dir is None:
+    if ans is None:
-        _cache_dir = _get_cache_dir()
+        ans = cache_dir.ans = _get_cache_dir()
-    return _cache_dir
+    return ans
 
 # plugins {{{
 

src/calibre/ptempfile.py

@@ -5,11 +5,11 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 Provides platform independent temporary files that persist even after
 being closed.
 """
-import tempfile, os, atexit
+import tempfile, os, atexit, errno
 from future_builtins import map
 
 from calibre.constants import (__version__, __appname__, filesystem_encoding,
-        get_unicode_windows_env_var, iswindows, get_windows_temp_path, isosx)
+        get_unicode_windows_env_var, iswindows, get_windows_temp_path, isosx, cache_dir)
 
 
 def cleanup(path):
@@ -293,3 +293,93 @@ def better_mktemp(*args, **kwargs):
     os.close(fd)
     return path
 
+
+TDIR_LOCK = 'tdir-lock'
+
+if iswindows:
+    def lock_tdir(path):
+        return lopen(os.path.join(path, TDIR_LOCK), 'wb')
+
+    def remove_tdir(path, lock_file):
+        lock_file.close()
+        remove_dir(path)
+
+    def is_tdir_locked(path):
+        try:
+            with lopen(os.path.join(path, TDIR_LOCK), 'wb'):
+                pass
+        except EnvironmentError:
+            return True
+        return False
+else:
+    import fcntl
+
+    def lock_tdir(path):
+        from calibre.utils.ipc import eintr_retry_call
+        lf = os.path.join(path, TDIR_LOCK)
+        f = lopen(lf, 'w')
+        eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+        return f
+
+    def remove_tdir(path, lock_file):
+        lock_file.close()
+        remove_dir(path)
+
+    def is_tdir_locked(path):
+        from calibre.utils.ipc import eintr_retry_call
+        lf = os.path.join(path, TDIR_LOCK)
+        f = lopen(lf, 'w')
+        try:
+            eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
+            eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_UN)
+            return False
+        except EnvironmentError:
+            return True
+        finally:
+            f.close()
+
+
+def tdirs_in(b):
+    try:
+        tdirs = os.listdir(b)
+    except EnvironmentError as e:
+        if e.errno != errno.ENOENT:
+            raise
+        tdirs = ()
+    for x in tdirs:
+        x = os.path.join(b, x)
+        if os.path.isdir(x):
+            yield x
+
+
+def clean_tdirs_in(b):
+    # Remove any stale tdirs left by previous program crashes
+    for q in tdirs_in(b):
+        if not is_tdir_locked(q):
+            remove_dir(q)
+
+
+def tdir_in_cache(base):
+    ''' Create a temp dir inside cache_dir/base. The created dir is robust
+    against application crashes. i.e. it will be cleaned up the next time the
+    application starts, even if it was left behind by a previous crash. '''
+    b = os.path.join(cache_dir(), base)
+    try:
+        os.makedirs(b)
+    except EnvironmentError as e:
+        if e.errno != errno.EEXIST:
+            raise
+    if b not in tdir_in_cache.scanned:
+        tdir_in_cache.scanned.add(b)
+        try:
+            clean_tdirs_in(b)
+        except Exception:
+            import traceback
+            traceback.print_exc()
+    tdir = _make_dir('', '', b)
+    lock_data = lock_tdir(tdir)
+    atexit.register(remove_tdir, tdir, lock_data)
+    return tdir
+
+
+tdir_in_cache.scanned = set()

src/calibre/utils/test_lock.py

@@ -13,8 +13,9 @@ import time
 import unittest
 from threading import Thread
 
-from calibre.constants import fcntl, iswindows
+from calibre.constants import cache_dir, fcntl, iswindows
-from calibre.utils.lock import ExclusiveFile, unix_open, create_single_instance_mutex
+from calibre.ptempfile import clean_tdirs_in, is_tdir_locked, tdir_in_cache, tdirs_in
+from calibre.utils.lock import ExclusiveFile, create_single_instance_mutex, unix_open
 
 
 def FastFailEF(name):
@@ -59,8 +60,11 @@ class IPCLockTest(unittest.TestCase):
         self.cwd = os.getcwd()
         self.tdir = tempfile.mkdtemp()
         os.chdir(self.tdir)
+        self.original_cache_dir = cache_dir()
+        cache_dir.ans = self.tdir
 
     def tearDown(self):
+        cache_dir.ans = self.original_cache_dir
         os.chdir(self.cwd)
         for i in range(100):
             try:
@@ -127,6 +131,22 @@ class IPCLockTest(unittest.TestCase):
         self.assertIsNotNone(release_mutex)
         release_mutex()
 
+    def test_tdir_in_cache_dir(self):
+        child = run_worker('calibre.utils.test_lock', 'other4')
+        tdirs = []
+        while not tdirs:
+            tdirs = list(tdirs_in('t'))
+        self.assertTrue(is_tdir_locked(tdirs[0]))
+        c2 = run_worker('calibre.utils.test_lock', 'other5')
+        c2.wait()
+        self.assertTrue(is_tdir_locked(tdirs[0]))
+        child.kill(), child.wait()
+        self.assertTrue(os.path.exists(tdirs[0]))
+        self.assertFalse(is_tdir_locked(tdirs[0]))
+        clean_tdirs_in('t')
+        self.assertFalse(os.path.exists(tdirs[0]))
+        self.assertFalse(os.listdir('t'))
+
 
 def other1():
     e = ExclusiveFile('test')
@@ -147,6 +167,17 @@ def other3():
     time.sleep(30)
 
 
+def other4():
+    cache_dir.ans = os.getcwdu()
+    tdir_in_cache('t')
+    time.sleep(30)
+
+
+def other5():
+    cache_dir.ans = os.getcwdu()
+    tdir_in_cache('t')
+
+
 def find_tests():
     return unittest.defaultTestLoader.loadTestsFromTestCase(IPCLockTest)