From 231f5423dd96acef3d7dcef8f5a3395f90a4a259 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 27 Jan 2018 20:21:40 +0530 Subject: [PATCH] Code to manage temp dirs inside the cache directory The temp dirs are cleaned up on subsequent application starts even if left behind by a previous crash. And they persist as long as the process that created them runs. --- src/calibre/constants.py | 11 ++-- src/calibre/ptempfile.py | 94 +++++++++++++++++++++++++++++++++- src/calibre/utils/test_lock.py | 35 ++++++++++++- 3 files changed, 129 insertions(+), 11 deletions(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index fb4a7edc75..eb41b5dff0 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -95,9 +95,6 @@ def debug(): DEBUG = True -_cache_dir = None - - def _get_cache_dir(): confcache = os.path.join(config_dir, u'caches') if isportable: @@ -131,10 +128,10 @@ def _get_cache_dir(): def cache_dir(): - global _cache_dir - if _cache_dir is None: - _cache_dir = _get_cache_dir() - return _cache_dir + ans = getattr(cache_dir, 'ans', None) + if ans is None: + ans = cache_dir.ans = _get_cache_dir() + return ans # plugins {{{ diff --git a/src/calibre/ptempfile.py b/src/calibre/ptempfile.py index 5a8babecf6..34376ca3e1 100644 --- a/src/calibre/ptempfile.py +++ b/src/calibre/ptempfile.py @@ -5,11 +5,11 @@ __copyright__ = '2008, Kovid Goyal ' Provides platform independent temporary files that persist even after being closed. """ -import tempfile, os, atexit +import tempfile, os, atexit, errno from future_builtins import map from calibre.constants import (__version__, __appname__, filesystem_encoding, - get_unicode_windows_env_var, iswindows, get_windows_temp_path, isosx) + get_unicode_windows_env_var, iswindows, get_windows_temp_path, isosx, cache_dir) def cleanup(path): @@ -293,3 +293,93 @@ def better_mktemp(*args, **kwargs): os.close(fd) return path + +TDIR_LOCK = 'tdir-lock' + +if iswindows: + def lock_tdir(path): + return lopen(os.path.join(path, TDIR_LOCK), 'wb') + + def remove_tdir(path, lock_file): + lock_file.close() + remove_dir(path) + + def is_tdir_locked(path): + try: + with lopen(os.path.join(path, TDIR_LOCK), 'wb'): + pass + except EnvironmentError: + return True + return False +else: + import fcntl + + def lock_tdir(path): + from calibre.utils.ipc import eintr_retry_call + lf = os.path.join(path, TDIR_LOCK) + f = lopen(lf, 'w') + eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + return f + + def remove_tdir(path, lock_file): + lock_file.close() + remove_dir(path) + + def is_tdir_locked(path): + from calibre.utils.ipc import eintr_retry_call + lf = os.path.join(path, TDIR_LOCK) + f = lopen(lf, 'w') + try: + eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + eintr_retry_call(fcntl.lockf, f.fileno(), fcntl.LOCK_UN) + return False + except EnvironmentError: + return True + finally: + f.close() + + +def tdirs_in(b): + try: + tdirs = os.listdir(b) + except EnvironmentError as e: + if e.errno != errno.ENOENT: + raise + tdirs = () + for x in tdirs: + x = os.path.join(b, x) + if os.path.isdir(x): + yield x + + +def clean_tdirs_in(b): + # Remove any stale tdirs left by previous program crashes + for q in tdirs_in(b): + if not is_tdir_locked(q): + remove_dir(q) + + +def tdir_in_cache(base): + ''' Create a temp dir inside cache_dir/base. The created dir is robust + against application crashes. i.e. it will be cleaned up the next time the + application starts, even if it was left behind by a previous crash. ''' + b = os.path.join(cache_dir(), base) + try: + os.makedirs(b) + except EnvironmentError as e: + if e.errno != errno.EEXIST: + raise + if b not in tdir_in_cache.scanned: + tdir_in_cache.scanned.add(b) + try: + clean_tdirs_in(b) + except Exception: + import traceback + traceback.print_exc() + tdir = _make_dir('', '', b) + lock_data = lock_tdir(tdir) + atexit.register(remove_tdir, tdir, lock_data) + return tdir + + +tdir_in_cache.scanned = set() diff --git a/src/calibre/utils/test_lock.py b/src/calibre/utils/test_lock.py index cc8eff6620..9a257360a1 100644 --- a/src/calibre/utils/test_lock.py +++ b/src/calibre/utils/test_lock.py @@ -13,8 +13,9 @@ import time import unittest from threading import Thread -from calibre.constants import fcntl, iswindows -from calibre.utils.lock import ExclusiveFile, unix_open, create_single_instance_mutex +from calibre.constants import cache_dir, fcntl, iswindows +from calibre.ptempfile import clean_tdirs_in, is_tdir_locked, tdir_in_cache, tdirs_in +from calibre.utils.lock import ExclusiveFile, create_single_instance_mutex, unix_open def FastFailEF(name): @@ -59,8 +60,11 @@ class IPCLockTest(unittest.TestCase): self.cwd = os.getcwd() self.tdir = tempfile.mkdtemp() os.chdir(self.tdir) + self.original_cache_dir = cache_dir() + cache_dir.ans = self.tdir def tearDown(self): + cache_dir.ans = self.original_cache_dir os.chdir(self.cwd) for i in range(100): try: @@ -127,6 +131,22 @@ class IPCLockTest(unittest.TestCase): self.assertIsNotNone(release_mutex) release_mutex() + def test_tdir_in_cache_dir(self): + child = run_worker('calibre.utils.test_lock', 'other4') + tdirs = [] + while not tdirs: + tdirs = list(tdirs_in('t')) + self.assertTrue(is_tdir_locked(tdirs[0])) + c2 = run_worker('calibre.utils.test_lock', 'other5') + c2.wait() + self.assertTrue(is_tdir_locked(tdirs[0])) + child.kill(), child.wait() + self.assertTrue(os.path.exists(tdirs[0])) + self.assertFalse(is_tdir_locked(tdirs[0])) + clean_tdirs_in('t') + self.assertFalse(os.path.exists(tdirs[0])) + self.assertFalse(os.listdir('t')) + def other1(): e = ExclusiveFile('test') @@ -147,6 +167,17 @@ def other3(): time.sleep(30) +def other4(): + cache_dir.ans = os.getcwdu() + tdir_in_cache('t') + time.sleep(30) + + +def other5(): + cache_dir.ans = os.getcwdu() + tdir_in_cache('t') + + def find_tests(): return unittest.defaultTestLoader.loadTestsFromTestCase(IPCLockTest)