From 492f2da250f95d79d5d859bdf997696d39a7a0f2 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sun, 12 Oct 2014 11:21:07 +0200 Subject: [PATCH 1/3] Prevent the metadata cache writer from being killed before it finished when calibre is shut down. This ensures that the cache is not truncated. --- .../devices/smart_device_app/driver.py | 87 ++++++++++++------- 1 file changed, 57 insertions(+), 30 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 74d410dd9d..4e02773820 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -364,7 +364,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return total_elapsed = time.time() - self.debug_start_time elapsed = time.time() - self.debug_time - print('SMART_DEV (%7.2f:%7.3f) %s'%(total_elapsed, elapsed, + prints('SMART_DEV (%7.2f:%7.3f) %s'%(total_elapsed, elapsed, inspect.stack()[1][3]), end='') for a in args: try: @@ -803,36 +803,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): traceback.print_exc() def _write_metadata_cache(self): - from calibre.utils.date import now self._debug() - from calibre.utils.config import to_json - cache_file_name = os.path.join(cache_dir(), - 'wireless_device_' + self.device_uuid + - '_metadata_cache.json') - try: - purged = 0 - count = 0 - with open(cache_file_name, mode='wb') as fd: - for key,book in self.device_book_cache.iteritems(): - if (now() - book['last_used']).days > self.PURGE_CACHE_ENTRIES_DAYS: - purged += 1 - continue - json_metadata = defaultdict(dict) - json_metadata[key]['book'] = self.json_codec.encode_book_metadata(book['book']) - json_metadata[key]['last_used'] = book['last_used'] - result = json.dumps(json_metadata, indent=2, default=to_json) - fd.write("%0.7d\n"%(len(result)+1)) - fd.write(result) - fd.write('\n') - count += 1 - self._debug('wrote', count, 'entries, purged', purged, 'entries') - except: - traceback.print_exc() - try: - if os.path.exists(cache_file_name): - os.remove(cache_file_name) - except: - traceback.print_exc() + # Write the cache in a non-daemon thread so that if calibre is closed then + # the write will finish. Wait for it here to simulate a method call + writer = CacheWriter(self.device_uuid, self.device_book_cache, + self.PURGE_CACHE_ENTRIES_DAYS, self.json_codec) + writer.start() + writer.join() def _make_metadata_cache_key(self, uuid, lpath_or_ext): key = None @@ -1953,4 +1930,54 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): def is_running(self): return getattr(self, 'listen_socket', None) is not None +class CacheWriter(Thread): + def __init__(self, device_uuid, device_book_cache, purge_delay, json_codec): + Thread.__init__(self) + # This thread must not be set as daemon. It seems that commonly people + # disconnect their device then close calibre immediately. If there are a + # lot of books on the device then writing the cache can take a long + # time, for example writing a 6000 book cache can take 12 seconds on one + # mac. We want to let this finish. + self.daemon = False + self.device_uuid = device_uuid + self.device_book_cache = device_book_cache + self.purge_delay = purge_delay + self.json_codec = json_codec + + # Do the imports here to avoid violating the restrictions decribed in + # section 16.2.9 of the thread module in the python 2.7.8 docs + from calibre.utils.config import to_json + self.to_json = to_json + from calibre.utils.date import now + self.now = now() + + def run(self): + cache_file_name = os.path.join(cache_dir(), + 'wireless_device_' + self.device_uuid + '_metadata_cache.json') + try: + purged = 0 + count = 0 + with open(cache_file_name, mode='wb') as fd: + for key,book in self.device_book_cache.iteritems(): + if (self.now - book['last_used']).days > self.purge_delay: + purged += 1 + continue + json_metadata = defaultdict(dict) + json_metadata[key]['book'] = self.json_codec.encode_book_metadata(book['book']) + json_metadata[key]['last_used'] = book['last_used'] + result = json.dumps(json_metadata, indent=2, default=self.to_json) + fd.write("%0.7d\n"%(len(result)+1)) + fd.write(result) + fd.write('\n') + count += 1 + if DEBUG: + prints('SMART_DEV cache_writer', 'wrote', count, + 'entries, purged', purged, 'entries') + except: + traceback.print_exc() + try: + if os.path.exists(cache_file_name): + os.remove(cache_file_name) + except: + traceback.print_exc() From 44ae1fbb76ba66aca636677b13d7bb9098fb2615 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sun, 12 Oct 2014 13:18:03 +0200 Subject: [PATCH 2/3] Revert "Prevent the metadata cache writer from being killed before it finished when calibre is shut down. This ensures that the cache is not truncated." This reverts commit 492f2da250f95d79d5d859bdf997696d39a7a0f2. --- .../devices/smart_device_app/driver.py | 87 +++++++------------ 1 file changed, 30 insertions(+), 57 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 4e02773820..74d410dd9d 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -364,7 +364,7 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return total_elapsed = time.time() - self.debug_start_time elapsed = time.time() - self.debug_time - prints('SMART_DEV (%7.2f:%7.3f) %s'%(total_elapsed, elapsed, + print('SMART_DEV (%7.2f:%7.3f) %s'%(total_elapsed, elapsed, inspect.stack()[1][3]), end='') for a in args: try: @@ -803,13 +803,36 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): traceback.print_exc() def _write_metadata_cache(self): + from calibre.utils.date import now self._debug() - # Write the cache in a non-daemon thread so that if calibre is closed then - # the write will finish. Wait for it here to simulate a method call - writer = CacheWriter(self.device_uuid, self.device_book_cache, - self.PURGE_CACHE_ENTRIES_DAYS, self.json_codec) - writer.start() - writer.join() + from calibre.utils.config import to_json + cache_file_name = os.path.join(cache_dir(), + 'wireless_device_' + self.device_uuid + + '_metadata_cache.json') + try: + purged = 0 + count = 0 + with open(cache_file_name, mode='wb') as fd: + for key,book in self.device_book_cache.iteritems(): + if (now() - book['last_used']).days > self.PURGE_CACHE_ENTRIES_DAYS: + purged += 1 + continue + json_metadata = defaultdict(dict) + json_metadata[key]['book'] = self.json_codec.encode_book_metadata(book['book']) + json_metadata[key]['last_used'] = book['last_used'] + result = json.dumps(json_metadata, indent=2, default=to_json) + fd.write("%0.7d\n"%(len(result)+1)) + fd.write(result) + fd.write('\n') + count += 1 + self._debug('wrote', count, 'entries, purged', purged, 'entries') + except: + traceback.print_exc() + try: + if os.path.exists(cache_file_name): + os.remove(cache_file_name) + except: + traceback.print_exc() def _make_metadata_cache_key(self, uuid, lpath_or_ext): key = None @@ -1930,54 +1953,4 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): def is_running(self): return getattr(self, 'listen_socket', None) is not None -class CacheWriter(Thread): - def __init__(self, device_uuid, device_book_cache, purge_delay, json_codec): - Thread.__init__(self) - # This thread must not be set as daemon. It seems that commonly people - # disconnect their device then close calibre immediately. If there are a - # lot of books on the device then writing the cache can take a long - # time, for example writing a 6000 book cache can take 12 seconds on one - # mac. We want to let this finish. - self.daemon = False - self.device_uuid = device_uuid - self.device_book_cache = device_book_cache - self.purge_delay = purge_delay - self.json_codec = json_codec - - # Do the imports here to avoid violating the restrictions decribed in - # section 16.2.9 of the thread module in the python 2.7.8 docs - from calibre.utils.config import to_json - self.to_json = to_json - from calibre.utils.date import now - self.now = now() - - def run(self): - cache_file_name = os.path.join(cache_dir(), - 'wireless_device_' + self.device_uuid + '_metadata_cache.json') - try: - purged = 0 - count = 0 - with open(cache_file_name, mode='wb') as fd: - for key,book in self.device_book_cache.iteritems(): - if (self.now - book['last_used']).days > self.purge_delay: - purged += 1 - continue - json_metadata = defaultdict(dict) - json_metadata[key]['book'] = self.json_codec.encode_book_metadata(book['book']) - json_metadata[key]['last_used'] = book['last_used'] - result = json.dumps(json_metadata, indent=2, default=self.to_json) - fd.write("%0.7d\n"%(len(result)+1)) - fd.write(result) - fd.write('\n') - count += 1 - if DEBUG: - prints('SMART_DEV cache_writer', 'wrote', count, - 'entries, purged', purged, 'entries') - except: - traceback.print_exc() - try: - if os.path.exists(cache_file_name): - os.remove(cache_file_name) - except: - traceback.print_exc() From c0c5cc31a615ad62d761c4088823892d38eec213 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Sun, 12 Oct 2014 13:46:00 +0200 Subject: [PATCH 3/3] Alternate implementation that doesn't require the thread. Instead write the cache whenever sync_booklists is called, meaning the cache always be "almost right". Also, use a tmp file and atomic rename to prevent truncating the file if calibre dies while it is being written. --- .../devices/smart_device_app/driver.py | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 74d410dd9d..2047b2b6b4 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -803,18 +803,18 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): traceback.print_exc() def _write_metadata_cache(self): - from calibre.utils.date import now self._debug() + from calibre.utils.date import now + now_ = now() from calibre.utils.config import to_json - cache_file_name = os.path.join(cache_dir(), - 'wireless_device_' + self.device_uuid + - '_metadata_cache.json') try: purged = 0 count = 0 - with open(cache_file_name, mode='wb') as fd: + prefix = os.path.join(cache_dir(), + 'wireless_device_' + self.device_uuid + '_metadata_cache') + with open(prefix + '.tmp', mode='wb') as fd: for key,book in self.device_book_cache.iteritems(): - if (now() - book['last_used']).days > self.PURGE_CACHE_ENTRIES_DAYS: + if (now_ - book['last_used']).days > self.PURGE_CACHE_ENTRIES_DAYS: purged += 1 continue json_metadata = defaultdict(dict) @@ -825,14 +825,12 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): fd.write(result) fd.write('\n') count += 1 - self._debug('wrote', count, 'entries, purged', purged, 'entries') + self._debug('wrote', count, 'entries, purged', purged, 'entries') + + from calibre.utils.filenames import atomic_rename + atomic_rename(fd.name, prefix + '.json') except: traceback.print_exc() - try: - if os.path.exists(cache_file_name): - os.remove(cache_file_name) - except: - traceback.print_exc() def _make_metadata_cache_key(self, uuid, lpath_or_ext): key = None @@ -1392,6 +1390,9 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): except: self._debug('failed to set local copy of _last_read_date_') traceback.print_exc() + # Write the cache here so that if we are interrupted on disconnect then the + # almost-latest info will be available. + self._write_metadata_cache() @synchronous('sync_lock') def eject(self):