From e34d7bdee724b12c33aae76f7ee9626766e6bbb0 Mon Sep 17 00:00:00 2001 From: Charles Haley Date: Tue, 1 Oct 2013 19:17:25 +0200 Subject: [PATCH] Use much less memory when reading and writing cache --- .../devices/smart_device_app/driver.py | 90 ++++++++++++++----- 1 file changed, 70 insertions(+), 20 deletions(-) diff --git a/src/calibre/devices/smart_device_app/driver.py b/src/calibre/devices/smart_device_app/driver.py index 3be0878aa4..270353ffac 100644 --- a/src/calibre/devices/smart_device_app/driver.py +++ b/src/calibre/devices/smart_device_app/driver.py @@ -36,6 +36,7 @@ from calibre.library.server import server_config as content_server_config from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.ipc import eintr_retry_call from calibre.utils.config_base import tweaks +from calibre.utils.config import to_json, from_json from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as unpublish_zeroconf, get_all_ips) @@ -717,32 +718,75 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): return None def _read_metadata_cache(self): +# cache_file_name = os.path.join(cache_dir(), +# 'device_drivers_' + self.__class__.__name__ + +# '_metadata_cache.pickle') +# if os.path.exists(cache_file_name): +# with open(cache_file_name, mode='rb') as fd: +# json_metadata = cPickle.load(fd) +# for uuid,json_book in json_metadata.iteritems(): +# book = self.json_codec.raw_to_book(json_book['book'], SDBook, self.PREFIX) +# self.known_uuids[uuid]['book'] = book +# self.known_uuids[uuid]['last_used'] = json_book['last_used'] +# lpath = book.get('lpath') +# if lpath in self.known_metadata: +# self.known_uuids.pop(uuid, None) +# else: +# self.known_metadata[lpath] = book + cache_file_name = os.path.join(cache_dir(), 'device_drivers_' + self.__class__.__name__ + - '_metadata_cache.pickle') - if os.path.exists(cache_file_name): - with open(cache_file_name, mode='rb') as fd: - json_metadata = cPickle.load(fd) - for uuid,json_book in json_metadata.iteritems(): - book = self.json_codec.raw_to_book(json_book['book'], SDBook, self.PREFIX) - self.known_uuids[uuid]['book'] = book - self.known_uuids[uuid]['last_used'] = json_book['last_used'] - lpath = book.get('lpath') - if lpath in self.known_metadata: - self.known_uuids.pop(uuid, None) - else: - self.known_metadata[lpath] = book + '_metadata_cache.json') + self.known_uuids = defaultdict(dict) + self.known_metadata = {} + with open(cache_file_name, mode='rb') as fd: + while True: + try: + rec_len = fd.readline() + if len(rec_len) != 8: + break + raw = fd.read(int(rec_len)) + book = json.loads(raw.decode('utf-8'), object_hook=from_json) + uuid = book.keys()[0] + metadata = self.json_codec.raw_to_book(book[uuid]['book'], + SDBook, self.PREFIX) + book[uuid]['book'] = metadata + self.known_uuids.update(book) + + lpath = metadata.get('lpath') + if lpath in self.known_metadata: + self.known_uuids.pop(uuid, None) + else: + self.known_metadata[lpath] = metadata + except: + traceback.print_exc() def _write_metadata_cache(self): +# cache_file_name = os.path.join(cache_dir(), +# 'device_drivers_' + self.__class__.__name__ + +# '_metadata_cache.pickle') +# json_metadata = defaultdict(dict) +# for uuid,book in self.known_uuids.iteritems(): +# json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book']) +# json_metadata[uuid]['last_used'] = book['last_used'] +# with open(cache_file_name, mode='wb') as fd: +# cPickle.dump(json_metadata, fd, -1) + cache_file_name = os.path.join(cache_dir(), 'device_drivers_' + self.__class__.__name__ + - '_metadata_cache.pickle') - json_metadata = defaultdict(dict) - for uuid,book in self.known_uuids.iteritems(): - json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book']) - json_metadata[uuid]['last_used'] = book['last_used'] + '_metadata_cache.json') with open(cache_file_name, mode='wb') as fd: - cPickle.dump(json_metadata, fd, -1) + try: + for uuid,book in self.known_uuids.iteritems(): + json_metadata = defaultdict(dict) + json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book']) + json_metadata[uuid]['last_used'] = book['last_used'] + result = json.dumps(json_metadata, indent=2, default=to_json) + fd.write("%0.7d\n"%(len(result)+1)) + fd.write(result) + fd.write('\n') + except: + traceback.print_exc() def _set_known_metadata(self, book, remove=False): from calibre.utils.date import now @@ -757,7 +801,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin): if key: self.known_uuids.pop(key, None) else: - new_book = self.known_metadata[lpath] = book.deepcopy() + # Check if we have another UUID with the same lpath. If so, remove it + existing_uuid = self.known_metadata.get(lpath, {}).get('uuid', None) + if existing_uuid: + self.known_uuids.pop(existing_uuid + ext, None) + + new_book = book.deepcopy() + self.known_metadata[lpath] = new_book if key: self.known_uuids[key]['book'] = new_book self.known_uuids[key]['last_used'] = now()