Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-07 10:14:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Use much less memory when reading and writing cache

Browse Source
This commit is contained in:
Charles Haley 2013-10-01 19:17:25 +02:00
parent ee8c9d5c91
commit e34d7bdee7
1 changed files with 70 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

90
src/calibre/devices/smart_device_app/driver.py
View File

@ -36,6 +36,7 @@ from calibre.library.server import server_config as content_server_config
from calibre.ptempfile import PersistentTemporaryFile from calibre.ptempfile import PersistentTemporaryFile
from calibre.utils.ipc import eintr_retry_call from calibre.utils.ipc import eintr_retry_call
from calibre.utils.config_base import tweaks from calibre.utils.config_base import tweaks
from calibre.utils.config import to_json, from_json
from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to
from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as
unpublish_zeroconf, get_all_ips) unpublish_zeroconf, get_all_ips)
@ -717,32 +718,75 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
return None return None
def _read_metadata_cache(self): def _read_metadata_cache(self):
# cache_file_name = os.path.join(cache_dir(),
# 'device_drivers_' + self.__class__.__name__ +
# '_metadata_cache.pickle')
# if os.path.exists(cache_file_name):
# with open(cache_file_name, mode='rb') as fd:
# json_metadata = cPickle.load(fd)
# for uuid,json_book in json_metadata.iteritems():
# book = self.json_codec.raw_to_book(json_book['book'], SDBook, self.PREFIX)
# self.known_uuids[uuid]['book'] = book
# self.known_uuids[uuid]['last_used'] = json_book['last_used']
# lpath = book.get('lpath')
# if lpath in self.known_metadata:
# self.known_uuids.pop(uuid, None)
# else:
# self.known_metadata[lpath] = book
cache_file_name = os.path.join(cache_dir(), cache_file_name = os.path.join(cache_dir(),
'device_drivers_' + self.__class__.__name__ + 'device_drivers_' + self.__class__.__name__ +
'_metadata_cache.pickle') '_metadata_cache.json')
if os.path.exists(cache_file_name): self.known_uuids = defaultdict(dict)
with open(cache_file_name, mode='rb') as fd: self.known_metadata = {}
json_metadata = cPickle.load(fd) with open(cache_file_name, mode='rb') as fd:
for uuid,json_book in json_metadata.iteritems(): while True:
book = self.json_codec.raw_to_book(json_book['book'], SDBook, self.PREFIX) try:
self.known_uuids[uuid]['book'] = book rec_len = fd.readline()
self.known_uuids[uuid]['last_used'] = json_book['last_used'] if len(rec_len) != 8:
lpath = book.get('lpath') break
if lpath in self.known_metadata: raw = fd.read(int(rec_len))
self.known_uuids.pop(uuid, None) book = json.loads(raw.decode('utf-8'), object_hook=from_json)
else: uuid = book.keys()[0]
self.known_metadata[lpath] = book metadata = self.json_codec.raw_to_book(book[uuid]['book'],
SDBook, self.PREFIX)
book[uuid]['book'] = metadata
self.known_uuids.update(book)
lpath = metadata.get('lpath')
if lpath in self.known_metadata:
self.known_uuids.pop(uuid, None)
else:
self.known_metadata[lpath] = metadata
except:
traceback.print_exc()
def _write_metadata_cache(self): def _write_metadata_cache(self):
# cache_file_name = os.path.join(cache_dir(),
# 'device_drivers_' + self.__class__.__name__ +
# '_metadata_cache.pickle')
# json_metadata = defaultdict(dict)
# for uuid,book in self.known_uuids.iteritems():
# json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
# json_metadata[uuid]['last_used'] = book['last_used']
# with open(cache_file_name, mode='wb') as fd:
# cPickle.dump(json_metadata, fd, -1)
cache_file_name = os.path.join(cache_dir(), cache_file_name = os.path.join(cache_dir(),
'device_drivers_' + self.__class__.__name__ + 'device_drivers_' + self.__class__.__name__ +
'_metadata_cache.pickle') '_metadata_cache.json')
json_metadata = defaultdict(dict)
for uuid,book in self.known_uuids.iteritems():
json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
json_metadata[uuid]['last_used'] = book['last_used']
with open(cache_file_name, mode='wb') as fd: with open(cache_file_name, mode='wb') as fd:
cPickle.dump(json_metadata, fd, -1) try:
for uuid,book in self.known_uuids.iteritems():
json_metadata = defaultdict(dict)
json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
json_metadata[uuid]['last_used'] = book['last_used']
result = json.dumps(json_metadata, indent=2, default=to_json)
fd.write("%0.7d\n"%(len(result)+1))
fd.write(result)
fd.write('\n')
except:
traceback.print_exc()
def _set_known_metadata(self, book, remove=False): def _set_known_metadata(self, book, remove=False):
from calibre.utils.date import now from calibre.utils.date import now
@ -757,7 +801,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
if key: if key:
self.known_uuids.pop(key, None) self.known_uuids.pop(key, None)
else: else:
new_book = self.known_metadata[lpath] = book.deepcopy() # Check if we have another UUID with the same lpath. If so, remove it
existing_uuid = self.known_metadata.get(lpath, {}).get('uuid', None)
if existing_uuid:
self.known_uuids.pop(existing_uuid + ext, None)
new_book = book.deepcopy()
self.known_metadata[lpath] = new_book
if key: if key:
self.known_uuids[key]['book'] = new_book self.known_uuids[key]['book'] = new_book
self.known_uuids[key]['last_used'] = now() self.known_uuids[key]['last_used'] = now()