Use much less memory when reading and writing cache

src/calibre/devices/smart_device_app/driver.py

@@ -36,6 +36,7 @@ from calibre.library.server import server_config as content_server_config
 from calibre.ptempfile import PersistentTemporaryFile
 from calibre.utils.ipc import eintr_retry_call
 from calibre.utils.config_base import tweaks
+from calibre.utils.config import to_json, from_json
 from calibre.utils.filenames import ascii_filename as sanitize, shorten_components_to
 from calibre.utils.mdns import (publish as publish_zeroconf, unpublish as
         unpublish_zeroconf, get_all_ips)
@@ -717,32 +718,75 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             return None
 
     def _read_metadata_cache(self):
+#         cache_file_name = os.path.join(cache_dir(),
+#                            'device_drivers_' + self.__class__.__name__ +
+#                                 '_metadata_cache.pickle')
+#         if os.path.exists(cache_file_name):
+#             with open(cache_file_name, mode='rb') as fd:
+#                 json_metadata = cPickle.load(fd)
+#             for uuid,json_book in json_metadata.iteritems():
+#                 book = self.json_codec.raw_to_book(json_book['book'], SDBook, self.PREFIX)
+#                 self.known_uuids[uuid]['book'] = book
+#                 self.known_uuids[uuid]['last_used'] = json_book['last_used']
+#                 lpath = book.get('lpath')
+#                 if lpath in self.known_metadata:
+#                     self.known_uuids.pop(uuid, None)
+#                 else:
+#                     self.known_metadata[lpath] = book
+
         cache_file_name = os.path.join(cache_dir(),
                            'device_drivers_' + self.__class__.__name__ +
-                                '_metadata_cache.pickle')
-        if os.path.exists(cache_file_name):
+                                '_metadata_cache.json')
+        self.known_uuids = defaultdict(dict)
+        self.known_metadata = {}
         with open(cache_file_name, mode='rb') as fd:
-                json_metadata = cPickle.load(fd)
-            for uuid,json_book in json_metadata.iteritems():
-                book = self.json_codec.raw_to_book(json_book['book'], SDBook, self.PREFIX)
-                self.known_uuids[uuid]['book'] = book
-                self.known_uuids[uuid]['last_used'] = json_book['last_used']
-                lpath = book.get('lpath')
+            while True:
+                try:
+                    rec_len = fd.readline()
+                    if len(rec_len) != 8:
+                        break
+                    raw = fd.read(int(rec_len))
+                    book = json.loads(raw.decode('utf-8'), object_hook=from_json)
+                    uuid = book.keys()[0]
+                    metadata = self.json_codec.raw_to_book(book[uuid]['book'],
+                                                        SDBook, self.PREFIX)
+                    book[uuid]['book'] = metadata
+                    self.known_uuids.update(book)
+
+                    lpath = metadata.get('lpath')
                     if lpath in self.known_metadata:
                         self.known_uuids.pop(uuid, None)
                     else:
-                    self.known_metadata[lpath] = book
+                        self.known_metadata[lpath] = metadata
+                except:
+                    traceback.print_exc()
 
     def _write_metadata_cache(self):
+#         cache_file_name = os.path.join(cache_dir(),
+#                            'device_drivers_' + self.__class__.__name__ +
+#                                 '_metadata_cache.pickle')
+#         json_metadata = defaultdict(dict)
+#         for uuid,book in self.known_uuids.iteritems():
+#             json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
+#             json_metadata[uuid]['last_used'] = book['last_used']
+#         with open(cache_file_name, mode='wb') as fd:
+#             cPickle.dump(json_metadata, fd, -1)
+
         cache_file_name = os.path.join(cache_dir(),
                            'device_drivers_' + self.__class__.__name__ +
-                                '_metadata_cache.pickle')
-        json_metadata = defaultdict(dict)
+                                '_metadata_cache.json')
+        with open(cache_file_name, mode='wb') as fd:
+            try:
                 for uuid,book in self.known_uuids.iteritems():
+                    json_metadata = defaultdict(dict)
                     json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
                     json_metadata[uuid]['last_used'] = book['last_used']
-        with open(cache_file_name, mode='wb') as fd:
-            cPickle.dump(json_metadata, fd, -1)
+                    result = json.dumps(json_metadata, indent=2, default=to_json)
+                    fd.write("%0.7d\n"%(len(result)+1))
+                    fd.write(result)
+                    fd.write('\n')
+            except:
+                traceback.print_exc()
 
     def _set_known_metadata(self, book, remove=False):
         from calibre.utils.date import now
@@ -757,7 +801,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             if key:
                 self.known_uuids.pop(key, None)
         else:
-            new_book = self.known_metadata[lpath] = book.deepcopy()
+            # Check if we have another UUID with the same lpath. If so, remove it
+            existing_uuid = self.known_metadata.get(lpath, {}).get('uuid', None)
+            if existing_uuid:
+                self.known_uuids.pop(existing_uuid + ext, None)
+
+            new_book = book.deepcopy()
+            self.known_metadata[lpath] = new_book
             if key:
                 self.known_uuids[key]['book'] = new_book
                 self.known_uuids[key]['last_used'] = now()