Merge branch 'master' of https://github.com/cbhaley/calibre

Add a tweak to control quality of thumbnails generated by the content
server.

Also improve cache usage performance of the wireless device driver.

resources/default_tweaks.py

@@ -550,3 +550,10 @@ highlight_virtual_library = 'yellow'
 # all available output formats to be present.
 restrict_output_formats = None
 
+#: Set the thumbnail image quality used by the content server
+# The quality of a thumbnail is largely controlled by the compression quality
+# used when creating it. Set this to a larger number to improve the quality.
+# Note that the thumbnails get much larger with larger compression quality
+# numbers.
+# The value can be between 50 and 99
+content_server_thumbnail_compression_quality = 75

src/calibre/devices/smart_device_app/driver.py

@@ -7,7 +7,7 @@ Created on 29 Jun 2012
 
 @author: charles
 '''
-import socket, select, json, os, traceback, time, sys, random, cPickle
+import socket, select, json, os, traceback, time, sys, random
 import posixpath
 from collections import defaultdict
 import hashlib, threading
@@ -686,10 +686,6 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
         key = uuid+ext
         if isinstance(lastmod, unicode):
             lastmod = parse_date(lastmod)
-#         if key in self.known_uuids:
-#             self._debug(key, lastmod, self.known_uuids[key].last_modified)
-#         else:
-#             self._debug(key, 'not in known uuids')
         if key in self.known_uuids and self.known_uuids[key]['book'].last_modified == lastmod:
             self.known_uuids[key]['last_used'] = now()
             return self.known_uuids[key]['book'].deepcopy()
@@ -717,32 +713,60 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             return None
 
     def _read_metadata_cache(self):
-        cache_file_name = os.path.join(cache_dir(),
+        from calibre.utils.config import from_json
+        try:
+            old_cache_file_name = os.path.join(cache_dir(),
                            'device_drivers_' + self.__class__.__name__ +
                                 '_metadata_cache.pickle')
-        if os.path.exists(cache_file_name):
-            with open(cache_file_name, mode='rb') as fd:
-                json_metadata = cPickle.load(fd)
-            for uuid,json_book in json_metadata.iteritems():
-                book = self.json_codec.raw_to_book(json_book['book'], SDBook, self.PREFIX)
-                self.known_uuids[uuid]['book'] = book
-                self.known_uuids[uuid]['last_used'] = json_book['last_used']
-                lpath = book.get('lpath')
-                if lpath in self.known_metadata:
-                    self.known_uuids.pop(uuid, None)
-                else:
-                    self.known_metadata[lpath] = book
+            if os.path.exists(old_cache_file_name):
+                os.remove(old_cache_file_name)
+        except:
+            pass
+
+        cache_file_name = os.path.join(cache_dir(),
+                           'device_drivers_' + self.__class__.__name__ +
+                                '_metadata_cache.json')
+        self.known_uuids = defaultdict(dict)
+        self.known_metadata = {}
+        with open(cache_file_name, mode='rb') as fd:
+            try:
+                while True:
+                    rec_len = fd.readline()
+                    if len(rec_len) != 8:
+                        break
+                    raw = fd.read(int(rec_len))
+                    book = json.loads(raw.decode('utf-8'), object_hook=from_json)
+                    uuid = book.keys()[0]
+                    metadata = self.json_codec.raw_to_book(book[uuid]['book'],
+                                                        SDBook, self.PREFIX)
+                    book[uuid]['book'] = metadata
+                    self.known_uuids.update(book)
+
+                    lpath = metadata.get('lpath')
+                    if lpath in self.known_metadata:
+                        self.known_uuids.pop(uuid, None)
+                    else:
+                        self.known_metadata[lpath] = metadata
+            except:
+                traceback.print_exc()
 
     def _write_metadata_cache(self):
+        from calibre.utils.config import to_json
         cache_file_name = os.path.join(cache_dir(),
                            'device_drivers_' + self.__class__.__name__ +
-                                '_metadata_cache.pickle')
-        json_metadata = defaultdict(dict)
-        for uuid,book in self.known_uuids.iteritems():
-            json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
-            json_metadata[uuid]['last_used'] = book['last_used']
+                                '_metadata_cache.json')
         with open(cache_file_name, mode='wb') as fd:
-            cPickle.dump(json_metadata, fd, -1)
+            try:
+                for uuid,book in self.known_uuids.iteritems():
+                    json_metadata = defaultdict(dict)
+                    json_metadata[uuid]['book'] = self.json_codec.encode_book_metadata(book['book'])
+                    json_metadata[uuid]['last_used'] = book['last_used']
+                    result = json.dumps(json_metadata, indent=2, default=to_json)
+                    fd.write("%0.7d\n"%(len(result)+1))
+                    fd.write(result)
+                    fd.write('\n')
+            except:
+                traceback.print_exc()
 
     def _set_known_metadata(self, book, remove=False):
         from calibre.utils.date import now
@@ -757,7 +781,13 @@ class SMART_DEVICE_APP(DeviceConfig, DevicePlugin):
             if key:
                 self.known_uuids.pop(key, None)
         else:
-            new_book = self.known_metadata[lpath] = book.deepcopy()
+            # Check if we have another UUID with the same lpath. If so, remove it
+            existing_uuid = self.known_metadata.get(lpath, {}).get('uuid', None)
+            if existing_uuid:
+                self.known_uuids.pop(existing_uuid + ext, None)
+
+            new_book = book.deepcopy()
+            self.known_metadata[lpath] = new_book
             if key:
                 self.known_uuids[key]['book'] = new_book
                 self.known_uuids[key]['last_used'] = now()

src/calibre/library/server/content.py

@@ -18,6 +18,7 @@ from calibre.utils.magick.draw import (save_cover_data_to, Image,
         thumbnail as generate_thumbnail)
 from calibre.utils.filenames import ascii_filename
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
+from calibre.utils.config import tweaks
 
 plugboard_content_server_value = 'content_server'
 plugboard_content_server_formats = ['epub', 'mobi', 'azw3']
@@ -175,8 +176,13 @@ class ContentServer(object):
             cherrypy.response.headers['Last-Modified'] = self.last_modified(updated)
 
             if thumbnail:
-                return generate_thumbnail(cover,
-                        width=thumb_width, height=thumb_height)[-1]
+                quality = tweaks['content_server_thumbnail_compression_quality']
+                if quality < 50:
+                    quality = 50
+                elif quality > 99:
+                    quality = 99
+                return generate_thumbnail(cover, width=thumb_width,
+                        height=thumb_height, compression_quality=quality)[-1]
 
             img = Image()
             img.load(cover)