Simplify implementation of cover caching and ensure cover browser is updated when covers are changed

src/calibre/gui2/actions.py

@@ -645,6 +645,8 @@ class EditMetadataAction(object): # {{{
         if x.exception is None:
             self.library_view.model().refresh_ids(
                 x.updated, cr)
+            if self.cover_flow:
+                self.cover_flow.dataChanged()
             if x.failures:
                 details = ['%s: %s'%(title, reason) for title,
                         reason in x.failures.values()]
@@ -689,7 +691,6 @@ class EditMetadataAction(object): # {{{
         if rows:
             current = self.library_view.currentIndex()
             m = self.library_view.model()
-            m.refresh_cover_cache(map(m.id, rows))
             if self.cover_flow:
                 self.cover_flow.dataChanged()
             m.current_changed(current, previous)
@@ -711,6 +712,8 @@ class EditMetadataAction(object): # {{{
             self.library_view.model().resort(reset=False)
             self.library_view.model().research()
             self.tags_view.recount()
+            if self.cover_flow:
+                self.cover_flow.dataChanged()
 
     # Merge books {{{
     def merge_books(self, safe_merge=False):

src/calibre/gui2/library/models.py

@@ -20,7 +20,8 @@ from calibre.utils.config import tweaks, prefs
 from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
-from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, REGEXP_MATCH
+from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
+    REGEXP_MATCH, CoverCache
 from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
 from calibre.constants import filesystem_encoding
@@ -149,21 +150,22 @@ class BooksModel(QAbstractTableModel): # {{{
         self.build_data_convertors()
         self.reset()
         self.database_changed.emit(db)
+        if self.cover_cache is not None:
+            self.cover_cache.stop()
+        self.cover_cache = CoverCache(db)
+        self.cover_cache.start()
+        def refresh_cover(event, ids):
+            if event == 'cover' and self.cover_cache is not None:
+                self.cover_cache.refresh(ids)
+        db.add_listener(refresh_cover)
 
     def refresh_ids(self, ids, current_row=-1):
         rows = self.db.refresh_ids(ids)
         if rows:
             self.refresh_rows(rows, current_row=current_row)
 
-    def refresh_cover_cache(self, ids):
-        if self.cover_cache:
-            self.cover_cache.refresh(ids)
-
     def refresh_rows(self, rows, current_row=-1):
         for row in rows:
-            if self.cover_cache:
-                id = self.db.id(row)
-                self.cover_cache.refresh([id])
             if row == current_row:
                 self.new_bookdisplay_data.emit(
                           self.get_book_display_info(row))
@@ -326,7 +328,7 @@ class BooksModel(QAbstractTableModel): # {{{
 
     def set_cache(self, idx):
         l, r = 0, self.count()-1
-        if self.cover_cache:
+        if self.cover_cache is not None:
             l = max(l, idx-self.buffer_size)
             r = min(r, idx+self.buffer_size)
             k = min(r-idx, idx-l)
@@ -494,11 +496,9 @@ class BooksModel(QAbstractTableModel): # {{{
         data = None
         try:
             id = self.db.id(row_number)
-            if self.cover_cache:
+            if self.cover_cache is not None:
                 img = self.cover_cache.cover(id)
-                if img:
+                if not img.isNull():
-                    if img.isNull():
-                        img = self.default_image
                     return img
             if not data:
                 data = self.db.cover(row_number)

src/calibre/gui2/ui.py

@@ -38,7 +38,6 @@ from calibre.gui2.dialogs.config import ConfigDialog
 
 from calibre.gui2.dialogs.book_info import BookInfo
 from calibre.library.database2 import LibraryDatabase2
-from calibre.library.caches import CoverCache
 from calibre.gui2.init import ToolbarMixin, LibraryViewMixin, LayoutMixin
 from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin
 from calibre.gui2.search_restriction_mixin import SearchRestrictionMixin
@@ -138,6 +137,7 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{
         self.restriction_in_effect = False
 
         self.progress_indicator = ProgressIndicator(self)
+        self.progress_indicator.pos = (0, 20)
         self.verbose = opts.verbose
         self.get_metadata = GetMetadata()
         self.upload_memory = {}
@@ -230,9 +230,6 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{
 
         if self.system_tray_icon.isVisible() and opts.start_in_tray:
             self.hide_windows()
-        self.cover_cache = CoverCache(self.library_path)
-        self.cover_cache.start()
-        self.library_view.model().cover_cache = self.cover_cache
         self.library_view.model().count_changed_signal.connect \
                                             (self.location_view.count_changed)
         if not gprefs.get('quick_start_guide_added', False):
@@ -606,9 +603,10 @@ class Main(MainWindow, Ui_MainWindow, DeviceMixin, ToolbarMixin, # {{{
         while self.spare_servers:
             self.spare_servers.pop().close()
         self.device_manager.keep_going = False
-        self.cover_cache.stop()
+        cc = self.library_view.model().cover_cache
+        if cc is not None:
+            cc.stop()
         self.hide_windows()
-        self.cover_cache.terminate()
         self.emailer.stop()
         try:
             try:

src/calibre/gui2/widgets.py

@@ -38,12 +38,16 @@ class ProgressIndicator(QWidget):
         self.status.setWordWrap(True)
         self.status.setAlignment(Qt.AlignHCenter|Qt.AlignTop)
         self.setVisible(False)
+        self.pos = None
 
     def start(self, msg=''):
         view = self.parent()
         pwidth, pheight = view.size().width(), view.size().height()
         self.resize(pwidth, min(pheight, 250))
-        self.move(0, (pheight-self.size().height())/2.)
+        if self.pos is None:
+            self.move(0, (pheight-self.size().height())/2.)
+        else:
+            self.move(self.pos[0], self.pos[1])
         self.pi.resize(self.pi.sizeHint())
         self.pi.move(int((self.size().width()-self.pi.size().width())/2.), 0)
         self.status.resize(self.size().width(), self.size().height()-self.pi.size().height()-10)

src/calibre/library/caches.py

@@ -6,11 +6,13 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
-import collections, glob, os, re, itertools, functools
+import re, itertools, functools
 from itertools import repeat
 from datetime import timedelta
+from threading import Thread, RLock
+from Queue import Queue, Empty
 
-from PyQt4.Qt import QThread, QReadWriteLock, QImage, Qt
+from PyQt4.Qt import QImage, Qt
 
 from calibre.utils.config import tweaks
 from calibre.utils.date import parse_date, now, UNDEFINED_DATE
@@ -19,120 +21,73 @@ from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import title_sort
 from calibre import fit_image
 
-class CoverCache(QThread):
+class CoverCache(Thread):
 
-    def __init__(self, library_path, parent=None):
+    def __init__(self, db):
-        QThread.__init__(self, parent)
+        Thread.__init__(self)
-        self.library_path = library_path
+        self.daemon = True
-        self.id_map = None
+        self.db = db
-        self.id_map_lock = QReadWriteLock()
+        self.load_queue = Queue()
-        self.load_queue = collections.deque()
-        self.load_queue_lock = QReadWriteLock(QReadWriteLock.Recursive)
-        self.cache = {}
-        self.cache_lock = QReadWriteLock()
-        self.id_map_stale = True
         self.keep_running = True
-
+        self.cache = {}
-    def build_id_map(self):
+        self.lock = RLock()
-        self.id_map_lock.lockForWrite()
+        self.null_image = QImage()
-        self.id_map = {}
-        for f in glob.glob(os.path.join(self.library_path, '*', '* (*)', 'cover.jpg')):
-            c = os.path.basename(os.path.dirname(f))
-            try:
-                id = int(re.search(r'\((\d+)\)', c[c.rindex('('):]).group(1))
-                self.id_map[id] = f
-            except:
-                continue
-        self.id_map_lock.unlock()
-        self.id_map_stale = False
-
-
-    def set_cache(self, ids):
-        self.cache_lock.lockForWrite()
-        already_loaded = set([])
-        for id in self.cache.keys():
-            if id in ids:
-                already_loaded.add(id)
-            else:
-                self.cache.pop(id)
-        self.cache_lock.unlock()
-        ids = [i for i in ids if i not in already_loaded]
-        self.load_queue_lock.lockForWrite()
-        self.load_queue = collections.deque(ids)
-        self.load_queue_lock.unlock()
-
-
-    def run(self):
-        while self.keep_running:
-            if self.id_map is None or self.id_map_stale:
-                self.build_id_map()
-            while True: # Load images from the load queue
-                self.load_queue_lock.lockForWrite()
-                try:
-                    id = self.load_queue.popleft()
-                except IndexError:
-                    break
-                finally:
-                    self.load_queue_lock.unlock()
-
-                self.cache_lock.lockForRead()
-                need = True
-                if id in self.cache.keys():
-                    need = False
-                self.cache_lock.unlock()
-                if not need:
-                    continue
-                path = None
-                self.id_map_lock.lockForRead()
-                if id in self.id_map.keys():
-                    path = self.id_map[id]
-                else:
-                    self.id_map_stale = True
-                self.id_map_lock.unlock()
-                if path and os.access(path, os.R_OK):
-                    try:
-                        img = QImage()
-                        data = open(path, 'rb').read()
-                        img.loadFromData(data)
-                        if img.isNull():
-                            continue
-                        scaled, nwidth, nheight = fit_image(img.width(),
-                                img.height(), 600, 800)
-                        if scaled:
-                            img = img.scaled(nwidth, nheight, Qt.KeepAspectRatio,
-                                    Qt.SmoothTransformation)
-                    except:
-                        continue
-                    self.cache_lock.lockForWrite()
-                    self.cache[id] = img
-                    self.cache_lock.unlock()
-
-            self.sleep(1)
 
     def stop(self):
         self.keep_running = False
 
-    def cover(self, id):
+    def _image_for_id(self, id_):
-        val = None
+        img = self.db.cover(id_, index_is_id=True, as_image=True)
-        if self.cache_lock.tryLockForRead(50):
+        if img is None:
-            val = self.cache.get(id, None)
+            img = QImage()
-            self.cache_lock.unlock()
+        if not img.isNull():
-        return val
+            scaled, nwidth, nheight = fit_image(img.width(),
+                    img.height(), 600, 800)
+            if scaled:
+                img = img.scaled(nwidth, nheight, Qt.KeepAspectRatio,
+                        Qt.SmoothTransformation)
+
+        return img
+
+    def run(self):
+        while self.keep_running:
+            try:
+                id_ = self.load_queue.get(True, 1)
+            except Empty:
+                continue
+            try:
+                img = self._image_for_id(id_)
+            except:
+                import traceback
+                traceback.print_exc()
+                continue
+            with self.lock:
+                self.cache[id_] = img
+
+    def set_cache(self, ids):
+        with self.lock:
+            already_loaded = set([])
+            for id in self.cache.keys():
+                if id in ids:
+                    already_loaded.add(id)
+                else:
+                    self.cache.pop(id)
+        for id_ in set(ids) - already_loaded:
+            self.load_queue.put(id_)
+
+    def cover(self, id_):
+        with self.lock:
+            return self.cache.get(id_, self.null_image)
 
     def clear_cache(self):
-        self.cache_lock.lockForWrite()
+        with self.lock:
-        self.cache = {}
+            self.cache = {}
-        self.cache_lock.unlock()
 
     def refresh(self, ids):
-        self.cache_lock.lockForWrite()
+        with self.lock:
-        for id in ids:
+            for id_ in ids:
-            self.cache.pop(id, None)
+                self.cache.pop(id_, None)
-        self.cache_lock.unlock()
+                self.load_queue.put(id_)
-        self.load_queue_lock.lockForWrite()
-        for id in ids:
-            self.load_queue.appendleft(id)
-        self.load_queue_lock.unlock()
 
 ### Global utility function for get_match here and in gui2/library.py
 CONTAINS_MATCH = 0

src/calibre/library/database2.py

@@ -6,7 +6,7 @@ __docformat__ = 'restructuredtext en'
 '''
 The database used to store ebook metadata
 '''
-import os, sys, shutil, cStringIO, glob,functools, traceback
+import os, sys, shutil, cStringIO, glob, time, functools, traceback
 from itertools import repeat
 from math import floor
 
@@ -440,12 +440,20 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if os.access(path, os.R_OK):
             if as_path:
                 return path
-            f = open(path, 'rb')
+            try:
+                f = open(path, 'rb')
+            except (IOError, OSError):
+                time.sleep(0.2)
+                f = open(path, 'rb')
             if as_image:
                 img = QImage()
                 img.loadFromData(f.read())
+                f.close()
                 return img
-            return f if as_file else f.read()
+            ans = f if as_file else f.read()
+            if ans is not f:
+                f.close()
+            return ans
 
     def get_metadata(self, idx, index_is_id=False, get_cover=False):
         '''
@@ -492,12 +500,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
         return os.access(path, os.R_OK)
 
-    def remove_cover(self, id):
+    def remove_cover(self, id, notify=True):
         path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
         if os.path.exists(path):
-            os.remove(path)
+            try:
+                os.remove(path)
+            except (IOError, OSError):
+                time.sleep(0.2)
+                os.remove(path)
+        if notify:
+            self.notify('cover', [id])
 
-    def set_cover(self, id, data):
+    def set_cover(self, id, data, notify=True):
         '''
         Set the cover for this book.
 
@@ -509,7 +523,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         else:
             if callable(getattr(data, 'read', None)):
                 data = data.read()
-            save_cover_data_to(data, path)
+            try:
+                save_cover_data_to(data, path)
+            except (IOError, OSError):
+                time.sleep(0.2)
+                save_cover_data_to(data, path)
+        if notify:
+            self.notify('cover', [id])
 
     def book_on_device(self, id):
         if callable(self.book_on_device_func):