Add thread to GUI for distributed metadata backup

src/calibre/gui2/library/models.py

@@ -21,7 +21,7 @@ from calibre.utils.date import dt_factory, qt_to_dt, isoformat
 from calibre.ebooks.metadata.meta import set_metadata as _set_metadata
 from calibre.utils.search_query_parser import SearchQueryParser
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
-    REGEXP_MATCH, CoverCache
+    REGEXP_MATCH, CoverCache, MetadataBackup
 from calibre.library.cli import parse_series_string
 from calibre import strftime, isbytestring, prepare_string_for_xml
 from calibre.constants import filesystem_encoding
@@ -153,6 +153,9 @@ class BooksModel(QAbstractTableModel): # {{{
             self.cover_cache.stop()
         self.cover_cache = CoverCache(db, FunctionDispatcher(self.db.cover))
         self.cover_cache.start()
+        self.metadata_backup = MetadataBackup(db,
+                FunctionDispatcher(self.db.dump_metadata))
+        self.metadata_backup.start()
         def refresh_cover(event, ids):
             if event == 'cover' and self.cover_cache is not None:
                 self.cover_cache.refresh(ids)

src/calibre/gui2/ui.py

@@ -551,6 +551,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, # {{{
         cc = self.library_view.model().cover_cache
         if cc is not None:
             cc.stop()
+        mb = self.library_view.model().metadata_backup
+        if mb is not None:
+            mb.stop()
+
         self.hide_windows()
         self.emailer.stop()
         try:

src/calibre/library/caches.py

@@ -21,7 +21,31 @@ from calibre.utils.pyparsing import ParseException
 from calibre.ebooks.metadata import title_sort
 from calibre import fit_image
 
-class CoverCache(Thread):
+class MetadataBackup(Thread): # {{{
+
+    def __init__(self, db, dump_func):
+        Thread.__init__(self)
+        self.daemon = True
+        self.db = db
+        self.dump_func = dump_func
+        self.keep_running = True
+
+    def stop(self):
+        self.keep_running = False
+
+    def run(self):
+        while self.keep_running:
+            try:
+                id_ = self.db.dirtied_queue.get(True, 5)
+            except Empty:
+                continue
+            # If there is an exception is dump_func, we
+            # have no way of knowing
+            self.dump_func([id_])
+
+# }}}
+
+class CoverCache(Thread): # {{{
 
     def __init__(self, db, cover_func):
         Thread.__init__(self)
@@ -90,6 +114,7 @@ class CoverCache(Thread):
             for id_ in ids:
                 self.cache.pop(id_, None)
                 self.load_queue.put(id_)
+# }}}
 
 ### Global utility function for get_match here and in gui2/library.py
 CONTAINS_MATCH = 0
@@ -107,7 +132,7 @@ def _match(query, value, matchkind):
             pass
     return False
 
-class ResultCache(SearchQueryParser):
+class ResultCache(SearchQueryParser): # {{{
 
     '''
     Stores sorted and filtered metadata in memory.
@@ -694,4 +719,5 @@ class SortKeyGenerator(object):
 
     # }}}
 
+# }}}
 

src/calibre/library/database2.py

@@ -9,6 +9,7 @@ The database used to store ebook metadata
 import os, sys, shutil, cStringIO, glob, time, functools, traceback, re
 from itertools import repeat
 from math import floor
+from Queue import Queue
 
 from PyQt4.QtGui import QImage
 
@@ -127,7 +128,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def __init__(self, library_path, row_factory=False):
         self.field_metadata = FieldMetadata()
-        self.dirtied_cache = set([])
+        self.dirtied_queue = Queue()
         if not os.path.exists(library_path):
             os.makedirs(library_path)
         self.listeners = set([])
@@ -340,7 +341,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                 loc=self.FIELD_MAP['sort']))
 
         d = self.conn.get('SELECT book FROM metadata_dirtied', all=True)
-        self.dirtied_cache.update(set([x[0] for x in d]))
+        for x in d:
+            self.dirtied_queue.put(x[0])
 
         self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
         self.refresh()
@@ -557,6 +559,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
 
     def dump_metadata(self, book_ids, remove_from_dirtied=True, commit=True):
         for book_id in book_ids:
+            if not self.data.has_id(book_id):
+                continue
             mi = self.get_metadata(book_id, index_is_id=True, get_cover=True)
             # Always set cover to cover.jpg. Even if cover doesn't exist,
             # no harm done. This way no need to call dirtied when
@@ -569,18 +573,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             if remove_from_dirtied:
                 self.conn.execute('DELETE FROM metadata_dirtied WHERE book=?',
                         (book_id,))
-                if book_id in self.dirtied_cache:
-                    self.dirtied_cache.remove(book_id)
         if commit:
             self.conn.commit()
 
     def dirtied(self, book_ids, commit=True):
         self.conn.executemany(
-            'INSERT OR REPLACE INTO metadata_dirtied VALUES (?)',
+            'INSERT OR REPLACE INTO metadata_dirtied (book) VALUES (?)',
                 [(x,) for x in book_ids])
         if commit:
             self.conn.commit()
-        self.dirtied.update(set(book_ids))
+        for x in book_ids:
+            self.dirtied_queue.put(x)
 
     def get_metadata(self, idx, index_is_id=False, get_cover=False):
         '''