Reduce startup times for users with huge calibre libraries by caching the cover check filesystem queries. The reduction will be most pronounced on filesystems that store unindexed directory names.

src/calibre/library/caches.py

@@ -670,7 +670,6 @@ class ResultCache(SearchQueryParser): # {{{
         for id in ids:
             try:
                 self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
-                self._data[id].append(db.has_cover(id, index_is_id=True))
                 self._data[id].append(db.book_on_device_string(id))
                 self._data[id].append(None)
                 if len(self.composites) > 0:
@@ -691,7 +690,6 @@ class ResultCache(SearchQueryParser): # {{{
         self._data.extend(repeat(None, max(ids)-len(self._data)+2))
         for id in ids:
             self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
-            self._data[id].append(db.has_cover(id, index_is_id=True))
             self._data[id].append(db.book_on_device_string(id))
             self._data[id].append(None)
             if len(self.composites) > 0:
@@ -721,7 +719,6 @@ class ResultCache(SearchQueryParser): # {{{
             self._data[r[0]] = r
         for item in self._data:
             if item is not None:
-                item.append(db.has_cover(item[0], index_is_id=True))
                 item.append(db.book_on_device_string(item[0]))
                 item.append(None)
                 if len(self.composites) > 0:

src/calibre/library/database2.py

@@ -226,7 +226,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'lccn',
              'pubdate',
              'flags',
-             'uuid'
+             'uuid',
+             'has_cover'
             ]
         lines = []
         for col in columns:
@@ -245,7 +246,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
              'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8,
              'publisher':9, 'series_index':10,
              'sort':11, 'author_sort':12, 'formats':13, 'isbn':14, 'path':15,
-             'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19}
+             'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19, 'cover':20}
 
         for k,v in self.FIELD_MAP.iteritems():
             self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
@@ -267,12 +268,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                             base,
                             prefer_custom=True)
 
-        self.FIELD_MAP['cover'] = base+1
+        self.field_metadata.set_field_record_index('cover',
-        self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
+                self.FIELD_MAP['cover'], prefer_custom=False)
-        self.FIELD_MAP['ondevice'] = base+2
+        self.FIELD_MAP['ondevice'] = base+1
-        self.field_metadata.set_field_record_index('ondevice', base+2, prefer_custom=False)
+        self.field_metadata.set_field_record_index('ondevice', base+1, prefer_custom=False)
-        self.FIELD_MAP['all_metadata'] = base+3
+        self.FIELD_MAP['all_metadata'] = base+2
-        self.field_metadata.set_field_record_index('all_metadata', base+3, prefer_custom=False)
+        self.field_metadata.set_field_record_index('all_metadata', base+2, prefer_custom=False)
 
         script = '''
         DROP VIEW IF EXISTS meta2;
@@ -332,7 +333,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         self.dirtied_cache = set([x[0] for x in d])
 
         self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
+        st = time.time()
         self.refresh()
+        print 'refresh time:', time.time() - st
         self.last_update_check = self.last_modified()
 
 
@@ -763,17 +766,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
                     identical_book_ids.add(book_id)
         return identical_book_ids
 
-    def has_cover(self, index, index_is_id=False):
+    def remove_cover(self, id, notify=True, commit=True):
-        id = index if index_is_id else self.id(index)
-        try:
-            path = os.path.join(self.abspath(id, index_is_id=True,
-                create_dirs=False), 'cover.jpg')
-        except:
-            # Can happen if path has not yet been set
-            return False
-        return os.path.exists(path)
-
-    def remove_cover(self, id, notify=True):
         path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
         if os.path.exists(path):
             try:
@@ -781,11 +774,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             except (IOError, OSError):
                 time.sleep(0.2)
                 os.remove(path)
+        self.conn.execute('UPDATE books SET has_cover=0 WHERE id=?', (id,))
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['cover'], False, row_is_id=True)
         if notify:
             self.notify('cover', [id])
 
-    def set_cover(self, id, data, notify=True):
+    def set_cover(self, id, data, notify=True, commit=True):
         '''
         Set the cover for this book.
 
@@ -802,6 +798,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             except (IOError, OSError):
                 time.sleep(0.2)
                 save_cover_data_to(data, path)
+        self.conn.execute('UPDATE books SET has_cover=1 WHERE id=?', (id,))
+        if commit:
+            self.conn.commit()
         self.data.set(id, self.FIELD_MAP['cover'], True, row_is_id=True)
         if notify:
             self.notify('cover', [id])
@@ -1273,11 +1272,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
         if mi.series:
             doit(self.set_series, id, mi.series, notify=False, commit=False)
         if mi.cover_data[1] is not None:
-            doit(self.set_cover, id, mi.cover_data[1]) # doesn't use commit
+            doit(self.set_cover, id, mi.cover_data[1], commit=False)
         elif mi.cover is not None:
             if os.access(mi.cover, os.R_OK):
                 with lopen(mi.cover, 'rb') as f:
-                    doit(self.set_cover, id, f)
+                    doit(self.set_cover, id, f, commit=False)
         if mi.tags:
             doit(self.set_tags, id, mi.tags, notify=False, commit=False)
         if mi.comments:
@@ -2291,7 +2290,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
             x['tags'] = [i.replace('|', ',').strip() for i in x['tags'].split(',')] if x['tags'] else []
             path = os.path.join(prefix, self.path(record[self.FIELD_MAP['id']], index_is_id=True))
             x['cover'] = os.path.join(path, 'cover.jpg')
-            if not self.has_cover(x['id'], index_is_id=True):
+            if not record[self.FIELD_MAP['cover']]:
                 x['cover'] = None
             formats = self.formats(record[self.FIELD_MAP['id']], index_is_id=True)
             if formats:

src/calibre/library/schema_upgrades.py

@@ -6,6 +6,8 @@ __license__   = 'GPL v3'
 __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
 __docformat__ = 'restructuredtext en'
 
+import os
+
 class SchemaUpgrade(object):
 
     def __init__(self):
@@ -409,3 +411,17 @@ class SchemaUpgrade(object):
         '''
         self.conn.executescript(script)
 
+    def upgrade_version_14(self):
+        'Cache has_cover'
+        self.conn.execute('ALTER TABLE books ADD COLUMN has_cover BOOL DEFAULT 0')
+        data = self.conn.get('SELECT id,path FROM books', all=True)
+        def has_cover(path):
+            if path:
+                path = os.path.join(self.library_path, path.replace('/', os.sep),
+                    'cover.jpg')
+                return os.path.exists(path)
+            return False
+
+        ids = [(x[0],) for x in data if has_cover(x[1])]
+        self.conn.executemany('UPDATE books SET has_cover=1 WHERE id=?', ids)
+

src/calibre/library/sqlite.py

@@ -34,10 +34,11 @@ sqlite.register_adapter(datetime, adapt_datetime)
 sqlite.register_converter('timestamp', convert_timestamp)
 
 def convert_bool(val):
-    return bool(int(val))
+    return val != '0'
 
 sqlite.register_adapter(bool, lambda x : 1 if x else 0)
 sqlite.register_converter('bool', convert_bool)
+sqlite.register_converter('BOOL', convert_bool)
 
 class DynamicFilter(object):