Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Reduce startup times for users with huge calibre libraries by caching the cover check filesystem queries. The reduction will be most pronounced on filesystems that store unindexed directory names.

Browse Source
This commit is contained in:
Kovid Goyal 2010-11-23 11:24:38 -07:00
parent 41ba44a793
commit 1080fdab8f
4 changed files with 40 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/calibre/library/caches.py
View File

@ -670,7 +670,6 @@ class ResultCache(SearchQueryParser): # {{{
for id in ids:
try:
self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
self._data[id].append(db.has_cover(id, index_is_id=True))
self._data[id].append(db.book_on_device_string(id))
self._data[id].append(None)
if len(self.composites) > 0:
@ -691,7 +690,6 @@ class ResultCache(SearchQueryParser): # {{{
self._data.extend(repeat(None, max(ids)-len(self._data)+2))
for id in ids:
self._data[id] = db.conn.get('SELECT * from meta2 WHERE id=?', (id,))[0]
self._data[id].append(db.has_cover(id, index_is_id=True))
self._data[id].append(db.book_on_device_string(id))
self._data[id].append(None)
if len(self.composites) > 0:
@ -721,7 +719,6 @@ class ResultCache(SearchQueryParser): # {{{
self._data[r[0]] = r
for item in self._data:
if item is not None:
item.append(db.has_cover(item[0], index_is_id=True))
item.append(db.book_on_device_string(item[0]))
item.append(None)
if len(self.composites) > 0:

45
src/calibre/library/database2.py
View File

@ -226,7 +226,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
'lccn',
'pubdate',
'flags',
'uuid'
'uuid',
'has_cover'
]
lines = []
for col in columns:
@ -245,7 +246,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
'size':4, 'rating':5, 'tags':6, 'comments':7, 'series':8,
'publisher':9, 'series_index':10,
'sort':11, 'author_sort':12, 'formats':13, 'isbn':14, 'path':15,
'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19}
'lccn':16, 'pubdate':17, 'flags':18, 'uuid':19, 'cover':20}
for k,v in self.FIELD_MAP.iteritems():
self.field_metadata.set_field_record_index(k, v, prefer_custom=False)
@ -267,12 +268,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
base,
prefer_custom=True)
self.FIELD_MAP['cover'] = base+1
self.field_metadata.set_field_record_index('cover', base+1, prefer_custom=False)
self.FIELD_MAP['ondevice'] = base+2
self.field_metadata.set_field_record_index('ondevice', base+2, prefer_custom=False)
self.FIELD_MAP['all_metadata'] = base+3
self.field_metadata.set_field_record_index('all_metadata', base+3, prefer_custom=False)
self.field_metadata.set_field_record_index('cover',
self.FIELD_MAP['cover'], prefer_custom=False)
self.FIELD_MAP['ondevice'] = base+1
self.field_metadata.set_field_record_index('ondevice', base+1, prefer_custom=False)
self.FIELD_MAP['all_metadata'] = base+2
self.field_metadata.set_field_record_index('all_metadata', base+2, prefer_custom=False)
script = '''
DROP VIEW IF EXISTS meta2;
@ -332,7 +333,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
self.dirtied_cache = set([x[0] for x in d])
self.refresh_ondevice = functools.partial(self.data.refresh_ondevice, self)
st = time.time()
self.refresh()
print 'refresh time:', time.time() - st
self.last_update_check = self.last_modified()
@ -763,17 +766,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
identical_book_ids.add(book_id)
return identical_book_ids
def has_cover(self, index, index_is_id=False):
id = index if index_is_id else self.id(index)
try:
path = os.path.join(self.abspath(id, index_is_id=True,
create_dirs=False), 'cover.jpg')
except:
# Can happen if path has not yet been set
return False
return os.path.exists(path)
def remove_cover(self, id, notify=True):
def remove_cover(self, id, notify=True, commit=True):
path = os.path.join(self.library_path, self.path(id, index_is_id=True), 'cover.jpg')
if os.path.exists(path):
try:
@ -781,11 +774,14 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
except (IOError, OSError):
time.sleep(0.2)
os.remove(path)
self.conn.execute('UPDATE books SET has_cover=0 WHERE id=?', (id,))
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['cover'], False, row_is_id=True)
if notify:
self.notify('cover', [id])
def set_cover(self, id, data, notify=True):
def set_cover(self, id, data, notify=True, commit=True):
'''
Set the cover for this book.
@ -802,6 +798,9 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
except (IOError, OSError):
time.sleep(0.2)
save_cover_data_to(data, path)
self.conn.execute('UPDATE books SET has_cover=1 WHERE id=?', (id,))
if commit:
self.conn.commit()
self.data.set(id, self.FIELD_MAP['cover'], True, row_is_id=True)
if notify:
self.notify('cover', [id])
@ -1273,11 +1272,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
if mi.series:
doit(self.set_series, id, mi.series, notify=False, commit=False)
if mi.cover_data[1] is not None:
doit(self.set_cover, id, mi.cover_data[1]) # doesn't use commit
doit(self.set_cover, id, mi.cover_data[1], commit=False)
elif mi.cover is not None:
if os.access(mi.cover, os.R_OK):
with lopen(mi.cover, 'rb') as f:
doit(self.set_cover, id, f)
doit(self.set_cover, id, f, commit=False)
if mi.tags:
doit(self.set_tags, id, mi.tags, notify=False, commit=False)
if mi.comments:
@ -2291,7 +2290,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns):
x['tags'] = [i.replace('|', ',').strip() for i in x['tags'].split(',')] if x['tags'] else []
path = os.path.join(prefix, self.path(record[self.FIELD_MAP['id']], index_is_id=True))
x['cover'] = os.path.join(path, 'cover.jpg')
if not self.has_cover(x['id'], index_is_id=True):
if not record[self.FIELD_MAP['cover']]:
x['cover'] = None
formats = self.formats(record[self.FIELD_MAP['id']], index_is_id=True)
if formats:

16
src/calibre/library/schema_upgrades.py
View File

@ -6,6 +6,8 @@ __license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import os
class SchemaUpgrade(object):
def __init__(self):
@ -409,3 +411,17 @@ class SchemaUpgrade(object):
'''
self.conn.executescript(script)
def upgrade_version_14(self):
'Cache has_cover'
self.conn.execute('ALTER TABLE books ADD COLUMN has_cover BOOL DEFAULT 0')
data = self.conn.get('SELECT id,path FROM books', all=True)
def has_cover(path):
if path:
path = os.path.join(self.library_path, path.replace('/', os.sep),
'cover.jpg')
return os.path.exists(path)
return False
ids = [(x[0],) for x in data if has_cover(x[1])]
self.conn.executemany('UPDATE books SET has_cover=1 WHERE id=?', ids)

3
src/calibre/library/sqlite.py
View File

@ -34,10 +34,11 @@ sqlite.register_adapter(datetime, adapt_datetime)
sqlite.register_converter('timestamp', convert_timestamp)
def convert_bool(val):
return bool(int(val))
return val != '0'
sqlite.register_adapter(bool, lambda x : 1 if x else 0)
sqlite.register_converter('bool', convert_bool)
sqlite.register_converter('BOOL', convert_bool)
class DynamicFilter(object):