From 26fcfc70f1603dea6e61eac174d88cbc4e027106 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 20 Jul 2013 10:07:49 +0530 Subject: [PATCH] Speed up book list rendering by using the new api to get values --- src/calibre/db/cache.py | 10 +++ src/calibre/db/fields.py | 12 +++- src/calibre/db/tables.py | 2 +- src/calibre/gui2/library/models.py | 111 ++++++++++++++++++++++++++++- 4 files changed, 132 insertions(+), 3 deletions(-) diff --git a/src/calibre/db/cache.py b/src/calibre/db/cache.py index 7cfe9bc4a6..a478a23664 100644 --- a/src/calibre/db/cache.py +++ b/src/calibre/db/cache.py @@ -318,6 +318,16 @@ class Cache(object): except (KeyError, IndexError): return default_value + @read_api + def fast_field_for(self, field_obj, book_id, default_value=None): + ' Same as field_for, except that it avoids the extra lookup to get the field object ' + if field_obj.is_composite: + return field_obj.get_value_with_cache(book_id, partial(self._get_metadata, get_user_categories=False)) + try: + return field_obj.for_book(book_id, default_value=default_value) + except (KeyError, IndexError): + return default_value + @read_api def composite_for(self, name, book_id, mi=None, default_value=''): try: diff --git a/src/calibre/db/fields.py b/src/calibre/db/fields.py index e028ff5d99..8c6d18a74d 100644 --- a/src/calibre/db/fields.py +++ b/src/calibre/db/fields.py @@ -23,6 +23,7 @@ class Field(object): is_many = False is_many_many = False + is_composite = False def __init__(self, name, table): self.name, self.table = name, table @@ -148,6 +149,8 @@ class OneToOneField(Field): class CompositeField(OneToOneField): + is_composite = True + def __init__(self, *args, **kwargs): OneToOneField.__init__(self, *args, **kwargs) @@ -229,6 +232,14 @@ class OnDeviceField(OneToOneField): self.is_multiple = False self.cache = {} self._lock = Lock() + self._metadata = { + 'table':None, 'column':None, 'datatype':'text', 'is_multiple':{}, + 'kind':'field', 'name':_('On Device'), 'search_terms':['ondevice'], + 'is_custom':False, 'is_category':False, 'is_csp': False, 'display':{}} + + @property + def metadata(self): + return self._metadata def clear_caches(self, book_ids=None): with self._lock: @@ -330,7 +341,6 @@ class ManyToManyField(Field): def __init__(self, *args, **kwargs): Field.__init__(self, *args, **kwargs) - self.alphabetical_sort = self.name != 'authors' def for_book(self, book_id, default_value=None): ids = self.table.book_col_map.get(book_id, ()) diff --git a/src/calibre/db/tables.py b/src/calibre/db/tables.py index 9b9ff4e9e0..3e1db9400f 100644 --- a/src/calibre/db/tables.py +++ b/src/calibre/db/tables.py @@ -113,7 +113,7 @@ class SizeTable(OneToOneTable): for row in db.conn.execute( 'SELECT books.id, (SELECT MAX(uncompressed_size) FROM data ' 'WHERE data.book=books.id) FROM books'): - self.book_col_map[row[0]] = self.unserialize(row[1]) + self.book_col_map[row[0]] = self.unserialize(row[1] or 0) def update_sizes(self, size_map): self.book_col_map.update(size_map) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 738b42a669..4a1a1a3e88 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -227,7 +227,10 @@ class BooksModel(QAbstractTableModel): # {{{ elif col in self.custom_columns: self.headers[col] = self.custom_columns[col]['name'] - self.build_data_convertors() + if hasattr(self.db, 'new_api'): + self.build_new_data_convertors() + else: + self.build_data_convertors() self.reset() self.database_changed.emit(db) self.stop_metadata_backup() @@ -634,6 +637,112 @@ class BooksModel(QAbstractTableModel): # {{{ img = self.default_image return img + def build_new_data_convertors(self): + + def renderer(field, decorator=False): + idfunc = self.db.id + fffunc = self.db.new_api.fast_field_for + field_obj = self.db.new_api.fields[field] + m = field_obj.metadata.copy() + if 'display' not in m: + m['display'] = {} + dt = m['datatype'] + + if decorator == 'bool': + bt = self.db.new_api.pref('bools_are_tristate') + bn = self.bool_no_icon + by = self.bool_yes_icon + def func(idx): + val = force_to_bool(fffunc(field_obj, idfunc(idx))) + if val is None: + return NONE if bt else bn + return by if val else bn + elif field == 'size': + sz_mult = 1.0/(1024**2) + def func(idx): + val = fffunc(field_obj, idfunc(idx), default_value=0) + ans = u'%.1f' % (val * sz_mult) + if val > 0 and ans == u'0.0': + ans = u'<0.1' + return QVariant(ans) + elif field == 'languages': + def func(idx): + return QVariant(', '.join(calibre_langcode_to_name(x) for x in fffunc(field_obj, idfunc(idx)))) + elif field == 'ondevice' and decorator: + by = self.bool_yes_icon + bb = self.bool_blank_icon + def func(idx): + return by if fffunc(field_obj, idfunc(idx)) else bb + elif dt in {'text', 'comments', 'composite', 'enumeration'}: + if m['is_multiple']: + jv = m['is_multiple']['list_to_ui'] + do_sort = field == 'tags' + if do_sort: + def func(idx): + return QVariant(jv.join(sorted(fffunc(field_obj, idfunc(idx), default_value=()), key=sort_key))) + else: + def func(idx): + return QVariant(jv.join(fffunc(field_obj, idfunc(idx), default_value=()))) + else: + if dt in {'text', 'composite', 'enumeration'} and m['display'].get('use_decorations', False): + def func(idx): + text = fffunc(field_obj, idfunc(idx)) + return QVariant(text) if force_to_bool(text) is None else NONE + else: + def func(idx): + return QVariant(fffunc(field_obj, idfunc(idx), default_value='')) + elif dt == 'datetime': + def func(idx): + return QVariant(fffunc(field_obj, idfunc(idx), default_value=UNDEFINED_QDATETIME)) + elif dt == 'rating': + def func(idx): + return QVariant(int(fffunc(field_obj, idfunc(idx), default_value=0)/2.0)) + elif dt == 'series': + sidx_field = self.db.new_api.fields[field + '_index'] + fffunc = self.db.new_api._fast_field_for + read_lock = self.db.new_api.read_lock + def func(idx): + book_id = idfunc(idx) + with read_lock: + series = fffunc(field_obj, book_id, default_value=False) + if series: + return QVariant('%s [%s]' % (series, fmt_sidx(fffunc(sidx_field, book_id, default_value=1.0)))) + return NONE + elif dt in {'int', 'float'}: + fmt = m['display'].get('number_format', None) + def func(idx): + val = fffunc(field_obj, idfunc(idx)) + if val is None: + return NONE + if fmt: + try: + return QVariant(fmt.format(val)) + except (TypeError, ValueError, AttributeError, IndexError): + pass + return QVariant(val) + else: + def func(idx): + return NONE + + return func + + self.dc = {f:renderer(f) for f in 'title authors size timestamp pubdate last_modified rating publisher tags series ondevice languages'.split()} + self.dc_decorator = {f:renderer(f, True) for f in ('ondevice',)} + + for col in self.custom_columns: + self.dc[col] = renderer(col) + m = self.custom_columns[col] + dt = m['datatype'] + mult = m['is_multiple'] + if dt in {'text', 'composite', 'enumeration'} and not mult and m['display'].get('use_decorations', False): + self.dc_decorator[col] = renderer(col, 'bool') + elif dt == 'bool': + self.dc_decorator[col] = renderer(col, 'bool') + + # build a index column to data converter map, to remove the string lookup in the data loop + self.column_to_dc_map = [self.dc[col] for col in self.column_map] + self.column_to_dc_decorator_map = [self.dc_decorator.get(col, None) for col in self.column_map] + def build_data_convertors(self): def authors(r, idx=-1): au = self.db.data[r][idx]