From 6eaa75527b5754cfbb8df833ad3375b724d51cfd Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 11 Sep 2010 21:01:26 +0100 Subject: [PATCH 1/5] resort maximum_resort_levels tweak implemented --- resources/default_tweaks.py | 7 +++++++ src/calibre/gui2/library/models.py | 4 ++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 66ee4d1471..9d9bc7651c 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -114,3 +114,10 @@ add_new_book_tags_when_importing_books = False # Set the maximum number of tags to show per book in the content server max_content_server_tags_shown=5 + +# Set the maximum number of sort 'levels' that calibre will use to resort the +# library after certain operations such as searches or device insertion. Each +# sort level adds a performance penalty. If the database is large (thousands of +# books) the penalty might be noticeable. If you are not concerned about multi- +# level sorts, and if you are seeing a slowdown, reduce the value of this tweak. +maximum_resort_levels = 5 \ No newline at end of file diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 8ad0cd6818..d2f38cc0a1 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -266,8 +266,8 @@ class BooksModel(QAbstractTableModel): # {{{ self.db.refresh(field=None) self.resort(reset=reset) - def resort(self, reset=True, history=5): # Bug report needed history=4 :) - for col,ord in reversed(self.sort_history[:history]): + def resort(self, reset=True): + for col,ord in reversed(self.sort_history[:tweaks['maximum_resort_levels']]): try: col = self.column_map.index(col) except ValueError: From 721e61ef2a1fd090566e232ff9ca65e37400fe44 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 11 Sep 2010 21:05:05 +0100 Subject: [PATCH 2/5] Clean up tweaks.py formatting (add blank lines) --- resources/default_tweaks.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index 9d9bc7651c..71bf2c6c37 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -120,4 +120,5 @@ max_content_server_tags_shown=5 # sort level adds a performance penalty. If the database is large (thousands of # books) the penalty might be noticeable. If you are not concerned about multi- # level sorts, and if you are seeing a slowdown, reduce the value of this tweak. -maximum_resort_levels = 5 \ No newline at end of file +maximum_resort_levels = 5 + From bcd0430791f44ec926910eeb8bb18d7cbbff5fc9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 12 Sep 2010 13:37:28 +0100 Subject: [PATCH 3/5] Starting from Kovid's multisort: 1) change _map_filtered to an ordered dict to make 'in' operations much faster 2) add a method to field_metadata to return a dict of database fields. 3) fix a couple of places where field_metadata needed to be used. 4) make changes so gui2.library.models.resort uses multisort --- src/calibre/gui2/library/models.py | 14 +++---- src/calibre/library/caches.py | 59 ++++++++++++++++----------- src/calibre/library/database2.py | 1 + src/calibre/library/field_metadata.py | 3 ++ 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index d2f38cc0a1..d18516493a 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -247,7 +247,7 @@ class BooksModel(QAbstractTableModel): # {{{ # the search and count records for restrictions self.searched.emit(True) - def sort(self, col, order, reset=True, update_history=True): + def sort(self, col, order, reset=True): if not self.db: return self.about_to_be_sorted.emit(self.db.id) @@ -258,8 +258,7 @@ class BooksModel(QAbstractTableModel): # {{{ self.clear_caches() self.reset() self.sorted_on = (label, order) - if update_history: - self.sort_history.insert(0, self.sorted_on) + self.sort_history.insert(0, self.sorted_on) self.sorting_done.emit(self.db.index) def refresh(self, reset=True): @@ -267,12 +266,9 @@ class BooksModel(QAbstractTableModel): # {{{ self.resort(reset=reset) def resort(self, reset=True): - for col,ord in reversed(self.sort_history[:tweaks['maximum_resort_levels']]): - try: - col = self.column_map.index(col) - except ValueError: - col = 0 - self.sort(col, ord, reset=False, update_history=False) + if not self.db: + return + self.db.multisort(self.sort_history[:tweaks['maximum_resort_levels']]) if reset: self.reset() diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 59d5b45d5f..c342d5ff15 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -20,6 +20,7 @@ from calibre.utils.search_query_parser import SearchQueryParser from calibre.utils.pyparsing import ParseException from calibre.ebooks.metadata import title_sort from calibre import fit_image +from calibre.utils.ordered_dict import OrderedDict class CoverCache(Thread): @@ -112,7 +113,8 @@ class ResultCache(SearchQueryParser): ''' def __init__(self, FIELD_MAP, field_metadata): self.FIELD_MAP = FIELD_MAP - self._map = self._map_filtered = self._data = [] + self._map = self._data = [] + self._map_filtered = OrderedDict() self.first_sort = True self.search_restriction = '' self.field_metadata = field_metadata @@ -122,14 +124,14 @@ class ResultCache(SearchQueryParser): self.build_numeric_relop_dict() def __getitem__(self, row): - return self._data[self._map_filtered[row]] + return self._data[self._map_filtered.keys()[row]] def __len__(self): return len(self._map_filtered) def __iter__(self): for id in self._map_filtered: - yield self._data[id] + yield id def iterall(self): for x in self._data: @@ -468,7 +470,7 @@ class ResultCache(SearchQueryParser): ans = self.search_getting_ids(query, self.search_restriction) if return_matches: return ans - self._map_filtered = ans + self._map_filtered = OrderedDict.fromkeys(ans, True) def search_getting_ids(self, query, search_restriction): q = '' @@ -480,7 +482,7 @@ class ResultCache(SearchQueryParser): q = u'%s (%s)' % (search_restriction, query) if not q: return list(self._map) - matches = sorted(self.parse(q)) + matches = self.parse(q) return [id for id in self._map if id in matches] def set_search_restriction(self, s): @@ -493,18 +495,18 @@ class ResultCache(SearchQueryParser): if id in self._map: self._map.remove(id) if id in self._map_filtered: - self._map_filtered.remove(id) + del self._map_filtered[id] def set(self, row, col, val, row_is_id=False): - id = row if row_is_id else self._map_filtered[row] + id = row if row_is_id else self._map_filtered.keys()[row] self._data[id][col] = val def get(self, row, col, row_is_id=False): - id = row if row_is_id else self._map_filtered[row] + id = row if row_is_id else self._map_filtered.keys()[row] return self._data[id][col] def index(self, id, cache=False): - x = self._map if cache else self._map_filtered + x = self._map if cache else self._map_filtered.keys() return x.index(id) def row(self, id): @@ -544,13 +546,18 @@ class ResultCache(SearchQueryParser): self._data[id].append(db.has_cover(id, index_is_id=True)) self._data[id].append(db.book_on_device_string(id)) self._map[0:0] = ids - self._map_filtered[0:0] = ids + mf = OrderedDict() + for id in ids: + mf[id] = True + for id in self._map_filtered: + mf[id] = True + self._map_filtered = mf def books_deleted(self, ids): for id in ids: self._data[id] = None if id in self._map: self._map.remove(id) - if id in self._map_filtered: self._map_filtered.remove(id) + if id in self._map_filtered: del self._map_filtered[id] def count(self): return len(self._map) @@ -573,7 +580,7 @@ class ResultCache(SearchQueryParser): self._map = [i[0] for i in self._data if i is not None] if field is not None: self.sort(field, ascending) - self._map_filtered = list(self._map) + self._map_filtered = OrderedDict.fromkeys(self._map, True) if self.search_restriction: self.search('', return_matches=False) @@ -644,10 +651,14 @@ class ResultCache(SearchQueryParser): self.FIELD_MAP['series_index'], library_order=tweaks['title_series_sorting'] == 'library_order') else: - fcmp = functools.partial(self.cmp, self.FIELD_MAP[field], + fcmp = functools.partial(self.cmp, self.field_metadata[field]['rec_index'], subsort=subsort, asstr=as_string) self._map.sort(cmp=fcmp, reverse=not ascending) - self._map_filtered = [id for id in self._map if id in self._map_filtered] + mf = OrderedDict() + for id in self._map: + if id in self._map_filtered: + mf[id] = True + self._map_filtered = mf def multisort(self, fields=[], subsort=False): fields = [(self.sanitize_field_name(x), bool(y)) for x, y in fields] @@ -655,7 +666,7 @@ class ResultCache(SearchQueryParser): fields += [('sort', True)] if not fields: fields = [('timestamp', False)] - keys = self.field_metadata.keys() + keys = self.field_metadata.field_keys() for f, order in fields: if f not in keys: raise ValueError(f + ' not an existing field name') @@ -665,7 +676,11 @@ class ResultCache(SearchQueryParser): self._map.sort(key=keyg, reverse=not fields[0][1]) else: self._map.sort(key=keyg) - self._map_filtered = [id for id in self._map if id in self._map_filtered] + mf = OrderedDict() + for id in self._map: + if id in self._map_filtered: + mf[id] = id + self._map_filtered = mf class SortKey(object): @@ -677,16 +692,14 @@ class SortKey(object): for i, ascending in enumerate(self.orders): ans = cmp(self.values[i], other.values[i]) if ans != 0: - if not ascending: - ans *= -1 - return ans + return ans * ascending return 0 class SortKeyGenerator(object): def __init__(self, fields, field_metadata, data): self.field_metadata = field_metadata - self.orders = [x[1] for x in fields] + self.orders = [-1 if x[1] else 1 for x in fields] self.entries = [(x[0], field_metadata[x[0]]) for x in fields] self.library_order = tweaks['title_series_sorting'] == 'library_order' self.data = data @@ -735,7 +748,7 @@ if __name__ == '__main__': db.refresh() - fields = db.field_metadata.keys() + fields = db.field_metadata.field_keys() print fields @@ -765,7 +778,7 @@ if __name__ == '__main__': print 'Running single sort differentials' for field in fields: if field in ('search', 'id', 'news', 'flags'): continue - print '\t', field + print '\t', field, db.field_metadata[field]['datatype'] old, new = test_single_sort(field) if old[1] != new[1] or old[2] != new[2]: print '\t\t', 'Sort failure!' @@ -797,7 +810,7 @@ if __name__ == '__main__': [('size', True), ('tags', True), ('author', False)], [('series', False), ('title', True)], [('size', True), ('tags', True), ('author', False), ('pubdate', - True), ('tags', False), ('formats', False), ('uuid', True)], + True), ('series', False), ('formats', False), ('uuid', True)], ]: print '\t', ms diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 4106f8c965..8a5ab75c3c 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -311,6 +311,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.search_getting_ids = self.data.search_getting_ids self.refresh = functools.partial(self.data.refresh, self) self.sort = self.data.sort + self.multisort = self.data.multisort self.index = self.data.index self.refresh_ids = functools.partial(self.data.refresh_ids, self) self.row = self.data.row diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 096dfa66fe..276a6ba971 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -335,6 +335,9 @@ class FieldMetadata(dict): def keys(self): return self._tb_cats.keys() + def field_keys(self): + return [k for k in self._tb_cats.keys() if self._tb_cats[k]['kind']=='field'] + def iterkeys(self): for key in self._tb_cats: yield key From 8b09f4c293e82ff797635320c42487d9be190831 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 12 Sep 2010 13:42:37 +0100 Subject: [PATCH 4/5] Restore the second 'tags' to the tests --- src/calibre/library/caches.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index c342d5ff15..882de975db 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -810,7 +810,7 @@ if __name__ == '__main__': [('size', True), ('tags', True), ('author', False)], [('series', False), ('title', True)], [('size', True), ('tags', True), ('author', False), ('pubdate', - True), ('series', False), ('formats', False), ('uuid', True)], + True), ('tags', False), ('formats', False), ('uuid', True)], ]: print '\t', ms From 5626418d1a6993b16f3d6a83c22a761a7490b7ee Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 12 Sep 2010 14:51:21 +0100 Subject: [PATCH 5/5] Correct regression in device handing -- sorting after sending a book. --- src/calibre/gui2/library/models.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index d18516493a..c746a5aa56 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -1024,6 +1024,11 @@ class DeviceBooksModel(BooksModel): # {{{ if reset: self.reset() + def resort(self, reset=True): + if self.sorted_on: + self.sort(self.column_map.index(self.sorted_on[0]), + self.sorted_on[1], reset=reset) + def columnCount(self, parent): if parent and parent.isValid(): return 0