From d2ff37b5a179ecf10041d8e51a810cb067820ca9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 Dec 2010 14:11:10 +0000 Subject: [PATCH 1/5] Improved get_categories -- approximately 6 times faster --- src/calibre/library/database2.py | 219 ++++++++++++++++++++++++++----- 1 file changed, 187 insertions(+), 32 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 1229b60577..0d301ccaff 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -14,7 +14,7 @@ from operator import itemgetter from PyQt4.QtGui import QImage - +from calibre import prints from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.library.database import LibraryDatabase @@ -1039,43 +1039,170 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tn=field['table'], col=field['link_column']), (id_,)) return set(x[0] for x in ans) +########## data structures for get_categories + CATEGORY_SORTS = ('name', 'popularity', 'rating') - def get_categories(self, sort='name', ids=None, icon_map=None): - self.books_list_filter.change([] if not ids else ids) + class TCat_Tag(object): - categories = {} + def __init__(self, name, sort): + self.n = name + self.s = sort + self.c = 0 + self.rt = 0 + self.rc = 0 + self.id = None + + def set_all(self, c, rt, rc, id): + self.c = c + self.rt = rt + self.rc = rc + self.id = id + + def __str__(self): + return unicode(self) + + def __unicode__(self): + return 'n=%s s=%s c=%d rt=%d rc=%d id=%s'%\ + (self.n, self.s, self.c, self.rt, self.rc, self.id) + + + def get_categories(self, sort='name', ids=None, icon_map=None): + start = time.time() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') + if sort not in self.CATEGORY_SORTS: + raise ValueError('sort ' + sort + ' not a valid value') + + self.books_list_filter.change([] if not ids else ids) + id_filter = None if not ids else frozenset(ids) tb_cats = self.field_metadata - #### First, build the standard and custom-column categories #### + tcategories = {} + tids = {} + md = [] + + # First, build the maps. We need a category->items map and an + # item -> (item_id, sort_val) map to use in the books loop for category in tb_cats.keys(): cat = tb_cats[category] - if not cat['is_category'] or cat['kind'] in ['user', 'search']: + if not cat['is_category'] or cat['kind'] in ['user', 'search'] \ + or category in ['news', 'formats']: continue - tn = cat['table'] - categories[category] = [] #reserve the position in the ordered list - if tn is None: # Nothing to do for the moment + # Get the ids for the item values + if not cat['is_custom']: + funcs = { + 'authors' : self.get_authors_with_ids, + 'series' : self.get_series_with_ids, + 'publisher': self.get_publishers_with_ids, + 'tags' : self.get_tags_with_ids, + 'rating' : self.get_ratings_with_ids, + } + func = funcs.get(category, None) + if func: + list = func() + else: + raise ValueError(category + ' has no get with ids function') + else: + list = self.get_custom_items_with_ids(label=cat['label']) + tids[category] = {} + if category == 'authors': + for l in list: + (id, val, sort_val) = (l[0], l[1], l[2]) + tids[category][val] = (id, sort_val) + else: + for l in list: + (id, val) = (l[0], l[1]) + tids[category][val] = (id, val) + # add an empty category to the category map + tcategories[category] = {} + # create a list of category/field_index for the books scan to use. + # This saves iterating through field_metadata for each book + md.append((category, cat['rec_index'], cat['is_multiple'])) + + print 'end phase "collection":', time.time() - start, 'seconds' + + # Now scan every book looking for category items. + # Code below is duplicated because it shaves off 10% of the loop time + id_dex = self.FIELD_MAP['id'] + rating_dex = self.FIELD_MAP['rating'] + for book in self.data.iterall(): + if id_filter and book[id_dex] not in id_filter: continue - cn = cat['column'] - if ids is None: - query = '''SELECT id, {0}, count, avg_rating, sort - FROM tag_browser_{1}'''.format(cn, tn) - else: - query = '''SELECT id, {0}, count, avg_rating, sort - FROM tag_browser_filtered_{1}'''.format(cn, tn) - if sort == 'popularity': - query += ' ORDER BY count DESC, sort ASC' - elif sort == 'name': - query += ' ORDER BY sort COLLATE icucollate' - else: - query += ' ORDER BY avg_rating DESC, sort ASC' - data = self.conn.get(query) + rating = book[rating_dex] + # We kept track of all possible category field_map positions above + for (cat, dex, mult) in md: + if book[dex] is None: + continue + if not mult: + val = book[dex] + try: + (item_id, sort_val) = tids[cat][val] # let exceptions fly + item = tcategories[cat].get(val, None) + if not item: + item = LibraryDatabase2.TCat_Tag(val, sort_val) + tcategories[cat][val] = item + item.c += 1 + item.id = item_id + if rating > 0: + item.rt += rating + item.rc += 1 + except: + prints('get_categories: item', val, 'is not in', cat, 'list!') + else: + vals = book[dex].split(mult) + for val in vals: + try: + (item_id, sort_val) = tids[cat][val] # let exceptions fly + item = tcategories[cat].get(val, None) + if not item: + item = LibraryDatabase2.TCat_Tag(val, sort_val) + tcategories[cat][val] = item + item.c += 1 + item.id = item_id + if rating > 0: + item.rt += rating + item.rc += 1 + except: + prints('get_categories: item', val, 'is not in', cat, 'list!') + + print 'end phase "books":', time.time() - start, 'seconds' + + # Now do news + tcategories['news'] = {} + cat = tb_cats['news'] + tn = cat['table'] + cn = cat['column'] + if ids is None: + query = '''SELECT id, {0}, count, avg_rating, sort + FROM tag_browser_{1}'''.format(cn, tn) + else: + query = '''SELECT id, {0}, count, avg_rating, sort + FROM tag_browser_filtered_{1}'''.format(cn, tn) + # results will be sorted later + data = self.conn.get(query) + for r in data: + item = LibraryDatabase2.TCat_Tag(r[1], r[1]) + item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) + tcategories['news'][r[1]] = item + + print 'end phase "news":', time.time() - start, 'seconds' + + # Build the real category list by iterating over the temporary copy + # and building the Tag instances. + categories = {} + for category in tb_cats.keys(): + if category not in tcategories: + continue + cat = tb_cats[category] + + # prepare the place where we will put the array of Tags + categories[category] = [] # icon_map is not None if get_categories is to store an icon and # possibly a tooltip in the tag structure. - icon, tooltip = None, '' + icon = None + tooltip = '' label = tb_cats.key_to_label(category) if icon_map: if not tb_cats.is_custom_field(category): @@ -1087,23 +1214,40 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip = self.custom_column_label_map[label]['name'] datatype = cat['datatype'] - avgr = itemgetter(3) - item_not_zero_func = lambda x: x[2] > 0 + avgr = lambda x: 0.0 if x.rc == 0 else x.rt/x.rc + # Duplicate the build of items below to avoid using a lambda func + # in the main Tag loop. Saves a few % if datatype == 'rating': - # eliminate the zero ratings line as well as count == 0 - item_not_zero_func = (lambda x: x[1] > 0 and x[2] > 0) formatter = (lambda x:u'\u2605'*int(x/2)) - avgr = itemgetter(1) + avgr = lambda x : x.n + # eliminate the zero ratings line as well as count == 0 + items = [v for v in tcategories[category].values() if v.c > 0 and v.n != 0] elif category == 'authors': # Clean up the authors strings to human-readable form formatter = (lambda x: x.replace('|', ',')) + items = [v for v in tcategories[category].values() if v.c > 0] else: formatter = (lambda x:unicode(x)) + items = [v for v in tcategories[category].values() if v.c > 0] - categories[category] = [Tag(formatter(r[1]), count=r[2], id=r[0], - avg=avgr(r), sort=r[4], icon=icon, + # sort the list + if sort == 'name': + kf = lambda x: sort_key(x.s) if isinstance(x.s, unicode) else x.s + reverse=False + elif sort == 'popularity': + kf = lambda x: x.c + reverse=True + else: + kf = avgr + reverse=True + items.sort(key=kf, reverse=reverse) + + categories[category] = [Tag(formatter(r.n), count=r.c, id=r.id, + avg=avgr(r), sort=r.s, icon=icon, tooltip=tooltip, category=category) - for r in data if item_not_zero_func(r)] + for r in items] + + print 'end phase "tags list":', time.time() - start, 'seconds' # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1189,8 +1333,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items + t = time.time() - start + print 'get_categories ran in:', t, 'seconds' + return categories + ############# End get_categories + def tags_older_than(self, tag, delta): tag = tag.lower().strip() now = nowf() @@ -1486,6 +1635,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # Note: we generally do not need to refresh_ids because library_view will # refresh everything. + def get_ratings_with_ids(self): + result = self.conn.get('SELECT id,rating FROM ratings') + if not result: + return [] + return result + def dirty_books_referencing(self, field, id, commit=True): # Get the list of books to dirty -- all books that reference the item table = self.field_metadata[field]['table'] From 8f2171033a814dc7a47daf7388fc95ed5d7f382e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 Dec 2010 18:05:54 +0000 Subject: [PATCH 2/5] Eliminate 2 superfluous calls to recount on startup -- 1 when not using a startup restriction and another immediately after initializing the model. --- src/calibre/gui2/tag_view.py | 1 + src/calibre/gui2/ui.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index d6c0156f13..f75061da12 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -101,6 +101,7 @@ class TagsView(QTreeView): # {{{ hidden_categories=self.hidden_categories, search_restriction=None, drag_drop_finished=self.drag_drop_finished) + self.pane_is_visible = True # because TagsModel.init did a recount self.sort_by = sort_by self.tag_match = tag_match self.db = db diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index cb25f75d4a..7279b7f8df 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -234,7 +234,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ ######################### Search Restriction ########################## SearchRestrictionMixin.__init__(self) - self.apply_named_search_restriction(db.prefs['gui_restriction']) + if db.prefs['gui_restriction']: + self.apply_named_search_restriction(db.prefs['gui_restriction']) ########################### Cover Flow ################################ From b7e9749610d47b3f762943e5d613bb09777971db Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 11:10:13 -0700 Subject: [PATCH 3/5] Use time.clock rather than time.time for timing --- src/calibre/library/database2.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 64d143ef3c..098cb04727 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1067,7 +1067,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_categories(self, sort='name', ids=None, icon_map=None): - start = last = time.time() + start = last = time.clock() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') if sort not in self.CATEGORY_SORTS: @@ -1119,8 +1119,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # This saves iterating through field_metadata for each book md.append((category, cat['rec_index'], cat['is_multiple'])) - print 'end phase "collection":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "collection":', time.clock() - last, 'seconds' + last = time.clock() # Now scan every book looking for category items. # Code below is duplicated because it shaves off 10% of the loop time @@ -1167,8 +1167,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: prints('get_categories: item', val, 'is not in', cat, 'list!') - print 'end phase "books":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "books":', time.clock() - last, 'seconds' + last = time.clock() # Now do news tcategories['news'] = {} @@ -1188,8 +1188,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) tcategories['news'][r[1]] = item - print 'end phase "news":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "news":', time.clock() - last, 'seconds' + last = time.clock() # Build the real category list by iterating over the temporary copy # and building the Tag instances. @@ -1256,8 +1256,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip=tooltip, category=category) for r in items] - print 'end phase "tags list":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "tags list":', time.clock() - last, 'seconds' + last = time.clock() # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1343,8 +1343,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items - print 'last phase ran in:', time.time() - last, 'seconds' - print 'get_categories ran in:', time.time() - start, 'seconds' + print 'last phase ran in:', time.clock() - last, 'seconds' + print 'get_categories ran in:', time.clock() - start, 'seconds' return categories From 88264f6650b09b345870b64dcb3b1c8d4ea00dd0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 11:26:41 -0700 Subject: [PATCH 4/5] Remove timing code from get_categories --- src/calibre/library/database2.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 098cb04727..33e4295f05 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1067,7 +1067,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_categories(self, sort='name', ids=None, icon_map=None): - start = last = time.clock() + #start = last = time.clock() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') if sort not in self.CATEGORY_SORTS: @@ -1119,8 +1119,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # This saves iterating through field_metadata for each book md.append((category, cat['rec_index'], cat['is_multiple'])) - print 'end phase "collection":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "collection":', time.clock() - last, 'seconds' + #last = time.clock() # Now scan every book looking for category items. # Code below is duplicated because it shaves off 10% of the loop time @@ -1167,8 +1167,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: prints('get_categories: item', val, 'is not in', cat, 'list!') - print 'end phase "books":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "books":', time.clock() - last, 'seconds' + #last = time.clock() # Now do news tcategories['news'] = {} @@ -1188,8 +1188,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) tcategories['news'][r[1]] = item - print 'end phase "news":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "news":', time.clock() - last, 'seconds' + #last = time.clock() # Build the real category list by iterating over the temporary copy # and building the Tag instances. @@ -1256,8 +1256,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip=tooltip, category=category) for r in items] - print 'end phase "tags list":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "tags list":', time.clock() - last, 'seconds' + #last = time.clock() # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1343,8 +1343,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items - print 'last phase ran in:', time.clock() - last, 'seconds' - print 'get_categories ran in:', time.clock() - start, 'seconds' + #print 'last phase ran in:', time.clock() - last, 'seconds' + #print 'get_categories ran in:', time.clock() - start, 'seconds' return categories From bd4fbc30603f5955b692b179eaf0237b589236da Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 12:45:03 -0700 Subject: [PATCH 5/5] Fix #7826 (Book Details window sizing on netbooks screen version) --- src/calibre/gui2/book_details.py | 7 ++++++- src/calibre/gui2/dialogs/book_info.py | 11 ++++++----- src/calibre/gui2/dialogs/book_info.ui | 2 +- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 5214f1a1d5..b7394b9dd1 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -253,7 +253,12 @@ class BookInfo(QWebView): % (left_pane, right_pane))) def mouseDoubleClickEvent(self, ev): - ev.ignore() + if self.width() - ev.x() < 25 or \ + self.height() - ev.y() < 25: + # Filter out double clicks on the scroll bar + ev.accept() + else: + ev.ignore() # }}} diff --git a/src/calibre/gui2/dialogs/book_info.py b/src/calibre/gui2/dialogs/book_info.py index df21314712..016f132c57 100644 --- a/src/calibre/gui2/dialogs/book_info.py +++ b/src/calibre/gui2/dialogs/book_info.py @@ -23,10 +23,6 @@ class BookInfo(QDialog, Ui_BookInfo): self.comments.sizeHint = self.comments_size_hint self.view_func = view_func - desktop = QCoreApplication.instance().desktop() - screen_height = desktop.availableGeometry().height() - 100 - self.resize(self.size().width(), screen_height) - self.view = view self.current_row = None @@ -40,8 +36,13 @@ class BookInfo(QDialog, Ui_BookInfo): self.fit_cover.stateChanged.connect(self.toggle_cover_fit) self.cover.resizeEvent = self.cover_view_resized + desktop = QCoreApplication.instance().desktop() + screen_height = desktop.availableGeometry().height() - 100 + self.resize(self.size().width(), screen_height) + + def comments_size_hint(self): - return QSize(350, 350) + return QSize(350, 250) def toggle_cover_fit(self, state): dynamic.set('book_info_dialog_fit_cover', self.fit_cover.isChecked()) diff --git a/src/calibre/gui2/dialogs/book_info.ui b/src/calibre/gui2/dialogs/book_info.ui index 7eb6ccd3d3..2902a2c917 100644 --- a/src/calibre/gui2/dialogs/book_info.ui +++ b/src/calibre/gui2/dialogs/book_info.ui @@ -7,7 +7,7 @@ 0 0 917 - 783 + 480