From d2ff37b5a179ecf10041d8e51a810cb067820ca9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 Dec 2010 14:11:10 +0000 Subject: [PATCH 1/9] Improved get_categories -- approximately 6 times faster --- src/calibre/library/database2.py | 219 ++++++++++++++++++++++++++----- 1 file changed, 187 insertions(+), 32 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 1229b60577..0d301ccaff 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -14,7 +14,7 @@ from operator import itemgetter from PyQt4.QtGui import QImage - +from calibre import prints from calibre.ebooks.metadata import title_sort, author_to_author_sort from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.library.database import LibraryDatabase @@ -1039,43 +1039,170 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tn=field['table'], col=field['link_column']), (id_,)) return set(x[0] for x in ans) +########## data structures for get_categories + CATEGORY_SORTS = ('name', 'popularity', 'rating') - def get_categories(self, sort='name', ids=None, icon_map=None): - self.books_list_filter.change([] if not ids else ids) + class TCat_Tag(object): - categories = {} + def __init__(self, name, sort): + self.n = name + self.s = sort + self.c = 0 + self.rt = 0 + self.rc = 0 + self.id = None + + def set_all(self, c, rt, rc, id): + self.c = c + self.rt = rt + self.rc = rc + self.id = id + + def __str__(self): + return unicode(self) + + def __unicode__(self): + return 'n=%s s=%s c=%d rt=%d rc=%d id=%s'%\ + (self.n, self.s, self.c, self.rt, self.rc, self.id) + + + def get_categories(self, sort='name', ids=None, icon_map=None): + start = time.time() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') + if sort not in self.CATEGORY_SORTS: + raise ValueError('sort ' + sort + ' not a valid value') + + self.books_list_filter.change([] if not ids else ids) + id_filter = None if not ids else frozenset(ids) tb_cats = self.field_metadata - #### First, build the standard and custom-column categories #### + tcategories = {} + tids = {} + md = [] + + # First, build the maps. We need a category->items map and an + # item -> (item_id, sort_val) map to use in the books loop for category in tb_cats.keys(): cat = tb_cats[category] - if not cat['is_category'] or cat['kind'] in ['user', 'search']: + if not cat['is_category'] or cat['kind'] in ['user', 'search'] \ + or category in ['news', 'formats']: continue - tn = cat['table'] - categories[category] = [] #reserve the position in the ordered list - if tn is None: # Nothing to do for the moment + # Get the ids for the item values + if not cat['is_custom']: + funcs = { + 'authors' : self.get_authors_with_ids, + 'series' : self.get_series_with_ids, + 'publisher': self.get_publishers_with_ids, + 'tags' : self.get_tags_with_ids, + 'rating' : self.get_ratings_with_ids, + } + func = funcs.get(category, None) + if func: + list = func() + else: + raise ValueError(category + ' has no get with ids function') + else: + list = self.get_custom_items_with_ids(label=cat['label']) + tids[category] = {} + if category == 'authors': + for l in list: + (id, val, sort_val) = (l[0], l[1], l[2]) + tids[category][val] = (id, sort_val) + else: + for l in list: + (id, val) = (l[0], l[1]) + tids[category][val] = (id, val) + # add an empty category to the category map + tcategories[category] = {} + # create a list of category/field_index for the books scan to use. + # This saves iterating through field_metadata for each book + md.append((category, cat['rec_index'], cat['is_multiple'])) + + print 'end phase "collection":', time.time() - start, 'seconds' + + # Now scan every book looking for category items. + # Code below is duplicated because it shaves off 10% of the loop time + id_dex = self.FIELD_MAP['id'] + rating_dex = self.FIELD_MAP['rating'] + for book in self.data.iterall(): + if id_filter and book[id_dex] not in id_filter: continue - cn = cat['column'] - if ids is None: - query = '''SELECT id, {0}, count, avg_rating, sort - FROM tag_browser_{1}'''.format(cn, tn) - else: - query = '''SELECT id, {0}, count, avg_rating, sort - FROM tag_browser_filtered_{1}'''.format(cn, tn) - if sort == 'popularity': - query += ' ORDER BY count DESC, sort ASC' - elif sort == 'name': - query += ' ORDER BY sort COLLATE icucollate' - else: - query += ' ORDER BY avg_rating DESC, sort ASC' - data = self.conn.get(query) + rating = book[rating_dex] + # We kept track of all possible category field_map positions above + for (cat, dex, mult) in md: + if book[dex] is None: + continue + if not mult: + val = book[dex] + try: + (item_id, sort_val) = tids[cat][val] # let exceptions fly + item = tcategories[cat].get(val, None) + if not item: + item = LibraryDatabase2.TCat_Tag(val, sort_val) + tcategories[cat][val] = item + item.c += 1 + item.id = item_id + if rating > 0: + item.rt += rating + item.rc += 1 + except: + prints('get_categories: item', val, 'is not in', cat, 'list!') + else: + vals = book[dex].split(mult) + for val in vals: + try: + (item_id, sort_val) = tids[cat][val] # let exceptions fly + item = tcategories[cat].get(val, None) + if not item: + item = LibraryDatabase2.TCat_Tag(val, sort_val) + tcategories[cat][val] = item + item.c += 1 + item.id = item_id + if rating > 0: + item.rt += rating + item.rc += 1 + except: + prints('get_categories: item', val, 'is not in', cat, 'list!') + + print 'end phase "books":', time.time() - start, 'seconds' + + # Now do news + tcategories['news'] = {} + cat = tb_cats['news'] + tn = cat['table'] + cn = cat['column'] + if ids is None: + query = '''SELECT id, {0}, count, avg_rating, sort + FROM tag_browser_{1}'''.format(cn, tn) + else: + query = '''SELECT id, {0}, count, avg_rating, sort + FROM tag_browser_filtered_{1}'''.format(cn, tn) + # results will be sorted later + data = self.conn.get(query) + for r in data: + item = LibraryDatabase2.TCat_Tag(r[1], r[1]) + item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) + tcategories['news'][r[1]] = item + + print 'end phase "news":', time.time() - start, 'seconds' + + # Build the real category list by iterating over the temporary copy + # and building the Tag instances. + categories = {} + for category in tb_cats.keys(): + if category not in tcategories: + continue + cat = tb_cats[category] + + # prepare the place where we will put the array of Tags + categories[category] = [] # icon_map is not None if get_categories is to store an icon and # possibly a tooltip in the tag structure. - icon, tooltip = None, '' + icon = None + tooltip = '' label = tb_cats.key_to_label(category) if icon_map: if not tb_cats.is_custom_field(category): @@ -1087,23 +1214,40 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip = self.custom_column_label_map[label]['name'] datatype = cat['datatype'] - avgr = itemgetter(3) - item_not_zero_func = lambda x: x[2] > 0 + avgr = lambda x: 0.0 if x.rc == 0 else x.rt/x.rc + # Duplicate the build of items below to avoid using a lambda func + # in the main Tag loop. Saves a few % if datatype == 'rating': - # eliminate the zero ratings line as well as count == 0 - item_not_zero_func = (lambda x: x[1] > 0 and x[2] > 0) formatter = (lambda x:u'\u2605'*int(x/2)) - avgr = itemgetter(1) + avgr = lambda x : x.n + # eliminate the zero ratings line as well as count == 0 + items = [v for v in tcategories[category].values() if v.c > 0 and v.n != 0] elif category == 'authors': # Clean up the authors strings to human-readable form formatter = (lambda x: x.replace('|', ',')) + items = [v for v in tcategories[category].values() if v.c > 0] else: formatter = (lambda x:unicode(x)) + items = [v for v in tcategories[category].values() if v.c > 0] - categories[category] = [Tag(formatter(r[1]), count=r[2], id=r[0], - avg=avgr(r), sort=r[4], icon=icon, + # sort the list + if sort == 'name': + kf = lambda x: sort_key(x.s) if isinstance(x.s, unicode) else x.s + reverse=False + elif sort == 'popularity': + kf = lambda x: x.c + reverse=True + else: + kf = avgr + reverse=True + items.sort(key=kf, reverse=reverse) + + categories[category] = [Tag(formatter(r.n), count=r.c, id=r.id, + avg=avgr(r), sort=r.s, icon=icon, tooltip=tooltip, category=category) - for r in data if item_not_zero_func(r)] + for r in items] + + print 'end phase "tags list":', time.time() - start, 'seconds' # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1189,8 +1333,13 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items + t = time.time() - start + print 'get_categories ran in:', t, 'seconds' + return categories + ############# End get_categories + def tags_older_than(self, tag, delta): tag = tag.lower().strip() now = nowf() @@ -1486,6 +1635,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # Note: we generally do not need to refresh_ids because library_view will # refresh everything. + def get_ratings_with_ids(self): + result = self.conn.get('SELECT id,rating FROM ratings') + if not result: + return [] + return result + def dirty_books_referencing(self, field, id, commit=True): # Get the list of books to dirty -- all books that reference the item table = self.field_metadata[field]['table'] From 2c652bdee745c36b504c761587b977b2d723624f Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:00:45 -0700 Subject: [PATCH 2/9] Science based medicine by BuzzKill --- .../recipes/science_based_medicine.recipe | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 resources/recipes/science_based_medicine.recipe diff --git a/resources/recipes/science_based_medicine.recipe b/resources/recipes/science_based_medicine.recipe new file mode 100644 index 0000000000..7aa28cb170 --- /dev/null +++ b/resources/recipes/science_based_medicine.recipe @@ -0,0 +1,42 @@ +#!/usr/bin/env python + +import re +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import Tag + +class SBM(BasicNewsRecipe): + title = 'Science Based Medicine' + __author__ = 'BuzzKill' + description = 'Exploring issues and controversies in the relationship between science and medicine' + oldest_article = 5 + max_articles_per_feed = 15 + no_stylesheets = True + use_embedded_content = False + encoding = 'utf-8' + publisher = 'SBM' + category = 'science, sbm, ebm, blog, pseudoscience' + language = 'en' + + lang = 'en-US' + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : lang + , 'pretty_print' : True + } + + keep_only_tags = [ + dict(name='a', attrs={'title':re.compile(r'Posts by.*', re.DOTALL|re.IGNORECASE)}), + dict(name='div', attrs={'class':'entry'}) + ] + + feeds = [(u'Science Based Medicine', u'http://www.sciencebasedmedicine.org/?feed=rss2')] + + def preprocess_html(self, soup): + mtag = Tag(soup,'meta',[('http-equiv','Content-Type'),('context','text/html; charset=utf-8')]) + soup.head.insert(0,mtag) + soup.html['lang'] = self.lang + return self.adeify_images(soup) + From 032890d06f69a92040884059ea5093c901414406 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:18:03 -0700 Subject: [PATCH 3/9] Fix Zeit Online --- resources/recipes/zeitde.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/zeitde.recipe b/resources/recipes/zeitde.recipe index 64345ea675..c757c0d5bd 100644 --- a/resources/recipes/zeitde.recipe +++ b/resources/recipes/zeitde.recipe @@ -60,7 +60,7 @@ class ZeitDe(BasicNewsRecipe): for tag in soup.findAll(name=['ul','li']): tag.name = 'div' - soup.html['xml:lang'] = self.lang + soup.html['xml:lang'] = self.language.replace('_', '-') soup.html['lang'] = self.lang mtag = '' soup.head.insert(0,mtag) From 240b371ad5becc0d53bca8c9e3fae33af93a41a1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:32:16 -0700 Subject: [PATCH 4/9] ... --- resources/recipes/zeitde.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/zeitde.recipe b/resources/recipes/zeitde.recipe index c757c0d5bd..389bdec670 100644 --- a/resources/recipes/zeitde.recipe +++ b/resources/recipes/zeitde.recipe @@ -61,7 +61,7 @@ class ZeitDe(BasicNewsRecipe): tag.name = 'div' soup.html['xml:lang'] = self.language.replace('_', '-') - soup.html['lang'] = self.lang + soup.html['lang'] = self.language.replace('_', '-') mtag = '' soup.head.insert(0,mtag) return soup From 65021a0f0e15efb85befb93a287220259a161eea Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:35:48 -0700 Subject: [PATCH 5/9] Allow device drivers to show feedback to users if their open() methods fail --- src/calibre/devices/errors.py | 5 +++++ src/calibre/devices/interface.py | 3 +++ src/calibre/gui2/device.py | 5 ++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/errors.py b/src/calibre/devices/errors.py index 7464d6635e..3d88eb741f 100644 --- a/src/calibre/devices/errors.py +++ b/src/calibre/devices/errors.py @@ -36,6 +36,11 @@ class UserFeedback(DeviceError): self.details = details self.msg = msg +class OpenFeedback(DeviceError): + def __init__(self, msg): + self.feedback_msg = msg + DeviceError.__init__(self, msg) + class DeviceBusy(ProtocolError): """ Raised when device is busy """ def __init__(self, uerr=""): diff --git a/src/calibre/devices/interface.py b/src/calibre/devices/interface.py index 48d751fc29..2a92f46e8d 100644 --- a/src/calibre/devices/interface.py +++ b/src/calibre/devices/interface.py @@ -216,6 +216,9 @@ class DevicePlugin(Plugin): an implementation of this function that should serve as a good example for USB Mass storage devices. + + This method can raise an OpenFeedback exception to display a message to + the user. ''' raise NotImplementedError() diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 008649f534..9d66f8fc0d 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -12,7 +12,7 @@ from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, \ from calibre.customize.ui import available_input_formats, available_output_formats, \ device_plugins from calibre.devices.interface import DevicePlugin -from calibre.devices.errors import UserFeedback +from calibre.devices.errors import UserFeedback, OpenFeedback from calibre.gui2.dialogs.choose_format import ChooseFormatDialog from calibre.utils.ipc.job import BaseJob from calibre.devices.scanner import DeviceScanner @@ -163,6 +163,9 @@ class DeviceManager(Thread): # {{{ dev.reset(detected_device=detected_device, report_progress=self.report_progress) dev.open() + except OpenFeedback, e: + self.open_feedback_slot(e.feedback_msg) + continue except: tb = traceback.format_exc() if DEBUG or tb not in self.reported_errors: From 0bd30a28d26e54c0b48e5ba3c311509744bbbd56 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 09:43:33 -0700 Subject: [PATCH 6/9] Show open feedback message in a separate dialog --- src/calibre/gui2/device.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 9d66f8fc0d..07bfeccc4f 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -122,7 +122,8 @@ def device_name_for_plugboards(device_class): class DeviceManager(Thread): # {{{ - def __init__(self, connected_slot, job_manager, open_feedback_slot, sleep_time=2): + def __init__(self, connected_slot, job_manager, open_feedback_slot, + open_feedback_msg, sleep_time=2): ''' :sleep_time: Time to sleep between device probes in secs ''' @@ -143,6 +144,7 @@ class DeviceManager(Thread): # {{{ self.ejected_devices = set([]) self.mount_connection_requests = Queue.Queue(0) self.open_feedback_slot = open_feedback_slot + self.open_feedback_msg = open_feedback_msg def report_progress(self, *args): pass @@ -164,7 +166,7 @@ class DeviceManager(Thread): # {{{ report_progress=self.report_progress) dev.open() except OpenFeedback, e: - self.open_feedback_slot(e.feedback_msg) + self.open_feedback_msg(dev.get_gui_name(), e.feedback_msg) continue except: tb = traceback.format_exc() @@ -597,11 +599,16 @@ class DeviceMixin(object): # {{{ _('Error communicating with device'), ' ') self.device_error_dialog.setModal(Qt.NonModal) self.device_manager = DeviceManager(Dispatcher(self.device_detected), - self.job_manager, Dispatcher(self.status_bar.show_message)) + self.job_manager, Dispatcher(self.status_bar.show_message), + Dispatcher(self.show_open_feedback)) self.device_manager.start() if tweaks['auto_connect_to_folder']: self.connect_to_folder_named(tweaks['auto_connect_to_folder']) + def show_open_feedback(self, devname, msg): + self.__of_dev_mem__ = d = info_dialog(self, devname, msg) + d.show() + def auto_convert_question(self, msg, autos): autos = u'\n'.join(map(unicode, map(force_unicode, autos))) return self.ask_a_yes_no_question( From 8f2171033a814dc7a47daf7388fc95ed5d7f382e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 14 Dec 2010 18:05:54 +0000 Subject: [PATCH 7/9] Eliminate 2 superfluous calls to recount on startup -- 1 when not using a startup restriction and another immediately after initializing the model. --- src/calibre/gui2/tag_view.py | 1 + src/calibre/gui2/ui.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index d6c0156f13..f75061da12 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -101,6 +101,7 @@ class TagsView(QTreeView): # {{{ hidden_categories=self.hidden_categories, search_restriction=None, drag_drop_finished=self.drag_drop_finished) + self.pane_is_visible = True # because TagsModel.init did a recount self.sort_by = sort_by self.tag_match = tag_match self.db = db diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index cb25f75d4a..7279b7f8df 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -234,7 +234,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ ######################### Search Restriction ########################## SearchRestrictionMixin.__init__(self) - self.apply_named_search_restriction(db.prefs['gui_restriction']) + if db.prefs['gui_restriction']: + self.apply_named_search_restriction(db.prefs['gui_restriction']) ########################### Cover Flow ################################ From b7e9749610d47b3f762943e5d613bb09777971db Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 11:10:13 -0700 Subject: [PATCH 8/9] Use time.clock rather than time.time for timing --- src/calibre/library/database2.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 64d143ef3c..098cb04727 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1067,7 +1067,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_categories(self, sort='name', ids=None, icon_map=None): - start = last = time.time() + start = last = time.clock() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') if sort not in self.CATEGORY_SORTS: @@ -1119,8 +1119,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # This saves iterating through field_metadata for each book md.append((category, cat['rec_index'], cat['is_multiple'])) - print 'end phase "collection":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "collection":', time.clock() - last, 'seconds' + last = time.clock() # Now scan every book looking for category items. # Code below is duplicated because it shaves off 10% of the loop time @@ -1167,8 +1167,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: prints('get_categories: item', val, 'is not in', cat, 'list!') - print 'end phase "books":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "books":', time.clock() - last, 'seconds' + last = time.clock() # Now do news tcategories['news'] = {} @@ -1188,8 +1188,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) tcategories['news'][r[1]] = item - print 'end phase "news":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "news":', time.clock() - last, 'seconds' + last = time.clock() # Build the real category list by iterating over the temporary copy # and building the Tag instances. @@ -1256,8 +1256,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip=tooltip, category=category) for r in items] - print 'end phase "tags list":', time.time() - last, 'seconds' - last = time.time() + print 'end phase "tags list":', time.clock() - last, 'seconds' + last = time.clock() # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1343,8 +1343,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items - print 'last phase ran in:', time.time() - last, 'seconds' - print 'get_categories ran in:', time.time() - start, 'seconds' + print 'last phase ran in:', time.clock() - last, 'seconds' + print 'get_categories ran in:', time.clock() - start, 'seconds' return categories From 88264f6650b09b345870b64dcb3b1c8d4ea00dd0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 14 Dec 2010 11:26:41 -0700 Subject: [PATCH 9/9] Remove timing code from get_categories --- src/calibre/library/database2.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 098cb04727..33e4295f05 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1067,7 +1067,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def get_categories(self, sort='name', ids=None, icon_map=None): - start = last = time.clock() + #start = last = time.clock() if icon_map is not None and type(icon_map) != TagsIcons: raise TypeError('icon_map passed to get_categories must be of type TagIcons') if sort not in self.CATEGORY_SORTS: @@ -1119,8 +1119,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): # This saves iterating through field_metadata for each book md.append((category, cat['rec_index'], cat['is_multiple'])) - print 'end phase "collection":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "collection":', time.clock() - last, 'seconds' + #last = time.clock() # Now scan every book looking for category items. # Code below is duplicated because it shaves off 10% of the loop time @@ -1167,8 +1167,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): except: prints('get_categories: item', val, 'is not in', cat, 'list!') - print 'end phase "books":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "books":', time.clock() - last, 'seconds' + #last = time.clock() # Now do news tcategories['news'] = {} @@ -1188,8 +1188,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.set_all(c=r[2], rt=r[2]*r[3], rc=r[2], id=r[0]) tcategories['news'][r[1]] = item - print 'end phase "news":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "news":', time.clock() - last, 'seconds' + #last = time.clock() # Build the real category list by iterating over the temporary copy # and building the Tag instances. @@ -1256,8 +1256,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): tooltip=tooltip, category=category) for r in items] - print 'end phase "tags list":', time.clock() - last, 'seconds' - last = time.clock() + #print 'end phase "tags list":', time.clock() - last, 'seconds' + #last = time.clock() # Needed for legacy databases that have multiple ratings that # map to n stars @@ -1343,8 +1343,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): icon_map['search'] = icon_map['search'] categories['search'] = items - print 'last phase ran in:', time.clock() - last, 'seconds' - print 'get_categories ran in:', time.clock() - start, 'seconds' + #print 'last phase ran in:', time.clock() - last, 'seconds' + #print 'get_categories ran in:', time.clock() - start, 'seconds' return categories