From 08b8ef818cafb806357bc279d614d17586ea1b19 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 20 Apr 2011 11:48:15 -0400 Subject: [PATCH] Store: SearchResult uses a different representation for DRM. Add Details download thread/pool. Change Amazon plugin to use details function instead of reading data in the search function. --- src/calibre/gui2/store/__init__.py | 13 ++- src/calibre/gui2/store/amazon_plugin.py | 27 +++--- src/calibre/gui2/store/search.py | 111 +++++++++++++++++------- src/calibre/gui2/store/search_result.py | 7 +- 4 files changed, 109 insertions(+), 49 deletions(-) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 26bafd2c95..d0ec0cc479 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -76,10 +76,16 @@ class StorePlugin(object): # {{{ return items as a generator. Don't be lazy with the search! Load as much data as possible in the - :class:`calibre.gui2.store.search_result.SearchResult` object. If you have to parse - multiple pages to get all of the data then do so. However, if data (such as cover_url) + :class:`calibre.gui2.store.search_result.SearchResult` object. + However, if data (such as cover_url) isn't available because the store does not display cover images then it's okay to ignore it. + + At the very least a :class:`calibre.gui2.store.search_result.SearchResult` + returned by this function must have the title, author and id. + + If you have to parse multiple pages to get all of the data then implement + :meth:`get_deatils` for retrieving additional information. Also, by default search results can only include ebooks. A plugin can offer users an option to include physical books in the search results but this must be @@ -96,6 +102,9 @@ class StorePlugin(object): # {{{ item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. ''' raise NotImplementedError() + + def get_details(self, search_result, timeout=60): + raise NotImplementedError() def get_settings(self): ''' diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 067e9e899e..2fc5f8ba1e 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -159,16 +159,6 @@ class AmazonKindleStore(StorePlugin): author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) author = author.split('by')[-1] price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) - - with closing(br.open(asin_href, timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'): - if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'): - drm = False - else: - drm = None - else: - drm = True counter -= 1 @@ -178,6 +168,21 @@ class AmazonKindleStore(StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = asin.strip() - s.drm = drm yield s + + def get_details(self, search_result, timeout): + url = 'http://amazon.com/dp/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//div[@class="content"]//li/b[contains(text(), "Simultaneous Device Usage")])'): + if idata.xpath('boolean(//div[@class="content"]//li[contains(., "Unlimited") and contains(b, "Simultaneous Device Usage")])'): + search_result.drm = SearchResult.DRM_UNLOCKED + else: + search_result.drm = SearchResult.DRM_UNKNOWN + else: + search_result.drm = SearchResult.DRM_LOCKED + + \ No newline at end of file diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 322680178b..52b0761a7d 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -21,6 +21,7 @@ from calibre import browser from calibre.gui2 import NONE from calibre.gui2.progress_indicator import ProgressIndicator from calibre.gui2.store.search_ui import Ui_Dialog +from calibre.gui2.store.search_result import SearchResult from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ REGEXP_MATCH from calibre.utils.config import DynamicConfig @@ -123,6 +124,8 @@ class SearchDialog(QDialog, Ui_Dialog): store_names = self.store_plugins.keys() if not store_names: return + # Remove all of our internal filtering logic from the query. + query = self.clean_query(query) shuffle(store_names) # Add plugins that the user has checked to the search pool's work queue. for n in store_names: @@ -134,6 +137,29 @@ class SearchDialog(QDialog, Ui_Dialog): self.search_pool.start_threads() self.pi.startAnimation() + def clean_query(self, query): + query = query.lower() + # Remove control modifiers. + query = query.replace('\\', '') + query = query.replace('!', '') + query = query.replace('=', '') + query = query.replace('~', '') + query = query.replace('>', '') + query = query.replace('<', '') + # Remove the prefix. + for loc in ( 'all', 'author', 'authors', 'title'): + query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) + # Remove the prefix and search text. + for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): + query = re.sub(r'%s:"[^"]"' % loc, '', query) + query = re.sub(r'%s:[^\s]*' % loc, '', query) + # Remove logic. + query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) + # Remove excess whitespace. + query = re.sub(r'\s{2,}', ' ', query) + query = query.strip() + return query + def save_state(self): self.config['store_search_geometry'] = self.saveGeometry() self.config['store_search_store_splitter_state'] = self.store_splitter.saveState() @@ -183,9 +209,9 @@ class SearchDialog(QDialog, Ui_Dialog): self.pi.stopAnimation() while self.search_pool.has_results(): - res = self.search_pool.get_result() + res, store_plugin = self.search_pool.get_result() if res: - self.results_view.model().add_result(res) + self.results_view.model().add_result(res, store_plugin) def open_store(self, index): result = self.results_view.model().get_result(index) @@ -307,37 +333,14 @@ class SearchThread(Thread): while self._run and not self.tasks.empty(): try: query, store_name, store_plugin, timeout = self.tasks.get() - query = self._clean_query(query) for res in store_plugin.search(query, timeout=timeout): if not self._run: return res.store_name = store_name - self.results.put(res) + self.results.put((res, store_plugin)) self.tasks.task_done() except: traceback.print_exc() - - def _clean_query(self, query): - query = query.lower() - # Remove control modifiers. - query = query.replace('\\', '') - query = query.replace('!', '') - query = query.replace('=', '') - query = query.replace('~', '') - query = query.replace('>', '') - query = query.replace('<', '') - # Remove the prefix. - for loc in ( 'all', 'author', 'authors', 'title'): - query = re.sub(r'%s:"?(?P[^\s"]+)"?' % loc, '\g', query) - # Remove the prefix and search text. - for loc in ('cover', 'drm', 'format', 'formats', 'price', 'store'): - query = re.sub(r'%s:"[^"]"' % loc, '', query) - query = re.sub(r'%s:[^\s]*' % loc, '', query) - # Remove logic. - query = re.sub(r'(^|\s)(and|not|or)(\s|$)', ' ', query) - # Remove excess whitespace. - query = re.sub(r'\s{2,}', ' ', query) - return query class CoverThreadPool(GenericDownloadThreadPool): @@ -381,6 +384,42 @@ class CoverThread(Thread): continue +class DetailsThreadPool(GenericDownloadThreadPool): + ''' + Once started all threads run until abort is called. + ''' + + def add_task(self, search_result, store_plugin, update_callback, timeout=10): + self.tasks.put((search_result, store_plugin, update_callback, timeout)) + + +class DetailsThread(Thread): + + def __init__(self, tasks, results): + Thread.__init__(self) + self.daemon = True + self.tasks = tasks + self.results = results + self._run = True + + def abort(self): + self._run = False + + def run(self): + while self._run: + try: + time.sleep(.1) + while not self.tasks.empty(): + if not self._run: + break + result, store_plugin, callback, timeout = self.tasks.get() + if result: + store_plugin.get_details(result, timeout) + callback() + self.tasks.task_done() + except: + continue + class Matches(QAbstractItemModel): HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('DRM'), _('Store')] @@ -402,9 +441,12 @@ class Matches(QAbstractItemModel): self.search_filter = SearchFilter() self.cover_pool = CoverThreadPool(CoverThread, 2) self.cover_pool.start_threads() + self.details_pool = DetailsThreadPool(DetailsThread, 4) + self.details_pool.start_threads() def closing(self): self.cover_pool.abort() + self.details_pool.abort() def clear_results(self): self.all_matches = [] @@ -414,13 +456,16 @@ class Matches(QAbstractItemModel): self.query = '' self.cover_pool.abort() self.cover_pool.start_threads() + self.details_pool.abort() + self.details_pool.start_threads() self.reset() - def add_result(self, result): + def add_result(self, result, store_plugin): self.layoutAboutToBeChanged.emit() self.all_matches.append(result) self.search_filter.add_search_result(result) self.cover_pool.add_task(result, self.filter_results) + self.details_pool.add_task(result, store_plugin, self.filter_results) self.filter_results() self.layoutChanged.emit() @@ -438,7 +483,7 @@ class Matches(QAbstractItemModel): else: self.matches = list(self.search_filter.universal_set()) self.reorder_matches() - self.layoutAboutToBeChanged.emit() + self.layoutChanged.emit() def set_query(self, query): self.query = query @@ -487,11 +532,11 @@ class Matches(QAbstractItemModel): p.loadFromData(result.cover_data) return QVariant(p) if col == 4: - if result.drm: + if result.drm == SearchResult.DRM_LOCKED: return QVariant(self.DRM_LOCKED_ICON) - if result.drm == False: + if result.drm == SearchResult.DRM_UNLOCKED: return QVariant(self.DRM_UNLOCKED_ICON) - else: + elif result.drm == SearchResult.DRM_UNKNOWN: return QVariant(self.DRM_UNKNOWN_ICON) elif role == Qt.SizeHintRole: return QSize(64, 64) @@ -596,7 +641,7 @@ class SearchFilter(SearchQueryParser): accessor = q[locvalue] if query == 'true': if locvalue == 'drm': - if accessor(sr) == True: + if accessor(sr) == SearchResult.DRM_LOCKED: matches.add(sr) else: if accessor(sr) is not None: @@ -604,7 +649,7 @@ class SearchFilter(SearchQueryParser): continue if query == 'false': if locvalue == 'drm': - if accessor(sr) == False: + if accessor(sr) == SearchResult.DRM_UNKNOWN: matches.add(sr) else: if accessor(sr) is None: diff --git a/src/calibre/gui2/store/search_result.py b/src/calibre/gui2/store/search_result.py index 25ff32d0ec..b817b68a77 100644 --- a/src/calibre/gui2/store/search_result.py +++ b/src/calibre/gui2/store/search_result.py @@ -8,6 +8,10 @@ __docformat__ = 'restructuredtext en' class SearchResult(object): + DRM_LOCKED = 1 + DRM_UNLOCKED = 2 + DRM_UNKNOWN = 3 + def __init__(self): self.store_name = '' self.cover_url = '' @@ -16,8 +20,5 @@ class SearchResult(object): self.author = '' self.price = '' self.detail_item = '' - # None = Unknown. - # True = Has DRM. - # False = Does not have DRM. self.drm = None self.formats = ''