From 2390a90e7b4d31c08b347287da27946cdd6c3fe0 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 20 Apr 2011 21:09:01 -0400 Subject: [PATCH] Store: Get DRM status for all stores. Load extra details in a separate thread. delay loading covers where the cover url is gotten via extra details. --- src/calibre/gui2/store/__init__.py | 2 +- .../gui2/store/baen_webscription_plugin.py | 2 +- src/calibre/gui2/store/bewrite_plugin.py | 23 ++++++----- src/calibre/gui2/store/bn_plugin.py | 1 + .../gui2/store/diesel_ebooks_plugin.py | 19 +++++---- src/calibre/gui2/store/ebooks_com_plugin.py | 40 +++++++++++-------- src/calibre/gui2/store/eharlequin_plugin.py | 29 ++++++++++---- src/calibre/gui2/store/feedbooks_plugin.py | 11 +++++ src/calibre/gui2/store/gutenberg_plugin.py | 2 +- src/calibre/gui2/store/kobo_plugin.py | 4 +- src/calibre/gui2/store/manybooks_plugin.py | 2 +- src/calibre/gui2/store/mobileread_plugin.py | 2 +- src/calibre/gui2/store/open_library_plugin.py | 1 + src/calibre/gui2/store/search.py | 20 ++++++++-- src/calibre/gui2/store/smashwords_plugin.py | 1 + 15 files changed, 110 insertions(+), 49 deletions(-) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index d0ec0cc479..2fc752ed55 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -104,7 +104,7 @@ class StorePlugin(object): # {{{ raise NotImplementedError() def get_details(self, search_result, timeout=60): - raise NotImplementedError() + pass def get_settings(self): ''' diff --git a/src/calibre/gui2/store/baen_webscription_plugin.py b/src/calibre/gui2/store/baen_webscription_plugin.py index 79b8751796..34ccc8f917 100644 --- a/src/calibre/gui2/store/baen_webscription_plugin.py +++ b/src/calibre/gui2/store/baen_webscription_plugin.py @@ -85,6 +85,6 @@ class BaenWebScriptionStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = id.strip() - s.drm = False + s.drm = SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/bewrite_plugin.py b/src/calibre/gui2/store/bewrite_plugin.py index 887390a3af..1f1cc59224 100644 --- a/src/calibre/gui2/store/bewrite_plugin.py +++ b/src/calibre/gui2/store/bewrite_plugin.py @@ -60,14 +60,6 @@ class BeWriteStore(BasicStoreConfig, StorePlugin): cover_url = '' price = '' - with closing(br.open(id.strip(), timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()')) - price = '$' + price.split('$')[-1] - cover_img = idata.xpath('//div[@id="content"]//img[1]/@src') - if cover_img: - cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0] - counter -= 1 s = SearchResult() @@ -76,6 +68,19 @@ class BeWriteStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = id.strip() - s.drm = False + s.drm = SearchResult.DRM_UNLOCKED yield s + + def get_details(self, search_result, timeout): + br = browser() + + with closing(br.open(search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()')) + price = '$' + price.split('$')[-1] + search_result.price = price.strip() + cover_img = idata.xpath('//div[@id="content"]//img[1]/@src') + if cover_img: + cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0] + search_result.cover_url = cover_url.strip() diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index 4da551fd92..aa74eebf54 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -78,5 +78,6 @@ class BNStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = id.strip() + s.drm = SearchResult.DRM_UNKNOWN yield s diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index e44896e1f8..93edfce272 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -74,13 +74,6 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): price_elem = data.xpath('//td[@class="price"]/text()') if price_elem: price = price_elem[0] - - with closing(br.open('http://www.diesel-ebooks.com/item/' + id.strip(), timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - if idata.xpath('boolean(//table[@class="format-info"]//tr[contains(th, "DRM") and contains(td, "No")])'): - drm = False - else: - drm = True counter -= 1 @@ -90,6 +83,16 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/item/' + id.strip() - s.drm = drm yield s + + def get_details(self, search_result, timeout): + url = 'http://www.diesel-ebooks.com/item/' + + br = browser() + with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//table[@class="format-info"]//tr[contains(th, "DRM") and contains(td, "No")])'): + search_result.drm = SearchResult.DRM_UNLOCKED + else: + search_result.drm = SearchResult.DRM_LOCKED diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index b1fc7e9c11..1405b2f33e 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -7,6 +7,7 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import random +import re import urllib2 from contextlib import closing @@ -63,20 +64,6 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): id = id.split('=')[-1] if not id: continue - - price = '' - with closing(br.open('http://www.ebooks.com/ebooks/book_display.asp?IID=' + id.strip(), timeout=timeout)) as fp: - pdoc = html.fromstring(fp.read()) - pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()') - if len(pdata) >= 2: - price = pdata[1] - drm = False - for sec in ('Printing', 'Copying', 'Lending'): - if pdoc.xpath('boolean(//div[@class="formatTableInner"]//table//tr[contains(th, "%s") and contains(td, "Off")])' % sec): - drm = True - break - if not price: - continue cover_url = ''.join(data.xpath('.//img[1]/@src')) @@ -94,8 +81,29 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): s.cover_url = cover_url s.title = title.strip() s.author = author.strip() - s.price = price.strip() s.detail_item = '?url=http://www.ebooks.com/cj.asp?IID=' + id.strip() + '&cjsku=' + id.strip() - s.drm = drm yield s + + def get_details(self, search_result, timeout): + url = 'http://www.ebooks.com/ebooks/book_display.asp?IID=' + + mo = re.search(r'\?IID=(?P\d+)', search_result.detail_item) + if mo: + id = mo.group('id') + if not id: + return + + price = _('Not Available') + br = browser() + with closing(br.open(url + id, timeout=timeout)) as nf: + pdoc = html.fromstring(nf.read()) + pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()') + if len(pdata) >= 2: + price = pdata[1] + search_result.drm = SearchResult.DRM_UNLOCKED + for sec in ('Printing', 'Copying', 'Lending'): + if pdoc.xpath('boolean(//div[@class="formatTableInner"]//table//tr[contains(th, "%s") and contains(td, "Off")])' % sec): + search_result.drm = SearchResult.DRM_LOCKED + break + search_result.price = price.strip() diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py index 4624732579..e77e6f11ce 100644 --- a/src/calibre/gui2/store/eharlequin_plugin.py +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -7,6 +7,7 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import random +import re import urllib2 from contextlib import closing @@ -68,12 +69,6 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): price = ''.join(data.xpath('.//div[@class="ourprice"]/font/text()')) cover_url = ''.join(data.xpath('.//a[@href="%s"]/img/@src' % id)) - with closing(br.open('http://ebooks.eharlequin.com/' + id.strip(), timeout=timeout/4)) as nf: - idata = html.fromstring(nf.read()) - drm = None - if idata.xpath('boolean(//div[@class="drm_head"])'): - drm = idata.xpath('boolean(//td[contains(., "Copy") and contains(., "not")])') - counter -= 1 s = SearchResult() @@ -82,6 +77,26 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '?url=http://ebooks.eharlequin.com/' + id.strip() - s.drm = drm yield s + + def get_details(self, search_result, timeout): + url = 'http://ebooks.eharlequin.com/en/ContentDetails.htm?ID=' + + mo = re.search(r'\?ID=(?P.+)', search_result.detail_item) + if mo: + id = mo.group('id') + if not id: + return + + + br = browser() + with closing(br.open(url + id, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + drm = SearchResult.DRM_UNKNOWN + if idata.xpath('boolean(//div[@class="drm_head"])'): + if idata.xpath('boolean(//td[contains(., "Copy") and contains(., "not")])'): + drm = SearchResult.DRM_LOCKED + else: + drm = SearchResult.DRM_UNLOCKED + search_result.drm = drm diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 12873f8bc9..67de97126e 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -90,3 +90,14 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin): s.detail_item = id.strip() yield s + + def get_details(self, search_result, timeout): + url = 'http://m.feedbooks.com/' + + br = browser() + with closing(br.open(url_slash_cleaner(url + search_result.detail_item), timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + if idata.xpath('boolean(//div[contains(@class, "m-description-long")]//p[contains(., "DRM") or contains(b, "Protection")])'): + search_result.drm = SearchResult.DRM_LOCKED + else: + search_result.drm = SearchResult.DRM_UNLOCKED diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 8166185ff5..0551c1b40b 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -79,6 +79,6 @@ class GutenbergStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/ebooks/' + id.strip() - s.drm = False + s.drm = SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/kobo_plugin.py b/src/calibre/gui2/store/kobo_plugin.py index d37e806c3f..e08094adf0 100644 --- a/src/calibre/gui2/store/kobo_plugin.py +++ b/src/calibre/gui2/store/kobo_plugin.py @@ -63,7 +63,7 @@ class KoboStore(BasicStoreConfig, StorePlugin): if not id: continue - price = ''.join(data.xpath('.//span[@class="SCOurPrice"]/strong/text()')) + price = ''.join(data.xpath('.//li[@class="OurPrice"]/strong/text()')) if not price: price = '$0.00' @@ -71,6 +71,7 @@ class KoboStore(BasicStoreConfig, StorePlugin): title = ''.join(data.xpath('.//div[@class="SCItemHeader"]/h1/a[1]/text()')) author = ''.join(data.xpath('.//div[@class="SCItemSummary"]/span/a[1]/text()')) + drm = data.xpath('boolean(.//span[@class="SCAvailibilityFormatsText" and contains(text(), "DRM")])') counter -= 1 @@ -80,5 +81,6 @@ class KoboStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() + s.drm = SearchResult.DRM_LOCKED if drm else SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 697a31eb85..aa5c45300a 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -89,6 +89,6 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/titles/' + id - s.drm = False + s.drm = SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 54f19cf814..3ac035d378 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -76,7 +76,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): matches = heapq.nlargest(max_results, matches) for score, book in matches: book.price = '$0.00' - book.drm = False + book.drm = SearchResult.DRM_UNLOCKED yield book def update_book_list(self, timeout=10): diff --git a/src/calibre/gui2/store/open_library_plugin.py b/src/calibre/gui2/store/open_library_plugin.py index 15b674f262..2bd38aae4f 100644 --- a/src/calibre/gui2/store/open_library_plugin.py +++ b/src/calibre/gui2/store/open_library_plugin.py @@ -68,5 +68,6 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = id.strip() + s.drm = SearchResult.DRM_UNKNOWN yield s diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 52b0761a7d..3c5d54afcf 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -415,9 +415,10 @@ class DetailsThread(Thread): result, store_plugin, callback, timeout = self.tasks.get() if result: store_plugin.get_details(result, timeout) - callback() + callback(result) self.tasks.task_done() except: + traceback.print_exc() continue class Matches(QAbstractItemModel): @@ -464,8 +465,12 @@ class Matches(QAbstractItemModel): self.layoutAboutToBeChanged.emit() self.all_matches.append(result) self.search_filter.add_search_result(result) - self.cover_pool.add_task(result, self.filter_results) - self.details_pool.add_task(result, store_plugin, self.filter_results) + if result.cover_url: + result.cover_queued = True + self.cover_pool.add_task(result, self.filter_results) + else: + result.cover_queued = False + self.details_pool.add_task(result, store_plugin, self.got_result_details) self.filter_results() self.layoutChanged.emit() @@ -485,6 +490,15 @@ class Matches(QAbstractItemModel): self.reorder_matches() self.layoutChanged.emit() + def got_result_details(self, result): + if not result.cover_queued and result.cover_url: + result.cover_queued = True + self.cover_pool.add_task(result, self.filter_results) + if result in self.matches: + row = self.matches.index(result) + self.dataChanged.emit(self.index(row, 0), self.index(row, self.columnCount() - 1)) + self.filter_results() + def set_query(self, query): self.query = query diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index 1806e9f4e1..43efe549cc 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -90,5 +90,6 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price.strip() s.detail_item = '/books/view/' + id.strip() + s.drm = SearchResult.DRM_UNLOCKED yield s