From b04eed00121f98fee1316cc3bff761e4f993c438 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Fri, 27 May 2011 13:00:31 +0100 Subject: [PATCH 1/4] New store: EBookShoppeUKStore. small correction to FoylesUKStore. fix to threading problem in search causing range errors. Disable waterstones. --- src/calibre/customize/builtins.py | 13 ++- .../gui2/store/ebookshoppe_uk_plugin.py | 97 +++++++++++++++++++ src/calibre/gui2/store/foyles_uk_plugin.py | 9 +- src/calibre/gui2/store/search/models.py | 2 + 4 files changed, 118 insertions(+), 3 deletions(-) create mode 100644 src/calibre/gui2/store/ebookshoppe_uk_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 4a970b4661..150ad269bb 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1393,6 +1393,16 @@ class StoreWoblinkStore(StoreBase): headquarters = 'PL' formats = ['EPUB'] +class StoreEBookShoppeUKStore(StoreBase): + name = 'ebookShoppe UK' + author = u'Charles Haley' + description = u'We made this website in an attempt to offer the widest range of UK eBooks possible across and as many formats as we could manage.' + actual_plugin = 'calibre.gui2.store.ebookshoppe_uk_plugin:EBookShoppeUKStore' + + drm_free_only = False + headquarters = 'UK' + formats = ['EPUB', 'PDF'] + plugins += [ StoreArchiveOrgStore, StoreAmazonKindleStore, @@ -1404,6 +1414,7 @@ plugins += [ StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, + StoreEBookShoppeUKStore, StoreEPubBuyDEStore, StoreEHarlequinStore, StoreFeedbooksStore, @@ -1421,7 +1432,7 @@ plugins += [ StorePragmaticBookshelfStore, StoreSmashwordsStore, StoreVirtualoStore, - StoreWaterstonesUKStore, + # StoreWaterstonesUKStore, StoreWeightlessBooksStore, StoreWizardsTowerBooksStore, StoreWoblinkStore diff --git a/src/calibre/gui2/store/ebookshoppe_uk_plugin.py b/src/calibre/gui2/store/ebookshoppe_uk_plugin.py new file mode 100644 index 0000000000..de5304da86 --- /dev/null +++ b/src/calibre/gui2/store/ebookshoppe_uk_plugin.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class EBookShoppeUKStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + url_details = 'http://www.awin1.com/cread.php?awinmid=1414&awinaffid=120917&clickref=&p={0}' + url = 'http://www.awin1.com/awclick.php?mid=2666&id=120917' + + if external or self.config.get('open_external', False): + if detail_item: + url = url_details.format(detail_item) + open_url(QUrl(url)) + else: + detail_url = None + if detail_item: + detail_url = url_details.format(detail_item) + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + + # reduce max_results because the filter will match everything. See the + # setting of 'author' below for more details + + def search(self, query, max_results=5, timeout=60): + url = 'http://www.ebookshoppe.com/search.php?search_query=' + urllib2.quote(query) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//ul[@class="ProductList"]/li'): + if counter <= 0: + break + + id = ''.join(data.xpath('./div[@class="ProductDetails"]/' + 'strong/a/@href')).strip() + if not id: + continue + cover_url = ''.join(data.xpath('./div[@class="ProductImage"]/a/img/@src')) + title = ''.join(data.xpath('./div[@class="ProductDetails"]/strong/a/text()')) + price = ''.join(data.xpath('./div[@class="ProductPriceRating"]/em/text()')) + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + # Set the author to the query terms to ensure that author + # queries match something when pruning searches. Of course, this + # means that all books will match. Sigh... + s.author = query + s.price = price + s.drm = SearchResult.DRM_UNLOCKED + s.detail_item = id + s.formats = '' + + # Call this here instead of later. Reason: painting then + # removing matches looks very strange. There are also issues + # with threading. Yes, this makes things take longer, but we + # will do the work anyway. + self.my_get_details(s, timeout) + + yield s + + def my_get_details(self, search_result, timeout): + br = browser() + with closing(br.open(search_result.detail_item, timeout=timeout)) as nf: + idata = html.fromstring(nf.read()) + author = ''.join(idata.xpath('//div[@id="ProductOtherDetails"]/dl/dd[1]/text()')) + if author: + search_result.author = author + formats = idata.xpath('//dl[@class="ProductAddToCart"]/dd/' + 'ul[@class="ProductOptionList"]/li/label/text()') + if formats: + search_result.formats = ', '.join(formats) + search_result.drm = SearchResult.DRM_UNKNOWN + return True \ No newline at end of file diff --git a/src/calibre/gui2/store/foyles_uk_plugin.py b/src/calibre/gui2/store/foyles_uk_plugin.py index 1a997cd671..fd670d2d85 100644 --- a/src/calibre/gui2/store/foyles_uk_plugin.py +++ b/src/calibre/gui2/store/foyles_uk_plugin.py @@ -23,12 +23,13 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog class FoylesUKStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): - url = 'http://www.awin1.com/cread.php?awinmid=1414&awinaffid=120917&clickref=&p=' + url = 'http://www.awin1.com/awclick.php?mid=1414&id=120917' + detail_url = 'http://www.awin1.com/cread.php?awinmid=1414&awinaffid=120917&clickref=&p=' url_redirect = 'http://www.foyles.co.uk' if external or self.config.get('open_external', False): if detail_item: - url = url + url_redirect + detail_item + url = detail_url + url_redirect + detail_item open_url(QUrl(url_slash_cleaner(url))) else: detail_url = None @@ -54,6 +55,10 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin): if not id: continue + # filter out the audio books + if not data.xpath('boolean(.//div[@class="Relative"]/ul/li[contains(text(), "ePub")])'): + continue + cover_url = ''.join(data.xpath('.//a[@class="Jacket"]/img/@src')) if cover_url: cover_url = 'http://www.foyles.co.uk' + cover_url diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index d7941480cc..64724be6aa 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -150,6 +150,8 @@ class Matches(QAbstractItemModel): def data(self, index, role): row, col = index.row(), index.column() + if row >= len(self.matches): + return NONE result = self.matches[row] if role == Qt.DisplayRole: if col == 1: From 40b073c60cd7bea1063677f89c2feed310206d6a Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 28 May 2011 13:54:57 +0100 Subject: [PATCH 2/4] Add heart to search screen --- src/calibre/customize/builtins.py | 8 +++++++- src/calibre/gui2/store/search/models.py | 18 ++++++++++++++++-- src/calibre/gui2/store/search_result.py | 5 +++-- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index ad587baf39..ba88d766dc 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1111,6 +1111,7 @@ class StoreAmazonKindleStore(StoreBase): drm_free_only = False headquarters = 'US' formats = ['KINDLE'] + affiliate = True class StoreAmazonDEKindleStore(StoreBase): name = 'Amazon DE Kindle' @@ -1121,6 +1122,7 @@ class StoreAmazonDEKindleStore(StoreBase): drm_free_only = False headquarters = 'DE' formats = ['KINDLE'] + affiliate = True class StoreAmazonUKKindleStore(StoreBase): name = 'Amazon UK Kindle' @@ -1131,6 +1133,7 @@ class StoreAmazonUKKindleStore(StoreBase): drm_free_only = False headquarters = 'UK' formats = ['KINDLE'] + affiliate = True class StoreArchiveOrgStore(StoreBase): name = 'Archive.org' @@ -1168,6 +1171,7 @@ class StoreBeamEBooksDEStore(StoreBase): drm_free_only = True headquarters = 'DE' formats = ['EPUB', 'MOBI', 'PDF'] + affiliate = True class StoreBeWriteStore(StoreBase): name = 'BeWrite Books' @@ -1233,6 +1237,7 @@ class StoreFoylesUKStore(StoreBase): drm_free_only = False headquarters = 'UK' formats = ['EPUB', 'PDF'] + affiliate = True class StoreGandalfStore(StoreBase): name = 'Gandalf' @@ -1402,6 +1407,7 @@ class StoreEBookShoppeUKStore(StoreBase): drm_free_only = False headquarters = 'UK' formats = ['EPUB', 'PDF'] + affiliate = True plugins += [ StoreArchiveOrgStore, @@ -1432,7 +1438,7 @@ plugins += [ StorePragmaticBookshelfStore, StoreSmashwordsStore, StoreVirtualoStore, - #StoreWaterstonesUKStore, + StoreWaterstonesUKStore, StoreWeightlessBooksStore, StoreWizardsTowerBooksStore, StoreWoblinkStore diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index 64724be6aa..797195e202 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -10,7 +10,7 @@ import re from operator import attrgetter from PyQt4.Qt import (Qt, QAbstractItemModel, QVariant, QPixmap, QModelIndex, QSize, - pyqtSignal) + pyqtSignal, QIcon) from calibre.gui2 import NONE from calibre.gui2.store.search_result import SearchResult @@ -33,7 +33,7 @@ class Matches(QAbstractItemModel): total_changed = pyqtSignal(int) - HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store')] + HEADERS = [_('Cover'), _('Title'), _('Price'), _('DRM'), _('Store'), _('')] HTML_COLS = (1, 4) def __init__(self, cover_thread_count=2, detail_thread_count=4): @@ -76,6 +76,7 @@ class Matches(QAbstractItemModel): self.reset() def add_result(self, result, store_plugin): + result.plugin = store_plugin if result not in self.all_matches: self.layoutAboutToBeChanged.emit() self.all_matches.append(result) @@ -175,6 +176,12 @@ class Matches(QAbstractItemModel): return QVariant(self.DRM_UNLOCKED_ICON) elif result.drm == SearchResult.DRM_UNKNOWN: return QVariant(self.DRM_UNKNOWN_ICON) + if col == 5: + if getattr(result.plugin.base_plugin, 'affiliate', False): + icon = QIcon() + icon.addFile(I('donate.png'), QSize(16, 16)) + return QVariant(icon) + return NONE elif role == Qt.ToolTipRole: if col == 1: return QVariant('

%s

' % result.title) @@ -189,6 +196,8 @@ class Matches(QAbstractItemModel): return QVariant('

' + _('The DRM status of this book could not be determined. There is a very high likelihood that this book is actually DRM restricted.') + '

') elif col == 4: return QVariant('

%s

' % result.formats) + elif col == 5: + return QVariant(_('Buying from this store supports a calibre developer')) elif role == Qt.SizeHintRole: return QSize(64, 64) return NONE @@ -208,6 +217,11 @@ class Matches(QAbstractItemModel): text = 'c' elif col == 4: text = result.store_name + elif col == 5: + if getattr(result.plugin.base_plugin, 'affiliate', False): + text = 'y' + else: + text = 'n' return text def sort(self, col, order, reset=True): diff --git a/src/calibre/gui2/store/search_result.py b/src/calibre/gui2/store/search_result.py index 7bf361157e..a3c6a5601e 100644 --- a/src/calibre/gui2/store/search_result.py +++ b/src/calibre/gui2/store/search_result.py @@ -7,11 +7,11 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' class SearchResult(object): - + DRM_LOCKED = 1 DRM_UNLOCKED = 2 DRM_UNKNOWN = 3 - + def __init__(self): self.store_name = '' self.cover_url = '' @@ -22,6 +22,7 @@ class SearchResult(object): self.detail_item = '' self.drm = None self.formats = '' + self.plugin = None def __eq__(self, other): return self.title == other.title and self.author == other.author and self.store_name == other.store_name From ddb7afffa9d846b2798e6c7a644d8dc052ebb7eb Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 28 May 2011 13:59:16 +0100 Subject: [PATCH 3/4] Improve handling of affiliate flag. --- src/calibre/gui2/store/search/models.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index 797195e202..44a993ef12 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -76,7 +76,7 @@ class Matches(QAbstractItemModel): self.reset() def add_result(self, result, store_plugin): - result.plugin = store_plugin + result.affiliate = getattr(store_plugin.base_plugin, 'affiliate', False) if result not in self.all_matches: self.layoutAboutToBeChanged.emit() self.all_matches.append(result) @@ -177,7 +177,7 @@ class Matches(QAbstractItemModel): elif result.drm == SearchResult.DRM_UNKNOWN: return QVariant(self.DRM_UNKNOWN_ICON) if col == 5: - if getattr(result.plugin.base_plugin, 'affiliate', False): + if result.affiliate: icon = QIcon() icon.addFile(I('donate.png'), QSize(16, 16)) return QVariant(icon) @@ -218,7 +218,7 @@ class Matches(QAbstractItemModel): elif col == 4: text = result.store_name elif col == 5: - if getattr(result.plugin.base_plugin, 'affiliate', False): + if result.affiliate: text = 'y' else: text = 'n' From 07716ef80130943906d2e2bac4ba3d241d9bde8b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sat, 28 May 2011 14:56:53 +0100 Subject: [PATCH 4/4] Move setting affiliate in the results structure. Add the heart to the list of stores. Initialize 'affiliate' in the thread. --- src/calibre/customize/__init__.py | 4 +++- src/calibre/gui2/actions/store.py | 9 +++++++-- src/calibre/gui2/store/search/download_thread.py | 7 ++++--- src/calibre/gui2/store/search/models.py | 6 ++++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 3d265aed1c..d087eb5351 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -615,7 +615,7 @@ class StoreBase(Plugin): # {{{ version = (1, 0, 1) actual_plugin = None - + # Does the store only distribute ebooks without DRM. drm_free_only = False # This is the 2 letter country code for the corporate @@ -623,6 +623,8 @@ class StoreBase(Plugin): # {{{ headquarters = '' # All formats the store distributes ebooks in. formats = [] + # Is this store on an affiliate program? + affiliate = False def load_actual_plugin(self, gui): ''' diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 6d9720548e..7f9b538bcf 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -8,7 +8,7 @@ __docformat__ = 'restructuredtext en' from functools import partial -from PyQt4.Qt import QMenu +from PyQt4.Qt import QMenu, QIcon, QSize from calibre.gui2 import error_dialog from calibre.gui2.actions import InterfaceAction @@ -32,8 +32,13 @@ class StoreAction(InterfaceAction): self.store_menu.addAction(_('Search for this book'), self.search_author_title) self.store_menu.addSeparator() self.store_list_menu = self.store_menu.addMenu(_('Stores')) + icon = QIcon() + icon.addFile(I('donate.png'), QSize(16, 16)) for n, p in sorted(self.gui.istores.items(), key=lambda x: x[0].lower()): - self.store_list_menu.addAction(n, partial(self.open_store, p)) + if p.base_plugin.affiliate: + self.store_list_menu.addAction(icon, n, partial(self.open_store, p)) + else: + self.store_list_menu.addAction(n, partial(self.open_store, p)) self.store_menu.addSeparator() self.store_menu.addAction(_('Choose stores'), self.choose) self.qaction.setMenu(self.store_menu) diff --git a/src/calibre/gui2/store/search/download_thread.py b/src/calibre/gui2/store/search/download_thread.py index 1fc74a5748..67b4224981 100644 --- a/src/calibre/gui2/store/search/download_thread.py +++ b/src/calibre/gui2/store/search/download_thread.py @@ -38,7 +38,7 @@ class GenericDownloadThreadPool(object): This must be implemented in a sub class and this function must be called at the end of the add_task function in the sub class. - + The implementation of this function (in this base class) starts any threads necessary to fill the pool if it is not already full. @@ -91,7 +91,7 @@ class SearchThreadPool(GenericDownloadThreadPool): sp = SearchThreadPool(3) sp.add_task(...) ''' - + def __init__(self, thread_count): GenericDownloadThreadPool.__init__(self, SearchThread, thread_count) @@ -120,6 +120,7 @@ class SearchThread(Thread): if not self._run: return res.store_name = store_name + res.affiliate = store_plugin.base_plugin.affiliate self.results.put((res, store_plugin)) self.tasks.task_done() except: @@ -167,7 +168,7 @@ class CoverThread(Thread): class DetailsThreadPool(GenericDownloadThreadPool): - + def __init__(self, thread_count): GenericDownloadThreadPool.__init__(self, DetailsThread, thread_count) diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index 44a993ef12..f5c24798f6 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -76,7 +76,6 @@ class Matches(QAbstractItemModel): self.reset() def add_result(self, result, store_plugin): - result.affiliate = getattr(store_plugin.base_plugin, 'affiliate', False) if result not in self.all_matches: self.layoutAboutToBeChanged.emit() self.all_matches.append(result) @@ -178,6 +177,8 @@ class Matches(QAbstractItemModel): return QVariant(self.DRM_UNKNOWN_ICON) if col == 5: if result.affiliate: + # For some reason the size(16, 16) is forgotten if the icon + # is a class attribute. Don't know why... icon = QIcon() icon.addFile(I('donate.png'), QSize(16, 16)) return QVariant(icon) @@ -197,7 +198,8 @@ class Matches(QAbstractItemModel): elif col == 4: return QVariant('

%s

' % result.formats) elif col == 5: - return QVariant(_('Buying from this store supports a calibre developer')) + if result.affiliate: + return QVariant(_('Buying from this store supports a calibre developer')) elif role == Qt.SizeHintRole: return QSize(64, 64) return NONE