From 2e2042d1fbeb92cc36647ee23d6daf735e2f187b Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 20 Mar 2012 13:59:05 +0100 Subject: [PATCH 1/3] Maintenance on amazon Europe to account for UTF8 and site changes. Maintenance on Waterstones and Foyles to account for site changes. Waterstones said that they would not pay commissions on ebooks, but they are in fact paying them. Turn on the affiliate flag. --- src/calibre/customize/builtins.py | 1 + src/calibre/gui2/store/stores/amazon_de_plugin.py | 8 +++++--- src/calibre/gui2/store/stores/amazon_es_plugin.py | 8 +++++--- src/calibre/gui2/store/stores/amazon_fr_plugin.py | 6 +++--- src/calibre/gui2/store/stores/amazon_it_plugin.py | 8 +++++--- src/calibre/gui2/store/stores/amazon_uk_plugin.py | 7 ++++--- src/calibre/gui2/store/stores/foyles_uk_plugin.py | 4 ---- src/calibre/gui2/store/stores/waterstones_uk_plugin.py | 2 +- 8 files changed, 24 insertions(+), 20 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 55742b3ee3..13cc2a6a33 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1538,6 +1538,7 @@ class StoreWaterstonesUKStore(StoreBase): headquarters = 'UK' formats = ['EPUB', 'PDF'] + affiliate = True class StoreWeightlessBooksStore(StoreBase): name = 'Weightless Books' diff --git a/src/calibre/gui2/store/stores/amazon_de_plugin.py b/src/calibre/gui2/store/stores/amazon_de_plugin.py index ea92839268..c42c7392a1 100644 --- a/src/calibre/gui2/store/stores/amazon_de_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_de_plugin.py @@ -41,7 +41,9 @@ class AmazonDEKindleStore(StorePlugin): counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read().decode('latin-1', 'replace')) + # doc = html.fromstring(f.read().decode('latin-1', 'replace')) + # Apparently amazon Europe is responding in UTF-8 now + doc = html.fromstring(f.read()) data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' format_xpath = './/span[@class="format"]/text()' @@ -65,8 +67,8 @@ class AmazonDEKindleStore(StorePlugin): cover_url = ''.join(data.xpath(cover_xpath)) - title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) - price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + title = ''.join(data.xpath('.//a[@class="title"]/text()')) + price = ''.join(data.xpath('.//span[@class="price"]/text()')) author = ''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()')) if author.startswith('von '): diff --git a/src/calibre/gui2/store/stores/amazon_es_plugin.py b/src/calibre/gui2/store/stores/amazon_es_plugin.py index d89c051d87..97abab61ed 100644 --- a/src/calibre/gui2/store/stores/amazon_es_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_es_plugin.py @@ -37,7 +37,9 @@ class AmazonESKindleStore(StorePlugin): counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read().decode('latin-1', 'replace')) + # doc = html.fromstring(f.read().decode('latin-1', 'replace')) + # Apparently amazon Europe is responding in UTF-8 now + doc = html.fromstring(f.read()) data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' format_xpath = './/span[@class="format"]/text()' @@ -61,8 +63,8 @@ class AmazonESKindleStore(StorePlugin): cover_url = ''.join(data.xpath(cover_xpath)) - title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) - price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + title = ''.join(data.xpath('.//a[@class="title"]/text()')) + price = ''.join(data.xpath('.//span[@class="price"]/text()')) author = unicode(''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()'))) if author.startswith('de '): author = author[3:] diff --git a/src/calibre/gui2/store/stores/amazon_fr_plugin.py b/src/calibre/gui2/store/stores/amazon_fr_plugin.py index ea4c80e50d..b98ba06117 100644 --- a/src/calibre/gui2/store/stores/amazon_fr_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_fr_plugin.py @@ -39,7 +39,7 @@ class AmazonFRKindleStore(StorePlugin): counter = max_results with closing(br.open(url, timeout=timeout)) as f: # doc = html.fromstring(f.read().decode('latin-1', 'replace')) - # Apparently amazon.fr is responding in UTF-8 now + # Apparently amazon Europe is responding in UTF-8 now doc = html.fromstring(f.read()) data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' @@ -64,8 +64,8 @@ class AmazonFRKindleStore(StorePlugin): cover_url = ''.join(data.xpath(cover_xpath)) - title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) - price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + title = ''.join(data.xpath('.//a[@class="title"]/text()')) + price = ''.join(data.xpath('.//span[@class="price"]/text()')) author = unicode(''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()'))) if author.startswith('de '): author = author[3:] diff --git a/src/calibre/gui2/store/stores/amazon_it_plugin.py b/src/calibre/gui2/store/stores/amazon_it_plugin.py index c62273deeb..23cde51555 100644 --- a/src/calibre/gui2/store/stores/amazon_it_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_it_plugin.py @@ -37,7 +37,9 @@ class AmazonITKindleStore(StorePlugin): counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read().decode('latin-1', 'replace')) + # doc = html.fromstring(f.read().decode('latin-1', 'replace')) + # Apparently amazon Europe is responding in UTF-8 now + doc = html.fromstring(f.read()) data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' format_xpath = './/span[@class="format"]/text()' @@ -61,8 +63,8 @@ class AmazonITKindleStore(StorePlugin): cover_url = ''.join(data.xpath(cover_xpath)) - title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) - price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + title = ''.join(data.xpath('.//a[@class="title"]/text()')) + price = ''.join(data.xpath('.//span[@class="price"]/text()')) author = unicode(''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()'))) if author.startswith('di '): author = author[3:] diff --git a/src/calibre/gui2/store/stores/amazon_uk_plugin.py b/src/calibre/gui2/store/stores/amazon_uk_plugin.py index ef15951d50..0d063a4a6e 100644 --- a/src/calibre/gui2/store/stores/amazon_uk_plugin.py +++ b/src/calibre/gui2/store/stores/amazon_uk_plugin.py @@ -38,7 +38,8 @@ class AmazonUKKindleStore(StorePlugin): counter = max_results with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read().decode('latin-1', 'replace')) + # Apparently amazon Europe is responding in UTF-8 now + doc = html.fromstring(f.read()) data_xpath = '//div[contains(@class, "result") and contains(@class, "product")]' format_xpath = './/span[@class="format"]/text()' @@ -62,8 +63,8 @@ class AmazonUKKindleStore(StorePlugin): cover_url = ''.join(data.xpath(cover_xpath)) - title = ''.join(data.xpath('.//div[@class="title"]/a/text()')) - price = ''.join(data.xpath('.//div[@class="newPrice"]/span/text()')) + title = ''.join(data.xpath('.//a[@class="title"]/text()')) + price = ''.join(data.xpath('.//span[@class="price"]/text()')) author = ''.join(data.xpath('.//div[@class="title"]/span[@class="ptBrand"]/text()')) if author.startswith('by '): diff --git a/src/calibre/gui2/store/stores/foyles_uk_plugin.py b/src/calibre/gui2/store/stores/foyles_uk_plugin.py index 0e5ccfad01..c7c236d200 100644 --- a/src/calibre/gui2/store/stores/foyles_uk_plugin.py +++ b/src/calibre/gui2/store/stores/foyles_uk_plugin.py @@ -60,10 +60,6 @@ class FoylesUKStore(BasicStoreConfig, StorePlugin): continue cover_url = ''.join(data.xpath('.//a[@class="Jacket"]/img/@src')) - if cover_url: - cover_url = 'http://www.foyles.co.uk' + cover_url - #print(cover_url) - title = ''.join(data.xpath('.//a[@class="Title"]/text()')) author = ', '.join(data.xpath('.//span[@class="Author"]/text()')) price = ''.join(data.xpath('./ul/li[@class="Strong"]/text()')) diff --git a/src/calibre/gui2/store/stores/waterstones_uk_plugin.py b/src/calibre/gui2/store/stores/waterstones_uk_plugin.py index a5065128ba..df17372d0a 100644 --- a/src/calibre/gui2/store/stores/waterstones_uk_plugin.py +++ b/src/calibre/gui2/store/stores/waterstones_uk_plugin.py @@ -57,7 +57,7 @@ class WaterstonesUKStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//div[@class="image"]/a/img/@src')) title = ''.join(data.xpath('./div/div/h2/a/text()')) author = ', '.join(data.xpath('.//p[@class="byAuthor"]/a/text()')) - price = ''.join(data.xpath('.//p[@class="price"]/span[@class="priceStandard"]/text()')) + price = ''.join(data.xpath('.//p[@class="price"]/span[@class="priceRed2"]/text()')) drm = data.xpath('boolean(.//td[@headers="productFormat" and contains(., "DRM")])') pdf = data.xpath('boolean(.//td[@headers="productFormat" and contains(., "PDF")])') epub = data.xpath('boolean(.//td[@headers="productFormat" and contains(., "EPUB")])') From b5e519b560e32156bab0e7ef1e27e77b9c458fb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Tue, 20 Mar 2012 21:53:29 +0100 Subject: [PATCH 2/3] empik no longer declines its participation in Calibre --- src/calibre/gui2/store/declined.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/gui2/store/declined.txt b/src/calibre/gui2/store/declined.txt index bd2aa296bd..b109d30d50 100644 --- a/src/calibre/gui2/store/declined.txt +++ b/src/calibre/gui2/store/declined.txt @@ -5,4 +5,3 @@ or asked not to be included in the store integration. * Indigo (http://www.chapters.indigo.ca/). * Libraria Rizzoli (http://libreriarizzoli.corriere.it/). * EPubBuy DE: reason: too much traffic for too little sales -* Empik (http://empik.com.pl). From 36de96f3fa02456f00b1fcc3f33000984a81fe6e Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 20 Mar 2012 21:08:33 -0400 Subject: [PATCH 3/3] Store: Fix B&N and Diesel. Enhance Kobo. Remove Wizards Tower as they removed search from their site. --- src/calibre/customize/builtins.py | 10 -- src/calibre/gui2/store/stores/bn_plugin.py | 2 +- .../gui2/store/stores/diesel_ebooks_plugin.py | 39 +++--- src/calibre/gui2/store/stores/kobo_plugin.py | 2 +- .../stores/wizards_tower_books_plugin.py | 118 ------------------ 5 files changed, 18 insertions(+), 153 deletions(-) delete mode 100644 src/calibre/gui2/store/stores/wizards_tower_books_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 55742b3ee3..db56bf351f 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1557,15 +1557,6 @@ class StoreWHSmithUKStore(StoreBase): headquarters = 'UK' formats = ['EPUB', 'PDF'] -class StoreWizardsTowerBooksStore(StoreBase): - name = 'Wizards Tower Books' - description = u'A science fiction and fantasy publisher. Concentrates mainly on making out-of-print works available once more as e-books, and helping other small presses exploit the e-book market. Also publishes a small number of limited-print-run anthologies with a view to encouraging diversity in the science fiction and fantasy field.' - actual_plugin = 'calibre.gui2.store.stores.wizards_tower_books_plugin:WizardsTowerBooksStore' - - drm_free_only = True - headquarters = 'UK' - formats = ['EPUB', 'MOBI'] - class StoreWoblinkStore(StoreBase): name = 'Woblink' author = u'Tomasz Długosz' @@ -1636,7 +1627,6 @@ plugins += [ StoreWaterstonesUKStore, StoreWeightlessBooksStore, StoreWHSmithUKStore, - StoreWizardsTowerBooksStore, StoreWoblinkStore, XinXiiStore, StoreZixoStore diff --git a/src/calibre/gui2/store/stores/bn_plugin.py b/src/calibre/gui2/store/stores/bn_plugin.py index aa30ffe677..ab3d39264f 100644 --- a/src/calibre/gui2/store/stores/bn_plugin.py +++ b/src/calibre/gui2/store/stores/bn_plugin.py @@ -62,7 +62,7 @@ class BNStore(BasicStoreConfig, StorePlugin): title = ''.join(data.xpath('.//p[@class="title"]//span[@class="name"]/text()')) author = ', '.join(data.xpath('.//ul[@class="contributors"]//li[position()>1]//a/text()')) - price = ''.join(data.xpath('.//table[@class="displayed-formats"]//a[@class="subtle"]/text()')) + price = ''.join(data.xpath('.//table[@class="displayed-formats"]//a[contains(@class, "bn-price")]/text()')) counter -= 1 diff --git a/src/calibre/gui2/store/stores/diesel_ebooks_plugin.py b/src/calibre/gui2/store/stores/diesel_ebooks_plugin.py index a6876f8840..ba2ba28475 100644 --- a/src/calibre/gui2/store/stores/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/stores/diesel_ebooks_plugin.py @@ -7,7 +7,8 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import random -import urllib2 +import re +import urllib from contextlib import closing from lxml import html @@ -33,7 +34,7 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): detail_url = None if detail_item: - detail_url = url + detail_item + aff_id + detail_url = detail_item + aff_id url = url + aff_id if external or self.config.get('open_external', False): @@ -45,33 +46,36 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): d.exec_() def search(self, query, max_results=10, timeout=60): - url = 'http://www.diesel-ebooks.com/index.php?page=seek&id[m]=&id[c]=scope%253Dinventory&id[q]=' + urllib2.quote(query) + url = 'http://www.diesel-ebooks.com/index.php?page=seek&id[m]=&id[c]=scope%253Dinventory&id[q]=' + urllib.quote_plus(query) br = browser() counter = max_results with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read()) - for data in doc.xpath('//div[@class="item clearfix"]'): - data = html.fromstring(html.tostring(data)) + for data in doc.xpath('//div[contains(@class, "item")]'): if counter <= 0: break id = ''.join(data.xpath('div[@class="cover"]/a/@href')) if not id or '/item/' not in id: continue - a, b, id = id.partition('/item/') cover_url = ''.join(data.xpath('div[@class="cover"]//img/@src')) - title = ''.join(data.xpath('.//div[@class="content"]//h2/text()')) - author = ''.join(data.xpath('//div[@class="content"]//div[@class="author"]/a/text()')) + title = ''.join(data.xpath('.//div[@class="content"]//h2/a/text()')) + author = ''.join(data.xpath('.//div[@class="content"]/span//a/text()')) price = '' - price_elem = data.xpath('//td[@class="price"]/text()') + price_elem = data.xpath('.//div[@class="price_fat"]//h1/text()') if price_elem: price = price_elem[0] - formats = ', '.join(data.xpath('.//td[@class="format"]/text()')) + formats = ', '.join(data.xpath('.//div[@class="book-info"]//text()')).strip() + a, b, formats = formats.partition('Format:') + drm = SearchResult.DRM_LOCKED + if 'drm free' not in formats.lower(): + drm = SearchResult.DRM_UNLOCKED + counter -= 1 @@ -80,19 +84,8 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): s.title = title.strip() s.author = author.strip() s.price = price.strip() - s.detail_item = '/item/' + id.strip() + s.detail_item = id.strip() s.formats = formats + s.drm = drm yield s - - def get_details(self, search_result, timeout): - url = 'http://www.diesel-ebooks.com/item/' - - br = browser() - with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: - idata = html.fromstring(nf.read()) - if idata.xpath('boolean(//table[@class="format-info"]//tr[contains(th, "DRM") and contains(td, "No")])'): - search_result.drm = SearchResult.DRM_UNLOCKED - else: - search_result.drm = SearchResult.DRM_LOCKED - return True diff --git a/src/calibre/gui2/store/stores/kobo_plugin.py b/src/calibre/gui2/store/stores/kobo_plugin.py index 9ec0e4b786..249d59ec5c 100644 --- a/src/calibre/gui2/store/stores/kobo_plugin.py +++ b/src/calibre/gui2/store/stores/kobo_plugin.py @@ -68,7 +68,7 @@ class KoboStore(BasicStoreConfig, StorePlugin): cover_url = ''.join(data.xpath('.//div[@class="SearchImageContainer"]//img[1]/@src')) title = ''.join(data.xpath('.//div[@class="SCItemHeader"]/h1/a[1]/text()')) - author = ''.join(data.xpath('.//div[@class="SCItemSummary"]/span/a[1]/text()')) + author = ', '.join(data.xpath('.//div[@class="SCItemSummary"]//span//a/text()')) drm = data.xpath('boolean(.//span[@class="SCAvailibilityFormatsText" and contains(text(), "DRM")])') counter -= 1 diff --git a/src/calibre/gui2/store/stores/wizards_tower_books_plugin.py b/src/calibre/gui2/store/stores/wizards_tower_books_plugin.py deleted file mode 100644 index 90966fc06a..0000000000 --- a/src/calibre/gui2/store/stores/wizards_tower_books_plugin.py +++ /dev/null @@ -1,118 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import (unicode_literals, division, absolute_import, print_function) - -__license__ = 'GPL 3' -__copyright__ = '2011, John Schember ' -__docformat__ = 'restructuredtext en' - -import urllib -from contextlib import closing - -from lxml import html - -from PyQt4.Qt import QUrl - -from calibre import browser, url_slash_cleaner -from calibre.gui2 import open_url -from calibre.gui2.store import StorePlugin -from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.search_result import SearchResult -from calibre.gui2.store.web_store_dialog import WebStoreDialog - -class WizardsTowerBooksStore(BasicStoreConfig, StorePlugin): - - url = 'http://www.wizardstowerbooks.com/' - - def open(self, parent=None, detail_item=None, external=False): - if detail_item: - detail_item = self.url + detail_item - - if external or self.config.get('open_external', False): - open_url(QUrl(url_slash_cleaner(detail_item))) - else: - d = WebStoreDialog(self.gui, self.url, parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(self.config.get('tags', '')) - d.exec_() - - def search(self, query, max_results=10, timeout=60): - url = 'http://www.wizardstowerbooks.com/search.html?for=' + urllib.quote(query) - - br = browser() - - counter = max_results - with closing(br.open(url, timeout=timeout)) as f: - doc = html.fromstring(f.read()) - if 'search.html' in f.geturl(): - for data in doc.xpath('//table[@class="gridp"]//td'): - if counter <= 0: - break - - id = ''.join(data.xpath('.//span[@class="prti"]/a/@href')) - id = id.strip() - if not id: - continue - - cover_url = ''.join(data.xpath('.//div[@class="prim"]/a/img/@src')) - cover_url = url_slash_cleaner(self.url + cover_url.strip()) - - price = ''.join(data.xpath('.//font[@class="selling_price"]//text()')) - price = price.strip() - if not price: - continue - - title = ''.join(data.xpath('.//span[@class="prti"]/a/b/text()')) - author = ''.join(data.xpath('.//p[@class="last"]/text()')) - a, b, author = author.partition(' by ') - - counter -= 1 - - s = SearchResult() - s.cover_url = cover_url - s.title = title.strip() - s.author = author.strip() - s.price = price.strip() - s.detail_item = id.strip() - s.drm = SearchResult.DRM_UNLOCKED - - yield s - # Exact match brought us to the books detail page. - else: - s = SearchResult() - - cover_url = ''.join(doc.xpath('//div[@id="image"]/a/img[@title="Zoom"]/@src')).strip() - s.cover_url = url_slash_cleaner(self.url + cover_url.strip()) - - s.title = ''.join(doc.xpath('//form[@name="details"]/h1/text()')).strip() - - authors = doc.xpath('//p[contains(., "Author:")]//text()') - author_index = None - for i, a in enumerate(authors): - if 'author' in a.lower(): - author_index = i + 1 - break - if author_index is not None and len(authors) > author_index: - a = authors[author_index] - a = a.replace(u'\xa0', '') - s.author = a.strip() - - s.price = ''.join(doc.xpath('//span[@id="price_selling"]//text()')).strip() - s.detail_item = f.geturl().replace(self.url, '').strip() - s.formats = ', '.join(doc.xpath('//select[@id="N1_"]//option//text()')) - s.drm = SearchResult.DRM_UNLOCKED - - yield s - - def get_details(self, search_result, timeout): - if search_result.formats: - return False - - br = browser() - with closing(br.open(url_slash_cleaner(self.url + search_result.detail_item), timeout=timeout)) as nf: - idata = html.fromstring(nf.read()) - - formats = ', '.join(idata.xpath('//select[@id="N1_"]//option//text()')) - search_result.formats = formats.upper() - - return True