From 95397a087a56ca2ca46d19e593a8441ed5e5bb37 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 22 Feb 2011 19:12:28 -0500 Subject: [PATCH 01/92] Beginnings of store. Start of Amazon Kindle store. Start of meta store search. --- src/calibre/customize/__init__.py | 20 +++++ src/calibre/customize/builtins.py | 13 ++- src/calibre/customize/ui.py | 13 ++- src/calibre/gui2/actions/store.py | 34 ++++++++ src/calibre/gui2/store/__init__.py | 0 src/calibre/gui2/store/amazon/__init__.py | 0 .../gui2/store/amazon/amazon_kindle_dialog.py | 45 ++++++++++ .../gui2/store/amazon/amazon_kindle_dialog.ui | 84 +++++++++++++++++++ .../gui2/store/amazon/amazon_plugin.py | 61 ++++++++++++++ src/calibre/gui2/store/amazon/web_control.py | 19 +++++ src/calibre/gui2/store/search.py | 74 ++++++++++++++++ src/calibre/gui2/store/search.ui | 37 ++++++++ 12 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 src/calibre/gui2/actions/store.py create mode 100644 src/calibre/gui2/store/__init__.py create mode 100644 src/calibre/gui2/store/amazon/__init__.py create mode 100644 src/calibre/gui2/store/amazon/amazon_kindle_dialog.py create mode 100644 src/calibre/gui2/store/amazon/amazon_kindle_dialog.ui create mode 100644 src/calibre/gui2/store/amazon/amazon_plugin.py create mode 100644 src/calibre/gui2/store/amazon/web_control.py create mode 100644 src/calibre/gui2/store/search.py create mode 100644 src/calibre/gui2/store/search.ui diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 1f44eb4ae2..598a5af8f9 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -581,3 +581,23 @@ class PreferencesPlugin(Plugin): # {{{ # }}} + +class StorePlugin(Plugin): # {{{ + + supported_platforms = ['windows', 'osx', 'linux'] + author = 'John Schember' + type = _('Stores') + + def open(self, parent=None, start_item=None): + ''' + Open a dialog for displaying the store. + start_item is a refernce unique to the store + plugin and opens to the item when specified. + ''' + raise NotImplementedError() + + def search(self, query, max_results=10): + raise NotImplementedError() + +# }}} + diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 6cfe915036..54dfc73242 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -805,6 +805,10 @@ class ActionTweakEpub(InterfaceActionBase): class ActionNextMatch(InterfaceActionBase): name = 'Next Match' actual_plugin = 'calibre.gui2.actions.next_match:NextMatchAction' + +class ActionStore(InterfaceActionBase): + name = 'Store' + actual_plugin = 'calibre.gui2.actions.store:StoreAction' plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, ActionConvert, ActionDelete, ActionEditMetadata, ActionView, @@ -812,7 +816,7 @@ plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, ActionRestart, ActionOpenFolder, ActionConnectShare, ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks, ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary, - ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch] + ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore] # }}} @@ -1037,3 +1041,10 @@ from calibre.ebooks.metadata.sources.google import GoogleBooks plugins += [GoogleBooks] # }}} + +# Store plugins {{{ +from calibre.gui2.store.amazon.amazon_plugin import AmazonKindleStore + +plugins += [AmazonKindleStore] + +# }}} diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index 5f67e23d92..ce20a96068 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -8,7 +8,7 @@ from contextlib import closing from calibre.customize import Plugin, CatalogPlugin, FileTypePlugin, \ MetadataReaderPlugin, MetadataWriterPlugin, \ InterfaceActionBase as InterfaceAction, \ - PreferencesPlugin + PreferencesPlugin, StorePlugin from calibre.customize.conversion import InputFormatPlugin, OutputFormatPlugin from calibre.customize.profiles import InputProfile, OutputProfile from calibre.customize.builtins import plugins as builtin_plugins @@ -277,6 +277,17 @@ def preferences_plugins(): yield plugin # }}} +# Store Plugins # {{{ + +def store_plugins(): + customization = config['plugin_customization'] + for plugin in _initialized_plugins: + if isinstance(plugin, StorePlugin): + if not is_disabled(plugin): + plugin.site_customization = customization.get(plugin.name, '') + yield plugin +# }}} + # Metadata read/write {{{ _metadata_readers = {} _metadata_writers = {} diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py new file mode 100644 index 0000000000..97094ad86e --- /dev/null +++ b/src/calibre/gui2/actions/store.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from functools import partial + +from PyQt4.Qt import Qt, QMenu, QToolButton, QDialog, QVBoxLayout + +from calibre.customize.ui import store_plugins +from calibre.gui2.actions import InterfaceAction + +class StoreAction(InterfaceAction): + + name = 'Store' + action_spec = (_('Store'), None, None, None) + + def genesis(self): + self.qaction.triggered.connect(self.search) + self.store_menu = QMenu() + self.store_menu.addAction(_('Search'), self.search) + self.store_menu.addSeparator() + for x in store_plugins(): + self.store_menu.addAction(x.name, partial(self.open_store, x)) + self.qaction.setMenu(self.store_menu) + + def search(self): + from calibre.gui2.store.search import SearchDialog + sd = SearchDialog(self.gui) + sd.exec_() + + def open_store(self, store_plugin): + store_plugin.open(self.gui) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/calibre/gui2/store/amazon/__init__.py b/src/calibre/gui2/store/amazon/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py b/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py new file mode 100644 index 0000000000..3d1f206eae --- /dev/null +++ b/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib + +from PyQt4.Qt import QDialog, QUrl + +from calibre.gui2.store.amazon.amazon_kindle_dialog_ui import Ui_Dialog + +class AmazonKindleDialog(QDialog, Ui_Dialog): + + ASTORE_URL = 'http://astore.amazon.com/josbl0e-20/' + + def __init__(self, parent=None, start_item=None): + QDialog.__init__(self, parent=parent) + self.setupUi(self) + + self.view.loadStarted.connect(self.load_started) + self.view.loadProgress.connect(self.load_progress) + self.view.loadFinished.connect(self.load_finished) + self.home.clicked.connect(self.go_home) + self.reload.clicked.connect(self.go_reload) + + self.go_home(start_item=start_item) + + def load_started(self): + self.progress.setValue(0) + + def load_progress(self, val): + self.progress.setValue(val) + + def load_finished(self): + self.progress.setValue(100) + + def go_home(self, checked=False, start_item=None): + url = self.ASTORE_URL + if start_item: + url += 'detail/' + urllib.quote(start_item) + self.view.load(QUrl(url)) + + def go_reload(self, checked=False): + self.view.reload() diff --git a/src/calibre/gui2/store/amazon/amazon_kindle_dialog.ui b/src/calibre/gui2/store/amazon/amazon_kindle_dialog.ui new file mode 100644 index 0000000000..fb4692fc5c --- /dev/null +++ b/src/calibre/gui2/store/amazon/amazon_kindle_dialog.ui @@ -0,0 +1,84 @@ + + + Dialog + + + + 0 + 0 + 681 + 615 + + + + Amazon Kindle Store + + + true + + + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + 0 + + + + + + about:blank + + + + + + + + + + + Home + + + + + + + Reload + + + + + + + 0 + + + %p% + + + + + + + + QWebView + QWidget +
QtWebKit/QWebView
+ + + NPWebView + QWebView +
web_control.h
+ + + + + diff --git a/src/calibre/gui2/store/amazon/amazon_plugin.py b/src/calibre/gui2/store/amazon/amazon_plugin.py new file mode 100644 index 0000000000..e056a174d8 --- /dev/null +++ b/src/calibre/gui2/store/amazon/amazon_plugin.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from calibre import browser +from calibre.customize import StorePlugin + +class AmazonKindleStore(StorePlugin): + + name = 'Amazon Kindle' + description = _('Buy Kindle books from Amazon') + + def open(self, parent=None, start_item=None): + from calibre.gui2.store.amazon.amazon_kindle_dialog import AmazonKindleDialog + d = AmazonKindleDialog(parent, start_item) + d = d.exec_() + + def search(self, query, max_results=10): + url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query) + br = browser() + + counter = max_results + with closing(br.open(url)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="productData"]'): + if counter <= 0: + break + + # Even though we are searching digital-text only Amazon will still + # put in results for non Kindle books (author pages). Se we need + # to explicitly check if the item is a Kindle book and ignore it + # if it isn't. + type = ''.join(data.xpath('//span[@class="format"]/text()')) + if 'kindle' not in type.lower(): + continue + + title = ''.join(data.xpath('div[@class="productTitle"]/a/text()')) + author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) + price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) + + # We must have an asin otherwise we can't easily reference the + # book later. + asin = data.xpath('div[@class="productTitle"]/a[1]') + if asin: + asin = asin[0].get('href', '') + m = re.search(r'/dp/(?P.+?)(/|$)', asin) + if m: + asin = m.group('asin') + else: + continue + + counter -= 1 + yield (title.strip(), author.strip(), price.strip(), asin.strip()) diff --git a/src/calibre/gui2/store/amazon/web_control.py b/src/calibre/gui2/store/amazon/web_control.py new file mode 100644 index 0000000000..d7f95343d2 --- /dev/null +++ b/src/calibre/gui2/store/amazon/web_control.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from PyQt4.Qt import QWebView, QWebPage + +class NPWebView(QWebView): + + def createWindow(self, type): + if type == QWebPage.WebBrowserWindow: + return self + else: + return None + + + + \ No newline at end of file diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py new file mode 100644 index 0000000000..b0cfb3c9b4 --- /dev/null +++ b/src/calibre/gui2/store/search.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from threading import Event, Thread +from Queue import Queue + +from PyQt4.Qt import QDialog, QTimer + +from calibre.customize.ui import store_plugins +from calibre.gui2.store.search_ui import Ui_Dialog + +class SearchDialog(QDialog, Ui_Dialog): + + def __init__(self, *args): + QDialog.__init__(self, *args) + self.setupUi(self) + + self.store_plugins = {} + self.running_threads = [] + self.results = Queue() + self.abort = Event() + self.checker = QTimer() + + for x in store_plugins(): + self.store_plugins[x.name] = x + + self.search.clicked.connect(self.do_search) + self.checker.timeout.connect(self.get_results) + + def do_search(self, checked=False): + # Stop all running threads. + self.checker.stop() + self.abort.set() + self.running_threads = [] + self.results = Queue() + self.abort = Event() + for n in self.store_plugins: + t = SearchThread(unicode(self.search_edit.text()), (n, self.store_plugins[n]), self.results, self.abort) + self.running_threads.append(t) + t.start() + if self.running_threads: + self.checker.start(100) + + def get_results(self): + running = False + for t in self.running_threads: + if t.is_alive(): + running = True + if not running: + self.checker.stop() + + while not self.results.empty(): + print self.results.get_nowait() + + +class SearchThread(Thread): + + def __init__(self, query, store, results, abort): + Thread.__init__(self) + self.daemon = True + self.query = query + self.store_name = store[0] + self.store_plugin = store[1] + self.results = results + self.abort = abort + + def run(self): + if self.abort.is_set(): + return + for res in self.store_plugin.search(self.query): + self.results.put((self.store_name, res)) diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui new file mode 100644 index 0000000000..46cac9864e --- /dev/null +++ b/src/calibre/gui2/store/search.ui @@ -0,0 +1,37 @@ + + + Dialog + + + + 0 + 0 + 616 + 545 + + + + calibre Store Search + + + true + + + + + + + + + Search + + + + + + + + + + + From 6d5d638fe3a0a8d768e41726f4d4bd700f0e8c0e Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 22 Feb 2011 19:28:23 -0500 Subject: [PATCH 02/92] Move abort and add timeout. --- src/calibre/customize/__init__.py | 2 +- .../gui2/store/amazon/amazon_plugin.py | 4 +- src/calibre/gui2/store/search.py | 43 ++++++++++++++----- 3 files changed, 36 insertions(+), 13 deletions(-) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 598a5af8f9..9419f6454c 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -596,7 +596,7 @@ class StorePlugin(Plugin): # {{{ ''' raise NotImplementedError() - def search(self, query, max_results=10): + def search(self, query, max_results=10, timeout=60): raise NotImplementedError() # }}} diff --git a/src/calibre/gui2/store/amazon/amazon_plugin.py b/src/calibre/gui2/store/amazon/amazon_plugin.py index e056a174d8..aca5602834 100644 --- a/src/calibre/gui2/store/amazon/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon/amazon_plugin.py @@ -23,12 +23,12 @@ class AmazonKindleStore(StorePlugin): d = AmazonKindleDialog(parent, start_item) d = d.exec_() - def search(self, query, max_results=10): + def search(self, query, max_results=10, timeout=60): url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query) br = browser() counter = max_results - with closing(br.open(url)) as f: + with closing(br.open(url, timeout=timeout)) as f: doc = html.fromstring(f.read()) for data in doc.xpath('//div[@class="productData"]'): if counter <= 0: diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index b0cfb3c9b4..a52706f05b 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -14,6 +14,9 @@ from calibre.gui2.store.search_ui import Ui_Dialog class SearchDialog(QDialog, Ui_Dialog): + HANG_TIME = 75000 # milliseconds seconds + TIMEOUT = 75 # seconds + def __init__(self, *args): QDialog.__init__(self, *args) self.setupUi(self) @@ -23,6 +26,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.results = Queue() self.abort = Event() self.checker = QTimer() + self.hang_check = 0 for x in store_plugins(): self.store_plugins[x.name] = x @@ -38,19 +42,28 @@ class SearchDialog(QDialog, Ui_Dialog): self.results = Queue() self.abort = Event() for n in self.store_plugins: - t = SearchThread(unicode(self.search_edit.text()), (n, self.store_plugins[n]), self.results, self.abort) + t = SearchThread(unicode(self.search_edit.text()), (n, self.store_plugins[n]), self.results, self.abort, self.TIMEOUT) self.running_threads.append(t) t.start() if self.running_threads: + self.hang_check = 0 self.checker.start(100) def get_results(self): - running = False - for t in self.running_threads: - if t.is_alive(): - running = True - if not running: + # We only want the search plugins to run + # a maximum set amount of time before giving up. + self.hang_check += 1 + if self.hang_check >= self.HANG_TIME: + self.abort.set() self.checker.stop() + else: + # Stop the checker if not threads are running. + running = False + for t in self.running_threads: + if t.is_alive(): + running = True + if not running: + self.checker.stop() while not self.results.empty(): print self.results.get_nowait() @@ -58,7 +71,7 @@ class SearchDialog(QDialog, Ui_Dialog): class SearchThread(Thread): - def __init__(self, query, store, results, abort): + def __init__(self, query, store, results, abort, timeout): Thread.__init__(self) self.daemon = True self.query = query @@ -66,9 +79,19 @@ class SearchThread(Thread): self.store_plugin = store[1] self.results = results self.abort = abort + self.timeout = timeout def run(self): - if self.abort.is_set(): - return - for res in self.store_plugin.search(self.query): + for res in self.store_plugin.search(self.query, timeout=self.timeout): + if self.abort.is_set(): + return self.results.put((self.store_name, res)) + +class SearchResult(object): + + def __init__(self): + self.title = '' + self.author = '' + self.price = '' + self.item_data = '' + self.plugin_name = '' From dd2c856dc072b81fc5bef7c46c26237274010679 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 22 Feb 2011 19:35:01 -0500 Subject: [PATCH 03/92] Comments. --- src/calibre/customize/__init__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 9419f6454c..2edb7d7fd1 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -597,6 +597,17 @@ class StorePlugin(Plugin): # {{{ raise NotImplementedError() def search(self, query, max_results=10, timeout=60): + ''' + Searches the store for items matching query. This should + return items as a generator. + + :param query: The string query search with. + :param max_results: The maximum number of results to return. + :param timeout: The maximum amount of time in seconds to spend download the search results. + + :return: A tuple (cover_url, title, author, price, start_item). The start_item is plugin + specific and is used in :meth:`open` to open to a specifc place in the store. + ''' raise NotImplementedError() # }}} From 00fd94e03392581cc3b88838eec2915af6a58bff Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 22 Feb 2011 21:27:25 -0500 Subject: [PATCH 04/92] Display search results in GUI. --- .../gui2/store/amazon/amazon_plugin.py | 2 +- src/calibre/gui2/store/search.py | 107 +++++++++++++++++- src/calibre/gui2/store/search.ui | 18 ++- 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/store/amazon/amazon_plugin.py b/src/calibre/gui2/store/amazon/amazon_plugin.py index aca5602834..1cbee5a6ee 100644 --- a/src/calibre/gui2/store/amazon/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon/amazon_plugin.py @@ -58,4 +58,4 @@ class AmazonKindleStore(StorePlugin): continue counter -= 1 - yield (title.strip(), author.strip(), price.strip(), asin.strip()) + yield ('', title.strip(), author.strip(), price.strip(), asin.strip()) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index a52706f05b..e4ae16d765 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -7,10 +7,13 @@ __docformat__ = 'restructuredtext en' from threading import Event, Thread from Queue import Queue -from PyQt4.Qt import QDialog, QTimer +from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \ + QModelIndex from calibre.customize.ui import store_plugins +from calibre.gui2 import NONE from calibre.gui2.store.search_ui import Ui_Dialog +from calibre.utils.icu import sort_key class SearchDialog(QDialog, Ui_Dialog): @@ -27,6 +30,9 @@ class SearchDialog(QDialog, Ui_Dialog): self.abort = Event() self.checker = QTimer() self.hang_check = 0 + + self.model = Matches() + self.results_view.setModel(self.model) for x in store_plugins(): self.store_plugins[x.name] = x @@ -41,6 +47,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.running_threads = [] self.results = Queue() self.abort = Event() + self.results_view.model().clear_results() for n in self.store_plugins: t = SearchThread(unicode(self.search_edit.text()), (n, self.store_plugins[n]), self.results, self.abort, self.TIMEOUT) self.running_threads.append(t) @@ -66,7 +73,17 @@ class SearchDialog(QDialog, Ui_Dialog): self.checker.stop() while not self.results.empty(): - print self.results.get_nowait() + res = self.results.get_nowait() + if res: + result = SearchResult() + result.store = res[0] + result.cover_url = res[1][0] + result.title = res[1][1] + result.author = res[1][2] + result.price = res[1][3] + result.item_data = res[1][4] + + self.results_view.model().add_result(result) class SearchThread(Thread): @@ -87,11 +104,95 @@ class SearchThread(Thread): return self.results.put((self.store_name, res)) + class SearchResult(object): def __init__(self): + self.cover_url = '' self.title = '' self.author = '' self.price = '' + self.store = '' self.item_data = '' - self.plugin_name = '' + + +class Matches(QAbstractItemModel): + + HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('Store')] + + def __init__(self): + QAbstractItemModel.__init__(self) + self.matches = [] + + def clear_results(self): + self.matches = [] + self.reset() + + def add_result(self, result): + self.matches.append(result) + self.reset() + #self.dataChanged.emit(self.createIndex(self.rowCount() - 1, 0), self.createIndex(self.rowCount() - 1, self.columnCount())) + + def index(self, row, column, parent=QModelIndex()): + return self.createIndex(row, column) + + def parent(self, index): + if not index.isValid() or index.internalId() == 0: + return QModelIndex() + return self.createIndex(0, 0) + + def rowCount(self, *args): + return len(self.matches) + + def columnCount(self, *args): + return 5 + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return NONE + text = '' + if orientation == Qt.Horizontal: + if section < len(self.HEADERS): + text = self.HEADERS[section] + return QVariant(text) + else: + return QVariant(section+1) + + def data(self, index, role): + row, col = index.row(), index.column() + result = self.matches[row] + if role == Qt.DisplayRole: + if col == 1: + return QVariant(result.title) + elif col == 2: + return QVariant(result.author) + elif col == 3: + return QVariant(result.price) + elif col == 4: + return QVariant(result.store) + return NONE + elif role == Qt.DecorationRole: + pass + return NONE + + def data_as_text(self, result, col): + text = '' + if col == 1: + text = result.title + elif col == 2: + text = result.author + elif col == 3: + text = result.price + elif col == 4: + text = result.store + return text + + def sort(self, col, order, reset=True): + if not self.matches: + return + descending = order == Qt.DescendingOrder + self.matches.sort(None, + lambda x: sort_key(unicode(self.data_as_text(x, col))), + descending) + if reset: + self.reset() diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui index 46cac9864e..a3b3ac674b 100644 --- a/src/calibre/gui2/store/search.ui +++ b/src/calibre/gui2/store/search.ui @@ -28,7 +28,23 @@ - + + + true + + + false + + + false + + + true + + + false + + From a827ea3ada098aba1e7c1df1ad2d58971ea96d58 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 22 Feb 2011 21:38:11 -0500 Subject: [PATCH 05/92] Open store when selecting search result. --- src/calibre/gui2/store/search.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index e4ae16d765..c8dd818d04 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -39,6 +39,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.search.clicked.connect(self.do_search) self.checker.timeout.connect(self.get_results) + self.results_view.activated.connect(self.open_store) def do_search(self, checked=False): # Stop all running threads. @@ -85,6 +86,10 @@ class SearchDialog(QDialog, Ui_Dialog): self.results_view.model().add_result(result) + def open_store(self, index): + result = self.results_view.model().get_result(index) + self.store_plugins[result.store].open(self, result.item_data) + class SearchThread(Thread): @@ -133,6 +138,13 @@ class Matches(QAbstractItemModel): self.reset() #self.dataChanged.emit(self.createIndex(self.rowCount() - 1, 0), self.createIndex(self.rowCount() - 1, self.columnCount())) + def get_result(self, index): + row = index.row() + if row < len(self.matches): + return self.matches[row] + else: + return None + def index(self, row, column, parent=QModelIndex()): return self.createIndex(row, column) From 673385edf83ff3d615814ea35a0c0340c2b7237d Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 22 Feb 2011 21:41:58 -0500 Subject: [PATCH 06/92] Don't start a search if there is nothing to search for. --- src/calibre/gui2/store/search.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index c8dd818d04..65fde3eac9 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -48,9 +48,16 @@ class SearchDialog(QDialog, Ui_Dialog): self.running_threads = [] self.results = Queue() self.abort = Event() + # Clear the visible results. self.results_view.model().clear_results() + + # Don't start a search if there is nothing to search for. + query = unicode(self.search_edit.text()) + if not query.strip(): + return + for n in self.store_plugins: - t = SearchThread(unicode(self.search_edit.text()), (n, self.store_plugins[n]), self.results, self.abort, self.TIMEOUT) + t = SearchThread(query, (n, self.store_plugins[n]), self.results, self.abort, self.TIMEOUT) self.running_threads.append(t) t.start() if self.running_threads: From 7ed397ca4de49d2daa2ee1cbd236d0ec8230c94c Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 23 Feb 2011 06:58:36 -0500 Subject: [PATCH 07/92] Better sorting of price. --- src/calibre/gui2/store/search.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 65fde3eac9..443b2a2847 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -4,6 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import re from threading import Event, Thread from Queue import Queue @@ -202,6 +203,9 @@ class Matches(QAbstractItemModel): text = result.author elif col == 3: text = result.price + if len(text) < 3 or text[-3] not in ('.', ','): + text += '00' + text = re.sub(r'\D', '', text) elif col == 4: text = result.store return text From deea9f48bc4e15cd863d7b4da979437469411fa5 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 24 Feb 2011 18:45:05 -0500 Subject: [PATCH 08/92] Untested store download job. --- src/calibre/customize/__init__.py | 2 +- src/calibre/gui2/actions/store.py | 4 +- .../gui2/store/amazon/amazon_kindle_dialog.py | 4 +- .../gui2/store/amazon/amazon_plugin.py | 4 +- src/calibre/gui2/store/search.py | 6 +- src/calibre/gui2/store_download.py | 183 ++++++++++++++++++ src/calibre/gui2/ui.py | 4 +- 7 files changed, 198 insertions(+), 9 deletions(-) create mode 100644 src/calibre/gui2/store_download.py diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 2edb7d7fd1..2c75b40494 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -588,7 +588,7 @@ class StorePlugin(Plugin): # {{{ author = 'John Schember' type = _('Stores') - def open(self, parent=None, start_item=None): + def open(self, gui, parent=None, start_item=None): ''' Open a dialog for displaying the store. start_item is a refernce unique to the store diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 97094ad86e..9066150684 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -27,8 +27,8 @@ class StoreAction(InterfaceAction): def search(self): from calibre.gui2.store.search import SearchDialog - sd = SearchDialog(self.gui) + sd = SearchDialog(self.gui, self.gui) sd.exec_() def open_store(self, store_plugin): - store_plugin.open(self.gui) + store_plugin.open(self.gui, self.gui) diff --git a/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py b/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py index 3d1f206eae..108e1cfc40 100644 --- a/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py +++ b/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py @@ -14,10 +14,12 @@ class AmazonKindleDialog(QDialog, Ui_Dialog): ASTORE_URL = 'http://astore.amazon.com/josbl0e-20/' - def __init__(self, parent=None, start_item=None): + def __init__(self, gui, parent=None, start_item=None): QDialog.__init__(self, parent=parent) self.setupUi(self) + self.gui = gui + self.view.loadStarted.connect(self.load_started) self.view.loadProgress.connect(self.load_progress) self.view.loadFinished.connect(self.load_finished) diff --git a/src/calibre/gui2/store/amazon/amazon_plugin.py b/src/calibre/gui2/store/amazon/amazon_plugin.py index 1cbee5a6ee..8c63bdafa5 100644 --- a/src/calibre/gui2/store/amazon/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon/amazon_plugin.py @@ -18,9 +18,9 @@ class AmazonKindleStore(StorePlugin): name = 'Amazon Kindle' description = _('Buy Kindle books from Amazon') - def open(self, parent=None, start_item=None): + def open(self, gui, parent=None, start_item=None): from calibre.gui2.store.amazon.amazon_kindle_dialog import AmazonKindleDialog - d = AmazonKindleDialog(parent, start_item) + d = AmazonKindleDialog(gui, parent, start_item) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 443b2a2847..ac6981a25f 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -21,9 +21,11 @@ class SearchDialog(QDialog, Ui_Dialog): HANG_TIME = 75000 # milliseconds seconds TIMEOUT = 75 # seconds - def __init__(self, *args): + def __init__(self, gui, *args): QDialog.__init__(self, *args) self.setupUi(self) + + self.gui = gui self.store_plugins = {} self.running_threads = [] @@ -96,7 +98,7 @@ class SearchDialog(QDialog, Ui_Dialog): def open_store(self, index): result = self.results_view.model().get_result(index) - self.store_plugins[result.store].open(self, result.item_data) + self.store_plugins[result.store].open(self.gui, self, result.item_data) class SearchThread(Thread): diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py new file mode 100644 index 0000000000..f9460df17b --- /dev/null +++ b/src/calibre/gui2/store_download.py @@ -0,0 +1,183 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import cStringIO +import os +import shutil +import time +from contextlib import closing +from threading import Thread +from Queue import Queue + +from calibre import browser +from calibre.ebooks import BOOK_EXTENSIONS +from calibre.gui2 import Dispatcher +from calibre.ptempfile import PersistentTemporaryFile +from calibre.utils.ipc.job import BaseJob + +class StoreDownloadJob(BaseJob): + + def __init__(self, callback, description, job_manager, db, url='', save_as_loc='', add_to_lib=True): + BaseJob.__init__(self, description) + self.exception = None + self.job_manager = job_manager + self.db = db + self.args = (url, save_as_loc, add_to_lib) + self.tmp_file_name = '' + self.callback = callback + self.log_path = None + self._log_file = cStringIO.StringIO() + self._log_file.write(self.description.encode('utf-8') + '\n') + + @property + def log_file(self): + if self.log_path is not None: + return open(self.log_path, 'rb') + return cStringIO.StringIO(self._log_file.getvalue()) + + def start_work(self): + self.start_time = time.time() + self.job_manager.changed_queue.put(self) + + def job_done(self): + self.duration = time.time() - self.start_time + self.percent = 1 + # Dump log onto disk + lf = PersistentTemporaryFile('store_log') + lf.write(self._log_file.getvalue()) + lf.close() + self.log_path = lf.name + self._log_file.close() + self._log_file = None + + self.job_manager.changed_queue.put(self) + + def log_write(self, what): + self._log_file.write(what) + +class StoreDownloader(Thread): + + def __init__(self, job_manager): + Thread.__init__(self) + self.daemon = True + self.jobs = Queue() + self.job_manager = job_manager + self._run = True + + def stop(self): + self._run = False + self.jobs.put(None) + + def run(self): + while self._run: + try: + job = self.jobs.get() + except: + break + if job is None or not self._run: + break + + failed, exc = False, None + job.start_work() + if job.kill_on_start: + job.log_write('Aborted\n') + job.failed = failed + job.killed = True + job.job_done() + continue + + try: + self._download(job) + self._add(job) + self._save_as(job) + break + except Exception, e: + if not self._run: + return + import traceback + failed = True + exc = e + job.log_write('\nSending failed...\n') + job.log_write(traceback.format_exc()) + + if not self._run: + break + + job.failed = failed + job.exception = exc + job.job_done() + try: + job.callback(job) + except: + import traceback + traceback.print_exc() + + def _download(self, job): + url, save_loc, add_to_lib = job.args + if not url: + raise Exception(_('No file specified to download.')) + if not save_loc and not add_to_lib: + # Nothing to do. + return + + br = browser() + + basename = br.geturl(url).split('/')[-1] + ext = os.path.splitext(basename)[1][1:].lower() + if ext not in BOOK_EXTENSIONS: + raise Exception(_('Not a valid ebook format.')) + + tf = PersistentTemporaryFile(suffix=basename) + with closing(br.urlopen(url)) as f: + tf.write(f.read()) + tf.close() + job.tmp_file_name = tf.name + + def _add(self, job): + url, save_loc, add_to_lib = job.args + if not add_to_lib and job.tmp_file_name: + return + + ext = os.path.splitext(job.tmp_file_name)[1:] + + from calibre.ebooks.metadata.meta import get_metadata + with open(job.tmp_file_name) as f: + mi = get_metadata(f, ext) + + job.db.add_books([job.tmp_file_name], [ext], [mi]) + + def _save_as(self, job): + url, save_loc, add_to_lib = job.args + if not save_loc and job.tmp_fie_name: + return + + shutil.copy(job.tmp_fie_name, save_loc) + + def download_from_store(self, callback, db, url='', save_as_loc='', add_to_lib=True): + description = _('Downloading %s') % url + job = StoreDownloadJob(callback, description, job_manager, db, url, save_as_loc, add_to_lib) + self.job_manager.add_job(job) + self.jobs.put(job) + + +class StoreDownloadMixin(object): + + def __init__(self): + self.store_downloader = StoreDownloader(self.job_manager) + self.store_downloader.start() + + def download_from_store(self, url='', save_as_loc='', add_to_lib=True): + self.store_downloader.download_from_store(Dispatcher(self.downloaded_from_store), self.library_view.model().db, url, save_as_loc, add_to_lib) + self.status_bar.show_message(_('Downloading') + ' ' + url, 3000) + + def downloaded_from_store(self, job): + if job.failed: + self.job_exception(job, dialog_title=_('Failed to download book')) + return + + self.status_bar.show_message(job.description + ' ' + _('finished'), 5000) + + diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 9b9308d253..61d8676cfd 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -33,6 +33,7 @@ from calibre.gui2.main_window import MainWindow from calibre.gui2.layout import MainWindowMixin from calibre.gui2.device import DeviceMixin from calibre.gui2.email import EmailMixin +from calibre.gui2.store_download import StoreDownloadMixin from calibre.gui2.jobs import JobManager, JobsDialog, JobsButton from calibre.gui2.init import LibraryViewMixin, LayoutMixin from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin @@ -89,7 +90,8 @@ class SystemTrayIcon(QSystemTrayIcon): # {{{ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin, - SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin + SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin, + StoreDownloadMixin ): 'The main GUI' From c890651806fa99b9a23d1c8c1323f0c4c849fdec Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 25 Feb 2011 08:47:09 -0500 Subject: [PATCH 09/92] web_control downloads ebook links. Add Project Gutenberg store plugin. --- src/calibre/customize/builtins.py | 5 +- src/calibre/gui2/store/amazon/__init__.py | 0 src/calibre/gui2/store/amazon/web_control.py | 19 ------- .../gui2/store/{amazon => }/amazon_plugin.py | 9 ++-- src/calibre/gui2/store/gutenberg_plugin.py | 50 +++++++++++++++++ src/calibre/gui2/store/web_control.py | 53 +++++++++++++++++++ ...n_kindle_dialog.py => web_store_dialog.py} | 26 ++++----- ...n_kindle_dialog.ui => web_store_dialog.ui} | 2 +- src/calibre/gui2/store_download.py | 11 ++-- src/calibre/gui2/ui.py | 1 + 10 files changed, 133 insertions(+), 43 deletions(-) delete mode 100644 src/calibre/gui2/store/amazon/__init__.py delete mode 100644 src/calibre/gui2/store/amazon/web_control.py rename src/calibre/gui2/store/{amazon => }/amazon_plugin.py (88%) create mode 100644 src/calibre/gui2/store/gutenberg_plugin.py create mode 100644 src/calibre/gui2/store/web_control.py rename src/calibre/gui2/store/{amazon/amazon_kindle_dialog.py => web_store_dialog.py} (62%) rename src/calibre/gui2/store/{amazon/amazon_kindle_dialog.ui => web_store_dialog.ui} (98%) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 22b337915c..38aff09eb0 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1041,8 +1041,9 @@ plugins += [GoogleBooks] # }}} # Store plugins {{{ -from calibre.gui2.store.amazon.amazon_plugin import AmazonKindleStore +from calibre.gui2.store.amazon_plugin import AmazonKindleStore +from calibre.gui2.store.gutenberg_plugin import GutenbergStore -plugins += [AmazonKindleStore] +plugins += [AmazonKindleStore, GutenbergStore] # }}} diff --git a/src/calibre/gui2/store/amazon/__init__.py b/src/calibre/gui2/store/amazon/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/calibre/gui2/store/amazon/web_control.py b/src/calibre/gui2/store/amazon/web_control.py deleted file mode 100644 index d7f95343d2..0000000000 --- a/src/calibre/gui2/store/amazon/web_control.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- - -__license__ = 'GPL 3' -__copyright__ = '2011, John Schember ' -__docformat__ = 'restructuredtext en' - -from PyQt4.Qt import QWebView, QWebPage - -class NPWebView(QWebView): - - def createWindow(self, type): - if type == QWebPage.WebBrowserWindow: - return self - else: - return None - - - - \ No newline at end of file diff --git a/src/calibre/gui2/store/amazon/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py similarity index 88% rename from src/calibre/gui2/store/amazon/amazon_plugin.py rename to src/calibre/gui2/store/amazon_plugin.py index 8c63bdafa5..7a3ba90ccf 100644 --- a/src/calibre/gui2/store/amazon/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -18,9 +18,12 @@ class AmazonKindleStore(StorePlugin): name = 'Amazon Kindle' description = _('Buy Kindle books from Amazon') + ASTORE_URL = 'http://astore.amazon.com/josbl0e-20/' + def open(self, gui, parent=None, start_item=None): - from calibre.gui2.store.amazon.amazon_kindle_dialog import AmazonKindleDialog - d = AmazonKindleDialog(gui, parent, start_item) + from calibre.gui2.store.web_store_dialog import WebStoreDialog + d = WebStoreDialog(gui, self.ASTORE_URL, parent, start_item) + d.setWindowTitle('Amazon Kindle Store') d = d.exec_() def search(self, query, max_results=10, timeout=60): @@ -58,4 +61,4 @@ class AmazonKindleStore(StorePlugin): continue counter -= 1 - yield ('', title.strip(), author.strip(), price.strip(), asin.strip()) + yield ('', title.strip(), author.strip(), price.strip(), '/detail/'+asin.strip()) diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py new file mode 100644 index 0000000000..c97e2c9ce3 --- /dev/null +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from calibre import browser +from calibre.customize import StorePlugin + +class GutenbergStore(StorePlugin): + + name = 'Project Gutenberg' + description = _('The first producer of free ebooks.') + + + def open(self, gui, parent=None, start_item=None): + from calibre.gui2.store.web_store_dialog import WebStoreDialog + d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, start_item) + d.setWindowTitle('Free eBooks by Project Gutenberg') + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'): + if counter <= 0: + break + + heading = ''.join(data.xpath('div[@class="jd"]/a//text()')) + title, _, author = heading.partition('by') + author = author.split('-')[0] + price = '$0.00' + + url = ''.join(data.xpath('span[@class="c"]/text()')) + id = url.split('/')[-1] + + counter -= 1 + yield ('', title.strip(), author.strip(), price.strip(), '/ebooks/' + id.strip()) + + diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py new file mode 100644 index 0000000000..f31b750e52 --- /dev/null +++ b/src/calibre/gui2/store/web_control.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest + +class NPWebView(QWebView): + + def __init__(self, *args): + QWebView.__init__(self, *args) + self.gui = None + + #self.setPage(NPWebPage()) + self.page().networkAccessManager().setCookieJar(QNetworkCookieJar()) + self.page().setForwardUnsupportedContent(True) + self.page().unsupportedContent.connect(self.start_download) + self.page().downloadRequested.connect(self.start_download) + self.page().networkAccessManager().sslErrors.connect(self.ignore_ssl_errors) + #self.page().networkAccessManager().finished.connect(self.fin) + + def createWindow(self, type): + if type == QWebPage.WebBrowserWindow: + return self + else: + return None + + def set_gui(self, gui): + self.gui = gui + + def start_download(self, request): + if not self.gui: + print 'no gui' + return + + url = unicode(request.url().toString()) + self.gui.download_from_store(url) + + def ignore_ssl_errors(self, reply, errors): + reply.ignoreSslErrors(errors) + + def fin(self, reply): + if reply.error(): + print 'error' + print reply.error() + #print reply.attribute(QNetworkRequest.HttpStatusCodeAttribute).toInt() + + +class NPWebPage(QWebPage): + + def userAgentForUrl(self, url): + return 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.102 Safari/534.13' diff --git a/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py b/src/calibre/gui2/store/web_store_dialog.py similarity index 62% rename from src/calibre/gui2/store/amazon/amazon_kindle_dialog.py rename to src/calibre/gui2/store/web_store_dialog.py index 108e1cfc40..467991e863 100644 --- a/src/calibre/gui2/store/amazon/amazon_kindle_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -8,39 +8,41 @@ import urllib from PyQt4.Qt import QDialog, QUrl -from calibre.gui2.store.amazon.amazon_kindle_dialog_ui import Ui_Dialog +from calibre.gui2.store.web_store_dialog_ui import Ui_Dialog -class AmazonKindleDialog(QDialog, Ui_Dialog): +class WebStoreDialog(QDialog, Ui_Dialog): - ASTORE_URL = 'http://astore.amazon.com/josbl0e-20/' - - def __init__(self, gui, parent=None, start_item=None): + def __init__(self, gui, base_url, parent=None, detail_item=None): QDialog.__init__(self, parent=parent) self.setupUi(self) self.gui = gui + self.base_url = base_url + self.view.set_gui(self.gui) self.view.loadStarted.connect(self.load_started) self.view.loadProgress.connect(self.load_progress) self.view.loadFinished.connect(self.load_finished) self.home.clicked.connect(self.go_home) self.reload.clicked.connect(self.go_reload) - self.go_home(start_item=start_item) - + self.go_home(detail_item=detail_item) + def load_started(self): self.progress.setValue(0) def load_progress(self, val): self.progress.setValue(val) - def load_finished(self): + def load_finished(self, ok=True): self.progress.setValue(100) + #if not ok: + # print 'Error' - def go_home(self, checked=False, start_item=None): - url = self.ASTORE_URL - if start_item: - url += 'detail/' + urllib.quote(start_item) + def go_home(self, checked=False, detail_item=None): + url = self.base_url + if detail_item: + url += '/' + urllib.quote(detail_item) self.view.load(QUrl(url)) def go_reload(self, checked=False): diff --git a/src/calibre/gui2/store/amazon/amazon_kindle_dialog.ui b/src/calibre/gui2/store/web_store_dialog.ui similarity index 98% rename from src/calibre/gui2/store/amazon/amazon_kindle_dialog.ui rename to src/calibre/gui2/store/web_store_dialog.ui index fb4692fc5c..a6cce24d33 100644 --- a/src/calibre/gui2/store/amazon/amazon_kindle_dialog.ui +++ b/src/calibre/gui2/store/web_store_dialog.ui @@ -11,7 +11,7 @@ - Amazon Kindle Store + true diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index 241ff2f10a..fe99ae6f14 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -93,7 +93,6 @@ class StoreDownloader(Thread): self._download(job) self._add(job) self._save_as(job) - break except Exception, e: if not self._run: return @@ -125,13 +124,13 @@ class StoreDownloader(Thread): br = browser() - basename = br.geturl(url).split('/')[-1] + basename = br.open(url).geturl().split('/')[-1] ext = os.path.splitext(basename)[1][1:].lower() if ext not in BOOK_EXTENSIONS: raise Exception(_('Not a valid ebook format.')) tf = PersistentTemporaryFile(suffix=basename) - with closing(br.urlopen(url)) as f: + with closing(br.open(url)) as f: tf.write(f.read()) tf.close() job.tmp_file_name = tf.name @@ -141,7 +140,7 @@ class StoreDownloader(Thread): if not add_to_lib and job.tmp_file_name: return - ext = os.path.splitext(job.tmp_file_name)[1:] + ext = os.path.splitext(job.tmp_file_name)[1][1:] from calibre.ebooks.metadata.meta import get_metadata with open(job.tmp_file_name) as f: @@ -151,14 +150,14 @@ class StoreDownloader(Thread): def _save_as(self, job): url, save_loc, add_to_lib = job.args - if not save_loc and job.tmp_fie_name: + if not save_loc and job.tmp_file_name: return shutil.copy(job.tmp_fie_name, save_loc) def download_from_store(self, callback, db, url='', save_as_loc='', add_to_lib=True): description = _('Downloading %s') % url - job = StoreDownloadJob(callback, description, job_manager, db, url, save_as_loc, add_to_lib) + job = StoreDownloadJob(callback, description, self.job_manager, db, url, save_as_loc, add_to_lib) self.job_manager.add_job(job) self.jobs.put(job) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 85f58be218..aa36279c85 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -166,6 +166,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ LayoutMixin.__init__(self) EmailMixin.__init__(self) + StoreDownloadMixin.__init__(self) DeviceMixin.__init__(self) self.progress_indicator = ProgressIndicator(self) From bcc406ee05f3791b773917d15b6b82b3ed32f4ea Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 25 Feb 2011 08:48:17 -0500 Subject: [PATCH 10/92] ... --- src/calibre/gui2/store/gutenberg_plugin.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index c97e2c9ce3..6904e27bcd 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -25,6 +25,8 @@ class GutenbergStore(StorePlugin): d = d.exec_() def search(self, query, max_results=10, timeout=60): + # Gutenberg's website does not allow searching both author and title. + # Using a google search so we can search on both fields at once. url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib2.quote(query) br = browser() From 77966b5071b40a182f0fa68d8f27f3fe16cfc02b Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 25 Feb 2011 22:00:08 -0500 Subject: [PATCH 11/92] Move cookies from web_control to StoreDownload --- src/calibre/gui2/store/web_control.py | 53 ++++++++++++++++++++++++++- src/calibre/gui2/store_download.py | 13 ++++--- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index f31b750e52..3b6337f597 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -4,7 +4,9 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest +from cookielib import Cookie, CookieJar + +from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString class NPWebView(QWebView): @@ -35,7 +37,7 @@ class NPWebView(QWebView): return url = unicode(request.url().toString()) - self.gui.download_from_store(url) + self.gui.download_from_store(url, self.get_cookies()) def ignore_ssl_errors(self, reply, errors): reply.ignoreSslErrors(errors) @@ -45,6 +47,53 @@ class NPWebView(QWebView): print 'error' print reply.error() #print reply.attribute(QNetworkRequest.HttpStatusCodeAttribute).toInt() + + def get_cookies(self): + cj = CookieJar() + + for c in self.page().networkAccessManager().cookieJar().allCookies(): + version = 0 + name = unicode(QString(c.name())) + value = unicode(QString(c.value())) + port = None + port_specified = False + domain = unicode(c.domain()) + if domain: + domain_specified = True + if domain.startswith('.'): + domain_initial_dot = True + else: + domain_initial_dot = False + else: + domain = None + domain_specified = False + path = unicode(c.path()) + if path: + path_specified = True + else: + path = None + path_specified = False + secure = c.isSecure() + expires = c.expirationDate().toMSecsSinceEpoch() / 1000 + discard = c.isSessionCookie() + comment = None + comment_url = None + rest = None + + cookie = Cookie(version, name, value, + port, port_specified, + domain, domain_specified, domain_initial_dot, + path, path_specified, + secure, + expires, + discard, + comment, + comment_url, + rest) + + cj.set_cookie(cookie) + + return cj class NPWebPage(QWebPage): diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index fe99ae6f14..71378c173f 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -8,6 +8,7 @@ import cStringIO import os import shutil import time +from cookielib import CookieJar from contextlib import closing from threading import Thread from Queue import Queue @@ -20,11 +21,12 @@ from calibre.utils.ipc.job import BaseJob class StoreDownloadJob(BaseJob): - def __init__(self, callback, description, job_manager, db, url='', save_as_loc='', add_to_lib=True): + def __init__(self, callback, description, job_manager, db, cookie_jar, url='', save_as_loc='', add_to_lib=True): BaseJob.__init__(self, description) self.exception = None self.job_manager = job_manager self.db = db + self.cookie_jar = cookie_jar self.args = (url, save_as_loc, add_to_lib) self.tmp_file_name = '' self.callback = callback @@ -123,6 +125,7 @@ class StoreDownloader(Thread): return br = browser() + br.set_cookiejar(job.cookie_jar) basename = br.open(url).geturl().split('/')[-1] ext = os.path.splitext(basename)[1][1:].lower() @@ -155,9 +158,9 @@ class StoreDownloader(Thread): shutil.copy(job.tmp_fie_name, save_loc) - def download_from_store(self, callback, db, url='', save_as_loc='', add_to_lib=True): + def download_from_store(self, callback, db, cookie_jar, url='', save_as_loc='', add_to_lib=True): description = _('Downloading %s') % url - job = StoreDownloadJob(callback, description, self.job_manager, db, url, save_as_loc, add_to_lib) + job = StoreDownloadJob(callback, description, self.job_manager, db, cookie_jar, url, save_as_loc, add_to_lib) self.job_manager.add_job(job) self.jobs.put(job) @@ -167,10 +170,10 @@ class StoreDownloadMixin(object): def __init__(self): self.store_downloader = StoreDownloader(self.job_manager) - def download_from_store(self, url='', save_as_loc='', add_to_lib=True): + def download_from_store(self, url='', cookie_jar=CookieJar(), save_as_loc='', add_to_lib=True): if not self.store_downloader.is_alive(): self.store_downloader.start() - self.store_downloader.download_from_store(Dispatcher(self.downloaded_from_store), self.library_view.model().db, url, save_as_loc, add_to_lib) + self.store_downloader.download_from_store(Dispatcher(self.downloaded_from_store), self.library_view.model().db, cookie_jar, url, save_as_loc, add_to_lib) self.status_bar.show_message(_('Downloading') + ' ' + url, 3000) def downloaded_from_store(self, job): From df8347a62b6e56b2ca1c485e7508a987c9ba672b Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 11:51:01 -0500 Subject: [PATCH 12/92] ManyBooks store. Gutenberg search filters invalid items quicker and ensures full id is not truncated. --- src/calibre/customize/builtins.py | 3 +- src/calibre/gui2/store/gutenberg_plugin.py | 16 ++++-- src/calibre/gui2/store/manybooks_plugin.py | 60 ++++++++++++++++++++++ 3 files changed, 74 insertions(+), 5 deletions(-) create mode 100644 src/calibre/gui2/store/manybooks_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 38aff09eb0..78fc0a4b9b 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1043,7 +1043,8 @@ plugins += [GoogleBooks] # Store plugins {{{ from calibre.gui2.store.amazon_plugin import AmazonKindleStore from calibre.gui2.store.gutenberg_plugin import GutenbergStore +from calibre.gui2.store.manybooks_plugin import ManyBooksStore -plugins += [AmazonKindleStore, GutenbergStore] +plugins += [AmazonKindleStore, GutenbergStore, ManyBooksStore] # }}} diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 6904e27bcd..e31fc1149d 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -38,14 +38,22 @@ class GutenbergStore(StorePlugin): if counter <= 0: break - heading = ''.join(data.xpath('div[@class="jd"]/a//text()')) + url = '' + url_a = data.xpath('div[@class="jd"]/a') + if url_a: + url_a = url_a[0] + url = url_a.get('href', None) + if url: + url = url.split('u=')[-1].split('&')[0] + if '/ebooks/' not in url: + continue + id = url.split('/')[-1] + + heading = ''.join(url_a.xpath('text()')) title, _, author = heading.partition('by') author = author.split('-')[0] price = '$0.00' - url = ''.join(data.xpath('span[@class="c"]/text()')) - id = url.split('/')[-1] - counter -= 1 yield ('', title.strip(), author.strip(), price.strip(), '/ebooks/' + id.strip()) diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py new file mode 100644 index 0000000000..79818b9123 --- /dev/null +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from calibre import browser +from calibre.customize import StorePlugin + +class ManyBooksStore(StorePlugin): + + name = 'ManyBooks' + description = _('The best ebooks at the best price: free!.') + + + def open(self, gui, parent=None, start_item=None): + from calibre.gui2.store.web_store_dialog import WebStoreDialog + d = WebStoreDialog(gui, 'http://manybooks.net/', parent, start_item) + d.setWindowTitle('Ad-free eBooks for your eBook reader') + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + # ManyBooks website separates results for title and author. + # Using a google search so we can search on both fields at once. + url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'): + if counter <= 0: + break + + url = '' + url_a = data.xpath('div[@class="jd"]/a') + if url_a: + url_a = url_a[0] + url = url_a.get('href', None) + if url: + url = url.split('u=')[-1][:-2] + if '/titles/' not in url: + continue + id = url.split('/')[-1] + + heading = ''.join(url_a.xpath('text()')) + title, _, author = heading.partition('by') + author = author.split('-')[0] + price = '$0.00' + + counter -= 1 + yield ('', title.strip(), author.strip(), price.strip(), '/titles/' + id.strip()) + + From 564a01e81a1d5e96912606f14621547c6d625a32 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 11:52:34 -0500 Subject: [PATCH 13/92] ... --- src/calibre/gui2/store/gutenberg_plugin.py | 2 -- src/calibre/gui2/store/manybooks_plugin.py | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index e31fc1149d..56edb4588e 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -56,5 +56,3 @@ class GutenbergStore(StorePlugin): counter -= 1 yield ('', title.strip(), author.strip(), price.strip(), '/ebooks/' + id.strip()) - - diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 79818b9123..4afceeeb63 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -26,6 +26,8 @@ class ManyBooksStore(StorePlugin): def search(self, query, max_results=10, timeout=60): # ManyBooks website separates results for title and author. + # It also doesn't do a clear job of references authors and + # secondary titles. Google is also faster. # Using a google search so we can search on both fields at once. url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib2.quote(query) @@ -56,5 +58,3 @@ class ManyBooksStore(StorePlugin): counter -= 1 yield ('', title.strip(), author.strip(), price.strip(), '/titles/' + id.strip()) - - From 210b7e48c00be1d9614618732664fbeee6331db7 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 11:55:16 -0500 Subject: [PATCH 14/92] Add back button to web store dialog and make the reload / back cleaner. --- src/calibre/gui2/store/web_store_dialog.py | 6 ++---- src/calibre/gui2/store/web_store_dialog.ui | 15 +++++++++++---- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py index 467991e863..818a5ec1b5 100644 --- a/src/calibre/gui2/store/web_store_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -24,7 +24,8 @@ class WebStoreDialog(QDialog, Ui_Dialog): self.view.loadProgress.connect(self.load_progress) self.view.loadFinished.connect(self.load_finished) self.home.clicked.connect(self.go_home) - self.reload.clicked.connect(self.go_reload) + self.reload.clicked.connect(self.view.reload) + self.back.clicked.connect(self.view.back) self.go_home(detail_item=detail_item) @@ -44,6 +45,3 @@ class WebStoreDialog(QDialog, Ui_Dialog): if detail_item: url += '/' + urllib.quote(detail_item) self.view.load(QUrl(url)) - - def go_reload(self, checked=False): - self.view.reload() diff --git a/src/calibre/gui2/store/web_store_dialog.ui b/src/calibre/gui2/store/web_store_dialog.ui index a6cce24d33..e2f15607ce 100644 --- a/src/calibre/gui2/store/web_store_dialog.ui +++ b/src/calibre/gui2/store/web_store_dialog.ui @@ -6,8 +6,8 @@ 0 0 - 681 - 615 + 962 + 656 @@ -17,7 +17,7 @@ true - + QFrame::StyledPanel @@ -55,7 +55,7 @@ - + 0 @@ -65,6 +65,13 @@ + + + + Back + + + From 93c2231f26df103c6c2e8231ca86b5cecc5bf9ec Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 13:29:52 -0500 Subject: [PATCH 15/92] Basic Feedbooks store plugin. --- src/calibre/customize/builtins.py | 3 +- src/calibre/gui2/store/feedbooks_plugin.py | 63 ++++++++++++++++++++++ src/calibre/gui2/store/gutenberg_plugin.py | 2 +- src/calibre/gui2/store/manybooks_plugin.py | 2 +- src/calibre/gui2/store/web_store_dialog.py | 7 ++- 5 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 src/calibre/gui2/store/feedbooks_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 78fc0a4b9b..76aaf3a8ce 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1043,8 +1043,9 @@ plugins += [GoogleBooks] # Store plugins {{{ from calibre.gui2.store.amazon_plugin import AmazonKindleStore from calibre.gui2.store.gutenberg_plugin import GutenbergStore +from calibre.gui2.store.feedbooks_plugin import FeedbooksStore from calibre.gui2.store.manybooks_plugin import ManyBooksStore -plugins += [AmazonKindleStore, GutenbergStore, ManyBooksStore] +plugins += [AmazonKindleStore, GutenbergStore, FeedbooksStore, ManyBooksStore] # }}} diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py new file mode 100644 index 0000000000..e1668a22a3 --- /dev/null +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from calibre import browser +from calibre.customize import StorePlugin + +class FeedbooksStore(StorePlugin): + + name = 'Feedbooks' + description = _('Read anywhere.') + + + def open(self, gui, parent=None, start_item=None): + from calibre.gui2.store.web_store_dialog import WebStoreDialog + d = WebStoreDialog(gui, 'http://m.feedbooks.com/', parent, start_item) + d.setWindowTitle('Feedbooks') + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://m.feedbooks.com/search?query=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//ul[@class="m-list"]//li'): + if counter <= 0: + break + data = html.fromstring(html.tostring(data)) + + id = '' + id_a = data.xpath('//a[@class="buy"]') + if id_a: + id = id_a[0].get('href', None) + id = id.split('/')[-2] + id = '/item/' + id + else: + id_a = data.xpath('//a[@class="download"]') + if id_a: + id = id_a[0].get('href', None) + id = id.split('/')[-1] + id = id.split('.')[0] + id = '/book/' + id + if not id: + continue + + title = ''.join(data.xpath('//h5/a/text()')) + author = ''.join(data.xpath('//h6/a/text()')) + price = ''.join(data.xpath('//a[@class="buy"]/text()')) + if not price: + price = '$0.00' + + counter -= 1 + yield ('', title.strip(), author.strip(), price.replace(' ', '').strip(), id.strip()) diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 56edb4588e..38b9af8012 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -21,7 +21,7 @@ class GutenbergStore(StorePlugin): def open(self, gui, parent=None, start_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, start_item) - d.setWindowTitle('Free eBooks by Project Gutenberg') + d.setWindowTitle('Project Gutenberg') d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 4afceeeb63..1ec6eeb500 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -21,7 +21,7 @@ class ManyBooksStore(StorePlugin): def open(self, gui, parent=None, start_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://manybooks.net/', parent, start_item) - d.setWindowTitle('Ad-free eBooks for your eBook reader') + d.setWindowTitle('ManyBooks') d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py index 818a5ec1b5..3910ca6912 100644 --- a/src/calibre/gui2/store/web_store_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -4,6 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import re import urllib from PyQt4.Qt import QDialog, QUrl @@ -37,11 +38,13 @@ class WebStoreDialog(QDialog, Ui_Dialog): def load_finished(self, ok=True): self.progress.setValue(100) - #if not ok: - # print 'Error' def go_home(self, checked=False, detail_item=None): url = self.base_url if detail_item: url += '/' + urllib.quote(detail_item) + # Reduce redundant /'s because some stores + # (Feedbooks) and server frameworks (cherrypy) + # choke on them. + url = re.sub(r'(? Date: Sat, 26 Feb 2011 15:42:51 -0500 Subject: [PATCH 16/92] Use StoreResult class for returning store results instead of a tuple. --- src/calibre/customize/__init__.py | 6 +++--- src/calibre/gui2/store/amazon_plugin.py | 16 +++++++++++++--- src/calibre/gui2/store/feedbooks_plugin.py | 15 ++++++++++++--- src/calibre/gui2/store/gutenberg_plugin.py | 15 ++++++++++++--- src/calibre/gui2/store/manybooks_plugin.py | 15 ++++++++++++--- src/calibre/gui2/store/search.py | 19 +------------------ 6 files changed, 53 insertions(+), 33 deletions(-) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 2c75b40494..c1e19bd543 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -588,7 +588,7 @@ class StorePlugin(Plugin): # {{{ author = 'John Schember' type = _('Stores') - def open(self, gui, parent=None, start_item=None): + def open(self, gui, parent=None, detail_item=None): ''' Open a dialog for displaying the store. start_item is a refernce unique to the store @@ -605,8 +605,8 @@ class StorePlugin(Plugin): # {{{ :param max_results: The maximum number of results to return. :param timeout: The maximum amount of time in seconds to spend download the search results. - :return: A tuple (cover_url, title, author, price, start_item). The start_item is plugin - specific and is used in :meth:`open` to open to a specifc place in the store. + :return: calibre.gui2.store.search_result.SearchResult object + item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. ''' raise NotImplementedError() diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 7a3ba90ccf..d3ab7c8911 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -12,6 +12,7 @@ from lxml import html from calibre import browser from calibre.customize import StorePlugin +from calibre.gui2.store.search_result import SearchResult class AmazonKindleStore(StorePlugin): @@ -20,9 +21,9 @@ class AmazonKindleStore(StorePlugin): ASTORE_URL = 'http://astore.amazon.com/josbl0e-20/' - def open(self, gui, parent=None, start_item=None): + def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, self.ASTORE_URL, parent, start_item) + d = WebStoreDialog(gui, self.ASTORE_URL, parent, detail_item) d.setWindowTitle('Amazon Kindle Store') d = d.exec_() @@ -47,6 +48,7 @@ class AmazonKindleStore(StorePlugin): title = ''.join(data.xpath('div[@class="productTitle"]/a/text()')) author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) + author = author.split('by')[-1] price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) # We must have an asin otherwise we can't easily reference the @@ -61,4 +63,12 @@ class AmazonKindleStore(StorePlugin): continue counter -= 1 - yield ('', title.strip(), author.strip(), price.strip(), '/detail/'+asin.strip()) + + s = SearchResult() + s.cover_url = '' + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/detail/' + asin.strip() + + yield s diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index e1668a22a3..c9669b94a4 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -11,6 +11,7 @@ from lxml import html from calibre import browser from calibre.customize import StorePlugin +from calibre.gui2.store.search_result import SearchResult class FeedbooksStore(StorePlugin): @@ -18,9 +19,9 @@ class FeedbooksStore(StorePlugin): description = _('Read anywhere.') - def open(self, gui, parent=None, start_item=None): + def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, 'http://m.feedbooks.com/', parent, start_item) + d = WebStoreDialog(gui, 'http://m.feedbooks.com/', parent, detail_item) d.setWindowTitle('Feedbooks') d = d.exec_() @@ -60,4 +61,12 @@ class FeedbooksStore(StorePlugin): price = '$0.00' counter -= 1 - yield ('', title.strip(), author.strip(), price.replace(' ', '').strip(), id.strip()) + + s = SearchResult() + s.cover_url = '' + s.title = title.strip() + s.author = author.strip() + s.price = price.replace(' ', '').strip() + s.detail_item = id.strip() + + yield s diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 38b9af8012..5d55844180 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -11,6 +11,7 @@ from lxml import html from calibre import browser from calibre.customize import StorePlugin +from calibre.gui2.store.search_result import SearchResult class GutenbergStore(StorePlugin): @@ -18,9 +19,9 @@ class GutenbergStore(StorePlugin): description = _('The first producer of free ebooks.') - def open(self, gui, parent=None, start_item=None): + def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, start_item) + d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, detail_item) d.setWindowTitle('Project Gutenberg') d = d.exec_() @@ -55,4 +56,12 @@ class GutenbergStore(StorePlugin): price = '$0.00' counter -= 1 - yield ('', title.strip(), author.strip(), price.strip(), '/ebooks/' + id.strip()) + + s = SearchResult() + s.cover_url = '' + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/ebooks/' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 1ec6eeb500..abb27206a9 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -11,6 +11,7 @@ from lxml import html from calibre import browser from calibre.customize import StorePlugin +from calibre.gui2.store.search_result import SearchResult class ManyBooksStore(StorePlugin): @@ -18,9 +19,9 @@ class ManyBooksStore(StorePlugin): description = _('The best ebooks at the best price: free!.') - def open(self, gui, parent=None, start_item=None): + def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, 'http://manybooks.net/', parent, start_item) + d = WebStoreDialog(gui, 'http://manybooks.net/', parent, detail_item) d.setWindowTitle('ManyBooks') d = d.exec_() @@ -57,4 +58,12 @@ class ManyBooksStore(StorePlugin): price = '$0.00' counter -= 1 - yield ('', title.strip(), author.strip(), price.strip(), '/titles/' + id.strip()) + + s = SearchResult() + s.cover_url = '' + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/titles/' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index ac6981a25f..5c07984cda 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -86,14 +86,8 @@ class SearchDialog(QDialog, Ui_Dialog): while not self.results.empty(): res = self.results.get_nowait() if res: - result = SearchResult() + result = res[1] result.store = res[0] - result.cover_url = res[1][0] - result.title = res[1][1] - result.author = res[1][2] - result.price = res[1][3] - result.item_data = res[1][4] - self.results_view.model().add_result(result) def open_store(self, index): @@ -120,17 +114,6 @@ class SearchThread(Thread): self.results.put((self.store_name, res)) -class SearchResult(object): - - def __init__(self): - self.cover_url = '' - self.title = '' - self.author = '' - self.price = '' - self.store = '' - self.item_data = '' - - class Matches(QAbstractItemModel): HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('Store')] From 668ee348ce2d5680ea5dd6e17c467b87dad4639a Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 15:44:38 -0500 Subject: [PATCH 17/92] Fix sorting by price. --- src/calibre/gui2/store/search.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 5c07984cda..df29181c4d 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -191,6 +191,7 @@ class Matches(QAbstractItemModel): if len(text) < 3 or text[-3] not in ('.', ','): text += '00' text = re.sub(r'\D', '', text) + text = text.rjust(6, '0') elif col == 4: text = result.store return text From 2ea80eed041eb763d3dba52260d3c5b1ce6d1b55 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 19:08:37 -0500 Subject: [PATCH 18/92] Don't lose user selection when updating item. Download covers in a separate thread and update dynamically. --- src/calibre/gui2/store/feedbooks_plugin.py | 7 +- src/calibre/gui2/store/search.py | 79 +++++++++++++++++++--- src/calibre/gui2/store/search.ui | 9 +++ 3 files changed, 84 insertions(+), 11 deletions(-) diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index c9669b94a4..2be8576515 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -59,11 +59,16 @@ class FeedbooksStore(StorePlugin): price = ''.join(data.xpath('//a[@class="buy"]/text()')) if not price: price = '$0.00' + cover_url = '' + cover_url_img = data.xpath('//img') + if cover_url_img: + cover_url = cover_url_img[0].get('src') + cover_url.split('?')[0] counter -= 1 s = SearchResult() - s.cover_url = '' + s.cover_url = cover_url s.title = title.strip() s.author = author.strip() s.price = price.replace(' ', '').strip() diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index df29181c4d..35e885869c 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -5,12 +5,15 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import re +import time +from contextlib import closing from threading import Event, Thread from Queue import Queue from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \ - QModelIndex + QModelIndex, QPixmap, QSize +from calibre import browser from calibre.customize.ui import store_plugins from calibre.gui2 import NONE from calibre.gui2.store.search_ui import Ui_Dialog @@ -88,11 +91,12 @@ class SearchDialog(QDialog, Ui_Dialog): if res: result = res[1] result.store = res[0] + self.results_view.model().add_result(result) def open_store(self, index): result = self.results_view.model().get_result(index) - self.store_plugins[result.store].open(self.gui, self, result.item_data) + self.store_plugins[result.store].open(self.gui, self, result.detail_item) class SearchThread(Thread): @@ -106,12 +110,52 @@ class SearchThread(Thread): self.results = results self.abort = abort self.timeout = timeout + self.br = browser() def run(self): - for res in self.store_plugin.search(self.query, timeout=self.timeout): - if self.abort.is_set(): - return - self.results.put((self.store_name, res)) + try: + for res in self.store_plugin.search(self.query, timeout=self.timeout): + if self.abort.is_set(): + return + #if res.cover_url: + # with closing(self.br.open(res.cover_url, timeout=15)) as f: + # res.cover_data = f.read() + self.results.put((self.store_name, res)) + except: + pass + + +class CoverDownloadThread(Thread): + + def __init__(self, items, update_callback, timeout=5): + Thread.__init__(self) + self.daemon = True + self.items = items + self.update_callback = update_callback + self.timeout = timeout + self.br = browser() + + self._run = True + + def abort(self): + self._run = False + + def is_running(self): + return self._run + + def run(self): + while self._run: + try: + time.sleep(.1) + if not self.items.empty(): + item = self.items.get_nowait() + if item and item.cover_url: + with closing(self.br.open(item.cover_url, timeout=self.timeout)) as f: + item.cover_data = f.read() + self.items.task_done() + self.update_callback(item) + except: + continue class Matches(QAbstractItemModel): @@ -121,15 +165,20 @@ class Matches(QAbstractItemModel): def __init__(self): QAbstractItemModel.__init__(self) self.matches = [] + self.cover_download_queue = Queue() + self.cover_download_thread = CoverDownloadThread(self.cover_download_queue, self.update_result) + self.cover_download_thread.start() def clear_results(self): self.matches = [] + #self.cover_download_queue.queue.clear() self.reset() def add_result(self, result): + self.layoutAboutToBeChanged.emit() self.matches.append(result) - self.reset() - #self.dataChanged.emit(self.createIndex(self.rowCount() - 1, 0), self.createIndex(self.rowCount() - 1, self.columnCount())) + self.cover_download_queue.put(result) + self.layoutChanged.emit() def get_result(self, index): row = index.row() @@ -138,6 +187,12 @@ class Matches(QAbstractItemModel): else: return None + def update_result(self, result): + if not result in self.matches: + return + self.layoutAboutToBeChanged.emit() + self.layoutChanged.emit() + def index(self, row, column, parent=QModelIndex()): return self.createIndex(row, column) @@ -150,7 +205,7 @@ class Matches(QAbstractItemModel): return len(self.matches) def columnCount(self, *args): - return 5 + return len(self.HEADERS) def headerData(self, section, orientation, role): if role != Qt.DisplayRole: @@ -177,7 +232,10 @@ class Matches(QAbstractItemModel): return QVariant(result.store) return NONE elif role == Qt.DecorationRole: - pass + if col == 0 and result.cover_data: + p = QPixmap() + p.loadFromData(result.cover_data) + return QVariant(p) return NONE def data_as_text(self, result, col): @@ -205,3 +263,4 @@ class Matches(QAbstractItemModel): descending) if reset: self.reset() + diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui index a3b3ac674b..9d14874f10 100644 --- a/src/calibre/gui2/store/search.ui +++ b/src/calibre/gui2/store/search.ui @@ -32,9 +32,18 @@ true + + + 32 + 32 + + false + + false + false From 8ccfab537c48722ef92fc965fd58f4a11f486735 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 19:38:15 -0500 Subject: [PATCH 19/92] Add search result class. Size columns for searching. --- src/calibre/gui2/store/search.py | 22 ++++++++++++++++++---- src/calibre/gui2/store/search_result.py | 17 +++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 src/calibre/gui2/store/search_result.py diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 35e885869c..39b3162c43 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -47,6 +47,22 @@ class SearchDialog(QDialog, Ui_Dialog): self.checker.timeout.connect(self.get_results) self.results_view.activated.connect(self.open_store) + self.resize_columns() + + def resize_columns(self): + total = 600 + # Cover + self.results_view.setColumnWidth(0, 85) + total = total - 85 + # Title + self.results_view.setColumnWidth(1,int(total*.35)) + # Author + self.results_view.setColumnWidth(2,int(total*.35)) + # Price + self.results_view.setColumnWidth(3, int(total*.10)) + # Store + self.results_view.setColumnWidth(4, int(total*.20)) + def do_search(self, checked=False): # Stop all running threads. self.checker.stop() @@ -117,9 +133,6 @@ class SearchThread(Thread): for res in self.store_plugin.search(self.query, timeout=self.timeout): if self.abort.is_set(): return - #if res.cover_url: - # with closing(self.br.open(res.cover_url, timeout=15)) as f: - # res.cover_data = f.read() self.results.put((self.store_name, res)) except: pass @@ -171,12 +184,13 @@ class Matches(QAbstractItemModel): def clear_results(self): self.matches = [] - #self.cover_download_queue.queue.clear() + self.cover_download_queue.queue.clear() self.reset() def add_result(self, result): self.layoutAboutToBeChanged.emit() self.matches.append(result) + self.cover_download_queue.put(result) self.layoutChanged.emit() diff --git a/src/calibre/gui2/store/search_result.py b/src/calibre/gui2/store/search_result.py new file mode 100644 index 0000000000..6517f23b3d --- /dev/null +++ b/src/calibre/gui2/store/search_result.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +class SearchResult(object): + + def __init__(self): + self.store = '' + self.cover_url = '' + self.cover_data = None + self.title = '' + self.author = '' + self.price = '' + self.store = '' + self.detail_item = '' From 5d92612f2eaa43c75c22e7aa60df0e84309b3352 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 20:03:20 -0500 Subject: [PATCH 20/92] ManyBooks get cover_url --- src/calibre/gui2/store/manybooks_plugin.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index abb27206a9..9a45ddc5b4 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -57,10 +57,19 @@ class ManyBooksStore(StorePlugin): author = author.split('-')[0] price = '$0.00' + cover_url = '' + with closing(br.open('http://manybooks.net/titles/%s' % id.strip(), timeout=timeout)) as f_i: + doc_i = html.fromstring(f_i.read()) + for img in doc_i.xpath('//img'): + src = img.get('src', None) + if src and src.endswith('-thumb.jpg'): + cover_url = src + print cover_url + counter -= 1 s = SearchResult() - s.cover_url = '' + s.cover_url = cover_url s.title = title.strip() s.author = author.strip() s.price = price.strip() From 3883a1d6a24fbe99f96258342e4197172a06e200 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 20:39:35 -0500 Subject: [PATCH 21/92] Make user agent consistant. Remove debugging code. --- src/calibre/__init__.py | 5 +++-- src/calibre/gui2/store/web_control.py | 16 +++++----------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 221f5911c6..61b8bd36f8 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -240,6 +240,8 @@ def get_parsed_proxy(typ='http', debug=True): prints('Using http proxy', str(ans)) return ans +USER_AGENT = 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101210 Gentoo Firefox/3.6.13' +USER_AGENT_MOBILE = 'Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016' def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None): ''' @@ -254,8 +256,7 @@ def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None): opener.set_handle_refresh(True, max_time=max_time, honor_time=honor_time) opener.set_handle_robots(False) if user_agent is None: - user_agent = ' Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016' if mobile_browser else \ - 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101210 Gentoo Firefox/3.6.13' + user_agent = USER_AGENT_MOBILE if mobile_browser else USER_AGENT opener.addheaders = [('User-agent', user_agent)] http_proxy = get_proxies().get('http', None) if http_proxy: diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index 3b6337f597..d679d2d45d 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -8,19 +8,20 @@ from cookielib import Cookie, CookieJar from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString +from calibre import USER_AGENT + class NPWebView(QWebView): def __init__(self, *args): QWebView.__init__(self, *args) self.gui = None - #self.setPage(NPWebPage()) + self.setPage(NPWebPage()) self.page().networkAccessManager().setCookieJar(QNetworkCookieJar()) self.page().setForwardUnsupportedContent(True) self.page().unsupportedContent.connect(self.start_download) self.page().downloadRequested.connect(self.start_download) self.page().networkAccessManager().sslErrors.connect(self.ignore_ssl_errors) - #self.page().networkAccessManager().finished.connect(self.fin) def createWindow(self, type): if type == QWebPage.WebBrowserWindow: @@ -33,7 +34,6 @@ class NPWebView(QWebView): def start_download(self, request): if not self.gui: - print 'no gui' return url = unicode(request.url().toString()) @@ -41,13 +41,7 @@ class NPWebView(QWebView): def ignore_ssl_errors(self, reply, errors): reply.ignoreSslErrors(errors) - - def fin(self, reply): - if reply.error(): - print 'error' - print reply.error() - #print reply.attribute(QNetworkRequest.HttpStatusCodeAttribute).toInt() - + def get_cookies(self): cj = CookieJar() @@ -99,4 +93,4 @@ class NPWebView(QWebView): class NPWebPage(QWebPage): def userAgentForUrl(self, url): - return 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Chrome/9.0.597.102 Safari/534.13' + return USER_AGENT From a1cdfeb4ba3f1573c311452baf0a8f0223281fc9 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 21:15:54 -0500 Subject: [PATCH 22/92] Construct manybooks cover url instead of parsing it as manybooks is very slow. --- src/calibre/gui2/store/amazon_plugin.py | 2 +- src/calibre/gui2/store/feedbooks_plugin.py | 2 +- src/calibre/gui2/store/gutenberg_plugin.py | 2 +- src/calibre/gui2/store/manybooks_plugin.py | 21 +++++++++++---------- src/calibre/gui2/store/web_control.py | 2 +- 5 files changed, 15 insertions(+), 14 deletions(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index d3ab7c8911..ab6fdce8d8 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -24,7 +24,7 @@ class AmazonKindleStore(StorePlugin): def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, self.ASTORE_URL, parent, detail_item) - d.setWindowTitle('Amazon Kindle Store') + d.setWindowTitle(self.name) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 2be8576515..5c5d430a05 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -22,7 +22,7 @@ class FeedbooksStore(StorePlugin): def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://m.feedbooks.com/', parent, detail_item) - d.setWindowTitle('Feedbooks') + d.setWindowTitle(self.name) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 5d55844180..6917165474 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -22,7 +22,7 @@ class GutenbergStore(StorePlugin): def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, detail_item) - d.setWindowTitle('Project Gutenberg') + d.setWindowTitle(self.name) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 9a45ddc5b4..1d68dc774b 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -4,6 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import re import urllib2 from contextlib import closing @@ -22,7 +23,7 @@ class ManyBooksStore(StorePlugin): def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://manybooks.net/', parent, detail_item) - d.setWindowTitle('ManyBooks') + d.setWindowTitle(self.name) d = d.exec_() def search(self, query, max_results=10, timeout=60): @@ -51,6 +52,7 @@ class ManyBooksStore(StorePlugin): if '/titles/' not in url: continue id = url.split('/')[-1] + id = id.strip() heading = ''.join(url_a.xpath('text()')) title, _, author = heading.partition('by') @@ -58,14 +60,13 @@ class ManyBooksStore(StorePlugin): price = '$0.00' cover_url = '' - with closing(br.open('http://manybooks.net/titles/%s' % id.strip(), timeout=timeout)) as f_i: - doc_i = html.fromstring(f_i.read()) - for img in doc_i.xpath('//img'): - src = img.get('src', None) - if src and src.endswith('-thumb.jpg'): - cover_url = src - print cover_url - + mo = re.match('^\D+', id) + if mo: + cover_name = mo.group() + cover_name = cover_name.replace('etext', '') + cover_id = id.split('.')[0] + cover_url = 'http://manybooks_images.s3.amazonaws.com/original_covers/' + id[0] + '/' + cover_name + '/' + cover_id + '-thumb.jpg' + counter -= 1 s = SearchResult() @@ -73,6 +74,6 @@ class ManyBooksStore(StorePlugin): s.title = title.strip() s.author = author.strip() s.price = price.strip() - s.detail_item = '/titles/' + id.strip() + s.detail_item = '/titles/' + id yield s diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index d679d2d45d..bd8efbe588 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -11,7 +11,7 @@ from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QSt from calibre import USER_AGENT class NPWebView(QWebView): - + def __init__(self, *args): QWebView.__init__(self, *args) self.gui = None From cd5e47d5ff50a5a0243e467e4730c95c136cd147 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 22:03:43 -0500 Subject: [PATCH 23/92] Disable Amazon plugin for the time being. Add Smashwords plugin. --- src/calibre/customize/builtins.py | 3 +- src/calibre/gui2/store/manybooks_plugin.py | 2 +- src/calibre/gui2/store/search.py | 4 +- src/calibre/gui2/store/smashwords_plugin.py | 74 +++++++++++++++++++++ src/calibre/gui2/store/web_store_dialog.py | 4 +- 5 files changed, 82 insertions(+), 5 deletions(-) create mode 100644 src/calibre/gui2/store/smashwords_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 76aaf3a8ce..451a1c9b62 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1045,7 +1045,8 @@ from calibre.gui2.store.amazon_plugin import AmazonKindleStore from calibre.gui2.store.gutenberg_plugin import GutenbergStore from calibre.gui2.store.feedbooks_plugin import FeedbooksStore from calibre.gui2.store.manybooks_plugin import ManyBooksStore +from calibre.gui2.store.smashwords_plugin import SmashwordsStore -plugins += [AmazonKindleStore, GutenbergStore, FeedbooksStore, ManyBooksStore] +plugins += [GutenbergStore, FeedbooksStore, ManyBooksStore, SmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 1d68dc774b..ac9cac5323 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -17,7 +17,7 @@ from calibre.gui2.store.search_result import SearchResult class ManyBooksStore(StorePlugin): name = 'ManyBooks' - description = _('The best ebooks at the best price: free!.') + description = _('The best ebooks at the best price: free!') def open(self, gui, parent=None, detail_item=None): diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 39b3162c43..82d6fc5e65 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -134,8 +134,8 @@ class SearchThread(Thread): if self.abort.is_set(): return self.results.put((self.store_name, res)) - except: - pass + except Exception as e: + print e class CoverDownloadThread(Thread): diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py new file mode 100644 index 0000000000..c4eb1bb741 --- /dev/null +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from calibre import browser +from calibre.customize import StorePlugin +from calibre.gui2.store.search_result import SearchResult + +class SmashwordsStore(StorePlugin): + + name = 'Smashwords' + description = _('Your ebook. Your way.') + + + def open(self, gui, parent=None, detail_item=None): + from calibre.gui2.store.web_store_dialog import WebStoreDialog + d = WebStoreDialog(gui, 'http://www.smashwords.com/?ref=usernone', parent, detail_item) + d.setWindowTitle(self.name) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.smashwords.com/books/search?query=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@id="pageCenterContent2"]//div[@class="bookCoverImg"]'): + if counter <= 0: + break + data = html.fromstring(html.tostring(data)) + + id = None + id_a = data.xpath('//a[@class="bookTitle"]') + if id_a: + id = id_a[0].get('href', None) + if id: + id = id.split('/')[-1] + if not id: + continue + + cover_url = '' + c_url = data.get('style', None) + if c_url: + mo = re.search(r'http://[^\'"]+', c_url) + if mo: + cover_url = mo.group() + + title = ''.join(data.xpath('//a[@class="bookTitle"]/text()')) + subnote = ''.join(data.xpath('//span[@class="subnote"]/text()')) + author = ''.join(data.xpath('//span[@class="subnote"]/a/text()')) + price = subnote.partition('$')[2] + price = price.split(u'\xa0')[0] + price = '$' + price + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/books/view/' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py index 3910ca6912..fcb294cd69 100644 --- a/src/calibre/gui2/store/web_store_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -42,7 +42,9 @@ class WebStoreDialog(QDialog, Ui_Dialog): def go_home(self, checked=False, detail_item=None): url = self.base_url if detail_item: - url += '/' + urllib.quote(detail_item) + url, q, ref = url.partition('?') + url = url + '/' + urllib.quote(detail_item) + q + ref + # Reduce redundant /'s because some stores # (Feedbooks) and server frameworks (cherrypy) # choke on them. From a097755a5c8f03b17740a0268b82ae570d5662c1 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 26 Feb 2011 22:17:10 -0500 Subject: [PATCH 24/92] Resize images to fit in cell size. --- src/calibre/gui2/store/search.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 82d6fc5e65..1e94e172ba 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -18,6 +18,7 @@ from calibre.customize.ui import store_plugins from calibre.gui2 import NONE from calibre.gui2.store.search_ui import Ui_Dialog from calibre.utils.icu import sort_key +from calibre.utils.magick.draw import thumbnail class SearchDialog(QDialog, Ui_Dialog): @@ -134,8 +135,8 @@ class SearchThread(Thread): if self.abort.is_set(): return self.results.put((self.store_name, res)) - except Exception as e: - print e + except: + pass class CoverDownloadThread(Thread): @@ -160,11 +161,14 @@ class CoverDownloadThread(Thread): while self._run: try: time.sleep(.1) - if not self.items.empty(): + while not self.items.empty(): + if not self._run: + break item = self.items.get_nowait() if item and item.cover_url: with closing(self.br.open(item.cover_url, timeout=self.timeout)) as f: item.cover_data = f.read() + item.cover_data = thumbnail(item.cover_data, 64, 64)[2] self.items.task_done() self.update_callback(item) except: @@ -250,6 +254,8 @@ class Matches(QAbstractItemModel): p = QPixmap() p.loadFromData(result.cover_data) return QVariant(p) + elif role == Qt.SizeHintRole: + return QSize(64, 64) return NONE def data_as_text(self, result, col): From 15816210ca4b854b6806774859130b9c3c33128c Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 07:24:17 -0500 Subject: [PATCH 25/92] Fix saving to disk. Offer to save file to disk when user wants to download a non support ebook file. --- src/calibre/gui2/store/web_control.py | 25 ++++++++++++++++++++++--- src/calibre/gui2/store_download.py | 16 ++++++++++------ 2 files changed, 32 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index bd8efbe588..0c9109b140 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -4,11 +4,14 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import os from cookielib import Cookie, CookieJar -from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString +from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString, \ + QFileDialog -from calibre import USER_AGENT +from calibre import USER_AGENT, browser +from calibre.ebooks import BOOK_EXTENSIONS class NPWebView(QWebView): @@ -37,7 +40,23 @@ class NPWebView(QWebView): return url = unicode(request.url().toString()) - self.gui.download_from_store(url, self.get_cookies()) + cj = self.get_cookies() + + br = browser() + br.set_cookiejar(cj) + + basename = br.open(url).geturl().split('/')[-1] + ext = os.path.splitext(basename)[1][1:].lower() + if ext not in BOOK_EXTENSIONS: + home = os.getenv('USERPROFILE') or os.getenv('HOME') + name = QFileDialog.getSaveFileName(self, + _('File is not a supported ebook type. Save to disk?'), + os.path.join(home, basename), + '*.*') + if name: + self.gui.download_from_store(url, cj, name, False) + else: + self.gui.download_from_store(url, cj) def ignore_ssl_errors(self, reply, errors): reply.ignoreSslErrors(errors) diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index 71378c173f..7b0d68d454 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -55,6 +55,11 @@ class StoreDownloadJob(BaseJob): self._log_file.close() self._log_file = None + try: + os.remove(self.tmp_file_name) + except: + pass + self.job_manager.changed_queue.put(self) def log_write(self, what): @@ -126,12 +131,8 @@ class StoreDownloader(Thread): br = browser() br.set_cookiejar(job.cookie_jar) - - basename = br.open(url).geturl().split('/')[-1] - ext = os.path.splitext(basename)[1][1:].lower() - if ext not in BOOK_EXTENSIONS: - raise Exception(_('Not a valid ebook format.')) + basename = br.open(url).geturl().split('/')[-1] tf = PersistentTemporaryFile(suffix=basename) with closing(br.open(url)) as f: tf.write(f.read()) @@ -142,6 +143,9 @@ class StoreDownloader(Thread): url, save_loc, add_to_lib = job.args if not add_to_lib and job.tmp_file_name: return + ext = os.path.splitext(job.tmp_file_name)[1][1:].lower() + if ext not in BOOK_EXTENSIONS: + raise Exception(_('Not a support ebook format.')) ext = os.path.splitext(job.tmp_file_name)[1][1:] @@ -156,7 +160,7 @@ class StoreDownloader(Thread): if not save_loc and job.tmp_file_name: return - shutil.copy(job.tmp_fie_name, save_loc) + shutil.copy(job.tmp_file_name, save_loc) def download_from_store(self, callback, db, cookie_jar, url='', save_as_loc='', add_to_lib=True): description = _('Downloading %s') % url From 7036d1acc1d0fb7351b92be7ac751471c523c662 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 07:26:15 -0500 Subject: [PATCH 26/92] Use expanduser instead of env vars. --- src/calibre/gui2/store/web_control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index 0c9109b140..6b149cdd32 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -48,7 +48,7 @@ class NPWebView(QWebView): basename = br.open(url).geturl().split('/')[-1] ext = os.path.splitext(basename)[1][1:].lower() if ext not in BOOK_EXTENSIONS: - home = os.getenv('USERPROFILE') or os.getenv('HOME') + home = os.path.expanduser('~') name = QFileDialog.getSaveFileName(self, _('File is not a supported ebook type. Save to disk?'), os.path.join(home, basename), From 498b79e6d5901d4d2f57a270ba74b7952c8a31b9 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 11:05:30 -0500 Subject: [PATCH 27/92] Add tags to downloaded items. --- src/calibre/gui2/store/feedbooks_plugin.py | 1 + src/calibre/gui2/store/gutenberg_plugin.py | 1 + src/calibre/gui2/store/manybooks_plugin.py | 1 + src/calibre/gui2/store/search.ui | 42 ++++++++++++++++++++- src/calibre/gui2/store/smashwords_plugin.py | 1 + src/calibre/gui2/store/web_control.py | 6 ++- src/calibre/gui2/store/web_store_dialog.py | 3 ++ src/calibre/gui2/store/web_store_dialog.ui | 28 +++++++++++++- src/calibre/gui2/store_download.py | 22 ++++++----- 9 files changed, 92 insertions(+), 13 deletions(-) diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 5c5d430a05..9a3fb163df 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -23,6 +23,7 @@ class FeedbooksStore(StorePlugin): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://m.feedbooks.com/', parent, detail_item) d.setWindowTitle(self.name) + d.set_tags(self.name + ',' + _('store')) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 6917165474..a039ae2c66 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -23,6 +23,7 @@ class GutenbergStore(StorePlugin): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, detail_item) d.setWindowTitle(self.name) + d.set_tags(self.name + ',' + _('store')) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index ac9cac5323..3f85c6bec3 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -24,6 +24,7 @@ class ManyBooksStore(StorePlugin): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://manybooks.net/', parent, detail_item) d.setWindowTitle(self.name) + d.set_tags(self.name + ',' + _('store')) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui index 9d14874f10..2ee02b4632 100644 --- a/src/calibre/gui2/store/search.ui +++ b/src/calibre/gui2/store/search.ui @@ -55,8 +55,48 @@ + + + + QDialogButtonBox::Close + + + - + + + buttonBox + accepted() + Dialog + accept() + + + 307 + 524 + + + 307 + 272 + + + + + buttonBox + rejected() + Dialog + accept() + + + 307 + 524 + + + 307 + 272 + + + + diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index c4eb1bb741..5b84ce58ad 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -24,6 +24,7 @@ class SmashwordsStore(StorePlugin): from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://www.smashwords.com/?ref=usernone', parent, detail_item) d.setWindowTitle(self.name) + d.set_tags(self.name + ',' + _('store')) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index 6b149cdd32..e73279ce58 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -18,6 +18,7 @@ class NPWebView(QWebView): def __init__(self, *args): QWebView.__init__(self, *args) self.gui = None + self.tags = '' self.setPage(NPWebPage()) self.page().networkAccessManager().setCookieJar(QNetworkCookieJar()) @@ -35,6 +36,9 @@ class NPWebView(QWebView): def set_gui(self, gui): self.gui = gui + def set_tags(self, tags): + self.tags = tags + def start_download(self, request): if not self.gui: return @@ -56,7 +60,7 @@ class NPWebView(QWebView): if name: self.gui.download_from_store(url, cj, name, False) else: - self.gui.download_from_store(url, cj) + self.gui.download_from_store(url, cj, tags=self.tags) def ignore_ssl_errors(self, reply, errors): reply.ignoreSslErrors(errors) diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py index fcb294cd69..0c032e26e0 100644 --- a/src/calibre/gui2/store/web_store_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -30,6 +30,9 @@ class WebStoreDialog(QDialog, Ui_Dialog): self.go_home(detail_item=detail_item) + def set_tags(self, tags): + self.view.set_tags(tags) + def load_started(self): self.progress.setValue(0) diff --git a/src/calibre/gui2/store/web_store_dialog.ui b/src/calibre/gui2/store/web_store_dialog.ui index e2f15607ce..b89b9305be 100644 --- a/src/calibre/gui2/store/web_store_dialog.ui +++ b/src/calibre/gui2/store/web_store_dialog.ui @@ -17,7 +17,7 @@ true - + QFrame::StyledPanel @@ -72,6 +72,13 @@ + + + + Close + + + @@ -87,5 +94,22 @@ - + + + close + clicked() + Dialog + accept() + + + 917 + 635 + + + 480 + 327 + + + + diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index 7b0d68d454..b3c5e6279a 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -21,13 +21,13 @@ from calibre.utils.ipc.job import BaseJob class StoreDownloadJob(BaseJob): - def __init__(self, callback, description, job_manager, db, cookie_jar, url='', save_as_loc='', add_to_lib=True): + def __init__(self, callback, description, job_manager, db, cookie_jar, url='', save_as_loc='', add_to_lib=True, tags=[]): BaseJob.__init__(self, description) self.exception = None self.job_manager = job_manager self.db = db self.cookie_jar = cookie_jar - self.args = (url, save_as_loc, add_to_lib) + self.args = (url, save_as_loc, add_to_lib, tags) self.tmp_file_name = '' self.callback = callback self.log_path = None @@ -122,7 +122,7 @@ class StoreDownloader(Thread): traceback.print_exc() def _download(self, job): - url, save_loc, add_to_lib = job.args + url, save_loc, add_to_lib, tags = job.args if not url: raise Exception(_('No file specified to download.')) if not save_loc and not add_to_lib: @@ -140,7 +140,7 @@ class StoreDownloader(Thread): job.tmp_file_name = tf.name def _add(self, job): - url, save_loc, add_to_lib = job.args + url, save_loc, add_to_lib, tags = job.args if not add_to_lib and job.tmp_file_name: return ext = os.path.splitext(job.tmp_file_name)[1][1:].lower() @@ -152,19 +152,20 @@ class StoreDownloader(Thread): from calibre.ebooks.metadata.meta import get_metadata with open(job.tmp_file_name) as f: mi = get_metadata(f, ext) + mi.tags.extend(tags) job.db.add_books([job.tmp_file_name], [ext], [mi]) def _save_as(self, job): - url, save_loc, add_to_lib = job.args + url, save_loc, add_to_lib, tags = job.args if not save_loc and job.tmp_file_name: return shutil.copy(job.tmp_file_name, save_loc) - def download_from_store(self, callback, db, cookie_jar, url='', save_as_loc='', add_to_lib=True): + def download_from_store(self, callback, db, cookie_jar, url='', save_as_loc='', add_to_lib=True, tags=[]): description = _('Downloading %s') % url - job = StoreDownloadJob(callback, description, self.job_manager, db, cookie_jar, url, save_as_loc, add_to_lib) + job = StoreDownloadJob(callback, description, self.job_manager, db, cookie_jar, url, save_as_loc, add_to_lib, tags) self.job_manager.add_job(job) self.jobs.put(job) @@ -174,10 +175,13 @@ class StoreDownloadMixin(object): def __init__(self): self.store_downloader = StoreDownloader(self.job_manager) - def download_from_store(self, url='', cookie_jar=CookieJar(), save_as_loc='', add_to_lib=True): + def download_from_store(self, url='', cookie_jar=CookieJar(), save_as_loc='', add_to_lib=True, tags=[]): if not self.store_downloader.is_alive(): self.store_downloader.start() - self.store_downloader.download_from_store(Dispatcher(self.downloaded_from_store), self.library_view.model().db, cookie_jar, url, save_as_loc, add_to_lib) + if tags: + if isinstance(tags, basestring): + tags = tags.split(',') + self.store_downloader.download_from_store(Dispatcher(self.downloaded_from_store), self.library_view.model().db, cookie_jar, url, save_as_loc, add_to_lib, tags) self.status_bar.show_message(_('Downloading') + ' ' + url, 3000) def downloaded_from_store(self, job): From 1633c8f8d0c83c61e907441f0eb55a95d418a143 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 14:25:43 -0500 Subject: [PATCH 28/92] Have amazon kindle plugin open astore in external browser --- src/calibre/customize/builtins.py | 2 +- src/calibre/gui2/store/amazon_plugin.py | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 451a1c9b62..f37d7292ef 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1047,6 +1047,6 @@ from calibre.gui2.store.feedbooks_plugin import FeedbooksStore from calibre.gui2.store.manybooks_plugin import ManyBooksStore from calibre.gui2.store.smashwords_plugin import SmashwordsStore -plugins += [GutenbergStore, FeedbooksStore, ManyBooksStore, SmashwordsStore] +plugins += [AmazonKindleStore, GutenbergStore, FeedbooksStore, ManyBooksStore, SmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index ab6fdce8d8..18fd7bbe2c 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -10,6 +10,8 @@ from contextlib import closing from lxml import html +from PyQt4.Qt import QUrl + from calibre import browser from calibre.customize import StorePlugin from calibre.gui2.store.search_result import SearchResult @@ -19,13 +21,12 @@ class AmazonKindleStore(StorePlugin): name = 'Amazon Kindle' description = _('Buy Kindle books from Amazon') - ASTORE_URL = 'http://astore.amazon.com/josbl0e-20/' - def open(self, gui, parent=None, detail_item=None): - from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, self.ASTORE_URL, parent, detail_item) - d.setWindowTitle(self.name) - d = d.exec_() + from calibre.gui2 import open_url + astore_link = 'http://astore.amazon.com/josbl0e-20' + if detail_item: + astore_link += detail_item + open_url(QUrl(astore_link)) def search(self, query, max_results=10, timeout=60): url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query) From 81b734c5c17ab8025589c57d2b578287b6f2daec Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 16:27:38 -0500 Subject: [PATCH 29/92] Amazon Kindle store return cover_url --- src/calibre/gui2/store/amazon_plugin.py | 47 ++++++++++++--------- src/calibre/gui2/store/gutenberg_plugin.py | 2 +- src/calibre/gui2/store/manybooks_plugin.py | 2 +- src/calibre/gui2/store/smashwords_plugin.py | 7 ++- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 18fd7bbe2c..00965f31e2 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -47,29 +47,38 @@ class AmazonKindleStore(StorePlugin): if 'kindle' not in type.lower(): continue - title = ''.join(data.xpath('div[@class="productTitle"]/a/text()')) - author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) - author = author.split('by')[-1] - price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) - # We must have an asin otherwise we can't easily reference the # book later. - asin = data.xpath('div[@class="productTitle"]/a[1]') - if asin: - asin = asin[0].get('href', '') - m = re.search(r'/dp/(?P.+?)(/|$)', asin) + asin_href = None + asin_a = data.xpath('div[@class="productTitle"]/a[1]') + if asin_a: + asin_href = asin_a[0].get('href', '') + m = re.search(r'/dp/(?P.+?)(/|$)', asin_href) if m: asin = m.group('asin') else: continue + else: + continue + + cover_url = '' + if asin_href: + cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href) + if cover_img: + cover_url = cover_img[0] + + title = ''.join(data.xpath('div[@class="productTitle"]/a/text()')) + author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) + author = author.split('by')[-1] + price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) - counter -= 1 - - s = SearchResult() - s.cover_url = '' - s.title = title.strip() - s.author = author.strip() - s.price = price.strip() - s.detail_item = '/detail/' + asin.strip() - - yield s + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/detail/' + asin.strip() + + yield s diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index a039ae2c66..8d7649f814 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -52,7 +52,7 @@ class GutenbergStore(StorePlugin): id = url.split('/')[-1] heading = ''.join(url_a.xpath('text()')) - title, _, author = heading.partition('by') + title, _, author = heading.partition('by ') author = author.split('-')[0] price = '$0.00' diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 3f85c6bec3..807f8fe562 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -56,7 +56,7 @@ class ManyBooksStore(StorePlugin): id = id.strip() heading = ''.join(url_a.xpath('text()')) - title, _, author = heading.partition('by') + title, _, author = heading.partition('by ') author = author.split('-')[0] price = '$0.00' diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index 5b84ce58ad..31bce64398 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -4,6 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import random import re import urllib2 from contextlib import closing @@ -22,7 +23,11 @@ class SmashwordsStore(StorePlugin): def open(self, gui, parent=None, detail_item=None): from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, 'http://www.smashwords.com/?ref=usernone', parent, detail_item) + aff_id = 'usernone' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + aff_id = 'kovidgoyal' + d = WebStoreDialog(gui, 'http://www.smashwords.com/?ref=%s' % aff_id, parent, detail_item) d.setWindowTitle(self.name) d.set_tags(self.name + ',' + _('store')) d = d.exec_() From d22139cabcd98f7e99dd54c7c555b992a47ad9aa Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 16:29:57 -0500 Subject: [PATCH 30/92] Fix Feedbooks displaying author --- src/calibre/gui2/store/feedbooks_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 9a3fb163df..45d6f87fdc 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -55,8 +55,8 @@ class FeedbooksStore(StorePlugin): if not id: continue - title = ''.join(data.xpath('//h5/a/text()')) - author = ''.join(data.xpath('//h6/a/text()')) + title = ''.join(data.xpath('//h5//a/text()')) + author = ''.join(data.xpath('//h6//a/text()')) price = ''.join(data.xpath('//a[@class="buy"]/text()')) if not price: price = '$0.00' From 9724d661735340fe05cd566b8216652d74df9340 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 17:26:26 -0500 Subject: [PATCH 31/92] Use affiliate id instead of astore. --- src/calibre/gui2/store/amazon_plugin.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 00965f31e2..3fd0e665cc 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -23,10 +23,12 @@ class AmazonKindleStore(StorePlugin): def open(self, gui, parent=None, detail_item=None): from calibre.gui2 import open_url - astore_link = 'http://astore.amazon.com/josbl0e-20' + aff_id = {'tag': 'josbl0e-cpb-20'} + store_link = 'http://www.amazon.com/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id if detail_item: - astore_link += detail_item - open_url(QUrl(astore_link)) + aff_id['asin'] = detail_item + store_link = 'http://www.amazon.com/dp/%(asin)s/?tag=%(tag)s' % aff_id + open_url(QUrl(store_link)) def search(self, query, max_results=10, timeout=60): url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query) @@ -79,6 +81,6 @@ class AmazonKindleStore(StorePlugin): s.title = title.strip() s.author = author.strip() s.price = price.strip() - s.detail_item = '/detail/' + asin.strip() + s.detail_item = asin.strip() yield s From e741fb3aeb705df5dbd3ce526dc999c500d19cff Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 19:34:04 -0500 Subject: [PATCH 32/92] ... --- src/calibre/gui2/store/amazon_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 3fd0e665cc..526103d9f9 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -19,7 +19,7 @@ from calibre.gui2.store.search_result import SearchResult class AmazonKindleStore(StorePlugin): name = 'Amazon Kindle' - description = _('Buy Kindle books from Amazon') + description = _('Kindle books from Amazon') def open(self, gui, parent=None, detail_item=None): from calibre.gui2 import open_url From 0eb9dbb4a491404a590053d5538f3b75f74e3795 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 20:20:58 -0500 Subject: [PATCH 33/92] Store search: Ability to control what stores are searched. --- src/calibre/gui2/store/search.py | 49 +++++++++- src/calibre/gui2/store/search.ui | 154 ++++++++++++++++++------------- 2 files changed, 135 insertions(+), 68 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 1e94e172ba..7c74ee288a 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -11,7 +11,8 @@ from threading import Event, Thread from Queue import Queue from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \ - QModelIndex, QPixmap, QSize + QModelIndex, QPixmap, QSize, QCheckBox, QVBoxLayout, QHBoxLayout, \ + QPushButton from calibre import browser from calibre.customize.ui import store_plugins @@ -41,12 +42,30 @@ class SearchDialog(QDialog, Ui_Dialog): self.model = Matches() self.results_view.setModel(self.model) + stores_group_layout = QVBoxLayout() + self.stores_group.setLayout(stores_group_layout) for x in store_plugins(): self.store_plugins[x.name] = x - + cbox = QCheckBox(x.name) + cbox.setChecked(True) + stores_group_layout.addWidget(cbox) + setattr(self, 'store_check_' + x.name, cbox) + + store_button_layout = QHBoxLayout() + stores_group_layout.addLayout(store_button_layout) + self.select_all_stores = QPushButton(_('All')) + self.select_invert_stores = QPushButton(_('Invert')) + self.select_none_stores = QPushButton(_('None')) + store_button_layout.addWidget(self.select_all_stores) + store_button_layout.addWidget(self.select_invert_stores) + store_button_layout.addWidget(self.select_none_stores) + self.search.clicked.connect(self.do_search) self.checker.timeout.connect(self.get_results) self.results_view.activated.connect(self.open_store) + self.select_all_stores.clicked.connect(self.stores_select_all) + self.select_invert_stores.clicked.connect(self.stores_select_invert) + self.select_none_stores.clicked.connect(self.stores_select_none) self.resize_columns() @@ -80,9 +99,10 @@ class SearchDialog(QDialog, Ui_Dialog): return for n in self.store_plugins: - t = SearchThread(query, (n, self.store_plugins[n]), self.results, self.abort, self.TIMEOUT) - self.running_threads.append(t) - t.start() + if getattr(self, 'store_check_' + n).isChecked(): + t = SearchThread(query, (n, self.store_plugins[n]), self.results, self.abort, self.TIMEOUT) + self.running_threads.append(t) + t.start() if self.running_threads: self.hang_check = 0 self.checker.start(100) @@ -115,6 +135,25 @@ class SearchDialog(QDialog, Ui_Dialog): result = self.results_view.model().get_result(index) self.store_plugins[result.store].open(self.gui, self, result.detail_item) + def get_store_checks(self): + checks = [] + for x in self.store_plugins: + check = getattr(self, 'store_check_' + x, None) + if check: + checks.append(check) + return checks + + def stores_select_all(self): + for check in self.get_store_checks(): + check.setChecked(True) + + def stores_select_invert(self): + for check in self.get_store_checks(): + check.setChecked(not check.isChecked()) + + def stores_select_none(self): + for check in self.get_store_checks(): + check.setChecked(False) class SearchThread(Thread): diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui index 2ee02b4632..19ff13300a 100644 --- a/src/calibre/gui2/store/search.ui +++ b/src/calibre/gui2/store/search.ui @@ -6,8 +6,8 @@ 0 0 - 616 - 545 + 937 + 669 @@ -16,81 +16,109 @@ true - - - + + + + + + + Query: + + + + + + + + + + Search + + + + - - - - Search + + + + Qt::Horizontal + + + Stores + + + + + + 2 + 0 + + + + true + + + + 32 + 32 + + + + false + + + false + + + false + + + true + + + false + + - - - - true - - - - 32 - 32 - - - - false - - - false - - - false - - - true - - - false - - - - - - - QDialogButtonBox::Close - - + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Close + + + + - buttonBox - accepted() + close + clicked() Dialog accept() - 307 - 524 - - - 307 - 272 - - - - - buttonBox - rejected() - Dialog - accept() - - - 307 - 524 + 526 + 525 307 From 84211bae545f67831c0363ba43053d50113a43c5 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 20:45:31 -0500 Subject: [PATCH 34/92] Layout check boxes better. --- src/calibre/gui2/store/search.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 7c74ee288a..8dadda7189 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -51,6 +51,7 @@ class SearchDialog(QDialog, Ui_Dialog): stores_group_layout.addWidget(cbox) setattr(self, 'store_check_' + x.name, cbox) + stores_group_layout.addStretch() store_button_layout = QHBoxLayout() stores_group_layout.addLayout(store_button_layout) self.select_all_stores = QPushButton(_('All')) From 4038afdec2e6d4e76aa08081d7e4d0f86ec7b8db Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 27 Feb 2011 21:08:44 -0500 Subject: [PATCH 35/92] Add rational for the functioning and opening of the amazon store. --- src/calibre/gui2/store/amazon_plugin.py | 83 +++++++++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 526103d9f9..49e0b98c77 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -22,6 +22,89 @@ class AmazonKindleStore(StorePlugin): description = _('Kindle books from Amazon') def open(self, gui, parent=None, detail_item=None): + ''' + Amazon comes with a number of difficulties. + + QWebView has major issues with Amazon.com. The largest of + issues is it simply doesn't work on a number of pages. + + When connecting to a number parts of Amazon.com (Kindle library + for instance) QNetworkAccessManager fails to connect with a + NetworkError of 399 - ProtocolFailure. The strange thing is, + when I check QNetworkRequest.HttpStatusCodeAttribute when the + 399 error is returned the status code is 200 (Ok). However, once + the QNetworkAccessManager decides there was a NetworkError it + does not download the page from Amazon. So I can't even set the + HTML in the QWebView myself. + + There is http://bugreports.qt.nokia.com/browse/QTWEBKIT-259 an + open bug about the issue but it is not correct. We can set the + useragent (Arora does) to something else and the above issue + will persist. This http://developer.qt.nokia.com/forums/viewthread/793 + gives a bit more information about the issue but as of now (27/Feb/2011) + there is no solution or work around. + + We cannot change the The linkDelegationPolicy to allow us to avoid + QNetworkAccessManager because it only works links. Forms aren't + included so the same issue persists on any part of the site (login) + that use a form to load a new page. + + Using an aStore was evaluated but I've decided against using it. + There are three major issues with an aStore. Because checkout is + handled by sending the user to Amazon we can't put it in a QWebView. + If we're sending the user to Amazon sending them there directly is + nicer. Also, we cannot put the aStore in a QWebView and let it open the + redirection the users default browser because the cookies with the + shopping cart won't transfer. + + Another issue with the aStore is how it handles the referral. It only + counts the referral for the items in the shopping card / the item + that directed the user to Amazon. Kindle books do not use the shopping + cart and send the user directly to Amazon for the purchase. In this + instance we would only get referral credit for the one book that the + aStore directs to Amazon that the user buys. Any other purchases we + won't get credit for. + + The last issue with the aStore is performance. Even though it's an + Amazon site it's alow. So much slower than Amazon.com that it makes + me not want to browse books using it. The look and feel are lesser + issues. So is the fact that it almost seems like the purchase is + with calibre. This can cause some support issues because we can't + do much for issues with Amazon.com purchase hiccups. + + Another option that was evaluated was the Product Advertising API. + The reasons against this are complexity. It would take a lot of work + to basically re-create Amazon.com within calibre. The Product + Advertising API is also designed with being run on a server not + in an app. The signing keys would have to be made avaliable to ever + calibre user which means bad things could be done with our account. + + The Product Advertising API also assumes the same browser for easy + shopping cart transfer to Amazon. With QWebView not working and there + not being an easy way to transfer cookies between a QWebView and the + users default browser this won't work well. + + We could create our own website on the calibre server and create an + Amazon Product Advertising API store. However, this goes back to the + complexity argument. Why spend the time recreating Amazon.com + + The final and largest issue against using the Product Advertising API + is the Efficiency Guidelines: + + "Each account used to access the Product Advertising API will be allowed + an initial usage limit of 2,000 requests per hour. Each account will + receive an additional 500 requests per hour (up to a maximum of 25,000 + requests per hour) for every $1 of shipped item revenue driven per hour + in a trailing 30-day period. Usage thresholds are recalculated daily based + on revenue performance." + + With over two million users a limit of 2,000 request per hour could + render our store unusable for no other reason than Amazon rate + limiting our traffic. + + The best (I use the term lightly here) solution is to open Amazon.com + in the users default browser and set the affiliate id as part of the url. + ''' from calibre.gui2 import open_url aff_id = {'tag': 'josbl0e-cpb-20'} store_link = 'http://www.amazon.com/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id From 1deb7783a171bebc522b5af7bef277cfa374e97e Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 11:40:02 -0500 Subject: [PATCH 36/92] Put store selection in a scroll area. --- src/calibre/gui2/store/search.py | 9 --- src/calibre/gui2/store/search.ui | 120 ++++++++++++++++++++++++------- 2 files changed, 93 insertions(+), 36 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 8dadda7189..960b8e25c7 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -50,16 +50,7 @@ class SearchDialog(QDialog, Ui_Dialog): cbox.setChecked(True) stores_group_layout.addWidget(cbox) setattr(self, 'store_check_' + x.name, cbox) - stores_group_layout.addStretch() - store_button_layout = QHBoxLayout() - stores_group_layout.addLayout(store_button_layout) - self.select_all_stores = QPushButton(_('All')) - self.select_invert_stores = QPushButton(_('Invert')) - self.select_none_stores = QPushButton(_('None')) - store_button_layout.addWidget(self.select_all_stores) - store_button_layout.addWidget(self.select_invert_stores) - store_button_layout.addWidget(self.select_none_stores) self.search.clicked.connect(self.do_search) self.checker.timeout.connect(self.get_results) diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui index 19ff13300a..077163dafa 100644 --- a/src/calibre/gui2/store/search.ui +++ b/src/calibre/gui2/store/search.ui @@ -39,46 +39,112 @@ - + Qt::Horizontal - + Stores + + + + + true + + + + + 0 + 0 + 215 + 116 + + + + + + + + + + + All + + + + + + + Invert + + + + + + + None + + + + + + - + - + 2 0 - - true - - - - 32 - 32 - - - - false - - - false - - - false - - - true - - - false + + Qt::Horizontal + + + Qt::Horizontal + + + + + 1 + 0 + + + + + 0 + 0 + + + + true + + + + 32 + 32 + + + + false + + + false + + + false + + + true + + + false + + + From 588adb70e4d598fa4dfd95d315cfc06e6923008d Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 11:44:13 -0500 Subject: [PATCH 37/92] Add Kovids Amazon id. --- src/calibre/gui2/store/amazon_plugin.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 49e0b98c77..7086cfba8f 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -4,6 +4,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import random import re import urllib2 from contextlib import closing @@ -107,6 +108,9 @@ class AmazonKindleStore(StorePlugin): ''' from calibre.gui2 import open_url aff_id = {'tag': 'josbl0e-cpb-20'} + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + aff_id['tag'] = 'calibrebs-20' store_link = 'http://www.amazon.com/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id if detail_item: aff_id['asin'] = detail_item From 98a0211463450e098707c4b781db952a4f20f518 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 19:20:27 -0500 Subject: [PATCH 38/92] Use 4 thread threadpool for downloading search results. --- src/calibre/customize/__init__.py | 7 +- src/calibre/customize/builtins.py | 1 + src/calibre/gui2/store/feedbooks_plugin.py | 19 ++++ src/calibre/gui2/store/gutenberg_plugin.py | 23 +++- src/calibre/gui2/store/manybooks_plugin.py | 19 ++++ src/calibre/gui2/store/search.py | 119 +++++++++++++------- src/calibre/gui2/store/search_result.py | 3 +- src/calibre/gui2/store/smashwords_plugin.py | 19 ++++ 8 files changed, 165 insertions(+), 45 deletions(-) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index c1e19bd543..f49c8496c5 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -586,7 +586,9 @@ class StorePlugin(Plugin): # {{{ supported_platforms = ['windows', 'osx', 'linux'] author = 'John Schember' - type = _('Stores') + type = _('Store') + # This needs to be changed to (0, 8, 0) + minimum_calibre_version = (0, 4, 118) def open(self, gui, parent=None, detail_item=None): ''' @@ -609,6 +611,9 @@ class StorePlugin(Plugin): # {{{ item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. ''' raise NotImplementedError() + + def get_settings(self): + raise NotImplementedError() # }}} diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index f37d7292ef..913b5d3c7b 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -806,6 +806,7 @@ class ActionNextMatch(InterfaceActionBase): class ActionStore(InterfaceActionBase): name = 'Store' + author = 'John Schember' actual_plugin = 'calibre.gui2.actions.store:StoreAction' plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 45d6f87fdc..57f9e2d472 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -76,3 +76,22 @@ class FeedbooksStore(StorePlugin): s.detail_item = id.strip() yield s + + def customization_help(self, gui=False): + return 'Customize the behavior of this store.' + + def config_widget(self): + from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget + return BasicStoreConfigWidget(self) + + def save_settings(self, config_widget): + from calibre.gui2.store.basic_config_widget import save_settings + save_settings(config_widget) + + def get_settings(self): + from calibre.gui2 import gprefs + settings = {} + + settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') + + return settings diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 8d7649f814..952f7f512f 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -17,13 +17,13 @@ class GutenbergStore(StorePlugin): name = 'Project Gutenberg' description = _('The first producer of free ebooks.') - def open(self, gui, parent=None, detail_item=None): + settings = self.get_settings() from calibre.gui2.store.web_store_dialog import WebStoreDialog d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, detail_item) d.setWindowTitle(self.name) - d.set_tags(self.name + ',' + _('store')) + d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() def search(self, query, max_results=10, timeout=60): @@ -66,3 +66,22 @@ class GutenbergStore(StorePlugin): s.detail_item = '/ebooks/' + id.strip() yield s + + def customization_help(self, gui=False): + return 'Customize the behavior of this store.' + + def config_widget(self): + from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget + return BasicStoreConfigWidget(self) + + def save_settings(self, config_widget): + from calibre.gui2.store.basic_config_widget import save_settings + save_settings(config_widget) + + def get_settings(self): + from calibre.gui2 import gprefs + settings = {} + + settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') + + return settings diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 807f8fe562..d2531cc7b1 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -78,3 +78,22 @@ class ManyBooksStore(StorePlugin): s.detail_item = '/titles/' + id yield s + + def customization_help(self, gui=False): + return 'Customize the behavior of this store.' + + def config_widget(self): + from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget + return BasicStoreConfigWidget(self) + + def save_settings(self, config_widget): + from calibre.gui2.store.basic_config_widget import save_settings + save_settings(config_widget) + + def get_settings(self): + from calibre.gui2 import gprefs + settings = {} + + settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') + + return settings diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 960b8e25c7..0d4bcb8252 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -25,6 +25,8 @@ class SearchDialog(QDialog, Ui_Dialog): HANG_TIME = 75000 # milliseconds seconds TIMEOUT = 75 # seconds + SEARCH_THREAD_TOTAL = 4 + COVER_DOWNLOAD_THREAD_TOTAL = 2 def __init__(self, gui, *args): QDialog.__init__(self, *args) @@ -33,9 +35,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.gui = gui self.store_plugins = {} - self.running_threads = [] - self.results = Queue() - self.abort = Event() + self.search_thread_pool = SearchThreadPool(self.SEARCH_THREAD_TOTAL) self.checker = QTimer() self.hang_check = 0 @@ -78,10 +78,7 @@ class SearchDialog(QDialog, Ui_Dialog): def do_search(self, checked=False): # Stop all running threads. self.checker.stop() - self.abort.set() - self.running_threads = [] - self.results = Queue() - self.abort = Event() + self.search_thread_pool.abort() # Clear the visible results. self.results_view.model().clear_results() @@ -92,40 +89,32 @@ class SearchDialog(QDialog, Ui_Dialog): for n in self.store_plugins: if getattr(self, 'store_check_' + n).isChecked(): - t = SearchThread(query, (n, self.store_plugins[n]), self.results, self.abort, self.TIMEOUT) - self.running_threads.append(t) - t.start() - if self.running_threads: + self.search_thread_pool.add_task(query, n, self.store_plugins[n], self.TIMEOUT) + if self.search_thread_pool.has_tasks(): self.hang_check = 0 self.checker.start(100) + self.search_thread_pool.start_threads() def get_results(self): # We only want the search plugins to run # a maximum set amount of time before giving up. self.hang_check += 1 if self.hang_check >= self.HANG_TIME: - self.abort.set() + self.search_thread_pool.abort() self.checker.stop() else: # Stop the checker if not threads are running. - running = False - for t in self.running_threads: - if t.is_alive(): - running = True - if not running: + if not self.search_thread_pool.threads_running(): self.checker.stop() - while not self.results.empty(): - res = self.results.get_nowait() + while self.search_thread_pool.has_results(): + res = self.search_thread_pool.get_result_no_wait() if res: - result = res[1] - result.store = res[0] - - self.results_view.model().add_result(result) + self.results_view.model().add_result(res) def open_store(self, index): result = self.results_view.model().get_result(index) - self.store_plugins[result.store].open(self.gui, self, result.detail_item) + self.store_plugins[result.store_name].open(self.gui, self, result.detail_item) def get_store_checks(self): checks = [] @@ -147,27 +136,77 @@ class SearchDialog(QDialog, Ui_Dialog): for check in self.get_store_checks(): check.setChecked(False) + +class SearchThreadPool(object): + + def __init__(self, thread_count): + self.tasks = Queue() + self.results = Queue() + self.threads = [] + self.thread_count = thread_count + + def start_threads(self): + for i in range(self.thread_count): + t = SearchThread(self.tasks, self.results) + self.threads.append(t) + t.start() + + def abort(self): + self.tasks = Queue() + self.results = Queue() + self.threads = [] + for t in self.threads: + t.abort() + + def add_task(self, query, store_name, store_plugin, timeout): + self.tasks.put((query, store_name, store_plugin, timeout)) + + def has_tasks(self): + return not self.tasks.empty() + + def get_result(self): + return self.results.get() + + def get_result_no_wait(self): + return self.results.get_nowait() + + def result_count(self): + return len(self.results) + + def has_results(self): + return not self.results.empty() + + def threads_running(self): + for t in self.threads: + if t.is_alive(): + return True + return False + + class SearchThread(Thread): - def __init__(self, query, store, results, abort, timeout): + def __init__(self, tasks, results): Thread.__init__(self) self.daemon = True - self.query = query - self.store_name = store[0] - self.store_plugin = store[1] + self.tasks = tasks self.results = results - self.abort = abort - self.timeout = timeout - self.br = browser() + self._run = True + + def abort(self): + self._run = False def run(self): - try: - for res in self.store_plugin.search(self.query, timeout=self.timeout): - if self.abort.is_set(): - return - self.results.put((self.store_name, res)) - except: - pass + while self._run and not self.tasks.empty(): + try: + query, store_name, store_plugin, timeout = self.tasks.get() + for res in store_plugin.search(query, timeout=timeout): + if not self._run: + return + res.store_name = store_name + self.results.put(res) + self.tasks.task_done() + except: + pass class CoverDownloadThread(Thread): @@ -278,7 +317,7 @@ class Matches(QAbstractItemModel): elif col == 3: return QVariant(result.price) elif col == 4: - return QVariant(result.store) + return QVariant(result.store_name) return NONE elif role == Qt.DecorationRole: if col == 0 and result.cover_data: @@ -302,7 +341,7 @@ class Matches(QAbstractItemModel): text = re.sub(r'\D', '', text) text = text.rjust(6, '0') elif col == 4: - text = result.store + text = result.store_name return text def sort(self, col, order, reset=True): diff --git a/src/calibre/gui2/store/search_result.py b/src/calibre/gui2/store/search_result.py index 6517f23b3d..7eb2987a78 100644 --- a/src/calibre/gui2/store/search_result.py +++ b/src/calibre/gui2/store/search_result.py @@ -7,11 +7,10 @@ __docformat__ = 'restructuredtext en' class SearchResult(object): def __init__(self): - self.store = '' + self.store_name = '' self.cover_url = '' self.cover_data = None self.title = '' self.author = '' self.price = '' - self.store = '' self.detail_item = '' diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index 31bce64398..5724b65669 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -78,3 +78,22 @@ class SmashwordsStore(StorePlugin): s.detail_item = '/books/view/' + id.strip() yield s + + def customization_help(self, gui=False): + return 'Customize the behavior of this store.' + + def config_widget(self): + from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget + return BasicStoreConfigWidget(self) + + def save_settings(self, config_widget): + from calibre.gui2.store.basic_config_widget import save_settings + save_settings(config_widget) + + def get_settings(self): + from calibre.gui2 import gprefs + settings = {} + + settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') + + return settings From 93f855abe989d8f0bd053ad0ef70336928cfaf47 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 19:45:05 -0500 Subject: [PATCH 39/92] User generic thread pool base class. Use 2 thread thread pool for downloading covers. --- src/calibre/gui2/store/search.py | 121 +++++++++++++++++++------------ 1 file changed, 73 insertions(+), 48 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 0d4bcb8252..272dd6d5f1 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -21,12 +21,12 @@ from calibre.gui2.store.search_ui import Ui_Dialog from calibre.utils.icu import sort_key from calibre.utils.magick.draw import thumbnail -class SearchDialog(QDialog, Ui_Dialog): +HANG_TIME = 75000 # milliseconds seconds +TIMEOUT = 75 # seconds +SEARCH_THREAD_TOTAL = 4 +COVER_DOWNLOAD_THREAD_TOTAL = 2 - HANG_TIME = 75000 # milliseconds seconds - TIMEOUT = 75 # seconds - SEARCH_THREAD_TOTAL = 4 - COVER_DOWNLOAD_THREAD_TOTAL = 2 +class SearchDialog(QDialog, Ui_Dialog): def __init__(self, gui, *args): QDialog.__init__(self, *args) @@ -35,7 +35,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.gui = gui self.store_plugins = {} - self.search_thread_pool = SearchThreadPool(self.SEARCH_THREAD_TOTAL) + self.search_pool = SearchThreadPool(SearchThread, SEARCH_THREAD_TOTAL) self.checker = QTimer() self.hang_check = 0 @@ -78,7 +78,7 @@ class SearchDialog(QDialog, Ui_Dialog): def do_search(self, checked=False): # Stop all running threads. self.checker.stop() - self.search_thread_pool.abort() + self.search_pool.abort() # Clear the visible results. self.results_view.model().clear_results() @@ -89,26 +89,26 @@ class SearchDialog(QDialog, Ui_Dialog): for n in self.store_plugins: if getattr(self, 'store_check_' + n).isChecked(): - self.search_thread_pool.add_task(query, n, self.store_plugins[n], self.TIMEOUT) - if self.search_thread_pool.has_tasks(): + self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT) + if self.search_pool.has_tasks(): self.hang_check = 0 self.checker.start(100) - self.search_thread_pool.start_threads() + self.search_pool.start_threads() def get_results(self): # We only want the search plugins to run # a maximum set amount of time before giving up. self.hang_check += 1 - if self.hang_check >= self.HANG_TIME: - self.search_thread_pool.abort() + if self.hang_check >= HANG_TIME: + self.search_pool.abort() self.checker.stop() else: # Stop the checker if not threads are running. - if not self.search_thread_pool.threads_running(): + if not self.search_pool.threads_running(): self.checker.stop() - while self.search_thread_pool.has_results(): - res = self.search_thread_pool.get_result_no_wait() + while self.search_pool.has_results(): + res = self.search_pool.get_result_no_wait() if res: self.results_view.model().add_result(res) @@ -137,17 +137,22 @@ class SearchDialog(QDialog, Ui_Dialog): check.setChecked(False) -class SearchThreadPool(object): +class GenericDownloadThreadPool(object): + ''' + add_task must be implemented in a subclass. + ''' - def __init__(self, thread_count): + def __init__(self, thread_type, thread_count): + self.thread_type = thread_type + self.thread_count = thread_count + self.tasks = Queue() self.results = Queue() self.threads = [] - self.thread_count = thread_count def start_threads(self): for i in range(self.thread_count): - t = SearchThread(self.tasks, self.results) + t = self.thread_type(self.tasks, self.results) self.threads.append(t) t.start() @@ -157,9 +162,6 @@ class SearchThreadPool(object): self.threads = [] for t in self.threads: t.abort() - - def add_task(self, query, store_name, store_plugin, timeout): - self.tasks.put((query, store_name, store_plugin, timeout)) def has_tasks(self): return not self.tasks.empty() @@ -181,7 +183,27 @@ class SearchThreadPool(object): if t.is_alive(): return True return False + + +class SearchThreadPool(GenericDownloadThreadPool): + ''' + Threads will run until there is no work or + abort is called. Create and start new threads + using start_threads(). Reset by calling abort(). + Example: + sp = SearchThreadPool(SearchThread, 3) + add tasks using add_task(...) + sp.start_threads() + all threads have finished. + sp.abort() + add tasks using add_task(...) + sp.start_threads() + ''' + + def add_task(self, query, store_name, store_plugin, timeout): + self.tasks.put((query, store_name, store_plugin, timeout)) + class SearchThread(Thread): @@ -209,38 +231,43 @@ class SearchThread(Thread): pass -class CoverDownloadThread(Thread): +class CoverThreadPool(GenericDownloadThreadPool): + ''' + Once started all threads run until abort is called. + ''' - def __init__(self, items, update_callback, timeout=5): + def add_task(self, search_result, update_callback, timeout=5): + self.tasks.put((search_result, update_callback, timeout)) + + +class CoverThread(Thread): + + def __init__(self, tasks, results): Thread.__init__(self) self.daemon = True - self.items = items - self.update_callback = update_callback - self.timeout = timeout - self.br = browser() - + self.tasks = tasks + self.results = results self._run = True + self.br = browser() + def abort(self): self._run = False - - def is_running(self): - return self._run def run(self): while self._run: try: time.sleep(.1) - while not self.items.empty(): + while not self.tasks.empty(): if not self._run: break - item = self.items.get_nowait() - if item and item.cover_url: - with closing(self.br.open(item.cover_url, timeout=self.timeout)) as f: - item.cover_data = f.read() - item.cover_data = thumbnail(item.cover_data, 64, 64)[2] - self.items.task_done() - self.update_callback(item) + result, callback, timeout = self.tasks.get() + if result and result.cover_url: + with closing(self.br.open(result.cover_url, timeout=timeout)) as f: + result.cover_data = f.read() + result.cover_data = thumbnail(result.cover_data, 64, 64)[2] + callback() + self.tasks.task_done() except: continue @@ -252,20 +279,20 @@ class Matches(QAbstractItemModel): def __init__(self): QAbstractItemModel.__init__(self) self.matches = [] - self.cover_download_queue = Queue() - self.cover_download_thread = CoverDownloadThread(self.cover_download_queue, self.update_result) - self.cover_download_thread.start() + self.cover_pool = CoverThreadPool(CoverThread, 2) + self.cover_pool.start_threads() def clear_results(self): self.matches = [] - self.cover_download_queue.queue.clear() + self.cover_pool.abort() + self.cover_pool.start_threads() self.reset() def add_result(self, result): self.layoutAboutToBeChanged.emit() self.matches.append(result) - self.cover_download_queue.put(result) + self.cover_pool.add_task(result, self.update_result) self.layoutChanged.emit() def get_result(self, index): @@ -275,9 +302,7 @@ class Matches(QAbstractItemModel): else: return None - def update_result(self, result): - if not result in self.matches: - return + def update_result(self): self.layoutAboutToBeChanged.emit() self.layoutChanged.emit() From a30f916813989c705a92e0b7b0c46582fbd386b6 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 20:06:23 -0500 Subject: [PATCH 40/92] ... --- src/calibre/gui2/store/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 272dd6d5f1..d3fcd84eca 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -267,7 +267,7 @@ class CoverThread(Thread): result.cover_data = f.read() result.cover_data = thumbnail(result.cover_data, 64, 64)[2] callback() - self.tasks.task_done() + self.tasks.task_done() except: continue From d4593675262a3257f510d9dd72acae5a01760b2a Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 20:06:56 -0500 Subject: [PATCH 41/92] ... --- src/calibre/gui2/store/search.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index d3fcd84eca..0d0c10a3cb 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -159,9 +159,9 @@ class GenericDownloadThreadPool(object): def abort(self): self.tasks = Queue() self.results = Queue() - self.threads = [] for t in self.threads: t.abort() + self.threads = [] def has_tasks(self): return not self.tasks.empty() From 0b565f39752362a5a2e936fb8bfcb3b4d3d95c67 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 20:14:39 -0500 Subject: [PATCH 42/92] Randomize the order in which plugins are added to the search queue. This way plugins with a lower letter do not have an unfair advantage. --- src/calibre/gui2/store/search.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 0d0c10a3cb..d77cf23bf8 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -7,7 +7,8 @@ __docformat__ = 'restructuredtext en' import re import time from contextlib import closing -from threading import Event, Thread +from random import shuffle +from threading import Thread from Queue import Queue from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \ @@ -87,7 +88,11 @@ class SearchDialog(QDialog, Ui_Dialog): if not query.strip(): return - for n in self.store_plugins: + store_names = self.store_plugins.keys() + if not store_names: + return + shuffle(store_names) + for n in store_names: if getattr(self, 'store_check_' + n).isChecked(): self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT) if self.search_pool.has_tasks(): From 8e1fba87a35d65f7dadfc83e810c9a9aa6f6894d Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 20:16:10 -0500 Subject: [PATCH 43/92] ... --- src/calibre/gui2/store/search.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index d77cf23bf8..f12c1fac4b 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -88,6 +88,10 @@ class SearchDialog(QDialog, Ui_Dialog): if not query.strip(): return + # Plugins are in alphebetic order. Randomize the + # order of plugin names. This way plugins closer + # to a don't have an unfair advantage over + # plugins further from a. store_names = self.store_plugins.keys() if not store_names: return From 06e3186391ce4a7e5e71d90b9be98d49ef93d273 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 28 Feb 2011 20:37:43 -0500 Subject: [PATCH 44/92] Add documentation. Fix various issues. --- src/calibre/customize/__init__.py | 31 ++++++++++++++++++++++----- src/calibre/gui2/store/search.py | 17 ++++++++++++++- src/calibre/gui2/store/web_control.py | 1 + src/calibre/gui2/store_download.py | 15 ++++++++----- 4 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index f49c8496c5..5198969430 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -590,11 +590,23 @@ class StorePlugin(Plugin): # {{{ # This needs to be changed to (0, 8, 0) minimum_calibre_version = (0, 4, 118) - def open(self, gui, parent=None, detail_item=None): + def open(self, gui, parent=None, detail_item=None, external=False): ''' - Open a dialog for displaying the store. - start_item is a refernce unique to the store - plugin and opens to the item when specified. + Open the store. + + :param gui: The main GUI. This will be used to have the job + system start downloading an item from the store. + + :param parent: The parent of the store dialog. This is used + to create modal dialogs. + + :param detail_item: A plugin specific reference to an item + in the store that the user should be shown. + + :param external: When False open an internal dialog with the + store. When True open the users default browser to the store's + web site. :param:`detail_item` should still be respected when external + is True. ''' raise NotImplementedError() @@ -607,12 +619,21 @@ class StorePlugin(Plugin): # {{{ :param max_results: The maximum number of results to return. :param timeout: The maximum amount of time in seconds to spend download the search results. - :return: calibre.gui2.store.search_result.SearchResult object + :return: :class:`calibre.gui2.store.search_result.SearchResult` objects item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. ''' raise NotImplementedError() def get_settings(self): + ''' + This is only useful for plugins that implement + :attr:`config_widget` that is the only way to save + settings. This is used by plugins to get the saved + settings and apply when necessary. + + :return: A dictionary filled with the settings used + by this plugin. + ''' raise NotImplementedError() # }}} diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index f12c1fac4b..89e2cf2696 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -33,16 +33,24 @@ class SearchDialog(QDialog, Ui_Dialog): QDialog.__init__(self, *args) self.setupUi(self) + # We pass this on to the store plugins so they can + # tell the gui's job system to start downloading an + # item. self.gui = gui + # We keep a cache of store plugins and reference them by name. self.store_plugins = {} self.search_pool = SearchThreadPool(SearchThread, SEARCH_THREAD_TOTAL) + # Check for results and hung threads. self.checker = QTimer() self.hang_check = 0 self.model = Matches() self.results_view.setModel(self.model) + # Add check boxes for each store so the user + # can disable searching specific stores on a + # per search basis. stores_group_layout = QVBoxLayout() self.stores_group.setLayout(stores_group_layout) for x in store_plugins(): @@ -96,6 +104,7 @@ class SearchDialog(QDialog, Ui_Dialog): if not store_names: return shuffle(store_names) + # Add plugins that the user has checked to the search pool's work queue. for n in store_names: if getattr(self, 'store_check_' + n).isChecked(): self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT) @@ -117,7 +126,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.checker.stop() while self.search_pool.has_results(): - res = self.search_pool.get_result_no_wait() + res = self.search_pool.get_result() if res: self.results_view.model().add_result(res) @@ -126,6 +135,9 @@ class SearchDialog(QDialog, Ui_Dialog): self.store_plugins[result.store_name].open(self.gui, self, result.detail_item) def get_store_checks(self): + ''' + Returns a list of QCheckBox's for each store. + ''' checks = [] for x in self.store_plugins: check = getattr(self, 'store_check_' + x, None) @@ -158,6 +170,9 @@ class GenericDownloadThreadPool(object): self.tasks = Queue() self.results = Queue() self.threads = [] + + def add_task(self): + raise NotImplementedError() def start_threads(self): for i in range(self.thread_count): diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index e73279ce58..295fde8b37 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -68,6 +68,7 @@ class NPWebView(QWebView): def get_cookies(self): cj = CookieJar() + # Translate Qt cookies to cookielib cookies for use by mechanize. for c in self.page().networkAccessManager().cookieJar().allCookies(): version = 0 name = unicode(QString(c.name())) diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index b3c5e6279a..55f39c9ad0 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -47,6 +47,13 @@ class StoreDownloadJob(BaseJob): def job_done(self): self.duration = time.time() - self.start_time self.percent = 1 + + try: + os.remove(self.tmp_file_name) + except: + import traceback + self.log_write(traceback.format_exc()) + # Dump log onto disk lf = PersistentTemporaryFile('store_log') lf.write(self._log_file.getvalue()) @@ -55,11 +62,6 @@ class StoreDownloadJob(BaseJob): self._log_file.close() self._log_file = None - try: - os.remove(self.tmp_file_name) - except: - pass - self.job_manager.changed_queue.put(self) def log_write(self, what): @@ -97,8 +99,11 @@ class StoreDownloader(Thread): continue try: + job.percent = .1 self._download(job) + job.percent = .7 self._add(job) + job.percent = .8 self._save_as(job) except Exception, e: if not self._run: From 84d8dc78c17f102217a44968960f4d6dab65c593 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 1 Mar 2011 07:48:33 -0500 Subject: [PATCH 45/92] Move store plugins to use a GUI wrapper plugin to keep the plugin system happy. Fix segfault with cover download thread pool. --- src/calibre/customize/__init__.py | 75 ++++++++------------- src/calibre/customize/builtins.py | 34 ++++++++-- src/calibre/customize/ui.py | 4 +- src/calibre/gui2/actions/store.py | 9 ++- src/calibre/gui2/store/__init__.py | 75 +++++++++++++++++++++ src/calibre/gui2/store/amazon_plugin.py | 9 +-- src/calibre/gui2/store/feedbooks_plugin.py | 35 ++-------- src/calibre/gui2/store/gutenberg_plugin.py | 38 +++-------- src/calibre/gui2/store/manybooks_plugin.py | 41 +++-------- src/calibre/gui2/store/search.py | 29 ++++---- src/calibre/gui2/store/smashwords_plugin.py | 36 ++-------- src/calibre/gui2/ui.py | 37 +++++++++- 12 files changed, 219 insertions(+), 203 deletions(-) diff --git a/src/calibre/customize/__init__.py b/src/calibre/customize/__init__.py index 5198969430..9f44f6972f 100644 --- a/src/calibre/customize/__init__.py +++ b/src/calibre/customize/__init__.py @@ -581,60 +581,41 @@ class PreferencesPlugin(Plugin): # {{{ # }}} - -class StorePlugin(Plugin): # {{{ - +class StoreBase(Plugin): # {{{ + supported_platforms = ['windows', 'osx', 'linux'] author = 'John Schember' type = _('Store') # This needs to be changed to (0, 8, 0) minimum_calibre_version = (0, 4, 118) - - def open(self, gui, parent=None, detail_item=None, external=False): + + actual_plugin = None + actual_plugin_object = None + + def load_actual_plugin(self, gui): ''' - Open the store. - - :param gui: The main GUI. This will be used to have the job - system start downloading an item from the store. - - :param parent: The parent of the store dialog. This is used - to create modal dialogs. - - :param detail_item: A plugin specific reference to an item - in the store that the user should be shown. - - :param external: When False open an internal dialog with the - store. When True open the users default browser to the store's - web site. :param:`detail_item` should still be respected when external - is True. + This method must return the actual interface action plugin object. ''' - raise NotImplementedError() - - def search(self, query, max_results=10, timeout=60): - ''' - Searches the store for items matching query. This should - return items as a generator. + mod, cls = self.actual_plugin.split(':') + self.actual_plugin_object = getattr(__import__(mod, fromlist=['1'], level=0), cls)(gui, self.name) + return self.actual_plugin_object + + def customization_help(self, gui=False): + if self.actual_plugin_object: + return self.actual_plugin_object.customization_help(gui) + else: + raise NotImplementedError() - :param query: The string query search with. - :param max_results: The maximum number of results to return. - :param timeout: The maximum amount of time in seconds to spend download the search results. - - :return: :class:`calibre.gui2.store.search_result.SearchResult` objects - item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. - ''' - raise NotImplementedError() - - def get_settings(self): - ''' - This is only useful for plugins that implement - :attr:`config_widget` that is the only way to save - settings. This is used by plugins to get the saved - settings and apply when necessary. - - :return: A dictionary filled with the settings used - by this plugin. - ''' - raise NotImplementedError() + def config_widget(self): + if self.actual_plugin_object: + return self.actual_plugin_object.config_widget() + else: + raise NotImplementedError() + + def save_settings(self, config_widget): + if self.actual_plugin_object: + return self.actual_plugin_object.save_settings(config_widget) + else: + raise NotImplementedError() # }}} - diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 913b5d3c7b..53d3f0dd08 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal ' import textwrap, os, glob, functools, re from calibre import guess_type from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \ - MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase + MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf @@ -1042,12 +1042,32 @@ plugins += [GoogleBooks] # }}} # Store plugins {{{ -from calibre.gui2.store.amazon_plugin import AmazonKindleStore -from calibre.gui2.store.gutenberg_plugin import GutenbergStore -from calibre.gui2.store.feedbooks_plugin import FeedbooksStore -from calibre.gui2.store.manybooks_plugin import ManyBooksStore -from calibre.gui2.store.smashwords_plugin import SmashwordsStore +class StoreAmazonKindleStore(StoreBase): + name = 'Amazon Kindle' + description = _('Kindle books from Amazon') + actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore' -plugins += [AmazonKindleStore, GutenbergStore, FeedbooksStore, ManyBooksStore, SmashwordsStore] + +class StoreGutenbergStore(StoreBase): + name = 'Project Gutenberg' + description = _('The first producer of free ebooks.') + actual_plugin = 'calibre.gui2.store.gutenberg_plugin:GutenbergStore' + +class StoreFeedbooksStore(StoreBase): + name = 'Feedbooks' + description = _('Read anywhere.') + actual_plugin = 'calibre.gui2.store.feedbooks_plugin:FeedbooksStore' + +class StoreManyBooksStore(StoreBase): + name = 'ManyBooks' + description = _('The best ebooks at the best price: free!') + actual_plugin = 'calibre.gui2.store.manybooks_plugin:ManyBooksStore' + +class StoreSmashwordsStore(StoreBase): + name = 'Smashwords' + description = _('Your ebook. Your way.') + actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' + +plugins += [StoreAmazonKindleStore, StoreGutenbergStore, StoreFeedbooksStore, StoreManyBooksStore, StoreSmashwordsStore] # }}} diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index bbe24125b8..8e7aa50951 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -8,7 +8,7 @@ from contextlib import closing from calibre.customize import Plugin, CatalogPlugin, FileTypePlugin, \ MetadataReaderPlugin, MetadataWriterPlugin, \ InterfaceActionBase as InterfaceAction, \ - PreferencesPlugin, StorePlugin + PreferencesPlugin, StoreBase from calibre.customize.conversion import InputFormatPlugin, OutputFormatPlugin from calibre.customize.profiles import InputProfile, OutputProfile from calibre.customize.builtins import plugins as builtin_plugins @@ -283,7 +283,7 @@ def preferences_plugins(): def store_plugins(): customization = config['plugin_customization'] for plugin in _initialized_plugins: - if isinstance(plugin, StorePlugin): + if isinstance(plugin, StoreBase): if not is_disabled(plugin): plugin.site_customization = customization.get(plugin.name, '') yield plugin diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 9066150684..0a14bce30b 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -8,7 +8,6 @@ from functools import partial from PyQt4.Qt import Qt, QMenu, QToolButton, QDialog, QVBoxLayout -from calibre.customize.ui import store_plugins from calibre.gui2.actions import InterfaceAction class StoreAction(InterfaceAction): @@ -21,14 +20,14 @@ class StoreAction(InterfaceAction): self.store_menu = QMenu() self.store_menu.addAction(_('Search'), self.search) self.store_menu.addSeparator() - for x in store_plugins(): - self.store_menu.addAction(x.name, partial(self.open_store, x)) + for n, p in self.gui.istores.items(): + self.store_menu.addAction(n, partial(self.open_store, p)) self.qaction.setMenu(self.store_menu) def search(self): from calibre.gui2.store.search import SearchDialog - sd = SearchDialog(self.gui, self.gui) + sd = SearchDialog(self.gui.istores, self.gui) sd.exec_() def open_store(self, store_plugin): - store_plugin.open(self.gui, self.gui) + store_plugin.open(self.gui) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index e69de29bb2..882c81597b 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -0,0 +1,75 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +class StorePlugin(object): # {{{ + + def __init__(self, gui, name): + self.gui = gui + self.name = name + self.base_plugin = None + + def open(self, gui, parent=None, detail_item=None, external=False): + ''' + Open the store. + + :param gui: The main GUI. This will be used to have the job + system start downloading an item from the store. + + :param parent: The parent of the store dialog. This is used + to create modal dialogs. + + :param detail_item: A plugin specific reference to an item + in the store that the user should be shown. + + :param external: When False open an internal dialog with the + store. When True open the users default browser to the store's + web site. :param:`detail_item` should still be respected when external + is True. + ''' + raise NotImplementedError() + + def search(self, query, max_results=10, timeout=60): + ''' + Searches the store for items matching query. This should + return items as a generator. + + :param query: The string query search with. + :param max_results: The maximum number of results to return. + :param timeout: The maximum amount of time in seconds to spend download the search results. + + :return: :class:`calibre.gui2.store.search_result.SearchResult` objects + item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. + ''' + raise NotImplementedError() + + def get_settings(self): + ''' + This is only useful for plugins that implement + :attr:`config_widget` that is the only way to save + settings. This is used by plugins to get the saved + settings and apply when necessary. + + :return: A dictionary filled with the settings used + by this plugin. + ''' + raise NotImplementedError() + + def do_genesis(self): + self.genesis() + + def genesis(self): + pass + + def config_widget(self): + raise NotImplementedError() + + def save_settings(self, config_widget): + raise NotImplementedError() + + def customization_help(self, gui=False): + raise NotImplementedError() + +# }}} \ No newline at end of file diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 7086cfba8f..68f2daf2ea 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -14,15 +14,13 @@ from lxml import html from PyQt4.Qt import QUrl from calibre import browser -from calibre.customize import StorePlugin +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin from calibre.gui2.store.search_result import SearchResult class AmazonKindleStore(StorePlugin): - name = 'Amazon Kindle' - description = _('Kindle books from Amazon') - - def open(self, gui, parent=None, detail_item=None): + def open(self, parent=None, detail_item=None, external=False): ''' Amazon comes with a number of difficulties. @@ -106,7 +104,6 @@ class AmazonKindleStore(StorePlugin): The best (I use the term lightly here) solution is to open Amazon.com in the users default browser and set the affiliate id as part of the url. ''' - from calibre.gui2 import open_url aff_id = {'tag': 'josbl0e-cpb-20'} # Use Kovid's affiliate id 30% of the time. if random.randint(1, 10) in (1, 2, 3): diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 57f9e2d472..121b193806 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -10,18 +10,14 @@ from contextlib import closing from lxml import html from calibre import browser -from calibre.customize import StorePlugin -from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.web_store_dialog import WebStoreDialog -class FeedbooksStore(StorePlugin): +class FeedbooksStore(BasicStoreConfig, StorePlugin): - name = 'Feedbooks' - description = _('Read anywhere.') - - - def open(self, gui, parent=None, detail_item=None): - from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, 'http://m.feedbooks.com/', parent, detail_item) + def open(self, parent=None, detail_item=None, external=False): + d = WebStoreDialog(self.gui, 'http://m.feedbooks.com/', parent, detail_item) d.setWindowTitle(self.name) d.set_tags(self.name + ',' + _('store')) d = d.exec_() @@ -76,22 +72,3 @@ class FeedbooksStore(StorePlugin): s.detail_item = id.strip() yield s - - def customization_help(self, gui=False): - return 'Customize the behavior of this store.' - - def config_widget(self): - from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget - return BasicStoreConfigWidget(self) - - def save_settings(self, config_widget): - from calibre.gui2.store.basic_config_widget import save_settings - save_settings(config_widget) - - def get_settings(self): - from calibre.gui2 import gprefs - settings = {} - - settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') - - return settings diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 952f7f512f..c263d544ea 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -10,18 +10,16 @@ from contextlib import closing from lxml import html from calibre import browser -from calibre.customize import StorePlugin +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog -class GutenbergStore(StorePlugin): - - name = 'Project Gutenberg' - description = _('The first producer of free ebooks.') +class GutenbergStore(BasicStoreConfig, StorePlugin): - def open(self, gui, parent=None, detail_item=None): + def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, detail_item) + d = WebStoreDialog(self.gui, 'http://m.gutenberg.org/', parent, detail_item) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() @@ -51,8 +49,9 @@ class GutenbergStore(StorePlugin): continue id = url.split('/')[-1] - heading = ''.join(url_a.xpath('text()')) - title, _, author = heading.partition('by ') + url_a = html.fromstring(html.tostring(url_a)) + heading = ''.join(url_a.xpath('//text()')) + title, _, author = heading.rpartition('by ') author = author.split('-')[0] price = '$0.00' @@ -66,22 +65,3 @@ class GutenbergStore(StorePlugin): s.detail_item = '/ebooks/' + id.strip() yield s - - def customization_help(self, gui=False): - return 'Customize the behavior of this store.' - - def config_widget(self): - from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget - return BasicStoreConfigWidget(self) - - def save_settings(self, config_widget): - from calibre.gui2.store.basic_config_widget import save_settings - save_settings(config_widget) - - def get_settings(self): - from calibre.gui2 import gprefs - settings = {} - - settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') - - return settings diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index d2531cc7b1..e1eb5eaa13 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -11,18 +11,15 @@ from contextlib import closing from lxml import html from calibre import browser -from calibre.customize import StorePlugin +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult - -class ManyBooksStore(StorePlugin): - - name = 'ManyBooks' - description = _('The best ebooks at the best price: free!') - +from calibre.gui2.store.web_store_dialog import WebStoreDialog - def open(self, gui, parent=None, detail_item=None): - from calibre.gui2.store.web_store_dialog import WebStoreDialog - d = WebStoreDialog(gui, 'http://manybooks.net/', parent, detail_item) +class ManyBooksStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + d = WebStoreDialog(self.gui, 'http://manybooks.net/', parent, detail_item) d.setWindowTitle(self.name) d.set_tags(self.name + ',' + _('store')) d = d.exec_() @@ -55,8 +52,9 @@ class ManyBooksStore(StorePlugin): id = url.split('/')[-1] id = id.strip() - heading = ''.join(url_a.xpath('text()')) - title, _, author = heading.partition('by ') + url_a = html.fromstring(html.tostring(url_a)) + heading = ''.join(url_a.xpath('//text()')) + title, _, author = heading.rpartition('by ') author = author.split('-')[0] price = '$0.00' @@ -78,22 +76,3 @@ class ManyBooksStore(StorePlugin): s.detail_item = '/titles/' + id yield s - - def customization_help(self, gui=False): - return 'Customize the behavior of this store.' - - def config_widget(self): - from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget - return BasicStoreConfigWidget(self) - - def save_settings(self, config_widget): - from calibre.gui2.store.basic_config_widget import save_settings - save_settings(config_widget) - - def get_settings(self): - from calibre.gui2 import gprefs - settings = {} - - settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') - - return settings diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 89e2cf2696..0f4a82ee5b 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -16,7 +16,6 @@ from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \ QPushButton from calibre import browser -from calibre.customize.ui import store_plugins from calibre.gui2 import NONE from calibre.gui2.store.search_ui import Ui_Dialog from calibre.utils.icu import sort_key @@ -29,17 +28,12 @@ COVER_DOWNLOAD_THREAD_TOTAL = 2 class SearchDialog(QDialog, Ui_Dialog): - def __init__(self, gui, *args): + def __init__(self, istores, *args): QDialog.__init__(self, *args) self.setupUi(self) - - # We pass this on to the store plugins so they can - # tell the gui's job system to start downloading an - # item. - self.gui = gui # We keep a cache of store plugins and reference them by name. - self.store_plugins = {} + self.store_plugins = istores self.search_pool = SearchThreadPool(SearchThread, SEARCH_THREAD_TOTAL) # Check for results and hung threads. self.checker = QTimer() @@ -53,12 +47,11 @@ class SearchDialog(QDialog, Ui_Dialog): # per search basis. stores_group_layout = QVBoxLayout() self.stores_group.setLayout(stores_group_layout) - for x in store_plugins(): - self.store_plugins[x.name] = x - cbox = QCheckBox(x.name) + for x in self.store_plugins: + cbox = QCheckBox(x) cbox.setChecked(True) stores_group_layout.addWidget(cbox) - setattr(self, 'store_check_' + x.name, cbox) + setattr(self, 'store_check_' + x, cbox) stores_group_layout.addStretch() self.search.clicked.connect(self.do_search) @@ -122,7 +115,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.checker.stop() else: # Stop the checker if not threads are running. - if not self.search_pool.threads_running(): + if not self.search_pool.threads_running() and not self.search_pool.has_tasks(): self.checker.stop() while self.search_pool.has_results(): @@ -132,7 +125,7 @@ class SearchDialog(QDialog, Ui_Dialog): def open_store(self, index): result = self.results_view.model().get_result(index) - self.store_plugins[result.store_name].open(self.gui, self, result.detail_item) + self.store_plugins[result.store_name].open(self, result.detail_item) def get_store_checks(self): ''' @@ -156,6 +149,10 @@ class SearchDialog(QDialog, Ui_Dialog): def stores_select_none(self): for check in self.get_store_checks(): check.setChecked(False) + + def closeEvent(self, e): + self.model.closing() + QDialog.closeEvent(self, e) class GenericDownloadThreadPool(object): @@ -305,6 +302,9 @@ class Matches(QAbstractItemModel): self.matches = [] self.cover_pool = CoverThreadPool(CoverThread, 2) self.cover_pool.start_threads() + + def closing(self): + self.cover_pool.abort() def clear_results(self): self.matches = [] @@ -315,7 +315,6 @@ class Matches(QAbstractItemModel): def add_result(self, result): self.layoutAboutToBeChanged.emit() self.matches.append(result) - self.cover_pool.add_task(result, self.update_result) self.layoutChanged.emit() diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index 5724b65669..dc9c308ad5 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -12,22 +12,19 @@ from contextlib import closing from lxml import html from calibre import browser -from calibre.customize import StorePlugin +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog -class SmashwordsStore(StorePlugin): - - name = 'Smashwords' - description = _('Your ebook. Your way.') - - - def open(self, gui, parent=None, detail_item=None): - from calibre.gui2.store.web_store_dialog import WebStoreDialog +class SmashwordsStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): aff_id = 'usernone' # Use Kovid's affiliate id 30% of the time. if random.randint(1, 10) in (1, 2, 3): aff_id = 'kovidgoyal' - d = WebStoreDialog(gui, 'http://www.smashwords.com/?ref=%s' % aff_id, parent, detail_item) + d = WebStoreDialog(self.gui, 'http://www.smashwords.com/?ref=%s' % aff_id, parent, detail_item) d.setWindowTitle(self.name) d.set_tags(self.name + ',' + _('store')) d = d.exec_() @@ -78,22 +75,3 @@ class SmashwordsStore(StorePlugin): s.detail_item = '/books/view/' + id.strip() yield s - - def customization_help(self, gui=False): - return 'Customize the behavior of this store.' - - def config_widget(self): - from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget - return BasicStoreConfigWidget(self) - - def save_settings(self, config_widget): - from calibre.gui2.store.basic_config_widget import save_settings - save_settings(config_widget) - - def get_settings(self): - from calibre.gui2 import gprefs - settings = {} - - settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') - - return settings diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index aa36279c85..629bea61af 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -22,7 +22,7 @@ from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import prefs, dynamic from calibre.utils.ipc.server import Server from calibre.library.database2 import LibraryDatabase2 -from calibre.customize.ui import interface_actions +from calibre.customize.ui import interface_actions, store_plugins from calibre.gui2 import error_dialog, GetMetadata, open_local_file, \ gprefs, max_available_height, config, info_dialog, Dispatcher, \ question_dialog @@ -102,6 +102,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.device_connected = None self.gui_debug = gui_debug self.iactions = OrderedDict() + # Actions for action in interface_actions(): if opts.ignore_plugins and action.plugin_path is not None: continue @@ -114,11 +115,24 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ if action.plugin_path is None: raise continue - ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action - self.add_iaction(ac) + # Stores + self.istores = OrderedDict() + for store in store_plugins(): + if opts.ignore_plugins and store.plugin_path is not None: + continue + try: + st = self.init_istore(store) + self.add_istore(st) + except: + # Ignore errors in loading user supplied plugins + import traceback + traceback.print_exc() + if store.plugin_path is None: + raise + continue def init_iaction(self, action): ac = action.load_actual_plugin(self) @@ -126,6 +140,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ ac.interface_action_base_plugin = action action.actual_iaction_plugin_loaded = True return ac + + def init_istore(self, store): + st = store.load_actual_plugin(self) + st.plugin_path = store.plugin_path + st.base_plugin = store + store.actual_istore_plugin_loaded = True + return st def add_iaction(self, ac): acmap = self.iactions @@ -134,6 +155,14 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ acmap[ac.name] = ac else: acmap[ac.name] = ac + + def add_istore(self, st): + stmap = self.istores + if st.name in stmap: + if st.priority >= stmap[st.name].priority: + stmap[st.name] = st + else: + stmap[st.name] = st def initialize(self, library_path, db, listener, actions, show_gui=True): @@ -155,6 +184,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ for ac in self.iactions.values(): ac.do_genesis() + for st in self.istores.values(): + st.do_genesis() MainWindowMixin.__init__(self, db) # Jobs Button {{{ From d931ac2fe2bdca8326896f6abb67919a4ccaae83 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 1 Mar 2011 12:21:18 -0500 Subject: [PATCH 46/92] ... --- src/calibre/gui2/store/search.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 0f4a82ee5b..d862e9e4c9 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -152,6 +152,7 @@ class SearchDialog(QDialog, Ui_Dialog): def closeEvent(self, e): self.model.closing() + self.search_pool.abort() QDialog.closeEvent(self, e) From e10b066b0380da058cbe44fbcd6a0a64705fd7c8 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 1 Mar 2011 13:15:56 -0500 Subject: [PATCH 47/92] Search dialog: properly stop thread pools. Restore window sizes. --- src/calibre/gui2/store/search.py | 37 +++++++++++++++++++++++++++----- src/calibre/gui2/store/search.ui | 2 +- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index d862e9e4c9..8e377c236d 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -13,11 +13,12 @@ from Queue import Queue from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \ QModelIndex, QPixmap, QSize, QCheckBox, QVBoxLayout, QHBoxLayout, \ - QPushButton + QPushButton, QString, QByteArray from calibre import browser from calibre.gui2 import NONE from calibre.gui2.store.search_ui import Ui_Dialog +from calibre.utils.config import DynamicConfig from calibre.utils.icu import sort_key from calibre.utils.magick.draw import thumbnail @@ -31,6 +32,8 @@ class SearchDialog(QDialog, Ui_Dialog): def __init__(self, istores, *args): QDialog.__init__(self, *args) self.setupUi(self) + + self.config = DynamicConfig('store_search') # We keep a cache of store plugins and reference them by name. self.store_plugins = istores @@ -60,8 +63,9 @@ class SearchDialog(QDialog, Ui_Dialog): self.select_all_stores.clicked.connect(self.stores_select_all) self.select_invert_stores.clicked.connect(self.stores_select_invert) self.select_none_stores.clicked.connect(self.stores_select_none) + self.finished.connect(self.dialog_closed) - self.resize_columns() + self.restore_state() def resize_columns(self): total = 600 @@ -105,7 +109,30 @@ class SearchDialog(QDialog, Ui_Dialog): self.hang_check = 0 self.checker.start(100) self.search_pool.start_threads() + + def save_state(self): + self.config['store_search_geometry'] = self.saveGeometry() + self.config['store_search_store_splitter_state'] = self.store_splitter.saveState() + self.config['store_search_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())] + + def restore_state(self): + geometry = self.config['store_search_geometry'] + if geometry: + self.restoreGeometry(geometry) + splitter_state = self.config['store_search_store_splitter_state'] + if splitter_state: + self.store_splitter.restoreState(splitter_state) + + results_cwidth = self.config['store_search_results_view_column_width'] + if results_cwidth: + for i, x in enumerate(results_cwidth): + if i >= self.model.columnCount(): + break + self.results_view.setColumnWidth(i, x) + else: + self.resize_columns() + def get_results(self): # We only want the search plugins to run # a maximum set amount of time before giving up. @@ -149,11 +176,11 @@ class SearchDialog(QDialog, Ui_Dialog): def stores_select_none(self): for check in self.get_store_checks(): check.setChecked(False) - - def closeEvent(self, e): + + def dialog_closed(self, result): self.model.closing() self.search_pool.abort() - QDialog.closeEvent(self, e) + self.save_state() class GenericDownloadThreadPool(object): diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui index 077163dafa..5c4b1a5c15 100644 --- a/src/calibre/gui2/store/search.ui +++ b/src/calibre/gui2/store/search.ui @@ -39,7 +39,7 @@ - + Qt::Horizontal From 02502674d24af08a13cae49f035227f3ab67e80a Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 1 Mar 2011 13:20:13 -0500 Subject: [PATCH 48/92] Store search: save check state of stores. --- src/calibre/gui2/store/search.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 8e377c236d..0155600d2d 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -114,7 +114,12 @@ class SearchDialog(QDialog, Ui_Dialog): self.config['store_search_geometry'] = self.saveGeometry() self.config['store_search_store_splitter_state'] = self.store_splitter.saveState() self.config['store_search_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())] - + + store_check = {} + for n in self.store_plugins: + store_check[n] = getattr(self, 'store_check_' + n).isChecked() + self.config['store_search_store_checked'] = store_check + def restore_state(self): geometry = self.config['store_search_geometry'] if geometry: @@ -132,6 +137,12 @@ class SearchDialog(QDialog, Ui_Dialog): self.results_view.setColumnWidth(i, x) else: self.resize_columns() + + store_check = self.config['store_search_store_checked'] + if store_check: + for n in store_check: + if hasattr(self, 'store_check_' + n): + getattr(self, 'store_check_' + n).setChecked(store_check[n]) def get_results(self): # We only want the search plugins to run From 0b0123f5db8fa5582209724c299b0ca3aaa7f8e1 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 1 Mar 2011 19:20:24 -0500 Subject: [PATCH 49/92] Fix feedbooks search. --- src/calibre/gui2/store/feedbooks_plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 121b193806..8f6c30f5c8 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -12,6 +12,7 @@ from lxml import html from calibre import browser from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog class FeedbooksStore(BasicStoreConfig, StorePlugin): From e0b6a9a7693bebe95b38dfb62a30181fe3c6f5fd Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 2 Mar 2011 07:27:04 -0500 Subject: [PATCH 50/92] Get tags from settings. --- src/calibre/gui2/store/feedbooks_plugin.py | 3 ++- src/calibre/gui2/store/manybooks_plugin.py | 3 ++- src/calibre/gui2/store/smashwords_plugin.py | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 8f6c30f5c8..e07920fc4d 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -18,9 +18,10 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog class FeedbooksStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() d = WebStoreDialog(self.gui, 'http://m.feedbooks.com/', parent, detail_item) d.setWindowTitle(self.name) - d.set_tags(self.name + ',' + _('store')) + d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index e1eb5eaa13..4ca7018d00 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -19,9 +19,10 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog class ManyBooksStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() d = WebStoreDialog(self.gui, 'http://manybooks.net/', parent, detail_item) d.setWindowTitle(self.name) - d.set_tags(self.name + ',' + _('store')) + d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() def search(self, query, max_results=10, timeout=60): diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index dc9c308ad5..3d27d39c04 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -20,13 +20,14 @@ from calibre.gui2.store.web_store_dialog import WebStoreDialog class SmashwordsStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() aff_id = 'usernone' # Use Kovid's affiliate id 30% of the time. if random.randint(1, 10) in (1, 2, 3): aff_id = 'kovidgoyal' d = WebStoreDialog(self.gui, 'http://www.smashwords.com/?ref=%s' % aff_id, parent, detail_item) d.setWindowTitle(self.name) - d.set_tags(self.name + ',' + _('store')) + d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() def search(self, query, max_results=10, timeout=60): From a6a0b6ece35827ae1376a0b3051619365861ca41 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 2 Mar 2011 07:53:09 -0500 Subject: [PATCH 51/92] Diesel ebooks plugin. --- src/calibre/customize/builtins.py | 16 +++-- src/calibre/gui2/store/basic_config.py | 47 ++++++++++++ src/calibre/gui2/store/basic_config_widget.ui | 31 ++++++++ .../gui2/store/diesel_ebooks_plugin.py | 71 +++++++++++++++++++ 4 files changed, 159 insertions(+), 6 deletions(-) create mode 100644 src/calibre/gui2/store/basic_config.py create mode 100644 src/calibre/gui2/store/basic_config_widget.ui create mode 100644 src/calibre/gui2/store/diesel_ebooks_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 53d3f0dd08..a70b320170 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1047,17 +1047,21 @@ class StoreAmazonKindleStore(StoreBase): description = _('Kindle books from Amazon') actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore' - -class StoreGutenbergStore(StoreBase): - name = 'Project Gutenberg' - description = _('The first producer of free ebooks.') - actual_plugin = 'calibre.gui2.store.gutenberg_plugin:GutenbergStore' +class StoreDieselEbooksStore(StoreBase): + name = 'Diesel eBooks' + description = _('World Famous eBook Store.') + actual_plugin = 'calibre.gui2.store.diesel_ebooks_plugin:DieselEbooksStore' class StoreFeedbooksStore(StoreBase): name = 'Feedbooks' description = _('Read anywhere.') actual_plugin = 'calibre.gui2.store.feedbooks_plugin:FeedbooksStore' +class StoreGutenbergStore(StoreBase): + name = 'Project Gutenberg' + description = _('The first producer of free ebooks.') + actual_plugin = 'calibre.gui2.store.gutenberg_plugin:GutenbergStore' + class StoreManyBooksStore(StoreBase): name = 'ManyBooks' description = _('The best ebooks at the best price: free!') @@ -1068,6 +1072,6 @@ class StoreSmashwordsStore(StoreBase): description = _('Your ebook. Your way.') actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' -plugins += [StoreAmazonKindleStore, StoreGutenbergStore, StoreFeedbooksStore, StoreManyBooksStore, StoreSmashwordsStore] +plugins += [StoreAmazonKindleStore, StoreDieselEbooksStore, StoreFeedbooksStore, StoreGutenbergStore, StoreManyBooksStore, StoreSmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/basic_config.py b/src/calibre/gui2/store/basic_config.py new file mode 100644 index 0000000000..1e8efc9175 --- /dev/null +++ b/src/calibre/gui2/store/basic_config.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from PyQt4.Qt import QWidget + +from calibre.gui2 import gprefs +from calibre.gui2.store.basic_config_widget_ui import Ui_Form + +def save_settings(config_widget): + tags = unicode(config_widget.tags.text()) + gprefs[config_widget.store.name + '_tags'] = tags + +class BasicStoreConfigWidget(QWidget, Ui_Form): + + def __init__(self, store): + QWidget.__init__(self) + self.setupUi(self) + + self.store = store + + self.load_setings() + + def load_setings(self): + settings = self.store.get_settings() + + self.tags.setText(settings.get(self.store.name + '_tags', '')) + +class BasicStoreConfig(object): + + def customization_help(self, gui=False): + return 'Customize the behavior of this store.' + + def config_widget(self): + return BasicStoreConfigWidget(self) + + def save_settings(self, config_widget): + save_settings(config_widget) + + def get_settings(self): + settings = {} + + settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') + + return settings diff --git a/src/calibre/gui2/store/basic_config_widget.ui b/src/calibre/gui2/store/basic_config_widget.ui new file mode 100644 index 0000000000..d4cf0fac65 --- /dev/null +++ b/src/calibre/gui2/store/basic_config_widget.ui @@ -0,0 +1,31 @@ + + + Form + + + + 0 + 0 + 501 + 46 + + + + Form + + + + + + Added Tags: + + + + + + + + + + + diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py new file mode 100644 index 0000000000..5ad22ba4cf --- /dev/null +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import urllib2 +from contextlib import closing + +from lxml import html + +from calibre import browser +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class DieselEbooksStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + aff_id = '2049' + # Use Kovid's affiliate id 30% of the time. + #if random.randint(1, 10) in (1, 2, 3): + #aff_id = '' + d = WebStoreDialog(self.gui, 'http://www.diesel-ebooks.com/?aid=%s' % aff_id, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.diesel-ebooks.com/index.php?page=seek&id[m]=&id[c]=scope%253Dinventory&id[q]=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="item clearfix"]'): + data = html.fromstring(html.tostring(data)) + if counter <= 0: + break + + id = ''.join(data.xpath('div[@class="cover"]/a/@href')) + if not id or '/item/' not in id: + continue + a, b, id = id.partition('/item/') + + cover_url = ''.join(data.xpath('div[@class="cover"]//img/@src')) + if cover_url.startswith('/'): + cover_url = cover_url[1:] + cover_url = 'http://www.diesel-ebooks.com/' + cover_url + + title = ''.join(data.xpath('//div[@class="content"]/h2/text()')) + author = ''.join(data.xpath('//div[@class="content"]//div[@class="author"]/a/text()')) + price = '' + price_elem = data.xpath('//td[@class="price"]/text()') + if price_elem: + price = price_elem[0] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/item/' + id.strip() + + yield s From df52819199bf9ae02ed4b5ec964bae9272a62181 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 2 Mar 2011 17:40:47 -0500 Subject: [PATCH 52/92] Kovid's aid for diesel. --- src/calibre/gui2/store/diesel_ebooks_plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index 5ad22ba4cf..87d1f8afab 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -22,8 +22,8 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): settings = self.get_settings() aff_id = '2049' # Use Kovid's affiliate id 30% of the time. - #if random.randint(1, 10) in (1, 2, 3): - #aff_id = '' + if random.randint(1, 10) in (1, 2, 3): + aff_id = '2053' d = WebStoreDialog(self.gui, 'http://www.diesel-ebooks.com/?aid=%s' % aff_id, parent, detail_item) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) From f64e279767b3c93002d5d742baabcb0300e4630b Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 2 Mar 2011 20:07:41 -0500 Subject: [PATCH 53/92] Add progress indicator to search dialog. --- src/calibre/gui2/store/search.py | 8 ++++++++ src/calibre/gui2/store/search.ui | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index 0155600d2d..a80201d1e4 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -17,6 +17,7 @@ from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \ from calibre import browser from calibre.gui2 import NONE +from calibre.gui2.progress_indicator import ProgressIndicator from calibre.gui2.store.search_ui import Ui_Dialog from calibre.utils.config import DynamicConfig from calibre.utils.icu import sort_key @@ -56,6 +57,10 @@ class SearchDialog(QDialog, Ui_Dialog): stores_group_layout.addWidget(cbox) setattr(self, 'store_check_' + x, cbox) stores_group_layout.addStretch() + + # Create and add the progress indicator + self.pi = ProgressIndicator(self, 24) + self.bottom_layout.insertWidget(0, self.pi) self.search.clicked.connect(self.do_search) self.checker.timeout.connect(self.get_results) @@ -109,6 +114,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.hang_check = 0 self.checker.start(100) self.search_pool.start_threads() + self.pi.startAnimation() def save_state(self): self.config['store_search_geometry'] = self.saveGeometry() @@ -151,10 +157,12 @@ class SearchDialog(QDialog, Ui_Dialog): if self.hang_check >= HANG_TIME: self.search_pool.abort() self.checker.stop() + self.pi.stopAnimation() else: # Stop the checker if not threads are running. if not self.search_pool.threads_running() and not self.search_pool.has_tasks(): self.checker.stop() + self.pi.stopAnimation() while self.search_pool.has_results(): res = self.search_pool.get_result() diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui index 5c4b1a5c15..16fc0c4deb 100644 --- a/src/calibre/gui2/store/search.ui +++ b/src/calibre/gui2/store/search.ui @@ -149,7 +149,7 @@ - + From 8e166a19d6c9cb4ef9e6ce51fc10e90a862fbd51 Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 3 Mar 2011 07:24:06 -0500 Subject: [PATCH 54/92] Option and implementation to open store in external browser. --- src/calibre/__init__.py | 6 ++++- src/calibre/gui2/store/basic_config.py | 3 +++ src/calibre/gui2/store/basic_config_widget.ui | 15 ++++++++--- .../gui2/store/diesel_ebooks_plugin.py | 25 +++++++++++++------ src/calibre/gui2/store/feedbooks_plugin.py | 21 ++++++++++++---- src/calibre/gui2/store/gutenberg_plugin.py | 21 ++++++++++++---- src/calibre/gui2/store/manybooks_plugin.py | 20 +++++++++++---- src/calibre/gui2/store/smashwords_plugin.py | 25 +++++++++++++------ src/calibre/gui2/store/web_store_dialog.py | 3 ++- 9 files changed, 104 insertions(+), 35 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 61b8bd36f8..f324a7e46c 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -473,7 +473,11 @@ def as_unicode(obj, enc=preferred_encoding): obj = repr(obj) return force_unicode(obj, enc=enc) - +def http_url_slash_cleaner(url): + ''' + Removes redundant /'s from url's. + ''' + return re.sub(r'(? 0 0 - 501 - 46 + 460 + 69 Form - + Added Tags: - + + + + + Open store in external web browswer + + + diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index 87d1f8afab..a213ffd5c4 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -10,7 +10,10 @@ from contextlib import closing from lxml import html -from calibre import browser +from PyQt4.Qt import QUrl + +from calibre import browser, http_url_slash_cleaner +from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult @@ -20,14 +23,22 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - aff_id = '2049' + url = 'http://www.diesel-ebooks.com/' + + aff_id = '?aid=2049' # Use Kovid's affiliate id 30% of the time. if random.randint(1, 10) in (1, 2, 3): - aff_id = '2053' - d = WebStoreDialog(self.gui, 'http://www.diesel-ebooks.com/?aid=%s' % aff_id, parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + aff_id = '?aid=2053' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(http_url_slash_cleaner(url + aff_id))) + else: + d = WebStoreDialog(self.gui, url + aff_id, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://www.diesel-ebooks.com/index.php?page=seek&id[m]=&id[c]=scope%253Dinventory&id[q]=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index e07920fc4d..ea27e0ca17 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -9,7 +9,10 @@ from contextlib import closing from lxml import html -from calibre import browser +from PyQt4.Qt import QUrl + +from calibre import browser, http_url_slash_cleaner +from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult @@ -19,10 +22,18 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - d = WebStoreDialog(self.gui, 'http://m.feedbooks.com/', parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + url = 'http://m.feedbooks.com/' + ext_url = 'http://feedbooks.com/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + ext_url = ext_url + detail_item + open_url(QUrl(http_url_slash_cleaner(ext_url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://m.feedbooks.com/search?query=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index c263d544ea..69df15c238 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -9,7 +9,10 @@ from contextlib import closing from lxml import html -from calibre import browser +from PyQt4.Qt import QUrl + +from calibre import browser, http_url_slash_cleaner +from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult @@ -19,10 +22,18 @@ class GutenbergStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - d = WebStoreDialog(self.gui, 'http://m.gutenberg.org/', parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + url = 'http://m.gutenberg.org/' + ext_url = 'http://gutenberg.org/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + ext_url = ext_url + detail_item + open_url(QUrl(http_url_slash_cleaner(ext_url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() def search(self, query, max_results=10, timeout=60): # Gutenberg's website does not allow searching both author and title. diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index 4ca7018d00..94c9a38863 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -10,7 +10,10 @@ from contextlib import closing from lxml import html -from calibre import browser +from PyQt4.Qt import QUrl + +from calibre import browser, http_url_slash_cleaner +from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult @@ -20,10 +23,17 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - d = WebStoreDialog(self.gui, 'http://manybooks.net/', parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + url = 'http://manybooks.net/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(http_url_slash_cleaner(url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() def search(self, query, max_results=10, timeout=60): # ManyBooks website separates results for title and author. diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index 3d27d39c04..5307cb0072 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -11,7 +11,10 @@ from contextlib import closing from lxml import html -from calibre import browser +from PyQt4.Qt import QUrl + +from calibre import browser, http_url_slash_cleaner +from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult @@ -21,14 +24,22 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - aff_id = 'usernone' + url = 'http://www.smashwords.com/' + + aff_id = '?ref=usernone' # Use Kovid's affiliate id 30% of the time. if random.randint(1, 10) in (1, 2, 3): - aff_id = 'kovidgoyal' - d = WebStoreDialog(self.gui, 'http://www.smashwords.com/?ref=%s' % aff_id, parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + aff_id = '?ref=kovidgoyal' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(http_url_slash_cleaner(url + aff_id))) + else: + d = WebStoreDialog(self.gui, url + aff_id, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://www.smashwords.com/books/search?query=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py index 0c032e26e0..a651cf3085 100644 --- a/src/calibre/gui2/store/web_store_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -9,6 +9,7 @@ import urllib from PyQt4.Qt import QDialog, QUrl +from calibre import http_url_slash_cleaner from calibre.gui2.store.web_store_dialog_ui import Ui_Dialog class WebStoreDialog(QDialog, Ui_Dialog): @@ -51,5 +52,5 @@ class WebStoreDialog(QDialog, Ui_Dialog): # Reduce redundant /'s because some stores # (Feedbooks) and server frameworks (cherrypy) # choke on them. - url = re.sub(r'(? Date: Fri, 4 Mar 2011 07:31:40 -0500 Subject: [PATCH 55/92] Rename function. --- src/calibre/__init__.py | 4 ++-- src/calibre/gui2/store/diesel_ebooks_plugin.py | 4 ++-- src/calibre/gui2/store/feedbooks_plugin.py | 4 ++-- src/calibre/gui2/store/gutenberg_plugin.py | 4 ++-- src/calibre/gui2/store/manybooks_plugin.py | 4 ++-- src/calibre/gui2/store/smashwords_plugin.py | 4 ++-- src/calibre/gui2/store/web_store_dialog.py | 4 ++-- 7 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index f324a7e46c..dbbb8b2296 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -473,11 +473,11 @@ def as_unicode(obj, enc=preferred_encoding): obj = repr(obj) return force_unicode(obj, enc=enc) -def http_url_slash_cleaner(url): +def url_slash_cleaner(url): ''' Removes redundant /'s from url's. ''' - return re.sub(r'(? Date: Fri, 4 Mar 2011 19:18:11 -0500 Subject: [PATCH 56/92] ebooks.com plugin. Change how detail items are handled. Fix diesel books search. --- src/calibre/customize/builtins.py | 7 +- .../gui2/store/diesel_ebooks_plugin.py | 13 ++- src/calibre/gui2/store/ebooks_com_plugin.py | 94 +++++++++++++++++++ src/calibre/gui2/store/feedbooks_plugin.py | 5 +- src/calibre/gui2/store/gutenberg_plugin.py | 5 +- src/calibre/gui2/store/manybooks_plugin.py | 10 +- src/calibre/gui2/store/smashwords_plugin.py | 12 ++- src/calibre/gui2/store/web_store_dialog.py | 14 +-- 8 files changed, 137 insertions(+), 23 deletions(-) create mode 100644 src/calibre/gui2/store/ebooks_com_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index a70b320170..f93359f234 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1052,6 +1052,11 @@ class StoreDieselEbooksStore(StoreBase): description = _('World Famous eBook Store.') actual_plugin = 'calibre.gui2.store.diesel_ebooks_plugin:DieselEbooksStore' +class StoreEbookscomStore(StoreBase): + name = 'eBooks.com' + description = _('The digital bookstore.') + actual_plugin = 'calibre.gui2.store.ebooks_com_plugin:EbookscomStore' + class StoreFeedbooksStore(StoreBase): name = 'Feedbooks' description = _('Read anywhere.') @@ -1072,6 +1077,6 @@ class StoreSmashwordsStore(StoreBase): description = _('Your ebook. Your way.') actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' -plugins += [StoreAmazonKindleStore, StoreDieselEbooksStore, StoreFeedbooksStore, StoreGutenbergStore, StoreManyBooksStore, StoreSmashwordsStore] +plugins += [StoreAmazonKindleStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreFeedbooksStore, StoreGutenbergStore, StoreManyBooksStore, StoreSmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index 9f2ee9dfcf..bae358a075 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -30,12 +30,15 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): if random.randint(1, 10) in (1, 2, 3): aff_id = '?aid=2053' + detail_url = None + if detail_item: + detail_url = url + detail_item + aff_id + url = url + aff_id + if external or settings.get(self.name + '_open_external', False): - if detail_item: - url = url + detail_item - open_url(QUrl(url_slash_cleaner(url + aff_id))) + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) else: - d = WebStoreDialog(self.gui, url + aff_id, parent, detail_item) + d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() @@ -63,7 +66,7 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): cover_url = cover_url[1:] cover_url = 'http://www.diesel-ebooks.com/' + cover_url - title = ''.join(data.xpath('//div[@class="content"]/h2/text()')) + title = ''.join(data.xpath('.//div[@class="content"]//h2/text()')) author = ''.join(data.xpath('//div[@class="content"]//div[@class="author"]/a/text()')) price = '' price_elem = data.xpath('//td[@class="price"]/text()') diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py new file mode 100644 index 0000000000..663cacf06b --- /dev/null +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class EbookscomStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + + m_url = 'http://www.dpbolvw.net/' + h_click = 'click-4879827-10364500' + d_click = 'click-4879827-10281551' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + #h_click = '' + #d_click = '' + pass + + url = m_url + h_click + detail_url = None + if detail_item: + detail_url = m_url + d_click + detail_item + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.ebooks.com/SearchApp/SearchResults.net?term=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="book_a" or @class="book_b"]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//a[1]/@href')) + id = id.split('=')[-1] + if not id: + continue + + price = '' + with closing(br.open('http://www.ebooks.com/ebooks/book_display.asp?IID=' + id.strip(), timeout=timeout)) as fp: + pdoc = html.fromstring(fp.read()) + pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()') + if len(pdata) >= 2: + price = pdata[1] + if not price: + continue + + cover_url = ''.join(data.xpath('.//img[1]/@src')) + + title = '' + author = '' + heading_a = data.xpath('.//a[1]/text()') + if heading_a: + title = heading_a[0] + if len(heading_a) >= 2: + author = heading_a[1] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '?url=http://www.ebooks.com/cj.asp?IID=' + id.strip() + '&cjsku=' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index 546e5cb389..e03b909f4d 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -30,7 +30,10 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin): ext_url = ext_url + detail_item open_url(QUrl(url_slash_cleaner(ext_url))) else: - d = WebStoreDialog(self.gui, url, parent, detail_item) + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 537ff9e274..2ccd969adb 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -30,7 +30,10 @@ class GutenbergStore(BasicStoreConfig, StorePlugin): ext_url = ext_url + detail_item open_url(QUrl(url_slash_cleaner(ext_url))) else: - d = WebStoreDialog(self.gui, url, parent, detail_item) + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index bc239dc96e..dbd328505a 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -24,13 +24,15 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() url = 'http://manybooks.net/' + + detail_url = None + if detail_item: + detail_url = url + detail_item if external or settings.get(self.name + '_open_external', False): - if detail_item: - url = url + detail_item - open_url(QUrl(url_slash_cleaner(url))) + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) else: - d = WebStoreDialog(self.gui, url, parent, detail_item) + d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index f1d93ec2af..db66d19cf8 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -31,12 +31,16 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin): if random.randint(1, 10) in (1, 2, 3): aff_id = '?ref=kovidgoyal' + detail_url = None + if detail_item: + detail_url = url + detail_item + aff_id + url = url + aff_id + if external or settings.get(self.name + '_open_external', False): - if detail_item: - url = url + detail_item - open_url(QUrl(url_slash_cleaner(url + aff_id))) + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) else: - d = WebStoreDialog(self.gui, url + aff_id, parent, detail_item) + print detail_url + d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) d = d.exec_() diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py index 5d7a338a0c..1924892666 100644 --- a/src/calibre/gui2/store/web_store_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -14,7 +14,7 @@ from calibre.gui2.store.web_store_dialog_ui import Ui_Dialog class WebStoreDialog(QDialog, Ui_Dialog): - def __init__(self, gui, base_url, parent=None, detail_item=None): + def __init__(self, gui, base_url, parent=None, detail_url=None): QDialog.__init__(self, parent=parent) self.setupUi(self) @@ -29,7 +29,7 @@ class WebStoreDialog(QDialog, Ui_Dialog): self.reload.clicked.connect(self.view.reload) self.back.clicked.connect(self.view.back) - self.go_home(detail_item=detail_item) + self.go_home(detail_url=detail_url) def set_tags(self, tags): self.view.set_tags(tags) @@ -43,11 +43,11 @@ class WebStoreDialog(QDialog, Ui_Dialog): def load_finished(self, ok=True): self.progress.setValue(100) - def go_home(self, checked=False, detail_item=None): - url = self.base_url - if detail_item: - url, q, ref = url.partition('?') - url = url + '/' + urllib.quote(detail_item) + q + ref + def go_home(self, checked=False, detail_url=None): + if detail_url: + url = detail_url + else: + url = self.base_url # Reduce redundant /'s because some stores # (Feedbooks) and server frameworks (cherrypy) From 1a46644c03792db222bbf77c14af230aa42745e9 Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 4 Mar 2011 20:04:10 -0500 Subject: [PATCH 57/92] Kobo store. --- src/calibre/customize/builtins.py | 7 ++- src/calibre/gui2/store/kobo_plugin.py | 83 +++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/store/kobo_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index f93359f234..2e470d3776 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1067,6 +1067,11 @@ class StoreGutenbergStore(StoreBase): description = _('The first producer of free ebooks.') actual_plugin = 'calibre.gui2.store.gutenberg_plugin:GutenbergStore' +class StoreKoboStore(StoreBase): + name = 'Kobo' + description = _('eReading: anytime. anyplace.') + actual_plugin = 'calibre.gui2.store.kobo_plugin:KoboStore' + class StoreManyBooksStore(StoreBase): name = 'ManyBooks' description = _('The best ebooks at the best price: free!') @@ -1077,6 +1082,6 @@ class StoreSmashwordsStore(StoreBase): description = _('Your ebook. Your way.') actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' -plugins += [StoreAmazonKindleStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreFeedbooksStore, StoreGutenbergStore, StoreManyBooksStore, StoreSmashwordsStore] +plugins += [StoreAmazonKindleStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreSmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/kobo_plugin.py b/src/calibre/gui2/store/kobo_plugin.py new file mode 100644 index 0000000000..e30827b9ea --- /dev/null +++ b/src/calibre/gui2/store/kobo_plugin.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class KoboStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + + m_url = 'http://www.dpbolvw.net/' + h_click = 'click-4879827-10755858' + d_click = 'click-4879827-10772898' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + #h_click = '' + #d_click = '' + pass + + url = m_url + h_click + detail_url = None + if detail_item: + detail_url = m_url + d_click + detail_item + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.kobobooks.com/search/search.html?q=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//ul[@class="SCShortCoverList"]/li'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//div[@class="SearchImageContainer"]/a[1]/@href')) + if not id: + continue + + price = ''.join(data.xpath('.//span[@class="SCOurPrice"]/strong/text()')) + if not price: + price = '$0.00' + + cover_url = ''.join(data.xpath('.//div[@class="SearchImageContainer"]//img[1]/@src')) + + title = ''.join(data.xpath('.//div[@class="SCItemHeader"]/h1/a[1]/text()')) + author = ''.join(data.xpath('.//div[@class="SCItemSummary"]/span/a[1]/text()')) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() + + yield s From 27996ca0d4c2efa6624599d6a65316a68ea7173b Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Mar 2011 16:34:30 -0500 Subject: [PATCH 58/92] Untested. Set proxy for web control. --- src/calibre/gui2/store/web_control.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index 295fde8b37..45396c6637 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -6,11 +6,12 @@ __docformat__ = 'restructuredtext en' import os from cookielib import Cookie, CookieJar +from urlparse import urlparse from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString, \ - QFileDialog + QFileDialog, QNetworkProxy -from calibre import USER_AGENT, browser +from calibre import USER_AGENT, browser, get_proxies from calibre.ebooks import BOOK_EXTENSIONS class NPWebView(QWebView): @@ -22,6 +23,18 @@ class NPWebView(QWebView): self.setPage(NPWebPage()) self.page().networkAccessManager().setCookieJar(QNetworkCookieJar()) + + http_proxy = get_proxies().get('http', None) + if http_proxy: + proxy_parts = urlparse(http_proxy) + proxy = QNetworkProxy() + proxy.setType(QNetworkProxy.HttpProxy) + proxy.setUser(proxy_parts.username) + proxy.setPassword(proxy_parts.password) + proxy.setHostName(proxy_parts.hostname) + proxy.setPort(proxy_parts.port) + self.page().networkAccessManager().setProxy(proxy) + self.page().setForwardUnsupportedContent(True) self.page().unsupportedContent.connect(self.start_download) self.page().downloadRequested.connect(self.start_download) From b29bd81cd881eacdcb0104fa422b4c1ffbb3bf8f Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Mar 2011 16:43:08 -0500 Subject: [PATCH 59/92] Store icon. --- resources/images/store.png | Bin 0 -> 26018 bytes src/calibre/gui2/actions/store.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 resources/images/store.png diff --git a/resources/images/store.png b/resources/images/store.png new file mode 100644 index 0000000000000000000000000000000000000000..947fb794b84224d22f2389c3f7f24be4d9b783d9 GIT binary patch literal 26018 zcmV(>K-j;DP)4a3O~MX_wfN`!!kO^cERiK0Yu$?bjjEBp0! zdzqH!JTvoMepx|y&<3%+q74ir zG~4!c*Ujmkr)j?5z&Q)=?x1PdPKihZ*Lz+pmh^(Y#-t-Irl7q;zKJA;?}Ue;JeM6;+dOTtL?URv2{IWm&-95 zOyh{e-Pbb2TMY1XA^PuWB1<5+m=VDZqm^>qcp|1*CBuq8kjbXa=IhZc0|s(BG0L*{kIt4XCmnr%tB$rYPVbIv=Pg+V}8*}W$zyx z%s=eMV)wa@%s>>iN~JB3Zi&Y=v668q6f!bAR+6EitYix*DU~vkNSb2m8i?#jvu#Ve zdKDnY}CYVwZ$}JLej=O>2$nut7Sj$dEWEls zYP3Xd*&-Q7bwBI*X5O+=j}=PkC%u?Fl(0;5Yr6$W z^(2vs%fpWxko)c%lkxGQQM%NaQ3C10dxaJc`&p07O(2W|?1nV2BZ zsr$y|BqaUN(IM&S%PQ%m6qh08*RQTfd8Y{pw4shY`T(9ANPHomlyu6J<<%+}prZ)M z^SZ7{PbmqNuah~}H|nyqT;cUJ_H6t&+LS~>C$niVE1GGLd3uLRVxWf=f@$u%?=#Uuli2=KX;m)tM$6nL>w@}pEy2#|5On!*+Zg@z?sW}pox@4mD{K`WbXDR)UhMwN|R$;*Fy`4 ztnQG@SW?H>O0`WX=ee*S?9Y%9<`;LQ2IdHVUFr8~2a)}&UcgUfvvzQWhMPW#{6?bhvD?M@9W)sCFSe?klpsrwtr zR4SjJnaPZpR{GfLM(dv&hJGrYPRIifOv%9m{gTOATu>(@T0Cw_Uw;l>dk3n%DygK& z`RdIM%!3O?ut_s)lMx)xqb8vJ4;uD8_W4{w3=n^Aafgf{dubhQ?Gid#F9<(Ohek~i z9QUv-lFBA!V7w1zLgUyNnZXf63v%P?s$4t2L_Y%0QgLki5B<*8pL?FUQ!X0~kw5CF z;lJ<<5J`PLznaDF@VzG|_w7`ir`xvwF-TSX@S!33zy}VibtjShQ0lPB?OZ;E@T?{c z5FF=u4u>I;4#t4Mq58)OG69KA$4|%*k%mw|FdMW&5E1PC@m}3$4*TechPidkr-K-c z)pd*t`<{^=NoJDNFjOL*rAy>>uX)vc4l+WP1YOG>VA3j1=uAI9qW%O`rJ>T`ddP>G?;p&|02+VcqMkp$D7SBI2>6N`fW!@n+$BExQ2M=)_&4UZWpTO8sV2q> zGSr{_q3WMqbbvcujiBpY4PfZ%h|fQ|VZtBirPL!d#CDGuU{=p@FvP+Av5grNGI{I9 zy3Ai)lxDR7=heI9In6KF&HCATy<9=0&}O9Yre^@s>v1EOo5|^Ja%5~Of8x-wsegU# z#^%8|K$?$!__z!VWJ9U-l{3dan@z}(W20gv46c8qx}8v>ZAkhzp1nwCOZ*spfY9-a zh}~rf9{0F^9ZuQuT9uwZA_F>xdE9FXz)=eEeeXb?Azh=|qV&RhS3byH<{^>L#I@CP zP&<$TfKSNiz5&s6Fen&ud47}Z(mPz@TpZ`wfjLv}A-ZfMWW0L*wrnn~!LN}ze1JdoJV>RQQ=Q^U@?BB!qPXCJ=EkmvSU`cItbi|MtR;v>cx> z@ZqB)GCW+6S6^9_1rVyIHzQNieSnJ#0DElW155K8vb|opi}-;9$Y;SbgRlXUkx|F? z4@=xKI1VS+d+Pq8iH{!tknlX`x;U@UMqe(SUzVjCw}CAv&ERQ@l%R+PjY+z#>KoH{nkL}6^6)gJ9xaqi;ld{@gODpU02eY3wC$dI z#u!pXl4m=+p&Vh@%a@jAadDg8{=p+70hj~gF0GWM)SD#?ys=RuICJ*W3g;Rd?*VK) zDc^eT5_18Myloa{#@uCqE<_ymXy;$OEn914W(>zi@{)q-ibL{UQSk5{4NXGlkZ|f5 z?%}(wEpAFOZ6QQVQR-c%jpsCQFGepBZa{`mGcGSx<>l{Ol6J#U9Cq1nH9ym~oh#L< zU3sH{BkgA>`Bpw}40-X`0o}@dHl0a-?(J`%)!zTU!wi2qw!16&x~UO}sn$DmaQ~n^`q%-UW8P;YbsQ@}>F55(2Eh^e#{*kp;+G1rG6jO}`%CJ@Z& z!cQ<5B0@`0Yb-BA+A#Lw)iv3HX6Z$k=YuHMudhiNaBm$pc5bOm#=CxVU7q{qWh8)h z!W@E-&<@c@CE?UUG)YH~j>2%T{6Ec;oQI0WQjg2t^&|+-#Q7&Fu6W7-ygVb~# zHh&vYM-5E4v9w7N#PoQmfLy+x&uevPfV@12(70!~pqQW++0Z}MO*x*;N**Eg8^Q#J zypca}W-_Vud2#=M9#4K^V6f+@kN(mLnHVog8pON+kjM`HRsdBBit0 zBncda;JB2FY=(feKi-vjYiSk`u?rWqPWY_8wV9YKU(_>OliCix)-``}PgMq>R%p zTc)PW0{b*HOg3#wDG!E1XsN4b(nO_>sYOtmpq`!m3Y^ZX0C=FpVDdV$4S&Fb37XzN zAk|6}wAN`ZwwAYK2eLujdZCcyBG(!=Ree|LNk|e7 z;19z-?rR!|KPEzGJw~+%sp+C~Y<_UwV@Rk7%X>{j(nsL~%tOk}RD!B`6ZUorg(Cm^ znR6uahdy)uh4CFCak6`!Qo=BrjKR6+ERwkbpkQgUp~mu_CQQLa6)rBUgD`z^;?RJcJ~=LF zOQVLk3}o=y?Jc^9&)!hH_+-4MYQT3)|wGI#hj@rh*yC!0%xsw6nY> z&9cgMcrtr@0`-X%IrH7~;rmIved{Wv8b3<@_+&3l zPUIViCpce|g%EVQ0ivye!HRiH7FVj$k9k^PG!%+Z%99o&1TqemK-zeZHH=wVYse4I zF3~wu4?7BGnBCWpl=q;F3?#u+gEGHTms>z9s}04tn(TB8)*r4cH$=~+WqM{n4&1X} zF1>gSX2|??Kc0Izm0J4iM&q*^YRP_%3_#Tnr?1&3;vX2_H~Q&%)0zFuZ#+oinrO|I zErSC&Izo|`?~Bj|cRE1@E)Ipn2XvcBu-b=SKBc-_v!^lKQjn`hhpg4{I2T)BD!7*c zpkqK?OwSf&r;?F|tI1zKb3y*omtJDbKZ#-z1E!(kHRa5!3o?M{pbu@Q(&T-?qy&xz z@Z3m5(H6UCFTH%5Rl_0*u!3!FblC7%@F%t@wE=G7lZo(*Xv#+*t?2{AeE0|ro6qXP zxtxdgX}K|3t9yL^-kd3KKRqFjJuoU&F!4=b7mElT^CJVYRbB*^P?cMAap@Z>%J9sv zEL>aAbR+(6QT2RP*DGgYu}|#KANV;i09Ai3_nDlg>H7~Hnt0#RYW;ox>{Abeh!!#Q zMvG-xkdmZaxUwpQzyn~11snDrUqIO3dt#JvxTEYn$Fl`ve4hkD*%kc>c};YXNg@P< zpV3!9qyZKA%*)I2NB?Dx@&6R0xm*c~LOmV#wp}@PY=ZZk03!Iz-&~M(2-WU`se;JE ziJX&1ADLynq3bC^YBW_H%_X^Pt9lS~_P)qVVa_8I^%uj}d~M-LC5+}N={_3n2ai46|sxR8-r zH%yH&F`Pf_yTtT?_R(fXM30Dzj=lml1@|fWd&0dKTFZ9G2&s%A#SAicd5es+3KjbY zfBL+9?QgC|)f?=uBNq_2bcN|N1Gt17K^czNBd4daUy&RbbbYf%N42+D>a@>(2r{;hcmLe(U3cT%Q0*pwp?zZo+VB*v+uo>1DxYBjC@cet z9*lTIGNE&K5MKY-V2*&)!b+VOzo}z*kPM6*R+gJD&aD%boIWrl{lhJ}dSQv$fsd-- zZrh_{X&E{}hV61)7M5%|b9qZfi*b4A*s#pQM1AARk{mxdBemgCw|D>isdp)YZR2od{dT09HQ&VrhIk)rBd*6GM-acYrV*Ej-%%_^Hf`hX;ixoSJ z|H%MskR*@eGFX^mAr(k{_YYSta_j~m6(^<(lFyiOb)hOhxVkAfmuqzBY-*7hzXwPo zpPxaWOqkR@bmmk72zb1%>2mnskUVf|LTb$p8wTvQo_sQL{uCO=FM+mKHsvY+B77J^ zcsQXNnAtQ4_mWuoT#_n%1zx=$B9-Y(5@BVY&lKerG604qW(HLS6HC&QDN8cYV0H*b z;YB&FQtznnk2!jSX|{T1ez($Bl9ZRKdxlNo8;^>JBCmo?$p^ z28y|aOpg^nI1nI{fQ?vfussnNV-Qzj6{ve>ReM!|5v!dr6S@CcdY%Ed*bybDnAAg=E+FM#(|Up5ic>O&Z8 zV2<_Grr7N=>j!%K3zF+E%9sE0MLCRU>7n~Z#Kybd`_N%|;klO-r2Lp}B|nsMtN*Rh z_@Xic&Kt-8lzsp{zmaK{Nx9YLK@;cCw>t2^?@OZSu94wrkZcGi3L#7{9R2!kci>n6dbxwef*?+{ckTy zleU@|0!y{Z)n-(@K{Ro2sK7kH!9$~jm%sb+BG1KcZp*=mVq}veD?&y_>`ou1-rNf5 z&vJ7*h%cO9krd7y+1G<-PAJd_Dtnz6gpNUmu1t-P`LA7B#-&_Kgm&AQo{1;U<%{wq z()3Lhk(}z(fJq!4!+Rr)Mi{zyYg1y%lq4-U)@w44aPbjD3{SlKpe){4k~?#&(0bPI z#I5$dPkEq`uLcoLj`_!D2;&Q+|lv1?F38jg-yWRWn zk~P$R8rz@EdKpvmr$o2jF*)9sI|O^mT6{aKzekI{wn2=~ELt5B%ms5i+EGXm$5`m! zg>)w{dvbM8E-sUU8A{^SJK*-sW>jkz5szxWiw*EN>i{IM+mymF#QINv>>j25dAx)S z-f2_-VW(^E8G`u?SJN2x!11v#g{zUswAo>^eb)agyG-*VA>lM#XJV@aqKpn?$tZYF zdg+yhLq@?^<~2lDkg!IECkPa_7zLz_@NPjaoV_ES-6s5uFP{cIeTGM78W<;eIGc1$R&m6GC%2A-f*d8B^BM?AQSLKNdH>Dr^&cw#^3o)NG_5e zBrJ$e!k}Xo%L_h_BC(+XLbV@xxE_d-l+zDP0+cDznL;Q>V&8&O*9j9vu+>!8*yMb4 z5ZP!37+a?-9pimUj~8l$oobusVOqHOzw+dXKoba4$YI>BOoK)S-Kw;i=YJIS3TFWq zm#Zzp$Qjt*8lD~C7Rxl&wpk#`>%i#T9v{piR?kuUfX1jauFzMBY<|S7g5Jd;EDfFI zfCYWRc3${Q9ov!HNFdD~>JwW_z&GIi_|8@e>!Zv`LVez*$<}%^EM8HWPg4>ZXH~k^BK=Vf-gvL(+`ebxpn-L@V@WF$_y`@r`aH!8vig7zz z(u!jd+k7bym-aA<`&~ak(T*G*k|XMiA0|soq-ER8*+aZQ7_EP46VeGZV@@qc~VSz}8 zgu+~b0cV6W32lSf81}6I{8@yQQaVH5ftRy|+nO?yYTFAPR~0TA!Pi5sLPk4aQU#yG zf9lE7`^Pk_DI~B2p|ab`N3!ugfrC9ZJH+(2=UP(7_1nue09_;UgBNaovTaxYqBmXK z!Mn1I?DaALJ%8B#K2OuekXU)wL#Jmz*nZEu) zlG47UaP+`?N26q@A-qQYq1qP`kAyY}z;G8@h7eIuSi&?jH4zEFZ>pCL74FZ)=K4oQ z&x_z^I-HCiW~ci&Z!dEG&%ZRc%Os(`;dV$=_HIy@!0E2R8QVbhj}bCkM0B!3H>$xo zWd=OFcc8vKu8W7Ya#=Gtu@BY6kYdki;a%e#BLNZ`GJagz%{IwKt*3Ak#}63}bH$U; z5)IN||7`!rrSt74eXp`4@{Xpu{p)6cQmLMF+o>_I=JAn{o}p|m5oFoornKs9EFAHD zf<6(*V+W34DPR%_P0i0|9h(_ssod~nzf=(^n1+RQ5y=$NU8fSQ z<(abo-~c~FogOdDZY*yfLt<$W=UP6v>vd>=O%wC>d$HJHPa%73V6YH3x@L z?I`|7FYE~co3Qz_J@UZ4Bl4~9&CAE1EQUFIG^kEo!dOIyO7;N z%ve|O+0)cEDjseI9v@^y_3$0nv&01zJlB2)=^)z%dZl_q0$e==^SC z1=+w{AuIV}h8aT>e-53TVtG2a2FL86dPvg)1k=HLW}u-PF8?2d(W(Alx^d;!^CC}G ze^`z{`*F@+KJTRBx;LR2=DkOb496WeD(Cii1AI!wP%xSe%bFbrHlNbhU1)BQDY(y6uBiRpJOWM5&4n#F zJku|ao|+&~)eJI$RM`__CHfEW2bdhw9QF*ADW8j7+Z9_8!ob1n2f;3?}FUS(X{zLl09(4mNQtd_EUZL^7kx(n?i* z9rai)mmKWv&04R+1U#k1+J4@P%V;*6Iyo{%>7@-s&(E4NnfPu^ozG`ym_+Ah9A0t( zIQgG_vE$yZhv4w;Szvqv>%n3{h=5}|FCoMC`gf(pJN6r!YfT`FxA5*g; z7#be!s)@ut4p|}}I zy-(Y0yEG}`b3~><$9-IA$_-g*W=#-`d32Q*GigPA42F3H6@vrpptKh zYj^1Hg!2;v$*3p|2>>>i3Tz1cu%}dFyxs+s(A{%*Sb6~zrGn%Uk2^lWCB*aXJyH*! zh31SB&iT6MVl*$1Of)dPOA z$IY1Ig~TWLL)%3nc^(}zZd)MSN8fiy6+yy|uG8l7I*eVoQx0vubZsa*N_(`=a0*0b zdX?jOSaCfFLFQ8nK#AajCqS4iOwm9db$7_i_XB9kX5wVPXyMT9!l&Oqe@Fh;x31Aq z-9OnwB(sIaaco*Z^+}FB;BGA_LM;M81IIfxyuCUq(M(Qsyebl=Ml&p z=QCm3U&1?l_w1s&hWY#J^IJqJF_zll?*ADiFh|iIhDpQoSmPIlcY7+ALmwrCkCi}l z9Y8Vx&09buKSCuXyf(xq7*UY9c^-cm8Sz@q;c+tT+JO%c;w>ie0|&x0$b|Ccxh|22a|K9L=Hr6}#dt0-H3Nt%^u~8x z$WRNpZUp%7M=fEQ2JY{|3Db!$c#cg~zgBL^<%MmIOT_gsjU8bR-5&|3?oTR5j-`)2 z)@6`j46jGWahyZz`Uu^b88Y%0A%>3f1#}lo-PLq_*SHx5t-L?AfDeh+G!u;Oa9uQo zKXa|r`f3y^`Zcb`|M_%q6veGSxC9?cLB_FDrkQybpio)e_Q#rlVJX70yM3v2a=6uT za$bz&l2^_xQDv5EE%a*P>4Kp^VV`{K`P*`9u}q%;T*u5-sh9|G0W{^DT!2{!rJWB; zVdfrDzuj}#*b_|0gAYh)_qv)nRM~-AjTV&z@)=e#|Bn)_vb=PDiCMV=V|gio5o3Ei z_NYbk?X~k!;i((;o@yZjcWW0wfbdf>9WBQF?wnmS@T2_zVI%6tIXG5lIKz7YkriQ> z7$-e~;oT2Q1ICE4;H21_Wr40M)V9Yu9S@Q_Y5X`JfcC$Za}%BhliHIxSgSeo?j;0k z&Rtjm^x0IZo6`N*LsJMjD)I_c>?V#UlW{K40y1mE$dq9iT2L+f@`dHdV?)<1|LD5Z z5^0H0Vgll3(k6Ro0rnx=@I2}Uc#PBXpKgEV;&1M^Ok-~wd^K9Dtm|?2(Ja=S&NqPziGDugb zRJF3JdPZbJqy4zIi{|xsx;EiB4MPd|k7a;(+>K*c9(KNn$ruCc0r%6Zhx(@jHKRVc zdiDx^0hd^Ss@b=ZKW~FU9NC6fJJcWK`A3KH*l&eGrUgzO=^Q0p*mVgnlxlkUT~s+n zXf_9@vyidK?Yn{>G!m)Svgz4#f4A)=@R5mvjP|Cvzn74An}n#RGqG1~-*tVAjp5p! z^F9HouBOm5nhW;K5oliI$uo5xkjKT~|3}?-$5?jV=lyPZW!{^*Gh1f+Y~wDM%PNr+ zDN!Q?mh3|;BB!9T9)4KnS0+Y@4b6|y`FZg6yU8x>X#s)^51xLJ1r&U z{Gv(i<}laHyrye@2yp_O0K8&Xv$HJ-ECy9>paj0xfseWeid+#luIkA%Es8)y+?w|f z0`oosbMJHi^A+0YsL%y_blIj=Y5h=EplO$we>XW(0mT(*Uq>W<4k~!SPo)AH(m-iY zq<%G-M4A*S=r`vH?pu+baEF$LR;)BibzbBgyEhNYW{9bID2DT8lypIF(MEt)*isDC z0M&?!ePfqz7;$8l6WOmmZ}Z{#8B{#^KO#qPqD~(n27dsy#Ony4T8z1{UV_N<&YM>w zk0P&-oKL3x;ylC>wSke|lta*r+zc4?0KT+2uEPPgy11!-^+?|^0tr2sNMaN^cgOZU z<>05#?BV&B&cml(V!r)FY+*4lfdcGag@TcLw*qHNfv(%jv*px`(^|I=C813aeB zYv&Dy=>yQ-PjyYAJK#Ly7BNBPsrj>7)DVY2e%SpF-8CvDme?y@Nj;+o-1g)gwOSpe z=l%3sq^xm+?*~PZea>}7z{rS=PY8y-)(i(53~V!dGHIg-HzxRU3uJ1?V4k3VqFaElrIaowyA3jsvG> z(kP*!(eoa2n6j6g4RQJV_N1(@m0M`!0kMP`m zebl2c(s+iBJ>0!_E$0Lo%4%K=Z>a6iRK)xTWc__8JWcqKvpx>4u3fw0UK0T*Ew2JmKI&91evo+7+D4Jb98*3g&(`O3^Qi0@(?b88y!3|OXSJYPoO8R8*wx? zDMYih-ynI>jEjt5kmw9y!B=a95;lm zgwQw7F7npY)1?rMmO-V}k}rJsDBZ0O-?5(6JFgy}rhY-SVaY=BXi0VfDRyie;`d#c zDQl`CaJ27VNA3*nky4AHR~S$%H+^dSxX=hFVw+Mpb$m_H2{M~mK%t^U{w{VEaqTEY z^!PV89l&cn#%6v6wX`&cQGG^V!KL|%hT6~QXna6-jsL(vM^G{-ZfWs4W@gaI9^+afy zuQ;@6r)!JC^!gct^#NwoPj9iqXN}Cy#{cf14`VqtCew*XpI_l?%3G{U~U1*;6R*!!DHuP9Z=qDg^tAY^TSg- z4|gpU$5zs+9CFtDYw2@k_23U~nlV^O1Jq3QTB0nF>J?()?`pkUQI4kUdo<*1U! z-F^UvVPWAQ;4yr$r3s*iyw+9;u3d;Jww?<&mnHkK2pyWU9aHrDOjVAbUxt|VpzE-) zWI{v^Q6E&_5GPYm;)!_~%&o?jB3mpVK<@f}fP6f?uA7IsMz%o+%xTSA7;4tl zJt1v)Dw$9s!5XtvJn@-K(llJcqW2Bi-1&(*mr%W`amG0&dc1_GS-N+#A zU>0b$9i_4$A$E;SxJA>Cg01KR7>C(CoKPT+Mi?Lv0E4il}xho2!ru| z=sQs|Lh0f>1{X1oo7FvrK7o>O&9(Hk40*e0!A!js%H_FQCr%Lh4FErndmQZ*M2OgD z1d$#^WhE3uvNv`GqfJ|OaG(%h$V}dglgnJRK|vVxX#@hotICN>F`-GzATwZXmD>K+ z^~nMXmu4$U7VA0dlwC35g0*+rrdHR8F_9-Hm%ufee8|LM#X?M#j;^|i5yJ+F^vDqK6DFCCRO^5c(TL1H-Tko{EOZ0g+*C zFSB{{c><1gfAA5oqXphGIAj2x`}!eYg!%ZMJ2&z(zWn5gSi?8?@%9Y^*s|DD+wSQQ z!-mHoqVQUZ0u{2wQ>WV`i+8rtqCyu$263th>PqpdwF7Aeq0uJ*?^Fnw!%r)e)O-5$ zF>F1$!oQ)!$0L1A&5ywaM4iLj28{ekiZ}zkS#ImifW`@Lr3D2FceCUK>RK&cV;_T~ zQj=9sswv=C*GGV+7cI3d2qLEG(zd#}fSSOzAJCLk`qEG_Ly;G0O`p`A$wKCv{tgnPnt%9RaGjB# zUbigw`UudFTnP5*0eONUOf(icVrWsdYztCVAKEu_>tVhz`2&d{)C(-rEs}(TeZ&Q( z%dFemy*07lk3;GE!V||j3 zKW@#)o~GWbiU9gIL?Z=p%2xRSmO=U5`@)Q0m7BP#Kg<0pQ{nN+L5tTRNR(_fM}ea( z!}+FiE_fd!KKYaffALNfdd8-_N$FCHJn&CYh z9Zq5p7iTJJB`~$P1d-&`GjnVfbtfEDM=mbW^|=huq<#OEk%FcETqJ@B6+FzfY(em) z7YKNHxv42p5sr2y@r|Q$!R_=omxoI|f}xhdk?wy?DpjYME!6S+St(~=D4koJXX%-w z%_T-)5zQ?ledK(X)N^JE&&zZ*H5E3tAY8C>UJ?&EIkWlViRl~{uV zv~C~*ESo{-)2FBFyHG^2vfMB$Z}gSga!7!}NZ(SaN#qzB$TKBN13~2ZLs#U1+eQ;P zm6|qR{po4>&d<;BJv4Yw{ir;KrDXPkx*664<{v$`Xv7S(9WSgb-VYP?wlUPhjr6g8 zpsQHJq-MRJ%JQ=)S35FQk;S@qO;&Fi0U$lPwSIFrHdQYj4HQX3*(#NcJ_dsWwA-Et z0K29yLsN65T*aKZ0CrY!B3<-~d#5hUpaIpKROgl@N}F_n`iU6TN)^{<@iAdLX>Xmm zJ_6Ww&KGeiSdO(cILL0>$`5hPm#V^1FKDZb`k{L!?Q}QIx<%_VV)MKAjiv#LWFo$? ze_Vd=KRzd){rVwR8i~cKU7U;WsZlJL>{heA1|IlL0PpiZn~;-;NP-1&tN{Yrjs~RO zULymj3)sr3#C-$skr4#CzUim#3&iOpeQ4xU&NJo&sK_ToE9M;V8t!wMr(ijG(eY!; zfL^5pQFe(0!oEigwWmwyG%FQpvOx;hEnJ2P+jN{-a7_e=@XF;HZ2)^Yih>gQ-6K3) zOf}BHbg9dYUhEM_ZNDey#10-RaNmCRto-KtcBt}i#dav;j-k=P7k_*@?E@qfB@?N1sR=JPd~PeyQ;uC;q4tlH@>>sXlix$K>fTL*q&)3jy+;0@d0Zn#_w`DSUpvv$ z>fTQt^(xAr5)nz7Jdq^pV7_ZLxXtTujR#tp?pOfI9B*SG%1M58Uy05FI!}nm9bAa_pycbSV!#COH93X`rd(eHSjILo!_ypH9#aC|wtRLc z9ox7H0L{xYFHc}0izUOcnTFC?nt$@}1(uKtOW1ozsPl7Wes0jfRvbF+037@e=N%`i z&d=9xlomb{`*>n4t?jkj^$U}WvUkrq<40tf{&mA%giHi&QK6y2$V?q{!-U))m*0L2 zD~A=cJ#}xM1%;fLt{T8lId=!Z zOr+sTcnh2pR0t4`TDMiz@*dQWl_+UBW zM=t6q^N4f~AmFte6AxrxvwBv8U&tr&9Veh zj)WUVylM`gQ5CsxBLw(%=r}9d`Y^Jau%I zvY!)dP_G!t89E)zxK8^5x;;eKGXJ<$>wCEdVkjigqFTC4MLLie5_YXL2}A&NaS42& z6wU0rkd`*T(uDW41_0WO_$@6U?f$@7IX5vavsbVT&6^jK{zAG$g-SP;;AO2wZFvTH zn`@-TkTC%x5b4vd?>0o93vcKOcHLYoNS08iR>=T}=%7tyaBS0%YM@ADxW;n3d{EVm zOLQiLk<0hBXU@q#dSp8zR_Lrn9UzoAqLyU7^kKLzxWStI5(>ma_yXB!^)XC5QM*i+Kg4wYP8vf%u4z;L{<=7XxlD*%r``U zcDvEUeN*VNJr5v|$6UC$M9SX?``l6mo?S!B|4^^DYaoHzfBEDrPv46iJarQ=nQ`xG z%g10ld3FQ;#i=<`M+3)kDV53Y${I1#xeYmgU8T+()^(7C)s<|?`t8+DiQ#wwB*7Z}^_Q3i=IK5KmItUCE zM&Yyoo{rOM-!K*s;e~3Tx)ny3DwXE6>#+EZ=T48$Q<2)RuHXC#I%q+_U;m{Q#jm5$ z;in(ou5rWC2^(oMCL+Qi$is=*D(&g}c8$o#?^q{&c?aG@4o2;u^_meha)IkmHTSzl zmnV?i!n6(m+<35Hlmk;jz(+|Qpa!^rg=_h5!~_)ZRSDa>_O3yvu^J+rwQJf~s#w;x zjq@zlJ#}&dh~ok()T@Xv^K(^t0`cY??wIQ>J5AhwV-6t9`-!F-wdN8Yp^QXG_3c;? zX@OZed!UqQHbCI(8vj&R&@Wc^cE#vlIx#1f;sWZho*YAt{--BT${XjF$m-^+ZN%+E zx_291NLO&II;=*X!d+uS^67gv%H#Xi$v`24Oy3OIY9Y_T1)z#KhXb61+XCO)+!pYx z#cW#s?P6ps47`sA0PPUS3<(p!?TCYb2Y|I{XPLnty;XCd^@4ZaGy+7j(FA+m$d2wJ zt{_qs)O@czk#jRsGCtYF+Z?DrD|st|KEU-6*;E2ZFyTvk;fkEO5eLwUU?vllAfIPy z^~O>W6BmUV4N(h7g#+zhSba;85E=_Rbv*z~i>qe#e(l+F@|lOmbgtgjk$}|y#DS@# zo!dY`K6d8@dF+;TvI;?zb^~krddH02?E+O{KD1Oi(5(Q_(c}M5l_Y*DkRA z-54)IBr@^kMz4ep2@gB|Rl$mcRuOM9M2ywzhr|Wor=cuL5zw zmrzdr87PDIfVH21(?`UVk=*LAFf4!>c?YbAF*J638WFz_-##i+a97{6eVD8?OqtTg zRXwpEf+A@r5Mn#Zq=G1-yEm8CD;~L0#;6!7N~9)PBVi1vNp1DPe&up=`;S1!>4LghnWh0BdEy2o>}9~6dXsoU(#(Rr8`mO$R*%;- zD%Av3kK@63Xa&>da(^Sj>u=E`Xt@>I(bU31=@Oo44)T&;k^nwP(xVde=eqn8uoCuw zGPlr5NC4#rf%+D@g4g56-L`$WOD8;Zc3BQ${5C{^77xk|Py1LcAffDV4*X|ik7?y1 zW_g+VtIblQh*a&|xv^h9ifWrRKp;Twnic`Sfcw`@^$LZE(dl$%cZyJqXJUTC$In^Yy(Wi6>o zU~!{529!FhAih=9{dA~%X&8*Ij?cyB;G%?H6yJLpXAXyqH2ct?L*6|DH z=g+U|&(s}9)FO_gaO_hvRTcpLFhl6@$yv_Fo^1%xt}3b+0@v$(<-`oVRc9f4A4P`H zy}MV(?2w6$pa03Y{P9P2GdVK}Ut6ZutXVDhRk*d`2PuWCp$OJQ1da}eF#yJE??K?` zQwUf8!&^t?1MtYb3*uoU7AK; zZXdk6mol~I2g=N+b zGC5=&0KW^L`#~VWeaK>pk>H-mHA<bvsMG# zEd#(8AW|HyZF2C}s1kE-$2JZUY0iv;z8o}ui)-y1KfAyJ&Yc?uSd%}*RYHVz>MN(`3=+ZPTqwUF8ekQbRmH{5T(@gepDR2f6r=NHf{i00hebqe6;*Gd=IABFnbVf8!pP| zK>n)UKh234b?5b0`OPp{QRj$7F~0NaB_y$U%2UecNT|If$fc4@)D{nW&9H!@8t@T@ z+MzMQTnY`y9Y~<@Apa*P=P~ydl|P8!6O!*MiTqjgRuF(e;JE{ z5ACG$#Y6(V|0n?e0U!Whk)nEo^7SqH58ro0yrB~qeG0mCioIwgVWy>v8Rj*y&p^cc z(az=W?L+dtH>Tyva#P#kmM8hJEBYiB< zn>agz7=Q;@1UT=%0F!-UWu;E`|5iDGfF^y*3a3EP2SERmD@zrGWHX6!l0aNHbc}Ta z;ftCplNhyFAgjVP?8I^M;tETh`tljs1(R(H*e#qoq=+|Q5gflfm*G!EP=cZY#N8Dxq#))R2HhimWT*czPL4jFZzHwPBZZeOrg^N z_NDnck8c@4$~TbVhlgk6J(#@Pv?wsDlAj%4>;N+Aww=2uSONDcrGPqX*Pv?sSu0JS zdcgyzkKDPY7Z$@h=mo3jVB$owd82V3k>!(jtd)mv8j>F#oW#Nsa{#_a&msWM;YF^V zXU?GpteP_-L8zNk@;Y@1Gl8=3u{c<1tSPlhRYE_CssHD;7oDh9d&?1CZ^N2F-}Zd# z_b1_CcmvC=-n_6<8?iDO*|>QXSMIc&jyT{y5=ROajQ~7O&6i$t*XiO8ab@rVhNNry z{I8#1lz;K~PWg9FjZ4i>s(NZ($jZxmwhStHt{OV3@8o-Z6LU>G zcb3|+6M1@C_45+}{WeW3YWKd01iYAl*Olo!w{>F?I~h5QB$}w@3EaNZ_>BFYmAfMH zUp$fjDsPbkm|UP<&n<~qug=btPUDv@x9Saeo0`d*yP_vlRDK&LR{O3AfTZ2HTKi9j zl~cB-0H~IafBM)QQo9!9x9;0SckfEwqbG6GP+s<|>rDeR7KSZ3%$lp(su|R8=wckE z%vV6yqC#VizHL3Mf^9>2CUay2;!k{ujkD2$$bb_{(Zz8)uB0x?Z(>QkJ!tcOVhIts z22DT(Z|!eR=Rn!}%DbxiDB*E5;c%`PX;q?zj-jmU$l&2LDQLELMwwlL^C z9Jhh#Pw4av_$zRMI63r>9vNfsW(i1e61u|=fE-OPDGc@KEJ{Rj)=3bem`3$_u`O_< z$AB;ej3Uh09>?Z39XKvHkKusAF$(e44keF~vjPGi-;uO)v0U*im}z!Sd^anBot0k1 z;UC@E&ug9@FH5-@5Ge@;9XK^jI*aBst}(eKyRh<*o)?9T_;&1sXuL5jrE*)&pPMJ= z0pKs<#vcWCZZbCE-`2H*V{EF|mlvTBz5oh87PK3ey;_|qPK%4x7?8|o6VZA5A-T0p zY~?d6>JK&BC!rA`IJr1(1ixub{D&W%1=aV+r|(|hA;}*j5Ois!*}0TiGaBrp(;I=48+)Xy)+%&0B2nKm>?PFQbfr-9NIUPikP} z`zIPTz$O4R7J5S6@7=ZMMVX+vn~=$#%Z%NAF*LCm~{rWz5wSQD8|W-pMU;?GxEqy!}2gpDoFwUmDi`3DrK7S+-ec4_f3b8jvXkm zVT>*-L2YeqLMn@46Uw(98Y{|l#gpgHSHw@L_0YM6IO=FTlUDdN;`1DlARTHMU6z>H zWdDv8ux~(QK-?cCpWxHVpnr5LI`UV5Y99nPaO3+XRmsA}0jO6A1O$9l# z*pgQ-Gl!NJAG@@K`o)Ifx>bN?*qlh*0lwZU;M-AJ1{a8p5n3)|{;D>h1o3+jQADc^ z;J4c)@mt?R+~#Phbg~lR^|m8`e)wnoO2xa3Q1Q=VPgBrpED7`h7f_-NngoVeB9@Ia zZ3;QIbqPeZT-OuO$yruNAPPCUO-B$4Yxn;1Gw0=>Ke|;$`*XbYMHG9Ui*pg|ZoqcC zfxD@n!+0;Ehq(Zi_Av-_21nkzrC0W?$;*K&4LLg76f6BVJ_8@eY}wHC2@_@NF=3X^ zZiq!?6}d?Skg}d&a?|eNLdBY`?^~GT2t?;M+c5&{_rvhTQmr-v_|Wcu+Y!KgeYF9_ z>lhwoe^1VuXrM3YV8Nwb89-sPQ_ODEJ@)N$GI8Z_o{KWMD#wnRATQ&?k`n}JShq_a z^2c(L4qvLsS6;g!fBM)~8OXc5+tw#p9RKOLikw?+cTUfi>WuMQiCi;=PL2uGvl#>A!3<*6 z*demB7>-748~1wP29Wtt%=R0#xC_Y9F!H}2*}2mq+l`cuw{iLSZiUdY{>egbZ?wN& zZEsq;t}uvSva1J}>Uj}%eQ_NpRpPkR3=2fiGM+e>%fEeQQdSzi{P@h0JidFROB56w0Nb%8lV&6NWUJig zX9vVXmC=(d`=a0-rsAjW2QpQKhdzV+fz^3gQ*h+?d{d6i0|u+3Pz5ky+f@-0h4drWDzI0$(DlOcbQu*p0 zhm=feSDe5${PY~^EOv<~Hs9-#@*Z%8yTIx{vaMHELErhs<%XPEpevVvP48to_gtw? zfGc40Imow#&lQGzCL(eMh~g#{0>V8Eiz}1sY9-Y@m6VSmUL8tb19m?Y@V5Vii2ZD3 zB|8atQ1ScaAb_?Cnzmg(W`*+osmt?6p$}A6_h(=`TZvr?B#21_gsjWD*)QMWdcT!s zn~9KpEChia4PZHu77NLUlC4&fxF2GC?uAQoVzws#>Y>fDrkH^uR+qnc<+4mHHy|FI z?r^Jm+ZDal2e@nN>$BM3HR#Ijp^PjwLOHeUCzefQek>C)RvOBr)9izbLnrSuR!HBy z%cig)R~L&cyi3eVH9r{fR%Kd?1VF3^z_;1}KIY6Tw_==m%u|5vV?FG^Mf9is$uk6nfukht|Q;Ts`skzlE2@``(5Z z^npxYUo>Wk^Bc%a`NSiS-g-;Lw=!>@T@utOiJi}45yU?F^Vw|I<%4Q6R9QYOadOGV zNsV!nN^l0-wW&ykw#Cs31?LhitCPbR86yF7tNJXuoR7ZWb{` z7AajXTd&g_`ZE?>youLv7N2#o5~SB9$k)VkOfJLRO|n55#(oX8jFU4-g9gj@$>O=L zl1CuY$DaZ9b2WI1IdZo|9B8Kp-#Z~VV@oJXC3f=EU|v?sQd?|-12j*ELF2!JD?U|T zwk~Sc|4Zipran-qtXx2#;|bsMfAsq6XU+^2Z2zu3t7)f(-~fK3MNGy6sxMY7kARNS z_Nz|D(TSJBaxmrylxyvHpN^kG#=%Hjx_Kul;CK_2L;KII$h$Z7%h+HpCR8eXEA7B6 zG2lPN4$PYUfvhEa5Lw$f;80&V4t9UB8sM``u8{?4?;6T-Y{m6{vkqW20b>SnNa0Pw zoXIPi^WZ)=lXebOOUIBH91&8!#l@t3pDVLc*|RWr2*iN$+p*$z5hnNl&PMLf%FfP9 z2JrtrIe_K_|138!(Ao(Jvi1cLSg> zNJj%Z`*s?2>jfgM%_BB|ZK)l~R6T_F2vez}6;E!{#T}Zh7MxS@QnpR;J?bm$QB(q= zz5owhi~0}lNmsc9|4M!Sssp$liP9JXiDEh51dM?+F;X=^{neT@kb~jHHh#{`!TaZV$4S;?7 z;3`0nY&`vDvnK4|I4lJKzUTn(9x9&d5yVu;b9&)Q#rnC=UIQ}-_j;{1;7VmxtqjHe zX~ZXfc6?5RfTtC}E%X2!VpcUDg^n*33QskBE4J@31fyN{#IIGPwOkUv+L(>J*4MyT zzPZvGJSOGOGROZ{69LFiKU3%%=(!Ep`l->4!;ih|{+qUE3k7%lN|gbZg$5otrepxl z&Sk^_fUJzx3y|bMJ%9jDbODu=skuQsT#H-;jVSQBQOZss8o*)pjTYs1ZXb~`_zUKn z4Ej*+$6wJ0zz=3ALI8M?8BlzU0T;qb1Dzaz`vlZw#5$I1ZB=p0jb&OXGJ#0tE8utp ze8AQ^@o{SYUSM(GR-V5gN- z{d&OvB{+g^Y#9J(^-AgY5BCpR-ptfeR5hvlp+4*%T$4om$?A3Npav{E1D8auoM-^m26Q(pgO5!x(cD^I;8N3_-v<9zc5Yt;>|>i z2*1a;4$p8^bh_AF)N)pst6Jfjc4a<$bzud^4lr3i@uq5h>E1>-P@yPeT?o0So#Ty(q*#56R0+@a(22Riwz&h;7K#Gs6iJ|?OVgvcDsQu%2{F)@N6Qnb-2J})zJau^N;k% zy_@v2j0z7ve14!|)Bik`1FdAt*+4+A0&nPp8=URX^sotk)=r?2A-e)LR8V%_MHdfH*Z)J9&4U6ee6b0YH zjd6eCwEEhcqD$^oTv`J>cA2vig=L4?=Om*5PlQuAjt zFYpO`#6wsR<9lpp4ms~(#P)qtTu}dD#+GH2b1hz3T8Y}N=Wz8CZ~?!t6gH=#`7gFa zy?z}LKqJ9JZg20-A=k3+0`Yu2SI9lkKiog&_7reX?`cI<8Do?9YP(>0y_S9soRI5-7|%x^km- zO!0Uh`$TFj>5ZB)oOfjE;=C+OFI|Lt_em?Vo`P0#$%;M4-k` zlQ7W%KCpKrE;cnB{KJSm?p@cLntiH?vtF~9!5@siY#Pa1X=A_+56nIX_@iR=40S7x zAUc$df@i~y2@AdZbF zabT=0$&AQQ6m?ncWjMF)uBvlV-Lu1iA`5ymf=voI{7w~S7sP&FovJz2ed_7y^ay#@ zH;I0-tNi}sTQ4v47k`rg;8F=-yE(k)cJ`lq3xYp`n!hS~{*SJIYybJB@L-`9cGQO* z^g)Hg?*0&S1-P&<&UM%&gN*6OCqrG8%pbjS0BgD91-!i<@cM2K&rix{qzllMh5sZI z79@2`1wk<`O!sKle$VF6-{8@C(|vAppwoe@*YzOC2mtN)dhP$aUw($gDs(%0i2{TG z?jGEb^*Y~*Ad`Q6Ah^*DY^g{yuaI!aiIWG&(Efd>{J^Xk8lZ7zYDX>&t=M1uu%f7eIB7*S- zfBpS~Om>MN`ahz8$L+(NbtVslMg$K|mN;29aE80j6Sf<`B9A@4<8e6hUDD@x5i+Lk z7&|2ab?ZM}h@v&YKfQmFH?s3}&OfXVZ)|E%uOg(1C^U4L#_Kx{pFKW9CVqN${Pf>M z@Don)ZSJ|>d@+!zzW=uon|7H6K$};IUwfVLvm3sw%(cEW-&3T&0O3z}_DA0)@AvFr zV5BJE2E%S?XpDg=T}OTw3A}rjOShi!$G7(I;=uquk8O(zqbn3O85HZAO`6z|ACkf{ zA~(Hpk5DDI@CmK0k}NoZ1=w6js0Q1Ql^i?y*`|!ZH)n1qjn>k zl5gvs{&cx6PC4sNAJ_geW7V_-OGmN?x;|opq?T-fhk!!ta`;r&IQe9XX1YX}Eg*F@ zZ>w|=<6o&Pe$M^=uXtztaj`p?`h(}Hg=p#*Chg>G?>lD`!R>by_wH%iE|UPZGynBB z`IS8DPa1Dl>Cg;%Q(HT0-6FbRht?uGL&nvsdC9A4+W#&ma<^x~NJv9+lnQ0eeG_y^-%Ke)PO~_U6&g?w{hr zwnDy-M5K7xObdz-F2BBGR@KT7IuXJ72Zr9Wu;j8mGWLY&5(z&GcDOvL__Xoe^ z-SKO>V_^&OZE-2riKTK)9C;Z}YbwKD3L) zz~-51D=M6$;Eaf8r#`|u67{Vm1CJ(_)P!N2Ev}^V>4837{Jt!ge;^?q8m38a)bCeO zO74|pM?G+_2b2zbdjljU;Nn>WFmDPyxRMK6SCryObZ4ze%nJ-~NL^WSV39=MT=WNXb?w9v@F#3%d_dlVX(Xs9Io@(h= z63|#pL719|5K2320)^14Q>Jp11T%-N)8OROA+p|4x= zz4sxBrw|IZE%`Q+ubx#sQYy%@lVl4G%Rbd%Cobj_oh`4$^Z8A!R)^ZeU4o9lT(D(N z&N?a!8czlgXHY9Z-heDUJ3k;IT%Gjbtwmm*0pe`|25DV-_GniDAXVK=U#D(Y-6L5^ zqckc~sp^cJ{esH!i5U+c@@VjBj!q}098A^m0z|c8u9Z?FiMqm~HgY4Je@XVsnwn&& zS*3m$?YdIVc&QRA6Q&o-gOx)Mi@`|jQJzZ8qT2|%h`BR&l zQGx-1!Nf2`n`rQnA^5_rZ|K^51Gq@-QW8i;Ly#(@B$`Ll`mQ!r)Uf89`u@!Hd!N&w zf6PUBPQBjLm42p$U#LpWY*a}N1Xd^|6%xZFs+z>9k)Woq1t-HC&4Lp@<~SP>?tHiV zduh#AaWWL^V;%U+Y7KRH3#7QP*pD z3n*`1@rlT|{f3}W+El(0 zFvV!7CSxn|Zpnq4v-k5%@KY+vIlP-^!WVGG0>)L8Kbx3qO_Z2aQc?vQWr89lfK}0r z5x}A_PEoBuzECJY zelDpY$$eE;5h7w$!;#oTvZf(25zGzP1h%#jHnnwrMcz~}E}N;rrr|uUgg`KjQtn?| zB@Cj5g|eR-m3E98$!1eV4%yJSKuuz9lI5nvDpcr&$=NC%0cpJlxMUHa-Q6DHRe9m$ zoR)QF2rG(}RmX)21tC)5WD`6`k|vUB6w*MQaU+Vg2%t`o(U4PtW3o;tdbP0B{xoy9 zY(S0WQY}G@Cb7`gU{s@l2t(8mK!6a~h=K%U5_Ml9TqNcQ*dQdy7!DwcU>_6|2_`Ue zqih0!f+_`43EBz))NuBNv=L}9h8j(1pfSK?;%t59Ztv8v)|cbJZDUlrh| ziZ?H19sq=U?b`M#-^MXdr#DPBJ+}SI*rr(8!WkPX@2n+jkX)qX7{(rjf_A^i6~_hVot3|?h}xRI1-uI zqH6Y;g0^GGXgF{y*p_&3FocFhqauZ*CRmt|d<+gYgeWnn3%-`vD^r|VPW_Key8kD= z1l$qagKlR3<%$3x;GSw0fgByZq_0f8P6tkwHl-MgSTAF$5ExaI6@~i^LUwQ0 zaOrw|P2L%PD8Mfi_wID$U+S}fZ6vr09KA_xVrcr&O|FJ>4&@FfSMgB9MAPH4MO70^ zFkA{Qof(CZY})n_j3v`@Vl){xBn(p4j`%JD5L%ywzWV=Sg{XpADXEen zfs&;5!zKxnO_crAMFUDqNQsb<*47}!c1gEBHjHE|PE{Va%*m#e6dBJ$DXyAIc`%gw znRs-GpVu<+YtmJr|F;f<6wl5D?&S3F?8%6#!w+fkaKUjhpzy_75Mx_&G<{BshD(J> zCUW9}g(E9__8FydER?dJSc0%xCJi=Y%qR@+pppoZ`-ZrtE(SrqqKMkVc4*|&8|JW0<{ zD-JCk^@AZTDw3EnT2kNow)>qLIrkgF*EDLpk`-O+ASyMbQbx8(sV;TtGFCy&zzeT} zvJcu4eOlJavr65S7do=-Xr;xcb3K}zDGtA*$b@e;!nk|4BmHY<{N)$GH%Er8Aht_I zH+Gyw@tqF{cm>tjhlJa=QJ?(RS{#dG!VN$cJxZ%H0-l46rWvn7tEb)1?`KQ(jDS6S zwVlEgBR#q3Q8`xNc%8SvXE42q@?c-__%ZC@>IKQar@#@O&1?W$vR}*i2DXI%_L=_w z0{A9DbWOK|-Tq!|J%;sO=}sxdO`4( Date: Sat, 5 Mar 2011 18:47:00 -0500 Subject: [PATCH 60/92] Open Library store. --- src/calibre/customize/builtins.py | 9 ++- src/calibre/gui2/store/open_library_plugin.py | 70 +++++++++++++++++++ 2 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/store/open_library_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 2e470d3776..b8c8c9885f 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1077,11 +1077,18 @@ class StoreManyBooksStore(StoreBase): description = _('The best ebooks at the best price: free!') actual_plugin = 'calibre.gui2.store.manybooks_plugin:ManyBooksStore' +class StoreOpenLibraryStore(StoreBase): + name = 'Open Library' + description = _('One web page for every book.') + actual_plugin = 'calibre.gui2.store.open_library_plugin:OpenLibraryStore' + class StoreSmashwordsStore(StoreBase): name = 'Smashwords' description = _('Your ebook. Your way.') actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' -plugins += [StoreAmazonKindleStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreSmashwordsStore] +plugins += [StoreAmazonKindleStore, StoreDieselEbooksStore, StoreEbookscomStore, + StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, + StoreOpenLibraryStore, StoreSmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/open_library_plugin.py b/src/calibre/gui2/store/open_library_plugin.py new file mode 100644 index 0000000000..c440a894ae --- /dev/null +++ b/src/calibre/gui2/store/open_library_plugin.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class OpenLibraryStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://openlibrary.org/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(url_slash_cleaner(url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://openlibrary.org/search?q=' + urllib2.quote(query) + '&has_fulltext=true' + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@id="searchResults"]/ul[@id="siteSearch"]/li'): + if counter <= 0: + break + + id = ''.join(data.xpath('./span[@class="bookcover"]/a/@href')) + if not id: + continue + cover_url = ''.join(data.xpath('./span[@class="bookcover"]/a/img/@src')) + + title = ''.join(data.xpath('.//h3[@class="booktitle"]/a[@class="results"]/text()')) + author = ''.join(data.xpath('.//span[@class="bookauthor"]/a/text()')) + price = '$0.00' + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = id.strip() + + yield s From 1ec3860ee65010bd21dcc9fd859b0f21e426a144 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 5 Mar 2011 21:16:06 -0500 Subject: [PATCH 61/92] Reload store plugins when they are enabled and disabled. --- src/calibre/gui2/actions/store.py | 4 +++ src/calibre/gui2/preferences/plugins.py | 6 ++++ src/calibre/gui2/ui.py | 38 +++++++++++++------------ 3 files changed, 30 insertions(+), 18 deletions(-) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index e552a51cb2..97653b072b 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -18,6 +18,10 @@ class StoreAction(InterfaceAction): def genesis(self): self.qaction.triggered.connect(self.search) self.store_menu = QMenu() + self.load_menu() + + def load_menu(self): + self.store_menu.clear() self.store_menu.addAction(_('Search'), self.search) self.store_menu.addSeparator() for n, p in self.gui.istores.items(): diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index acf42fee16..94f3bae3dc 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -217,6 +217,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.search.search.connect(self.find) self.next_button.clicked.connect(self.find_next) self.previous_button.clicked.connect(self.find_previous) + self.changed_signal.connect(self.reload_store_plugins) def find(self, query): idx = self._plugin_model.find(query) @@ -342,6 +343,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): plugin.name + _(' cannot be removed. It is a ' 'builtin plugin. Try disabling it instead.')).exec_() + def reload_store_plugins(self): + self.gui.load_store_plugins() + if self.gui.iactions.has_key('Store'): + self.gui.iactions['Store'].load_menu() + if __name__ == '__main__': from PyQt4.Qt import QApplication diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 751647edb6..455e56169e 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -118,10 +118,27 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action self.add_iaction(ac) - # Stores + self.load_store_plugins() + + def init_iaction(self, action): + ac = action.load_actual_plugin(self) + ac.plugin_path = action.plugin_path + ac.interface_action_base_plugin = action + action.actual_iaction_plugin_loaded = True + return ac + + def add_iaction(self, ac): + acmap = self.iactions + if ac.name in acmap: + if ac.priority >= acmap[ac.name].priority: + acmap[ac.name] = ac + else: + acmap[ac.name] = ac + + def load_store_plugins(self): self.istores = OrderedDict() for store in store_plugins(): - if opts.ignore_plugins and store.plugin_path is not None: + if self.opts.ignore_plugins and store.plugin_path is not None: continue try: st = self.init_istore(store) @@ -133,13 +150,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ if store.plugin_path is None: raise continue - - def init_iaction(self, action): - ac = action.load_actual_plugin(self) - ac.plugin_path = action.plugin_path - ac.interface_action_base_plugin = action - action.actual_iaction_plugin_loaded = True - return ac def init_istore(self, store): st = store.load_actual_plugin(self) @@ -147,15 +157,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ st.base_plugin = store store.actual_istore_plugin_loaded = True return st - - def add_iaction(self, ac): - acmap = self.iactions - if ac.name in acmap: - if ac.priority >= acmap[ac.name].priority: - acmap[ac.name] = ac - else: - acmap[ac.name] = ac - + def add_istore(self, st): stmap = self.istores if st.name in stmap: From 273ddfa2804335673a8a66a1d0d50eaeac2deba4 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Mar 2011 10:25:57 -0500 Subject: [PATCH 62/92] Change how downloads are added to the library to ensure FileType plugins are run over the additions. --- src/calibre/gui2/store_download.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index 55f39c9ad0..ab2d63506d 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -159,7 +159,8 @@ class StoreDownloader(Thread): mi = get_metadata(f, ext) mi.tags.extend(tags) - job.db.add_books([job.tmp_file_name], [ext], [mi]) + id = job.db.create_book_entry(mi) + job.db.add_format_with_hooks(id, ext.upper(), job.tmp_file_name, index_is_id=True) def _save_as(self, job): url, save_loc, add_to_lib, tags = job.args From c508275f3d768fa42b4393f05e33b54621c81660 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Mar 2011 11:19:09 -0500 Subject: [PATCH 63/92] Fix download issues. clean download code. add checks for aborting in more places. --- src/calibre/gui2/store_download.py | 46 ++++++++++++++++++------------ 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index ab2d63506d..b5d670be97 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -92,18 +92,25 @@ class StoreDownloader(Thread): failed, exc = False, None job.start_work() if job.kill_on_start: - job.log_write('Aborted\n') - job.failed = failed - job.killed = True - job.job_done() + self._abort_job(job) continue try: - job.percent = .1 self._download(job) - job.percent = .7 - self._add(job) + if not self._run: + self._abort_job(job) + return + job.percent = .8 + self._add(job) + if not self._run: + self._abort_job(job) + return + + job.percent = .9 + if not self._run: + self._abort_job(job) + return self._save_as(job) except Exception, e: if not self._run: @@ -125,7 +132,13 @@ class StoreDownloader(Thread): except: import traceback traceback.print_exc() - + + def _abort_job(self, job): + job.log_write('Aborted\n') + job.failed = False + job.killed = True + job.job_done() + def _download(self, job): url, save_loc, add_to_lib, tags = job.args if not url: @@ -137,23 +150,20 @@ class StoreDownloader(Thread): br = browser() br.set_cookiejar(job.cookie_jar) - basename = br.open(url).geturl().split('/')[-1] - tf = PersistentTemporaryFile(suffix=basename) - with closing(br.open(url)) as f: - tf.write(f.read()) - tf.close() - job.tmp_file_name = tf.name + with closing(br.open(url)) as r: + basename = r.geturl().split('/')[-1] + tf = PersistentTemporaryFile(suffix=basename) + tf.write(r.read()) + job.tmp_file_name = tf.name def _add(self, job): url, save_loc, add_to_lib, tags = job.args - if not add_to_lib and job.tmp_file_name: + if not add_to_lib or not job.tmp_file_name: return ext = os.path.splitext(job.tmp_file_name)[1][1:].lower() if ext not in BOOK_EXTENSIONS: raise Exception(_('Not a support ebook format.')) - ext = os.path.splitext(job.tmp_file_name)[1][1:] - from calibre.ebooks.metadata.meta import get_metadata with open(job.tmp_file_name) as f: mi = get_metadata(f, ext) @@ -164,7 +174,7 @@ class StoreDownloader(Thread): def _save_as(self, job): url, save_loc, add_to_lib, tags = job.args - if not save_loc and job.tmp_file_name: + if not save_loc or not job.tmp_file_name: return shutil.copy(job.tmp_file_name, save_loc) From d19787a475138708f74bb061389ffd507676d079 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Mar 2011 14:35:23 -0500 Subject: [PATCH 64/92] Bean WebScription plugin. --- src/calibre/customize/builtins.py | 7 +- .../gui2/store/baen_webscription_plugin.py | 87 +++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/store/baen_webscription_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index b8c8c9885f..3194a64de0 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1046,6 +1046,11 @@ class StoreAmazonKindleStore(StoreBase): name = 'Amazon Kindle' description = _('Kindle books from Amazon') actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore' + +class StoreBaenWebScriptionStore(StoreBase): + name = 'Baen WebScription' + description = _('Ebooks for readers.') + actual_plugin = 'calibre.gui2.store.baen_webscription_plugin:BaenWebScriptionStore' class StoreDieselEbooksStore(StoreBase): name = 'Diesel eBooks' @@ -1087,7 +1092,7 @@ class StoreSmashwordsStore(StoreBase): description = _('Your ebook. Your way.') actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' -plugins += [StoreAmazonKindleStore, StoreDieselEbooksStore, StoreEbookscomStore, +plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreOpenLibraryStore, StoreSmashwordsStore] diff --git a/src/calibre/gui2/store/baen_webscription_plugin.py b/src/calibre/gui2/store/baen_webscription_plugin.py new file mode 100644 index 0000000000..1e7c093d37 --- /dev/null +++ b/src/calibre/gui2/store/baen_webscription_plugin.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class BaenWebScriptionStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://www.webscription.net/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(url_slash_cleaner(url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.webscription.net/searchadv.aspx?IsSubmit=true&SearchTerm=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//table/tr/td/img[@src="skins/Skin_1/images/matchingproducts.gif"]/..//tr'): + if counter <= 0: + break + + id = ''.join(data.xpath('./td[1]/a/@href')) + if not id: + continue + + title = ''.join(data.xpath('./td[1]/a/text()')) + + author = '' + cover_url = '' + price = '' + + with closing(br.open('http://www.webscription.net/' + id.strip(), timeout=timeout/4)) as nf: + idata = html.fromstring(nf.read()) + author = ''.join(idata.xpath('//span[@class="ProductNameText"]/../b/text()')) + author = author.split('by ')[-1] + price = ''.join(idata.xpath('//span[@class="variantprice"]/text()')) + a, b, price = price.partition('$') + price = b + price + + pnum = '' + mo = re.search(r'p-(?P\d+)-', id.strip()) + if mo: + pnum = mo.group('num') + if pnum: + cover_url = 'http://www.webscription.net/' + ''.join(idata.xpath('//img[@id="ProductPic%s"]/@src' % pnum)) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = id.strip() + + yield s From 5e72c701871803e3f0b572359c2917893c12ffcc Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Mar 2011 20:02:51 -0500 Subject: [PATCH 65/92] BN store plugin. --- src/calibre/customize/builtins.py | 10 ++++- src/calibre/gui2/store/bn_plugin.py | 66 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) create mode 100644 src/calibre/gui2/store/bn_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 3194a64de0..a4ec9cfeee 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1052,6 +1052,11 @@ class StoreBaenWebScriptionStore(StoreBase): description = _('Ebooks for readers.') actual_plugin = 'calibre.gui2.store.baen_webscription_plugin:BaenWebScriptionStore' +class StoreBNStore(StoreBase): + name = 'Barnes and Noble' + description = _('Books, Textbooks, eBooks, Toys, Games and More.') + actual_plugin = 'calibre.gui2.store.bn_plugin:BNStore' + class StoreDieselEbooksStore(StoreBase): name = 'Diesel eBooks' description = _('World Famous eBook Store.') @@ -1092,8 +1097,9 @@ class StoreSmashwordsStore(StoreBase): description = _('Your ebook. Your way.') actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' -plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreDieselEbooksStore, StoreEbookscomStore, - StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, +plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore, + StoreDieselEbooksStore, StoreEbookscomStore, StoreFeedbooksStore, + StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreOpenLibraryStore, StoreSmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py new file mode 100644 index 0000000000..b4fedae987 --- /dev/null +++ b/src/calibre/gui2/store/bn_plugin.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class BNStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://www.barnesandnoble.com/ebooks/index.asp' + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results + url += urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//ul[contains(@class, "wgt-search-results-display")]/li[contains(@class, "search-result-item") and contains(@class, "nook-result-item")]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/@href')) + if not id: + continue + cover_url = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/img/@src')) + + title = ''.join(data.xpath('.//span[@class="product-title"]/a/text()')) + author = ', '.join(data.xpath('.//span[@class="contributers-line"]/a/text()')) + price = ''.join(data.xpath('.//span[contains(@class, "onlinePriceValue2")]/text()')) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = id.strip() + + yield s From 1d50ac5cd11344bdaecb2eb61f8a5174fd28cbb0 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 6 Mar 2011 20:18:06 -0500 Subject: [PATCH 66/92] Documentation. --- src/calibre/gui2/store/__init__.py | 49 +++++++++++++++++++++- src/calibre/gui2/store/web_store_dialog.py | 3 -- 2 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 882c81597b..c413fd02dc 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -5,6 +5,35 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' class StorePlugin(object): # {{{ + ''' + A plugin representing an online ebook repository (store). The store can + be a comercial store that sells ebooks or a source of free downloadable + ebooks. + + Note that this class is the base class for these plugins, however, to + integrate the plugin with calibre's plugin system, you have to make a + wrapper class that references the actual plugin. See the + :mod:`calibre.customize.builtins` module for examples. + + If two :class:`StorePlugin` objects have the same name, the one with higher + priority takes precedence. + + Sub-classes must implement :meth:`open`, and :meth:`search`. + + Regarding :meth:`open`. Most stores only make themselves available + though a web site thus most store plugins will open using + :class:`calibre.gui2.store.web_store_dialog.WebStoreDialog`. This will + open a modal window and display the store website in a QWebView. + + Sub-classes should implement and use the :meth:`genesis` if they require + plugin specific initialization. They should not override or otherwise + reimplement :meth:`__init__`. + + Once initialized, this plugin has access to the main calibre GUI via the + :attr:`gui` member. You can access other plugins by name, for example:: + + self.gui.istores['Amazon Kindle'] + ''' def __init__(self, gui, name): self.gui = gui @@ -36,12 +65,18 @@ class StorePlugin(object): # {{{ Searches the store for items matching query. This should return items as a generator. + Don't be lazy with the search! Load as much data as possible in the + :class:`calibre.gui2.store.search_result.SearchResult` object. If you have to parse + multiple pages to get all of the data then do so. However, if data (such as cover_url) + isn't available because the store does not display cover images then it's okay to + ignore it. + :param query: The string query search with. :param max_results: The maximum number of results to return. :param timeout: The maximum amount of time in seconds to spend download the search results. :return: :class:`calibre.gui2.store.search_result.SearchResult` objects - item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. + item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. ''' raise NotImplementedError() @@ -61,15 +96,27 @@ class StorePlugin(object): # {{{ self.genesis() def genesis(self): + ''' + Plugin specific initialization. + ''' pass def config_widget(self): + ''' + See :class:`calibre.customize.Plugin` for details. + ''' raise NotImplementedError() def save_settings(self, config_widget): + ''' + See :class:`calibre.customize.Plugin` for details. + ''' raise NotImplementedError() def customization_help(self, gui=False): + ''' + See :class:`calibre.customize.Plugin` for details. + ''' raise NotImplementedError() # }}} \ No newline at end of file diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py index 1924892666..b192446be0 100644 --- a/src/calibre/gui2/store/web_store_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -4,9 +4,6 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import re -import urllib - from PyQt4.Qt import QDialog, QUrl from calibre import url_slash_cleaner From 3ba9b28c94b9717808df3f0d4704256a316072c5 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 7 Mar 2011 07:29:12 -0500 Subject: [PATCH 67/92] More store plugin documentation. --- src/calibre/gui2/store/__init__.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index c413fd02dc..3c42588fe2 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -33,6 +33,14 @@ class StorePlugin(object): # {{{ :attr:`gui` member. You can access other plugins by name, for example:: self.gui.istores['Amazon Kindle'] + + Plugin authors can use affiliate programs within their plugin. The + distribution of money earned from a store plugin is 70/30. 70% going + to the pluin author / maintainer and 30% going to the calibre project. + + The easiest way to handle affiliate money payouts is to randomly select + between the author's affiliate id and calibre's affiliate id so that + 70% of the time the author's id is used. ''' def __init__(self, gui, name): @@ -71,6 +79,13 @@ class StorePlugin(object): # {{{ isn't available because the store does not display cover images then it's okay to ignore it. + Also, by default search results can only include ebooks. A plugin can offer users + an option to include physical books in the search results but this must be + disabled by default. + + If a store doesn't provide search on it's own use something like a site specific + google search to get search results for this funtion. + :param query: The string query search with. :param max_results: The maximum number of results to return. :param timeout: The maximum amount of time in seconds to spend download the search results. From fad6742d67825059c7cc7f8caee5060cd2d220bf Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 7 Mar 2011 07:35:21 -0500 Subject: [PATCH 68/92] future imports. --- src/calibre/gui2/actions/store.py | 2 ++ src/calibre/gui2/store/__init__.py | 2 ++ src/calibre/gui2/store/amazon_plugin.py | 2 ++ src/calibre/gui2/store/baen_webscription_plugin.py | 2 ++ src/calibre/gui2/store/basic_config.py | 2 ++ src/calibre/gui2/store/bn_plugin.py | 2 ++ src/calibre/gui2/store/diesel_ebooks_plugin.py | 2 ++ src/calibre/gui2/store/ebooks_com_plugin.py | 2 ++ src/calibre/gui2/store/feedbooks_plugin.py | 2 ++ src/calibre/gui2/store/gutenberg_plugin.py | 2 ++ src/calibre/gui2/store/kobo_plugin.py | 2 ++ src/calibre/gui2/store/manybooks_plugin.py | 2 ++ src/calibre/gui2/store/open_library_plugin.py | 2 ++ src/calibre/gui2/store/search.py | 2 ++ src/calibre/gui2/store/search_result.py | 2 ++ src/calibre/gui2/store/smashwords_plugin.py | 3 ++- src/calibre/gui2/store/web_control.py | 2 ++ src/calibre/gui2/store/web_store_dialog.py | 2 ++ src/calibre/gui2/store_download.py | 2 ++ 19 files changed, 38 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py index 97653b072b..f00497ad64 100644 --- a/src/calibre/gui2/actions/store.py +++ b/src/calibre/gui2/actions/store.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 3c42588fe2..73d0d0a8d4 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index 68f2daf2ea..0b42ee1308 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/baen_webscription_plugin.py b/src/calibre/gui2/store/baen_webscription_plugin.py index 1e7c093d37..9a590a5e57 100644 --- a/src/calibre/gui2/store/baen_webscription_plugin.py +++ b/src/calibre/gui2/store/baen_webscription_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/basic_config.py b/src/calibre/gui2/store/basic_config.py index 80db687314..88ee197146 100644 --- a/src/calibre/gui2/store/basic_config.py +++ b/src/calibre/gui2/store/basic_config.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index b4fedae987..872397f5f8 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index bae358a075..1e778d94df 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index 663cacf06b..da00aa2fee 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index e03b909f4d..f3589c1ad0 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 2ccd969adb..14ee1a7827 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/kobo_plugin.py b/src/calibre/gui2/store/kobo_plugin.py index e30827b9ea..5e5db4ec82 100644 --- a/src/calibre/gui2/store/kobo_plugin.py +++ b/src/calibre/gui2/store/kobo_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index dbd328505a..c5b9a6cf4a 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/open_library_plugin.py b/src/calibre/gui2/store/open_library_plugin.py index c440a894ae..d07e0de3bd 100644 --- a/src/calibre/gui2/store/open_library_plugin.py +++ b/src/calibre/gui2/store/open_library_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py index a80201d1e4..970aaf61d2 100644 --- a/src/calibre/gui2/store/search.py +++ b/src/calibre/gui2/store/search.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/search_result.py b/src/calibre/gui2/store/search_result.py index 7eb2987a78..6e0ed0b572 100644 --- a/src/calibre/gui2/store/search_result.py +++ b/src/calibre/gui2/store/search_result.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index db66d19cf8..67e1a3c852 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' @@ -39,7 +41,6 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin): if external or settings.get(self.name + '_open_external', False): open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) else: - print detail_url d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index 45396c6637..8d0fa8a150 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py index b192446be0..20fb016b6b 100644 --- a/src/calibre/gui2/store/web_store_dialog.py +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index b5d670be97..ade8403254 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import (unicode_literals, division, absolute_import, print_function) + __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' From bcfe58bebdd2e95c8f387e3cf864f06d15764440 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 7 Mar 2011 18:27:39 -0500 Subject: [PATCH 69/92] Fix Kobo aff id. Add Kobo and ebooks.com aff ids for Kovid. --- src/calibre/gui2/store/ebooks_com_plugin.py | 5 ++--- src/calibre/gui2/store/kobo_plugin.py | 7 +++---- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index da00aa2fee..05f4e58d26 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -31,9 +31,8 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): d_click = 'click-4879827-10281551' # Use Kovid's affiliate id 30% of the time. if random.randint(1, 10) in (1, 2, 3): - #h_click = '' - #d_click = '' - pass + h_click = 'click-4913808-10364500' + d_click = 'click-4913808-10281551' url = m_url + h_click detail_url = None diff --git a/src/calibre/gui2/store/kobo_plugin.py b/src/calibre/gui2/store/kobo_plugin.py index 5e5db4ec82..9941c02837 100644 --- a/src/calibre/gui2/store/kobo_plugin.py +++ b/src/calibre/gui2/store/kobo_plugin.py @@ -27,13 +27,12 @@ class KoboStore(BasicStoreConfig, StorePlugin): settings = self.get_settings() m_url = 'http://www.dpbolvw.net/' - h_click = 'click-4879827-10755858' + h_click = 'click-4879827-10762497' d_click = 'click-4879827-10772898' # Use Kovid's affiliate id 30% of the time. if random.randint(1, 10) in (1, 2, 3): - #h_click = '' - #d_click = '' - pass + h_click = 'click-4913808-10762497' + d_click = 'click-4913808-10772898' url = m_url + h_click detail_url = None From 07f0209fc6aa89ffc78be3c2e5c4d740f3a276fa Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 7 Mar 2011 20:56:54 -0500 Subject: [PATCH 70/92] Basic MobileRead store plugin. --- src/calibre/customize/builtins.py | 7 +- src/calibre/gui2/store/mobileread_plugin.py | 114 ++++++++++++++++++++ 2 files changed, 120 insertions(+), 1 deletion(-) create mode 100644 src/calibre/gui2/store/mobileread_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index a4ec9cfeee..8d0ec7ec1d 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1087,6 +1087,11 @@ class StoreManyBooksStore(StoreBase): description = _('The best ebooks at the best price: free!') actual_plugin = 'calibre.gui2.store.manybooks_plugin:ManyBooksStore' +class StoreMobileReadStore(StoreBase): + name = 'MobileRead' + description = _('Handcrafted with utmost care ;)') + actual_plugin = 'calibre.gui2.store.mobileread_plugin:MobileReadStore' + class StoreOpenLibraryStore(StoreBase): name = 'Open Library' description = _('One web page for every book.') @@ -1100,6 +1105,6 @@ class StoreSmashwordsStore(StoreBase): plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore, StoreDieselEbooksStore, StoreEbookscomStore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, - StoreOpenLibraryStore, StoreSmashwordsStore] + StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py new file mode 100644 index 0000000000..808514b344 --- /dev/null +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -0,0 +1,114 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import difflib +import heapq +import time +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.search_result import SearchResult +from calibre.utils.config import DynamicConfig + +class MobileReadStore(StorePlugin): + + def genesis(self): + self.config = DynamicConfig('store_' + self.name) + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://www.mobileread.com/' + open_url(QUrl(detail_item if detail_item else url)) + + def search(self, query, max_results=10, timeout=60): + books = self.get_book_list(timeout=timeout) + + matches = [] + s = difflib.SequenceMatcher(lambda x: x in ' \t,.') + s.set_seq2(query.lower()) + for x in books: + # Find the match ratio for each part of the book. + s.set_seq1(x.author.lower()) + a_ratio = s.ratio() + s.set_seq1(x.title.lower()) + t_ratio = s.ratio() + s.set_seq1(x.format.lower()) + f_ratio = s.ratio() + ratio = sorted([a_ratio, t_ratio, f_ratio])[-1] + # Store the highest match ratio with the book. + matches.append((ratio, x)) + + # Move the best scorers to head of list. + matches = heapq.nlargest(max_results, matches) + for score, book in matches: + s = SearchResult() + s.title = book.title + s.author = book.author + s.price = '$0.00' + s.detail_item = book.url + + yield s + + def update_book_list(self, timeout=10): + url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' + + last_download = self.config.get(self.name + '_last_download', None) + # Don't update the book list if our cache is less than one week old. + if last_download and (time.time() - last_download) < 604800: + return + + # Download the book list HTML file from MobileRead. + br = browser() + raw_data = None + with closing(br.open(url, timeout=timeout)) as f: + raw_data = f.read() + + if not raw_data: + return + + # Turn books listed in the HTML file into BookRef's. + books = [] + try: + data = html.fromstring(raw_data) + for book_data in data.xpath('//ul/li'): + book = BookRef() + book.url = ''.join(book_data.xpath('.//a/@href')) + book.format = ''.join(book_data.xpath('.//i/text()')) + book.format = book.format.strip() + + text = ''.join(book_data.xpath('.//a/text()')) + if ':' in text: + book.author, q, text = text.partition(':') + book.author = book.author.strip() + book.title = text.strip() + books.append(book) + except: + pass + + # Save the book list and it's create time. + if books: + self.config[self.name + '_last_download'] = time.time() + self.config[self.name + '_book_list'] = books + + def get_book_list(self, timeout=10): + self.update_book_list(timeout=timeout) + return self.config.get(self.name + '_book_list', []) + + +class BookRef(object): + + def __init__(self): + self.author = '' + self.title = '' + self.format = '' + self.url = '' From 4ddc69e0a6944f88158847e5b9a025f45e5e034e Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 8 Mar 2011 07:04:04 -0500 Subject: [PATCH 71/92] Ensure only one thread can run the update function for mobileread. --- src/calibre/gui2/store/mobileread_plugin.py | 77 +++++++++++---------- 1 file changed, 40 insertions(+), 37 deletions(-) diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 808514b344..b8b681b6b9 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -10,6 +10,7 @@ import difflib import heapq import time from contextlib import closing +from threading import RLock from lxml import html @@ -25,6 +26,7 @@ class MobileReadStore(StorePlugin): def genesis(self): self.config = DynamicConfig('store_' + self.name) + self.rlock = RLock() def open(self, parent=None, detail_item=None, external=False): url = 'http://www.mobileread.com/' @@ -60,45 +62,46 @@ class MobileReadStore(StorePlugin): yield s def update_book_list(self, timeout=10): - url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' + with self.rlock: + url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' + + last_download = self.config.get(self.name + '_last_download', None) + # Don't update the book list if our cache is less than one week old. + if last_download and (time.time() - last_download) < 604800: + return + + # Download the book list HTML file from MobileRead. + br = browser() + raw_data = None + with closing(br.open(url, timeout=timeout)) as f: + raw_data = f.read() + + if not raw_data: + return + + # Turn books listed in the HTML file into BookRef's. + books = [] + try: + data = html.fromstring(raw_data) + for book_data in data.xpath('//ul/li'): + book = BookRef() + book.url = ''.join(book_data.xpath('.//a/@href')) + book.format = ''.join(book_data.xpath('.//i/text()')) + book.format = book.format.strip() - last_download = self.config.get(self.name + '_last_download', None) - # Don't update the book list if our cache is less than one week old. - if last_download and (time.time() - last_download) < 604800: - return - - # Download the book list HTML file from MobileRead. - br = browser() - raw_data = None - with closing(br.open(url, timeout=timeout)) as f: - raw_data = f.read() - - if not raw_data: - return - - # Turn books listed in the HTML file into BookRef's. - books = [] - try: - data = html.fromstring(raw_data) - for book_data in data.xpath('//ul/li'): - book = BookRef() - book.url = ''.join(book_data.xpath('.//a/@href')) - book.format = ''.join(book_data.xpath('.//i/text()')) - book.format = book.format.strip() + text = ''.join(book_data.xpath('.//a/text()')) + if ':' in text: + book.author, q, text = text.partition(':') + book.author = book.author.strip() + book.title = text.strip() + books.append(book) + except: + pass - text = ''.join(book_data.xpath('.//a/text()')) - if ':' in text: - book.author, q, text = text.partition(':') - book.author = book.author.strip() - book.title = text.strip() - books.append(book) - except: - pass - - # Save the book list and it's create time. - if books: - self.config[self.name + '_last_download'] = time.time() - self.config[self.name + '_book_list'] = books + # Save the book list and it's create time. + if books: + self.config[self.name + '_last_download'] = time.time() + self.config[self.name + '_book_list'] = books def get_book_list(self, timeout=10): self.update_book_list(timeout=timeout) From 6d74a02f3f7aa33724a675a85aacead4f6c75be5 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 8 Mar 2011 07:55:33 -0500 Subject: [PATCH 72/92] MobileRead: more accurate and faster search. --- src/calibre/gui2/store/mobileread_plugin.py | 29 ++++++++++++--------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index b8b681b6b9..6e63ff054d 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -35,24 +35,29 @@ class MobileReadStore(StorePlugin): def search(self, query, max_results=10, timeout=60): books = self.get_book_list(timeout=timeout) + query = query.lower() + query_parts = query.split(' ') matches = [] - s = difflib.SequenceMatcher(lambda x: x in ' \t,.') - s.set_seq2(query.lower()) + s = difflib.SequenceMatcher() for x in books: - # Find the match ratio for each part of the book. - s.set_seq1(x.author.lower()) - a_ratio = s.ratio() - s.set_seq1(x.title.lower()) - t_ratio = s.ratio() - s.set_seq1(x.format.lower()) - f_ratio = s.ratio() - ratio = sorted([a_ratio, t_ratio, f_ratio])[-1] - # Store the highest match ratio with the book. - matches.append((ratio, x)) + ratio = 0 + t_string = '%s %s' % (x.author.lower(), x.title.lower()) + query_matches = [] + for q in query_parts: + if q in t_string: + query_matches.append(q) + for q in query_matches: + s.set_seq2(q) + for p in t_string.split(' '): + s.set_seq1(p) + ratio += s.ratio() + if ratio > 0: + matches.append((ratio, x)) # Move the best scorers to head of list. matches = heapq.nlargest(max_results, matches) for score, book in matches: + print('%s %s' % (score, book.title)) s = SearchResult() s.title = book.title s.author = book.author From 000258910460669f3f2268ebed148ddfc0e8baab Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 8 Mar 2011 19:00:02 -0500 Subject: [PATCH 73/92] Base BookRef on SearchResult so we can pass it on directly. --- src/calibre/gui2/store/mobileread_plugin.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 6e63ff054d..235b24e4eb 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -57,14 +57,8 @@ class MobileReadStore(StorePlugin): # Move the best scorers to head of list. matches = heapq.nlargest(max_results, matches) for score, book in matches: - print('%s %s' % (score, book.title)) - s = SearchResult() - s.title = book.title - s.author = book.author - s.price = '$0.00' - s.detail_item = book.url - - yield s + book.price = '$0.00' + yield book def update_book_list(self, timeout=10): with self.rlock: @@ -90,7 +84,7 @@ class MobileReadStore(StorePlugin): data = html.fromstring(raw_data) for book_data in data.xpath('//ul/li'): book = BookRef() - book.url = ''.join(book_data.xpath('.//a/@href')) + book.detail_item = ''.join(book_data.xpath('.//a/@href')) book.format = ''.join(book_data.xpath('.//i/text()')) book.format = book.format.strip() @@ -113,10 +107,9 @@ class MobileReadStore(StorePlugin): return self.config.get(self.name + '_book_list', []) -class BookRef(object): +class BookRef(SearchResult): def __init__(self): - self.author = '' - self.title = '' + SearchResult.__init__(self) + self.format = '' - self.url = '' From f00ea5e7af3f6848fd5b0ce3fa7a0cbbb7552631 Mon Sep 17 00:00:00 2001 From: John Schember Date: Tue, 8 Mar 2011 21:28:13 -0500 Subject: [PATCH 74/92] Mobileread: Basic config and open in embedded browser. Use Content-disposition header when avaliable to get filename. --- src/calibre/gui2/store/mobileread_plugin.py | 14 ++++++++++++-- src/calibre/gui2/store/web_control.py | 14 +++++++++++++- src/calibre/gui2/store_download.py | 13 ++++++++++++- 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 235b24e4eb..d8c8b540cf 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -19,18 +19,28 @@ from PyQt4.Qt import QUrl from calibre import browser from calibre.gui2 import open_url from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog from calibre.utils.config import DynamicConfig -class MobileReadStore(StorePlugin): +class MobileReadStore(BasicStoreConfig, StorePlugin): def genesis(self): self.config = DynamicConfig('store_' + self.name) self.rlock = RLock() def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() url = 'http://www.mobileread.com/' - open_url(QUrl(detail_item if detail_item else url)) + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(detail_item if detail_item else url)) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() def search(self, query, max_results=10, timeout=60): books = self.get_book_list(timeout=timeout) diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index 8d0fa8a150..4ec40bcf69 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -7,6 +7,7 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import os +import urllib2 from cookielib import Cookie, CookieJar from urlparse import urlparse @@ -63,8 +64,19 @@ class NPWebView(QWebView): br = browser() br.set_cookiejar(cj) + r = br.open(url) + + basename = '' + disposition = r.info().get('Content-disposition', '') + if 'filename' in disposition: + if 'filename*=' in disposition: + basename = disposition.split('filename*=')[-1].split('\'\'')[-1] + else: + basename = disposition.split('filename=')[-1] + basename = urllib2.unquote(basename) + if not basename: + basename = r.geturl().split('/')[-1] - basename = br.open(url).geturl().split('/')[-1] ext = os.path.splitext(basename)[1][1:].lower() if ext not in BOOK_EXTENSIONS: home = os.path.expanduser('~') diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/store_download.py index ade8403254..e101badfb3 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/store_download.py @@ -10,6 +10,7 @@ import cStringIO import os import shutil import time +import urllib2 from cookielib import CookieJar from contextlib import closing from threading import Thread @@ -153,7 +154,17 @@ class StoreDownloader(Thread): br.set_cookiejar(job.cookie_jar) with closing(br.open(url)) as r: - basename = r.geturl().split('/')[-1] + basename = '' + disposition = r.info().get('Content-disposition', '') + if 'filename' in disposition: + if 'filename*=' in disposition: + basename = disposition.split('filename*=')[-1].split('\'\'')[-1] + else: + basename = disposition.split('filename=')[-1] + basename = urllib2.unquote(basename) + if not basename: + basename = r.geturl().split('/')[-1] + tf = PersistentTemporaryFile(suffix=basename) tf.write(r.read()) job.tmp_file_name = tf.name From 602f7d11acb731b30f232a585104a510970faff5 Mon Sep 17 00:00:00 2001 From: John Schember Date: Wed, 9 Mar 2011 07:30:46 -0500 Subject: [PATCH 75/92] add function to get download file name from url. Send filename when starting download and use filename. --- src/calibre/__init__.py | 29 +++++++++++++++++ src/calibre/gui2/store/web_control.py | 26 ++++------------ src/calibre/gui2/store_download.py | 45 +++++++++++---------------- 3 files changed, 53 insertions(+), 47 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index dbbb8b2296..9b9b473b5b 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -5,7 +5,9 @@ __docformat__ = 'restructuredtext en' import uuid, sys, os, re, logging, time, \ __builtin__, warnings, multiprocessing +from contextlib import closing from urllib import getproxies +from urllib2 import unquote as urllib2_unquote __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) from htmlentitydefs import name2codepoint from math import floor @@ -479,6 +481,33 @@ def url_slash_cleaner(url): ''' return re.sub(r'(? Date: Wed, 9 Mar 2011 18:24:30 -0500 Subject: [PATCH 76/92] B&N affiliate code. --- src/calibre/gui2/store/bn_plugin.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index 872397f5f8..77b982af4a 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -6,6 +6,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import re import urllib2 from contextlib import closing @@ -24,7 +25,13 @@ class BNStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - url = 'http://www.barnesandnoble.com/ebooks/index.asp' + url = 'http://gan.doubleclick.net/gan_click?lid=41000000028437369&pubid=21000000000352219' + + if detail_item: + mo = re.search(r'(?<=/)(?P\d+)(?=/|$)', detail_item) + if mo: + isbn = mo.group('isbn') + detail_item = 'http://gan.doubleclick.net/gan_click?lid=41000000012871747&pid=' + isbn + '&adurl=' + detail_item + '&pubid=21000000000352219' if external or settings.get(self.name + '_open_external', False): open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url))) From b6af1d4e28975adec7902c61f938e1deed7f7b2d Mon Sep 17 00:00:00 2001 From: John Schember Date: Thu, 10 Mar 2011 20:31:34 -0500 Subject: [PATCH 77/92] BeWrite store. --- src/calibre/customize/builtins.py | 9 ++- src/calibre/gui2/store/bewrite_plugin.py | 81 ++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 src/calibre/gui2/store/bewrite_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 8d0ec7ec1d..e6c1b4bdbe 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1057,6 +1057,11 @@ class StoreBNStore(StoreBase): description = _('Books, Textbooks, eBooks, Toys, Games and More.') actual_plugin = 'calibre.gui2.store.bn_plugin:BNStore' +class StoreBeWriteStore(StoreBase): + name = 'BeWrite Books' + description = _('Publishers of fine books.') + actual_plugin = 'calibre.gui2.store.bewrite_plugin:BeWriteStore' + class StoreDieselEbooksStore(StoreBase): name = 'Diesel eBooks' description = _('World Famous eBook Store.') @@ -1103,8 +1108,8 @@ class StoreSmashwordsStore(StoreBase): actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore, - StoreDieselEbooksStore, StoreEbookscomStore, StoreFeedbooksStore, - StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, + StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, + StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore] # }}} diff --git a/src/calibre/gui2/store/bewrite_plugin.py b/src/calibre/gui2/store/bewrite_plugin.py new file mode 100644 index 0000000000..991abae3d6 --- /dev/null +++ b/src/calibre/gui2/store/bewrite_plugin.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class BeWriteStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://www.bewrite.net/mm5/merchant.mvc?Screen=SFNT' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(url_slash_cleaner(url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.bewrite.net/mm5/merchant.mvc?Search_Code=B&Screen=SRCH&Search=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@id="content"]//table/tr[position() > 1]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//a/@href')) + if not id: + continue + + heading = ''.join(data.xpath('./td[2]//text()')) + title, q, author = heading.partition('by ') + cover_url = '' + price = '' + + with closing(br.open(id.strip(), timeout=timeout/4)) as nf: + idata = html.fromstring(nf.read()) + price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()')) + price = '$' + price.split('$')[-1] + cover_img = idata.xpath('//div[@id="content"]//img[1]/@src') + if cover_img: + cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url.strip() + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = id.strip() + + yield s From 5a3667e7013da15e2608669eae996142acf93e02 Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 11 Mar 2011 06:58:52 -0500 Subject: [PATCH 78/92] Add Kovid's pubid for B&N. --- src/calibre/gui2/store/bn_plugin.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index 77b982af4a..d36d8e466e 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -6,6 +6,7 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' +import random import re import urllib2 from contextlib import closing @@ -25,13 +26,19 @@ class BNStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - url = 'http://gan.doubleclick.net/gan_click?lid=41000000028437369&pubid=21000000000352219' + + pub_id = '21000000000352219' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + pub_id = '21000000000352583' + + url = 'http://gan.doubleclick.net/gan_click?lid=41000000028437369&pubid=' + pub_id if detail_item: mo = re.search(r'(?<=/)(?P\d+)(?=/|$)', detail_item) if mo: isbn = mo.group('isbn') - detail_item = 'http://gan.doubleclick.net/gan_click?lid=41000000012871747&pid=' + isbn + '&adurl=' + detail_item + '&pubid=21000000000352219' + detail_item = 'http://gan.doubleclick.net/gan_click?lid=41000000012871747&pid=' + isbn + '&adurl=' + detail_item + '&pubid=' + pub_id if external or settings.get(self.name + '_open_external', False): open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url))) From 4a34af3ad68efdaf0465d9072b70f1f6082f8577 Mon Sep 17 00:00:00 2001 From: John Schember Date: Fri, 11 Mar 2011 21:01:00 -0500 Subject: [PATCH 79/92] Change how cookies are transfered. Use a Mozilla cookie txt file for moving cookies. This fixes and issue with nook library files not downloading from B&N. --- src/calibre/__init__.py | 11 +++++--- src/calibre/gui2/store/web_control.py | 39 ++++++++++++++++++++++++--- src/calibre/gui2/store_download.py | 22 ++++++++------- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 9b9b473b5b..605307d44b 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -481,13 +481,16 @@ def url_slash_cleaner(url): ''' return re.sub(r'(? Date: Fri, 11 Mar 2011 21:03:37 -0500 Subject: [PATCH 80/92] Remove unused code. --- src/calibre/gui2/store/web_control.py | 54 +-------------------------- 1 file changed, 1 insertion(+), 53 deletions(-) diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index f22b4c74b2..783594796c 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -7,14 +7,12 @@ __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' import os -import urllib2 -from cookielib import Cookie, CookieJar from urlparse import urlparse from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString, \ QFileDialog, QNetworkProxy -from calibre import USER_AGENT, browser, get_proxies, get_download_filename +from calibre import USER_AGENT, get_proxies, get_download_filename from calibre.ebooks import BOOK_EXTENSIONS from calibre.ptempfile import PersistentTemporaryFile @@ -106,56 +104,6 @@ class NPWebView(QWebView): cf.close() return cf.name - - ''' - def get_cookies(self): - cj = CookieJar() - - # Translate Qt cookies to cookielib cookies for use by mechanize. - for c in self.page().networkAccessManager().cookieJar().allCookies(): - version = 0 - name = unicode(QString(c.name())) - value = unicode(QString(c.value())) - port = None - port_specified = False - domain = unicode(c.domain()) - if domain: - domain_specified = True - if domain.startswith('.'): - domain_initial_dot = True - else: - domain_initial_dot = False - else: - domain = None - domain_specified = False - path = unicode(c.path()) - if path: - path_specified = True - else: - path = None - path_specified = False - secure = c.isSecure() - expires = c.expirationDate().toMSecsSinceEpoch() / 1000 - discard = c.isSessionCookie() - comment = None - comment_url = None - rest = None - - cookie = Cookie(version, name, value, - port, port_specified, - domain, domain_specified, domain_initial_dot, - path, path_specified, - secure, - expires, - discard, - comment, - comment_url, - rest) - - cj.set_cookie(cookie) - - return cj - ''' class NPWebPage(QWebPage): From 4f40bdfdf8eed791b5d66849040bbab9a977e7cd Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 12 Mar 2011 08:18:03 -0500 Subject: [PATCH 81/92] Rename store_download to ebook_download. --- .../{store_download.py => ebook_download.py} | 29 +++++++++---------- src/calibre/gui2/store/web_control.py | 4 +-- src/calibre/gui2/ui.py | 6 ++-- 3 files changed, 19 insertions(+), 20 deletions(-) rename src/calibre/gui2/{store_download.py => ebook_download.py} (87%) diff --git a/src/calibre/gui2/store_download.py b/src/calibre/gui2/ebook_download.py similarity index 87% rename from src/calibre/gui2/store_download.py rename to src/calibre/gui2/ebook_download.py index 74489156c8..b3468cde24 100644 --- a/src/calibre/gui2/store_download.py +++ b/src/calibre/gui2/ebook_download.py @@ -21,7 +21,7 @@ from calibre.gui2 import Dispatcher from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.ipc.job import BaseJob -class StoreDownloadJob(BaseJob): +class EbookDownloadJob(BaseJob): def __init__(self, callback, description, job_manager, db, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): BaseJob.__init__(self, description) @@ -57,7 +57,7 @@ class StoreDownloadJob(BaseJob): self.log_write(traceback.format_exc()) # Dump log onto disk - lf = PersistentTemporaryFile('store_log') + lf = PersistentTemporaryFile('ebook_download_log') lf.write(self._log_file.getvalue()) lf.close() self.log_path = lf.name @@ -69,7 +69,7 @@ class StoreDownloadJob(BaseJob): def log_write(self, what): self._log_file.write(what) -class StoreDownloader(Thread): +class EbookDownloader(Thread): def __init__(self, job_manager): Thread.__init__(self) @@ -120,7 +120,7 @@ class StoreDownloader(Thread): import traceback failed = True exc = e - job.log_write('\nSending failed...\n') + job.log_write('\nDownloading failed...\n') job.log_write(traceback.format_exc()) if not self._run: @@ -185,32 +185,31 @@ class StoreDownloader(Thread): shutil.copy(job.tmp_file_name, save_loc) - def download_from_store(self, callback, db, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): + def download_ebook(self, callback, db, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): description = _('Downloading %s') % filename if filename else url - job = StoreDownloadJob(callback, description, self.job_manager, db, cookie_file, url, filename, save_as_loc, add_to_lib, tags) + job = EbookDownloadJob(callback, description, self.job_manager, db, cookie_file, url, filename, save_as_loc, add_to_lib, tags) self.job_manager.add_job(job) self.jobs.put(job) -class StoreDownloadMixin(object): +class EbookDownloadMixin(object): def __init__(self): - self.store_downloader = StoreDownloader(self.job_manager) + self.ebook_downloader = EbookDownloader(self.job_manager) - def download_from_store(self, url='', cookie_file=None, filename='', save_as_loc='', add_to_lib=True, tags=[]): - if not self.store_downloader.is_alive(): - self.store_downloader.start() + def download_ebook(self, url='', cookie_file=None, filename='', save_as_loc='', add_to_lib=True, tags=[]): + if not self.ebook_downloader.is_alive(): + self.ebook_downloader.start() if tags: if isinstance(tags, basestring): tags = tags.split(',') - self.store_downloader.download_from_store(Dispatcher(self.downloaded_from_store), self.library_view.model().db, cookie_file, url, filename, save_as_loc, add_to_lib, tags) + self.ebook_downloader.download_ebook(Dispatcher(self.downloaded_ebook), self.library_view.model().db, cookie_file, url, filename, save_as_loc, add_to_lib, tags) self.status_bar.show_message(_('Downloading') + ' ' + filename if filename else url, 3000) - def downloaded_from_store(self, job): + def downloaded_ebook(self, job): if job.failed: - self.job_exception(job, dialog_title=_('Failed to download book')) + self.job_exception(job, dialog_title=_('Failed to download ebook')) return self.status_bar.show_message(job.description + ' ' + _('finished'), 5000) - diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index 783594796c..b7ab75975d 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -70,9 +70,9 @@ class NPWebView(QWebView): os.path.join(home, filename), '*.*') if name: - self.gui.download_from_store(url, cf, name, name, False) + self.gui.download_ebook(url, cf, name, name, False) else: - self.gui.download_from_store(url, cf, filename, tags=self.tags) + self.gui.download_ebook(url, cf, filename, tags=self.tags) def ignore_ssl_errors(self, reply, errors): reply.ignoreSslErrors(errors) diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 455e56169e..8bd7b6fe87 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -33,7 +33,7 @@ from calibre.gui2.main_window import MainWindow from calibre.gui2.layout import MainWindowMixin from calibre.gui2.device import DeviceMixin from calibre.gui2.email import EmailMixin -from calibre.gui2.store_download import StoreDownloadMixin +from calibre.gui2.ebook_download import EbookDownloadMixin from calibre.gui2.jobs import JobManager, JobsDialog, JobsButton from calibre.gui2.init import LibraryViewMixin, LayoutMixin from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin @@ -91,7 +91,7 @@ class SystemTrayIcon(QSystemTrayIcon): # {{{ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin, SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin, - StoreDownloadMixin + EbookDownloadMixin ): 'The main GUI' @@ -199,7 +199,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ LayoutMixin.__init__(self) EmailMixin.__init__(self) - StoreDownloadMixin.__init__(self) + EbookDownloadMixin.__init__(self) DeviceMixin.__init__(self) self.progress_indicator = ProgressIndicator(self) From 5ec3737e300861affcd3508c3134cec0017706ac Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 12 Mar 2011 08:32:48 -0500 Subject: [PATCH 82/92] Update library view when new books finish downloading. --- src/calibre/gui2/ebook_download.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/ebook_download.py b/src/calibre/gui2/ebook_download.py index b3468cde24..c8e56830f8 100644 --- a/src/calibre/gui2/ebook_download.py +++ b/src/calibre/gui2/ebook_download.py @@ -23,11 +23,11 @@ from calibre.utils.ipc.job import BaseJob class EbookDownloadJob(BaseJob): - def __init__(self, callback, description, job_manager, db, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): + def __init__(self, callback, description, job_manager, model, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): BaseJob.__init__(self, description) self.exception = None self.job_manager = job_manager - self.db = db + self.model = model self.cookie_file = cookie_file self.args = (url, filename, save_as_loc, add_to_lib, tags) self.tmp_file_name = '' @@ -175,8 +175,10 @@ class EbookDownloader(Thread): mi = get_metadata(f, ext) mi.tags.extend(tags) - id = job.db.create_book_entry(mi) - job.db.add_format_with_hooks(id, ext.upper(), job.tmp_file_name, index_is_id=True) + id = job.model.db.create_book_entry(mi) + job.model.db.add_format_with_hooks(id, ext.upper(), job.tmp_file_name, index_is_id=True) + job.model.books_added(1) + job.model.count_changed() def _save_as(self, job): url, filename, save_loc, add_to_lib, tags = job.args @@ -185,9 +187,9 @@ class EbookDownloader(Thread): shutil.copy(job.tmp_file_name, save_loc) - def download_ebook(self, callback, db, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): + def download_ebook(self, callback, model, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): description = _('Downloading %s') % filename if filename else url - job = EbookDownloadJob(callback, description, self.job_manager, db, cookie_file, url, filename, save_as_loc, add_to_lib, tags) + job = EbookDownloadJob(callback, description, self.job_manager, model, cookie_file, url, filename, save_as_loc, add_to_lib, tags) self.job_manager.add_job(job) self.jobs.put(job) @@ -203,7 +205,7 @@ class EbookDownloadMixin(object): if tags: if isinstance(tags, basestring): tags = tags.split(',') - self.ebook_downloader.download_ebook(Dispatcher(self.downloaded_ebook), self.library_view.model().db, cookie_file, url, filename, save_as_loc, add_to_lib, tags) + self.ebook_downloader.download_ebook(Dispatcher(self.downloaded_ebook), self.library_view.model(), cookie_file, url, filename, save_as_loc, add_to_lib, tags) self.status_bar.show_message(_('Downloading') + ' ' + filename if filename else url, 3000) def downloaded_ebook(self, job): From 9f7eb04762b1145ae45e9c7700138d9a79bf2817 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 12 Mar 2011 08:48:48 -0500 Subject: [PATCH 83/92] Send reference to GUI to store download jobs. This way db references are not stored in the download queue. This should prevent issues where the db or model reference is invalid because the user changed libraries during a download. --- src/calibre/gui2/ebook_download.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/ebook_download.py b/src/calibre/gui2/ebook_download.py index c8e56830f8..4ffff9978d 100644 --- a/src/calibre/gui2/ebook_download.py +++ b/src/calibre/gui2/ebook_download.py @@ -23,11 +23,11 @@ from calibre.utils.ipc.job import BaseJob class EbookDownloadJob(BaseJob): - def __init__(self, callback, description, job_manager, model, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): + def __init__(self, callback, description, job_manager, gui, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): BaseJob.__init__(self, description) self.exception = None self.job_manager = job_manager - self.model = model + self.gui = gui self.cookie_file = cookie_file self.args = (url, filename, save_as_loc, add_to_lib, tags) self.tmp_file_name = '' @@ -175,10 +175,10 @@ class EbookDownloader(Thread): mi = get_metadata(f, ext) mi.tags.extend(tags) - id = job.model.db.create_book_entry(mi) - job.model.db.add_format_with_hooks(id, ext.upper(), job.tmp_file_name, index_is_id=True) - job.model.books_added(1) - job.model.count_changed() + id = job.gui.library_view.model().db.create_book_entry(mi) + job.gui.library_view.model().db.add_format_with_hooks(id, ext.upper(), job.tmp_file_name, index_is_id=True) + job.gui.library_view.model().books_added(1) + job.gui.library_view.model().count_changed() def _save_as(self, job): url, filename, save_loc, add_to_lib, tags = job.args @@ -187,9 +187,9 @@ class EbookDownloader(Thread): shutil.copy(job.tmp_file_name, save_loc) - def download_ebook(self, callback, model, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): + def download_ebook(self, callback, gui, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): description = _('Downloading %s') % filename if filename else url - job = EbookDownloadJob(callback, description, self.job_manager, model, cookie_file, url, filename, save_as_loc, add_to_lib, tags) + job = EbookDownloadJob(callback, description, self.job_manager, gui, cookie_file, url, filename, save_as_loc, add_to_lib, tags) self.job_manager.add_job(job) self.jobs.put(job) @@ -205,7 +205,7 @@ class EbookDownloadMixin(object): if tags: if isinstance(tags, basestring): tags = tags.split(',') - self.ebook_downloader.download_ebook(Dispatcher(self.downloaded_ebook), self.library_view.model(), cookie_file, url, filename, save_as_loc, add_to_lib, tags) + self.ebook_downloader.download_ebook(Dispatcher(self.downloaded_ebook), self, cookie_file, url, filename, save_as_loc, add_to_lib, tags) self.status_bar.show_message(_('Downloading') + ' ' + filename if filename else url, 3000) def downloaded_ebook(self, job): From 6663715f7562661636dc5def70101258f2526d87 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 12 Mar 2011 14:38:30 -0500 Subject: [PATCH 84/92] Initalize variable. --- src/calibre/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index f25a6341ca..9408f7266a 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -520,6 +520,7 @@ def get_download_filename(url, cookie_file=None): br.set_cookiejar(cj) with closing(br.open(url)) as r: + filename = '' disposition = r.info().get('Content-disposition', '') for p in disposition.split(';'): if 'filename' in p: From 77a9964fac5607a25aba43d9a04a55804ab5a451 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 12 Mar 2011 14:39:00 -0500 Subject: [PATCH 85/92] ... --- src/calibre/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 9408f7266a..f25a6341ca 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -520,7 +520,6 @@ def get_download_filename(url, cookie_file=None): br.set_cookiejar(cj) with closing(br.open(url)) as r: - filename = '' disposition = r.info().get('Content-disposition', '') for p in disposition.split(';'): if 'filename' in p: From 5de66728072e29c385fd40eb8303af410be36bfc Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 12 Mar 2011 14:40:41 -0500 Subject: [PATCH 86/92] Ensure filename is always set and report any errors while trying to get the filename from a url. --- src/calibre/__init__.py | 39 ++++++++++++++++++++++----------------- 1 file changed, 22 insertions(+), 17 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index f25a6341ca..4d3e5076f6 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -519,23 +519,28 @@ def get_download_filename(url, cookie_file=None): cj.load(cookie_file) br.set_cookiejar(cj) - with closing(br.open(url)) as r: - disposition = r.info().get('Content-disposition', '') - for p in disposition.split(';'): - if 'filename' in p: - if '*=' in disposition: - parts = disposition.split('*=')[-1] - filename = parts.split('\'')[-1] - else: - filename = disposition.split('=')[-1] - if filename[0] in ('\'', '"'): - filename = filename[1:] - if filename[-1] in ('\'', '"'): - filename = filename[:-1] - filename = urllib2_unquote(filename) - break - if not filename: - filename = r.geturl().split('/')[-1] + try: + with closing(br.open(url)) as r: + disposition = r.info().get('Content-disposition', '') + for p in disposition.split(';'): + if 'filename' in p: + if '*=' in disposition: + parts = disposition.split('*=')[-1] + filename = parts.split('\'')[-1] + else: + filename = disposition.split('=')[-1] + if filename[0] in ('\'', '"'): + filename = filename[1:] + if filename[-1] in ('\'', '"'): + filename = filename[:-1] + filename = urllib2_unquote(filename) + break + except: + import traceback + traceback.print_exc() + + if not filename: + filename = r.geturl().split('/')[-1] return filename From 98921e4bfa0691fefd61ec8d5d8c19b7f3ce08a2 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 12 Mar 2011 22:15:45 -0500 Subject: [PATCH 87/92] eharlequin store plugin. --- src/calibre/customize/builtins.py | 6 ++ src/calibre/gui2/store/eharlequin_plugin.py | 86 +++++++++++++++++++++ 2 files changed, 92 insertions(+) create mode 100644 src/calibre/gui2/store/eharlequin_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 1a3e8b7511..9ee13b0a28 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1072,6 +1072,11 @@ class StoreEbookscomStore(StoreBase): description = _('The digital bookstore.') actual_plugin = 'calibre.gui2.store.ebooks_com_plugin:EbookscomStore' +class StoreEHarlequinStoretore(StoreBase): + name = 'eHarlequin' + description = _('entertain, enrich, inspire.') + actual_plugin = 'calibre.gui2.store.eharlequin_plugin:EHarlequinStore' + class StoreFeedbooksStore(StoreBase): name = 'Feedbooks' description = _('Read anywhere.') @@ -1109,6 +1114,7 @@ class StoreSmashwordsStore(StoreBase): plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore, StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, + StoreEHarlequinStoretore, StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore] diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py new file mode 100644 index 0000000000..241c73312b --- /dev/null +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class EHarlequinStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + + ''' + m_url = 'http://www.dpbolvw.net/' + h_click = 'click-4879827-10762497' + d_click = 'click-4879827-10772898' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + h_click = 'click-4913808-10762497' + d_click = 'click-4913808-10772898' + ''' + m_url = 'http://ebooks.eharlequin.com/' + h_click = '' + d_click = '' + + url = m_url + h_click + detail_url = None + if detail_item: + detail_url = m_url + d_click + detail_item + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d = d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://ebooks.eharlequin.com/BANGSearch.dll?Type=FullText&FullTextField=All&FullTextCriteria=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//table[not(.//@class="sidelink")]/tr[.//ul[@id="details"]]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//ul[@id="details"]/li[@id="title-results"]/a/@href')) + if not id: + continue + + title = ''.join(data.xpath('.//ul[@id="details"]/li[@id="title-results"]/a/text()')) + author = ''.join(data.xpath('.//ul[@id="details"]/li[@id="author"][1]//a/text()')) + price = ''.join(data.xpath('.//div[@class="ourprice"]/font/text()')) + cover_url = ''.join(data.xpath('.//a[@href="%s"]/img/@src' % id)) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = id.strip() + #s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() + + yield s From ad306d8f6778c5385b5e443dade17c416281b253 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sun, 13 Mar 2011 12:43:02 -0400 Subject: [PATCH 88/92] MobileRead store dialog to display avaliable books. --- src/calibre/gui2/store/mobileread_plugin.py | 190 +++++++++++++++++- .../gui2/store/mobileread_store_dialog.ui | 112 +++++++++++ 2 files changed, 296 insertions(+), 6 deletions(-) create mode 100644 src/calibre/gui2/store/mobileread_store_dialog.ui diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index d8c8b540cf..37f125f351 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -14,15 +14,18 @@ from threading import RLock from lxml import html -from PyQt4.Qt import QUrl +from PyQt4.Qt import Qt, QUrl, QDialog, QAbstractItemModel, QModelIndex, QVariant, \ + pyqtSignal from calibre import browser -from calibre.gui2 import open_url +from calibre.gui2 import open_url, NONE from calibre.gui2.store import StorePlugin from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.mobileread_store_dialog_ui import Ui_Dialog from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.web_store_dialog import WebStoreDialog from calibre.utils.config import DynamicConfig +from calibre.utils.icu import sort_key class MobileReadStore(BasicStoreConfig, StorePlugin): @@ -37,10 +40,15 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): if external or settings.get(self.name + '_open_external', False): open_url(QUrl(detail_item if detail_item else url)) else: - d = WebStoreDialog(self.gui, url, parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + if detail_item: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + else: + d = MobeReadStoreDialog(self, parent) + d.setWindowTitle(self.name) + d.exec_() def search(self, query, max_results=10, timeout=60): books = self.get_book_list(timeout=timeout) @@ -123,3 +131,173 @@ class BookRef(SearchResult): SearchResult.__init__(self) self.format = '' + + +class MobeReadStoreDialog(QDialog, Ui_Dialog): + + def __init__(self, plugin, *args): + QDialog.__init__(self, *args) + self.setupUi(self) + + self.plugin = plugin + + self.model = BooksModel() + self.results_view.setModel(self.model) + self.results_view.model().set_books(self.plugin.get_book_list()) + self.total.setText('%s' % self.model.rowCount()) + + self.results_view.activated.connect(self.open_store) + self.search_query.textChanged.connect(self.model.set_filter) + self.results_view.model().total_changed.connect(self.total.setText) + self.finished.connect(self.dialog_closed) + + self.restore_state() + + def open_store(self, index): + result = self.results_view.model().get_book(index) + if result: + self.plugin.open(self, result.detail_item) + + def restore_state(self): + geometry = self.plugin.config['store_mobileread_dialog_geometry'] + if geometry: + self.restoreGeometry(geometry) + + results_cwidth = self.plugin.config['store_mobileread_dialog_results_view_column_width'] + if results_cwidth: + for i, x in enumerate(results_cwidth): + if i >= self.results_view.model().columnCount(): + break + self.results_view.setColumnWidth(i, x) + else: + for i in xrange(self.results_view.model().columnCount()): + self.results_view.resizeColumnToContents(i) + + self.results_view.model().sort_col = self.plugin.config.get('store_mobileread_dialog_sort_col', 0) + self.results_view.model().sort_order = self.plugin.config.get('store_mobileread_dialog_sort_order', Qt.AscendingOrder) + self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) + + def save_state(self): + self.plugin.config['store_mobileread_dialog_geometry'] = self.saveGeometry() + self.plugin.config['store_mobileread_dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())] + self.plugin.config['store_mobileread_dialog_sort_col'] = self.results_view.model().sort_col + self.plugin.config['store_mobileread_dialog_sort_order'] = self.results_view.model().sort_order + + def dialog_closed(self, result): + self.save_state() + + +class BooksModel(QAbstractItemModel): + + total_changed = pyqtSignal(unicode) + + HEADERS = [_('Title'), _('Author(s)'), _('Format')] + + def __init__(self): + QAbstractItemModel.__init__(self) + self.books = [] + self.all_books = [] + self.filter = '' + self.sort_col = 0 + self.sort_order = Qt.AscendingOrder + + def set_books(self, books): + self.books = books + self.all_books = books + + self.sort(self.sort_col, self.sort_order) + + def get_book(self, index): + row = index.row() + if row < len(self.books): + return self.books[row] + else: + return None + + def set_filter(self, filter): + #self.layoutAboutToBeChanged.emit() + self.beginResetModel() + + self.filter = unicode(filter) + self.books = [] + if self.filter: + for b in self.all_books: + test = '%s %s %s' % (b.title, b.author, b.format) + test = test.lower() + include = True + for item in self.filter.split(' '): + item = item.lower() + if item not in test: + include = False + break + if include: + self.books.append(b) + else: + self.books = self.all_books + + self.sort(self.sort_col, self.sort_order, reset=False) + self.total_changed.emit('%s' % self.rowCount()) + + self.endResetModel() + #self.layoutChanged.emit() + + def index(self, row, column, parent=QModelIndex()): + return self.createIndex(row, column) + + def parent(self, index): + if not index.isValid() or index.internalId() == 0: + return QModelIndex() + return self.createIndex(0, 0) + + def rowCount(self, *args): + return len(self.books) + + def columnCount(self, *args): + return len(self.HEADERS) + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return NONE + text = '' + if orientation == Qt.Horizontal: + if section < len(self.HEADERS): + text = self.HEADERS[section] + return QVariant(text) + else: + return QVariant(section+1) + + def data(self, index, role): + row, col = index.row(), index.column() + result = self.books[row] + if role == Qt.DisplayRole: + if col == 0: + return QVariant(result.title) + elif col == 1: + return QVariant(result.author) + elif col == 2: + return QVariant(result.format) + return NONE + + def data_as_text(self, result, col): + text = '' + if col == 0: + text = result.title + elif col == 1: + text = result.author + elif col == 2: + text = result.format + return text + + def sort(self, col, order, reset=True): + self.sort_col = col + self.sort_order = order + + if not self.books: + return + descending = order == Qt.DescendingOrder + self.books.sort(None, + lambda x: sort_key(unicode(self.data_as_text(x, col))), + descending) + if reset: + self.reset() + diff --git a/src/calibre/gui2/store/mobileread_store_dialog.ui b/src/calibre/gui2/store/mobileread_store_dialog.ui new file mode 100644 index 0000000000..027d5994f0 --- /dev/null +++ b/src/calibre/gui2/store/mobileread_store_dialog.ui @@ -0,0 +1,112 @@ + + + Dialog + + + + 0 + 0 + 691 + 614 + + + + Dialog + + + + + + + + Search: + + + + + + + + + + + + true + + + false + + + false + + + true + + + false + + + false + + + + + + + + + Books: + + + + + + + 0 + + + + + + + Qt::Horizontal + + + + 308 + 20 + + + + + + + + Close + + + + + + + + + + + close_button + clicked() + Dialog + accept() + + + 440 + 432 + + + 245 + 230 + + + + + From f820b37d80669e5cda1d3f88a4a46dc7cb2a4844 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 14 Mar 2011 06:53:26 -0400 Subject: [PATCH 89/92] MobileRead dialog: Set sort indicator when resorting. --- src/calibre/gui2/store/mobileread_plugin.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 37f125f351..49c265d7fe 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -176,6 +176,7 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog): self.results_view.model().sort_col = self.plugin.config.get('store_mobileread_dialog_sort_col', 0) self.results_view.model().sort_order = self.plugin.config.get('store_mobileread_dialog_sort_order', Qt.AscendingOrder) self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) + self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order) def save_state(self): self.plugin.config['store_mobileread_dialog_geometry'] = self.saveGeometry() From af470120fbb86ae7580fe642b2893fcdb30cfa62 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 14 Mar 2011 06:55:51 -0400 Subject: [PATCH 90/92] Remove unnecessary assignment. --- src/calibre/gui2/store/baen_webscription_plugin.py | 2 +- src/calibre/gui2/store/bewrite_plugin.py | 2 +- src/calibre/gui2/store/bn_plugin.py | 2 +- src/calibre/gui2/store/diesel_ebooks_plugin.py | 2 +- src/calibre/gui2/store/ebooks_com_plugin.py | 2 +- src/calibre/gui2/store/eharlequin_plugin.py | 2 +- src/calibre/gui2/store/feedbooks_plugin.py | 2 +- src/calibre/gui2/store/gutenberg_plugin.py | 2 +- src/calibre/gui2/store/kobo_plugin.py | 2 +- src/calibre/gui2/store/manybooks_plugin.py | 2 +- src/calibre/gui2/store/open_library_plugin.py | 2 +- src/calibre/gui2/store/smashwords_plugin.py | 2 +- 12 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/store/baen_webscription_plugin.py b/src/calibre/gui2/store/baen_webscription_plugin.py index 9a590a5e57..46a7c1ec7c 100644 --- a/src/calibre/gui2/store/baen_webscription_plugin.py +++ b/src/calibre/gui2/store/baen_webscription_plugin.py @@ -38,7 +38,7 @@ class BaenWebScriptionStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://www.webscription.net/searchadv.aspx?IsSubmit=true&SearchTerm=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/bewrite_plugin.py b/src/calibre/gui2/store/bewrite_plugin.py index 991abae3d6..ffdb3cd4a2 100644 --- a/src/calibre/gui2/store/bewrite_plugin.py +++ b/src/calibre/gui2/store/bewrite_plugin.py @@ -38,7 +38,7 @@ class BeWriteStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://www.bewrite.net/mm5/merchant.mvc?Search_Code=B&Screen=SRCH&Search=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py index d36d8e466e..4da551fd92 100644 --- a/src/calibre/gui2/store/bn_plugin.py +++ b/src/calibre/gui2/store/bn_plugin.py @@ -46,7 +46,7 @@ class BNStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_item) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py index 1e778d94df..66c22f847f 100644 --- a/src/calibre/gui2/store/diesel_ebooks_plugin.py +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -43,7 +43,7 @@ class DieselEbooksStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://www.diesel-ebooks.com/index.php?page=seek&id[m]=&id[c]=scope%253Dinventory&id[q]=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py index 05f4e58d26..259e996ebe 100644 --- a/src/calibre/gui2/store/ebooks_com_plugin.py +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -45,7 +45,7 @@ class EbookscomStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://www.ebooks.com/SearchApp/SearchResults.net?term=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py index 241c73312b..fd95ab461b 100644 --- a/src/calibre/gui2/store/eharlequin_plugin.py +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -50,7 +50,7 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://ebooks.eharlequin.com/BANGSearch.dll?Type=FullText&FullTextField=All&FullTextCriteria=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py index f3589c1ad0..12873f8bc9 100644 --- a/src/calibre/gui2/store/feedbooks_plugin.py +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -38,7 +38,7 @@ class FeedbooksStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://m.feedbooks.com/search?query=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py index 14ee1a7827..8d04b6236d 100644 --- a/src/calibre/gui2/store/gutenberg_plugin.py +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -38,7 +38,7 @@ class GutenbergStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): # Gutenberg's website does not allow searching both author and title. diff --git a/src/calibre/gui2/store/kobo_plugin.py b/src/calibre/gui2/store/kobo_plugin.py index 9941c02837..d37e806c3f 100644 --- a/src/calibre/gui2/store/kobo_plugin.py +++ b/src/calibre/gui2/store/kobo_plugin.py @@ -45,7 +45,7 @@ class KoboStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://www.kobobooks.com/search/search.html?q=' + urllib2.quote(query) diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py index c5b9a6cf4a..72fe54c427 100644 --- a/src/calibre/gui2/store/manybooks_plugin.py +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -37,7 +37,7 @@ class ManyBooksStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): # ManyBooks website separates results for title and author. diff --git a/src/calibre/gui2/store/open_library_plugin.py b/src/calibre/gui2/store/open_library_plugin.py index d07e0de3bd..15b674f262 100644 --- a/src/calibre/gui2/store/open_library_plugin.py +++ b/src/calibre/gui2/store/open_library_plugin.py @@ -37,7 +37,7 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://openlibrary.org/search?q=' + urllib2.quote(query) + '&has_fulltext=true' diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py index 67e1a3c852..1806e9f4e1 100644 --- a/src/calibre/gui2/store/smashwords_plugin.py +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -44,7 +44,7 @@ class SmashwordsStore(BasicStoreConfig, StorePlugin): d = WebStoreDialog(self.gui, url, parent, detail_url) d.setWindowTitle(self.name) d.set_tags(settings.get(self.name + '_tags', '')) - d = d.exec_() + d.exec_() def search(self, query, max_results=10, timeout=60): url = 'http://www.smashwords.com/books/search?query=' + urllib2.quote(query) From 67633067d348804a132a948788d2069de127cbb8 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 14 Mar 2011 19:58:33 -0400 Subject: [PATCH 91/92] eharlequin aff ids. --- src/calibre/gui2/store/eharlequin_plugin.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py index fd95ab461b..1886671b0a 100644 --- a/src/calibre/gui2/store/eharlequin_plugin.py +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -26,19 +26,14 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() - ''' m_url = 'http://www.dpbolvw.net/' - h_click = 'click-4879827-10762497' - d_click = 'click-4879827-10772898' + h_click = 'click-4879827-534091' + d_click = 'click-4879827-10375439' # Use Kovid's affiliate id 30% of the time. if random.randint(1, 10) in (1, 2, 3): - h_click = 'click-4913808-10762497' - d_click = 'click-4913808-10772898' - ''' - m_url = 'http://ebooks.eharlequin.com/' - h_click = '' - d_click = '' - + h_click = 'click-4913808-534091' + d_click = 'click-4913808-10375439' + url = m_url + h_click detail_url = None if detail_item: @@ -80,7 +75,6 @@ class EHarlequinStore(BasicStoreConfig, StorePlugin): s.title = title.strip() s.author = author.strip() s.price = price.strip() - s.detail_item = id.strip() - #s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() + s.detail_item = '?url=http://ebooks.eharlequin.com/' + id.strip() yield s From 1837ae68f276d2319b62d635279fd7599f7436f7 Mon Sep 17 00:00:00 2001 From: John Schember Date: Mon, 11 Apr 2011 19:58:05 -0400 Subject: [PATCH 92/92] Store: Move to new job system classes for running ebook downloads. --- src/calibre/gui2/ebook_download.py | 207 +++++++---------------------- src/calibre/gui2/threaded_jobs.py | 2 +- 2 files changed, 49 insertions(+), 160 deletions(-) diff --git a/src/calibre/gui2/ebook_download.py b/src/calibre/gui2/ebook_download.py index 4ffff9978d..96cea4fe21 100644 --- a/src/calibre/gui2/ebook_download.py +++ b/src/calibre/gui2/ebook_download.py @@ -6,206 +6,96 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import cStringIO import os import shutil -import time from contextlib import closing from mechanize import MozillaCookieJar -from threading import Thread -from Queue import Queue from calibre import browser, get_download_filename from calibre.ebooks import BOOK_EXTENSIONS from calibre.gui2 import Dispatcher +from calibre.gui2.threaded_jobs import ThreadedJob from calibre.ptempfile import PersistentTemporaryFile -from calibre.utils.ipc.job import BaseJob -class EbookDownloadJob(BaseJob): - - def __init__(self, callback, description, job_manager, gui, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): - BaseJob.__init__(self, description) - self.exception = None - self.job_manager = job_manager - self.gui = gui - self.cookie_file = cookie_file - self.args = (url, filename, save_as_loc, add_to_lib, tags) - self.tmp_file_name = '' - self.callback = callback - self.log_path = None - self._log_file = cStringIO.StringIO() - self._log_file.write(self.description.encode('utf-8') + '\n') - - @property - def log_file(self): - if self.log_path is not None: - return open(self.log_path, 'rb') - return cStringIO.StringIO(self._log_file.getvalue()) - - def start_work(self): - self.start_time = time.time() - self.job_manager.changed_queue.put(self) - - def job_done(self): - self.duration = time.time() - self.start_time - self.percent = 1 - - try: - os.remove(self.tmp_file_name) - except: - import traceback - self.log_write(traceback.format_exc()) - - # Dump log onto disk - lf = PersistentTemporaryFile('ebook_download_log') - lf.write(self._log_file.getvalue()) - lf.close() - self.log_path = lf.name - self._log_file.close() - self._log_file = None - - self.job_manager.changed_queue.put(self) - - def log_write(self, what): - self._log_file.write(what) - -class EbookDownloader(Thread): +class EbookDownload(object): - def __init__(self, job_manager): - Thread.__init__(self) - self.daemon = True - self.jobs = Queue() - self.job_manager = job_manager - self._run = True + def __call__(self, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[], log=None, abort=None, notifications=None): + dfilename = '' + try: + dfilename = self._download(cookie_file, url, filename, save_loc, add_to_lib) + self._add(dfilename, gui, add_to_lib, tags) + self._save_as(dfilename, save_loc) + except Exception as e: + raise e + finally: + try: + if dfilename: + os.remove(dfilename) + except: + pass + + def _download(self, cookie_file, url, filename, save_loc, add_to_lib): + dfilename = '' - def stop(self): - self._run = False - self.jobs.put(None) - - def run(self): - while self._run: - try: - job = self.jobs.get() - except: - break - if job is None or not self._run: - break - - failed, exc = False, None - job.start_work() - if job.kill_on_start: - self._abort_job(job) - continue - - try: - self._download(job) - if not self._run: - self._abort_job(job) - return - - job.percent = .8 - self._add(job) - if not self._run: - self._abort_job(job) - return - - job.percent = .9 - if not self._run: - self._abort_job(job) - return - self._save_as(job) - except Exception, e: - if not self._run: - return - import traceback - failed = True - exc = e - job.log_write('\nDownloading failed...\n') - job.log_write(traceback.format_exc()) - - if not self._run: - break - - job.failed = failed - job.exception = exc - job.job_done() - try: - job.callback(job) - except: - import traceback - traceback.print_exc() - - def _abort_job(self, job): - job.log_write('Aborted\n') - job.failed = False - job.killed = True - job.job_done() - - def _download(self, job): - url, filename, save_loc, add_to_lib, tags = job.args if not url: raise Exception(_('No file specified to download.')) if not save_loc and not add_to_lib: # Nothing to do. - return + return dfilename if not filename: - filename = get_download_filename(url, job.cookie_file) + filename = get_download_filename(url, cookie_file) br = browser() - if job.cookie_file: + if cookie_file: cj = MozillaCookieJar() - cj.load(job.cookie_file) + cj.load(cookie_file) br.set_cookiejar(cj) with closing(br.open(url)) as r: tf = PersistentTemporaryFile(suffix=filename) tf.write(r.read()) - job.tmp_file_name = tf.name + dfilename = tf.name - def _add(self, job): - url, filename, save_loc, add_to_lib, tags = job.args - if not add_to_lib or not job.tmp_file_name: + return dfilename + + def _add(self, filename, gui, add_to_lib, tags): + if not add_to_lib or not filename: return - ext = os.path.splitext(job.tmp_file_name)[1][1:].lower() + ext = os.path.splitext(filename)[1][1:].lower() if ext not in BOOK_EXTENSIONS: raise Exception(_('Not a support ebook format.')) from calibre.ebooks.metadata.meta import get_metadata - with open(job.tmp_file_name) as f: + with open(filename) as f: mi = get_metadata(f, ext) mi.tags.extend(tags) - id = job.gui.library_view.model().db.create_book_entry(mi) - job.gui.library_view.model().db.add_format_with_hooks(id, ext.upper(), job.tmp_file_name, index_is_id=True) - job.gui.library_view.model().books_added(1) - job.gui.library_view.model().count_changed() + id = gui.library_view.model().db.create_book_entry(mi) + gui.library_view.model().db.add_format_with_hooks(id, ext.upper(), filename, index_is_id=True) + gui.library_view.model().books_added(1) + gui.library_view.model().count_changed() - def _save_as(self, job): - url, filename, save_loc, add_to_lib, tags = job.args - if not save_loc or not job.tmp_file_name: + def _save_as(self, dfilename, save_loc): + if not save_loc or not dfilename: return - - shutil.copy(job.tmp_file_name, save_loc) - - def download_ebook(self, callback, gui, cookie_file=None, url='', filename='', save_as_loc='', add_to_lib=True, tags=[]): - description = _('Downloading %s') % filename if filename else url - job = EbookDownloadJob(callback, description, self.job_manager, gui, cookie_file, url, filename, save_as_loc, add_to_lib, tags) - self.job_manager.add_job(job) - self.jobs.put(job) + shutil.copy(dfilename, save_loc) + + +gui_ebook_download = EbookDownload() + +def start_ebook_download(callback, job_manager, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[]): + description = _('Downloading %s') % filename if filename else url + job = ThreadedJob('ebook_download', description, gui_ebook_download, (gui, cookie_file, url, filename, save_loc, add_to_lib, tags), {}, callback, max_concurrent_count=2, killable=False) + job_manager.run_threaded_job(job) class EbookDownloadMixin(object): - - def __init__(self): - self.ebook_downloader = EbookDownloader(self.job_manager) - - def download_ebook(self, url='', cookie_file=None, filename='', save_as_loc='', add_to_lib=True, tags=[]): - if not self.ebook_downloader.is_alive(): - self.ebook_downloader.start() + + def download_ebook(self, url='', cookie_file=None, filename='', save_loc='', add_to_lib=True, tags=[]): if tags: if isinstance(tags, basestring): tags = tags.split(',') - self.ebook_downloader.download_ebook(Dispatcher(self.downloaded_ebook), self, cookie_file, url, filename, save_as_loc, add_to_lib, tags) + start_ebook_download(Dispatcher(self.downloaded_ebook), self.job_manager, self, cookie_file, url, filename, save_loc, add_to_lib, tags) self.status_bar.show_message(_('Downloading') + ' ' + filename if filename else url, 3000) def downloaded_ebook(self, job): @@ -214,4 +104,3 @@ class EbookDownloadMixin(object): return self.status_bar.show_message(job.description + ' ' + _('finished'), 5000) - diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py index f29baf4134..46bed3af5d 100644 --- a/src/calibre/gui2/threaded_jobs.py +++ b/src/calibre/gui2/threaded_jobs.py @@ -38,7 +38,7 @@ class ThreadedJob(BaseJob): :func: The function that actually does the work. This function *must* accept at least three keyword arguments: abort, log and notifications. abort is - An Event object. func should periodically check abort.is_set(0 and if + An Event object. func should periodically check abort.is_set() and if it is True, it should stop processing as soon as possible. notifications is a Queue. func should put progress notifications into it in the form of a tuple (frac, msg). frac is a number between 0 and 1 indicating