diff --git a/resources/images/store.png b/resources/images/store.png new file mode 100644 index 0000000000..947fb794b8 Binary files /dev/null and b/resources/images/store.png differ diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 2f457bf2bc..3aad3522be 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -5,7 +5,9 @@ __docformat__ = 'restructuredtext en' import uuid, sys, os, re, logging, time, random, \ __builtin__, warnings, multiprocessing +from contextlib import closing from urllib import getproxies +from urllib2 import unquote as urllib2_unquote __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) from htmlentitydefs import name2codepoint from math import floor @@ -290,6 +292,9 @@ def get_parsed_proxy(typ='http', debug=True): prints('Using http proxy', str(ans)) return ans +USER_AGENT = 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101210 Gentoo Firefox/3.6.13' +USER_AGENT_MOBILE = 'Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016' + def random_user_agent(): choices = [ 'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1', @@ -305,7 +310,6 @@ def random_user_agent(): #return choices[-1] return choices[random.randint(0, len(choices)-1)] - def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None): ''' Create a mechanize browser for web scraping. The browser handles cookies, @@ -319,8 +323,7 @@ def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None): opener.set_handle_refresh(True, max_time=max_time, honor_time=honor_time) opener.set_handle_robots(False) if user_agent is None: - user_agent = ' Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016' if mobile_browser else \ - 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101210 Gentoo Firefox/3.6.13' + user_agent = USER_AGENT_MOBILE if mobile_browser else USER_AGENT opener.addheaders = [('User-agent', user_agent)] http_proxy = get_proxies().get('http', None) if http_proxy: @@ -537,7 +540,46 @@ def as_unicode(obj, enc=preferred_encoding): obj = repr(obj) return force_unicode(obj, enc=enc) +def url_slash_cleaner(url): + ''' + Removes redundant /'s from url's. + ''' + return re.sub(r'(?' import textwrap, os, glob, functools, re from calibre import guess_type from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \ - MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase + MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf @@ -854,13 +854,18 @@ class ActionNextMatch(InterfaceActionBase): name = 'Next Match' actual_plugin = 'calibre.gui2.actions.next_match:NextMatchAction' +class ActionStore(InterfaceActionBase): + name = 'Store' + author = 'John Schember' + actual_plugin = 'calibre.gui2.actions.store:StoreAction' + plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, ActionConvert, ActionDelete, ActionEditMetadata, ActionView, ActionFetchNews, ActionSaveToDisk, ActionShowBookDetails, ActionRestart, ActionOpenFolder, ActionConnectShare, ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks, ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary, - ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch] + ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore] # }}} @@ -1093,4 +1098,81 @@ if test_eight_code: #}}} +# Store plugins {{{ +class StoreAmazonKindleStore(StoreBase): + name = 'Amazon Kindle' + description = _('Kindle books from Amazon') + actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore' +class StoreBaenWebScriptionStore(StoreBase): + name = 'Baen WebScription' + description = _('Ebooks for readers.') + actual_plugin = 'calibre.gui2.store.baen_webscription_plugin:BaenWebScriptionStore' + +class StoreBNStore(StoreBase): + name = 'Barnes and Noble' + description = _('Books, Textbooks, eBooks, Toys, Games and More.') + actual_plugin = 'calibre.gui2.store.bn_plugin:BNStore' + +class StoreBeWriteStore(StoreBase): + name = 'BeWrite Books' + description = _('Publishers of fine books.') + actual_plugin = 'calibre.gui2.store.bewrite_plugin:BeWriteStore' + +class StoreDieselEbooksStore(StoreBase): + name = 'Diesel eBooks' + description = _('World Famous eBook Store.') + actual_plugin = 'calibre.gui2.store.diesel_ebooks_plugin:DieselEbooksStore' + +class StoreEbookscomStore(StoreBase): + name = 'eBooks.com' + description = _('The digital bookstore.') + actual_plugin = 'calibre.gui2.store.ebooks_com_plugin:EbookscomStore' + +class StoreEHarlequinStoretore(StoreBase): + name = 'eHarlequin' + description = _('entertain, enrich, inspire.') + actual_plugin = 'calibre.gui2.store.eharlequin_plugin:EHarlequinStore' + +class StoreFeedbooksStore(StoreBase): + name = 'Feedbooks' + description = _('Read anywhere.') + actual_plugin = 'calibre.gui2.store.feedbooks_plugin:FeedbooksStore' + +class StoreGutenbergStore(StoreBase): + name = 'Project Gutenberg' + description = _('The first producer of free ebooks.') + actual_plugin = 'calibre.gui2.store.gutenberg_plugin:GutenbergStore' + +class StoreKoboStore(StoreBase): + name = 'Kobo' + description = _('eReading: anytime. anyplace.') + actual_plugin = 'calibre.gui2.store.kobo_plugin:KoboStore' + +class StoreManyBooksStore(StoreBase): + name = 'ManyBooks' + description = _('The best ebooks at the best price: free!') + actual_plugin = 'calibre.gui2.store.manybooks_plugin:ManyBooksStore' + +class StoreMobileReadStore(StoreBase): + name = 'MobileRead' + description = _('Ebooks handcrafted with the utmost care') + actual_plugin = 'calibre.gui2.store.mobileread_plugin:MobileReadStore' + +class StoreOpenLibraryStore(StoreBase): + name = 'Open Library' + description = _('One web page for every book.') + actual_plugin = 'calibre.gui2.store.open_library_plugin:OpenLibraryStore' + +class StoreSmashwordsStore(StoreBase): + name = 'Smashwords' + description = _('Your ebook. Your way.') + actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore' + +plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore, + StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore, + StoreEHarlequinStoretore, + StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore, + StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore] + +# }}} diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index e8011e9ad8..c58f36524e 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -7,7 +7,8 @@ import os, shutil, traceback, functools, sys from calibre.customize import (CatalogPlugin, FileTypePlugin, PluginNotFound, MetadataReaderPlugin, MetadataWriterPlugin, InterfaceActionBase as InterfaceAction, - PreferencesPlugin, platform, InvalidPlugin) + PreferencesPlugin, platform, InvalidPlugin, + StoreBase as Store) from calibre.customize.conversion import InputFormatPlugin, OutputFormatPlugin from calibre.customize.zipplugin import loader from calibre.customize.profiles import InputProfile, OutputProfile @@ -244,6 +245,17 @@ def preferences_plugins(): yield plugin # }}} +# Store Plugins # {{{ + +def store_plugins(): + customization = config['plugin_customization'] + for plugin in _initialized_plugins: + if isinstance(plugin, Store): + if not is_disabled(plugin): + plugin.site_customization = customization.get(plugin.name, '') + yield plugin +# }}} + # Metadata read/write {{{ _metadata_readers = {} _metadata_writers = {} diff --git a/src/calibre/gui2/actions/store.py b/src/calibre/gui2/actions/store.py new file mode 100644 index 0000000000..f00497ad64 --- /dev/null +++ b/src/calibre/gui2/actions/store.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from functools import partial + +from PyQt4.Qt import Qt, QMenu, QToolButton, QDialog, QVBoxLayout + +from calibre.gui2.actions import InterfaceAction + +class StoreAction(InterfaceAction): + + name = 'Store' + action_spec = (_('Store'), 'store.png', None, None) + + def genesis(self): + self.qaction.triggered.connect(self.search) + self.store_menu = QMenu() + self.load_menu() + + def load_menu(self): + self.store_menu.clear() + self.store_menu.addAction(_('Search'), self.search) + self.store_menu.addSeparator() + for n, p in self.gui.istores.items(): + self.store_menu.addAction(n, partial(self.open_store, p)) + self.qaction.setMenu(self.store_menu) + + def search(self): + from calibre.gui2.store.search import SearchDialog + sd = SearchDialog(self.gui.istores, self.gui) + sd.exec_() + + def open_store(self, store_plugin): + store_plugin.open(self.gui) diff --git a/src/calibre/gui2/ebook_download.py b/src/calibre/gui2/ebook_download.py new file mode 100644 index 0000000000..2fea67b9f0 --- /dev/null +++ b/src/calibre/gui2/ebook_download.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import os +import shutil +from contextlib import closing +from mechanize import MozillaCookieJar + +from calibre import browser, get_download_filename +from calibre.ebooks import BOOK_EXTENSIONS +from calibre.gui2 import Dispatcher +from calibre.gui2.threaded_jobs import ThreadedJob +from calibre.ptempfile import PersistentTemporaryFile + +class EbookDownload(object): + + def __call__(self, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[], log=None, abort=None, notifications=None): + dfilename = '' + try: + dfilename = self._download(cookie_file, url, filename, save_loc, add_to_lib) + self._add(dfilename, gui, add_to_lib, tags) + self._save_as(dfilename, save_loc) + except Exception as e: + raise e + finally: + try: + if dfilename: + os.remove(dfilename) + except: + pass + + def _download(self, cookie_file, url, filename, save_loc, add_to_lib): + dfilename = '' + + if not url: + raise Exception(_('No file specified to download.')) + if not save_loc and not add_to_lib: + # Nothing to do. + return dfilename + + if not filename: + filename = get_download_filename(url, cookie_file) + + br = browser() + if cookie_file: + cj = MozillaCookieJar() + cj.load(cookie_file) + br.set_cookiejar(cj) + with closing(br.open(url)) as r: + tf = PersistentTemporaryFile(suffix=filename) + tf.write(r.read()) + dfilename = tf.name + + return dfilename + + def _add(self, filename, gui, add_to_lib, tags): + if not add_to_lib or not filename: + return + ext = os.path.splitext(filename)[1][1:].lower() + if ext not in BOOK_EXTENSIONS: + raise Exception(_('Not a support ebook format.')) + + from calibre.ebooks.metadata.meta import get_metadata + with open(filename) as f: + mi = get_metadata(f, ext) + mi.tags.extend(tags) + + id = gui.library_view.model().db.create_book_entry(mi) + gui.library_view.model().db.add_format_with_hooks(id, ext.upper(), filename, index_is_id=True) + gui.library_view.model().books_added(1) + gui.library_view.model().count_changed() + + def _save_as(self, dfilename, save_loc): + if not save_loc or not dfilename: + return + shutil.copy(dfilename, save_loc) + + +gui_ebook_download = EbookDownload() + +def start_ebook_download(callback, job_manager, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[]): + description = _('Downloading %s') % filename if filename else url + job = ThreadedJob('ebook_download', description, gui_ebook_download, (gui, cookie_file, url, filename, save_loc, add_to_lib, tags), {}, callback, max_concurrent_count=2, killable=False) + job_manager.run_threaded_job(job) + + +class EbookDownloadMixin(object): + + def download_ebook(self, url='', cookie_file=None, filename='', save_loc='', add_to_lib=True, tags=[]): + if tags: + if isinstance(tags, basestring): + tags = tags.split(',') + start_ebook_download(Dispatcher(self.downloaded_ebook), self.job_manager, self, cookie_file, url, filename, save_loc, add_to_lib, tags) + self.status_bar.show_message(_('Downloading') + ' ' + filename if filename else url, 3000) + + def downloaded_ebook(self, job): + if job.failed: + self.job_exception(job, dialog_title=_('Failed to download ebook')) + return + + self.status_bar.show_message(job.description + ' ' + _('finished'), 5000) diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index 8151fe6021..79cd2b1ce4 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -218,6 +218,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.search.search.connect(self.find) self.next_button.clicked.connect(self.find_next) self.previous_button.clicked.connect(self.find_previous) + self.changed_signal.connect(self.reload_store_plugins) def find(self, query): idx = self._plugin_model.find(query) @@ -344,6 +345,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): plugin.name + _(' cannot be removed. It is a ' 'builtin plugin. Try disabling it instead.')).exec_() + def reload_store_plugins(self): + self.gui.load_store_plugins() + if self.gui.iactions.has_key('Store'): + self.gui.iactions['Store'].load_menu() + def check_for_add_to_toolbars(self, plugin): from calibre.gui2.preferences.toolbar import ConfigWidget from calibre.customize import InterfaceActionBase @@ -376,6 +382,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): installed_actions.append(plugin_action.name) gprefs['action-layout-'+key] = tuple(installed_actions) + if __name__ == '__main__': from PyQt4.Qt import QApplication app = QApplication([]) diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py new file mode 100644 index 0000000000..73d0d0a8d4 --- /dev/null +++ b/src/calibre/gui2/store/__init__.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +class StorePlugin(object): # {{{ + ''' + A plugin representing an online ebook repository (store). The store can + be a comercial store that sells ebooks or a source of free downloadable + ebooks. + + Note that this class is the base class for these plugins, however, to + integrate the plugin with calibre's plugin system, you have to make a + wrapper class that references the actual plugin. See the + :mod:`calibre.customize.builtins` module for examples. + + If two :class:`StorePlugin` objects have the same name, the one with higher + priority takes precedence. + + Sub-classes must implement :meth:`open`, and :meth:`search`. + + Regarding :meth:`open`. Most stores only make themselves available + though a web site thus most store plugins will open using + :class:`calibre.gui2.store.web_store_dialog.WebStoreDialog`. This will + open a modal window and display the store website in a QWebView. + + Sub-classes should implement and use the :meth:`genesis` if they require + plugin specific initialization. They should not override or otherwise + reimplement :meth:`__init__`. + + Once initialized, this plugin has access to the main calibre GUI via the + :attr:`gui` member. You can access other plugins by name, for example:: + + self.gui.istores['Amazon Kindle'] + + Plugin authors can use affiliate programs within their plugin. The + distribution of money earned from a store plugin is 70/30. 70% going + to the pluin author / maintainer and 30% going to the calibre project. + + The easiest way to handle affiliate money payouts is to randomly select + between the author's affiliate id and calibre's affiliate id so that + 70% of the time the author's id is used. + ''' + + def __init__(self, gui, name): + self.gui = gui + self.name = name + self.base_plugin = None + + def open(self, gui, parent=None, detail_item=None, external=False): + ''' + Open the store. + + :param gui: The main GUI. This will be used to have the job + system start downloading an item from the store. + + :param parent: The parent of the store dialog. This is used + to create modal dialogs. + + :param detail_item: A plugin specific reference to an item + in the store that the user should be shown. + + :param external: When False open an internal dialog with the + store. When True open the users default browser to the store's + web site. :param:`detail_item` should still be respected when external + is True. + ''' + raise NotImplementedError() + + def search(self, query, max_results=10, timeout=60): + ''' + Searches the store for items matching query. This should + return items as a generator. + + Don't be lazy with the search! Load as much data as possible in the + :class:`calibre.gui2.store.search_result.SearchResult` object. If you have to parse + multiple pages to get all of the data then do so. However, if data (such as cover_url) + isn't available because the store does not display cover images then it's okay to + ignore it. + + Also, by default search results can only include ebooks. A plugin can offer users + an option to include physical books in the search results but this must be + disabled by default. + + If a store doesn't provide search on it's own use something like a site specific + google search to get search results for this funtion. + + :param query: The string query search with. + :param max_results: The maximum number of results to return. + :param timeout: The maximum amount of time in seconds to spend download the search results. + + :return: :class:`calibre.gui2.store.search_result.SearchResult` objects + item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. + ''' + raise NotImplementedError() + + def get_settings(self): + ''' + This is only useful for plugins that implement + :attr:`config_widget` that is the only way to save + settings. This is used by plugins to get the saved + settings and apply when necessary. + + :return: A dictionary filled with the settings used + by this plugin. + ''' + raise NotImplementedError() + + def do_genesis(self): + self.genesis() + + def genesis(self): + ''' + Plugin specific initialization. + ''' + pass + + def config_widget(self): + ''' + See :class:`calibre.customize.Plugin` for details. + ''' + raise NotImplementedError() + + def save_settings(self, config_widget): + ''' + See :class:`calibre.customize.Plugin` for details. + ''' + raise NotImplementedError() + + def customization_help(self, gui=False): + ''' + See :class:`calibre.customize.Plugin` for details. + ''' + raise NotImplementedError() + +# }}} \ No newline at end of file diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py new file mode 100644 index 0000000000..0b42ee1308 --- /dev/null +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.search_result import SearchResult + +class AmazonKindleStore(StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + ''' + Amazon comes with a number of difficulties. + + QWebView has major issues with Amazon.com. The largest of + issues is it simply doesn't work on a number of pages. + + When connecting to a number parts of Amazon.com (Kindle library + for instance) QNetworkAccessManager fails to connect with a + NetworkError of 399 - ProtocolFailure. The strange thing is, + when I check QNetworkRequest.HttpStatusCodeAttribute when the + 399 error is returned the status code is 200 (Ok). However, once + the QNetworkAccessManager decides there was a NetworkError it + does not download the page from Amazon. So I can't even set the + HTML in the QWebView myself. + + There is http://bugreports.qt.nokia.com/browse/QTWEBKIT-259 an + open bug about the issue but it is not correct. We can set the + useragent (Arora does) to something else and the above issue + will persist. This http://developer.qt.nokia.com/forums/viewthread/793 + gives a bit more information about the issue but as of now (27/Feb/2011) + there is no solution or work around. + + We cannot change the The linkDelegationPolicy to allow us to avoid + QNetworkAccessManager because it only works links. Forms aren't + included so the same issue persists on any part of the site (login) + that use a form to load a new page. + + Using an aStore was evaluated but I've decided against using it. + There are three major issues with an aStore. Because checkout is + handled by sending the user to Amazon we can't put it in a QWebView. + If we're sending the user to Amazon sending them there directly is + nicer. Also, we cannot put the aStore in a QWebView and let it open the + redirection the users default browser because the cookies with the + shopping cart won't transfer. + + Another issue with the aStore is how it handles the referral. It only + counts the referral for the items in the shopping card / the item + that directed the user to Amazon. Kindle books do not use the shopping + cart and send the user directly to Amazon for the purchase. In this + instance we would only get referral credit for the one book that the + aStore directs to Amazon that the user buys. Any other purchases we + won't get credit for. + + The last issue with the aStore is performance. Even though it's an + Amazon site it's alow. So much slower than Amazon.com that it makes + me not want to browse books using it. The look and feel are lesser + issues. So is the fact that it almost seems like the purchase is + with calibre. This can cause some support issues because we can't + do much for issues with Amazon.com purchase hiccups. + + Another option that was evaluated was the Product Advertising API. + The reasons against this are complexity. It would take a lot of work + to basically re-create Amazon.com within calibre. The Product + Advertising API is also designed with being run on a server not + in an app. The signing keys would have to be made avaliable to ever + calibre user which means bad things could be done with our account. + + The Product Advertising API also assumes the same browser for easy + shopping cart transfer to Amazon. With QWebView not working and there + not being an easy way to transfer cookies between a QWebView and the + users default browser this won't work well. + + We could create our own website on the calibre server and create an + Amazon Product Advertising API store. However, this goes back to the + complexity argument. Why spend the time recreating Amazon.com + + The final and largest issue against using the Product Advertising API + is the Efficiency Guidelines: + + "Each account used to access the Product Advertising API will be allowed + an initial usage limit of 2,000 requests per hour. Each account will + receive an additional 500 requests per hour (up to a maximum of 25,000 + requests per hour) for every $1 of shipped item revenue driven per hour + in a trailing 30-day period. Usage thresholds are recalculated daily based + on revenue performance." + + With over two million users a limit of 2,000 request per hour could + render our store unusable for no other reason than Amazon rate + limiting our traffic. + + The best (I use the term lightly here) solution is to open Amazon.com + in the users default browser and set the affiliate id as part of the url. + ''' + aff_id = {'tag': 'josbl0e-cpb-20'} + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + aff_id['tag'] = 'calibrebs-20' + store_link = 'http://www.amazon.com/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id + if detail_item: + aff_id['asin'] = detail_item + store_link = 'http://www.amazon.com/dp/%(asin)s/?tag=%(tag)s' % aff_id + open_url(QUrl(store_link)) + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query) + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="productData"]'): + if counter <= 0: + break + + # Even though we are searching digital-text only Amazon will still + # put in results for non Kindle books (author pages). Se we need + # to explicitly check if the item is a Kindle book and ignore it + # if it isn't. + type = ''.join(data.xpath('//span[@class="format"]/text()')) + if 'kindle' not in type.lower(): + continue + + # We must have an asin otherwise we can't easily reference the + # book later. + asin_href = None + asin_a = data.xpath('div[@class="productTitle"]/a[1]') + if asin_a: + asin_href = asin_a[0].get('href', '') + m = re.search(r'/dp/(?P.+?)(/|$)', asin_href) + if m: + asin = m.group('asin') + else: + continue + else: + continue + + cover_url = '' + if asin_href: + cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href) + if cover_img: + cover_url = cover_img[0] + + title = ''.join(data.xpath('div[@class="productTitle"]/a/text()')) + author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) + author = author.split('by')[-1] + price = ''.join(data.xpath('div[@class="newPrice"]/span/text()')) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = asin.strip() + + yield s diff --git a/src/calibre/gui2/store/baen_webscription_plugin.py b/src/calibre/gui2/store/baen_webscription_plugin.py new file mode 100644 index 0000000000..46a7c1ec7c --- /dev/null +++ b/src/calibre/gui2/store/baen_webscription_plugin.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class BaenWebScriptionStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://www.webscription.net/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(url_slash_cleaner(url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.webscription.net/searchadv.aspx?IsSubmit=true&SearchTerm=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//table/tr/td/img[@src="skins/Skin_1/images/matchingproducts.gif"]/..//tr'): + if counter <= 0: + break + + id = ''.join(data.xpath('./td[1]/a/@href')) + if not id: + continue + + title = ''.join(data.xpath('./td[1]/a/text()')) + + author = '' + cover_url = '' + price = '' + + with closing(br.open('http://www.webscription.net/' + id.strip(), timeout=timeout/4)) as nf: + idata = html.fromstring(nf.read()) + author = ''.join(idata.xpath('//span[@class="ProductNameText"]/../b/text()')) + author = author.split('by ')[-1] + price = ''.join(idata.xpath('//span[@class="variantprice"]/text()')) + a, b, price = price.partition('$') + price = b + price + + pnum = '' + mo = re.search(r'p-(?P\d+)-', id.strip()) + if mo: + pnum = mo.group('num') + if pnum: + cover_url = 'http://www.webscription.net/' + ''.join(idata.xpath('//img[@id="ProductPic%s"]/@src' % pnum)) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = id.strip() + + yield s diff --git a/src/calibre/gui2/store/basic_config.py b/src/calibre/gui2/store/basic_config.py new file mode 100644 index 0000000000..88ee197146 --- /dev/null +++ b/src/calibre/gui2/store/basic_config.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from PyQt4.Qt import QWidget + +from calibre.gui2 import gprefs +from calibre.gui2.store.basic_config_widget_ui import Ui_Form + +def save_settings(config_widget): + gprefs[config_widget.store.name + '_open_external'] = config_widget.open_external.isChecked() + tags = unicode(config_widget.tags.text()) + gprefs[config_widget.store.name + '_tags'] = tags + +class BasicStoreConfigWidget(QWidget, Ui_Form): + + def __init__(self, store): + QWidget.__init__(self) + self.setupUi(self) + + self.store = store + + self.load_setings() + + def load_setings(self): + settings = self.store.get_settings() + + self.open_external.setChecked(settings.get(self.store.name + '_open_external')) + self.tags.setText(settings.get(self.store.name + '_tags', '')) + +class BasicStoreConfig(object): + + def customization_help(self, gui=False): + return 'Customize the behavior of this store.' + + def config_widget(self): + return BasicStoreConfigWidget(self) + + def save_settings(self, config_widget): + save_settings(config_widget) + + def get_settings(self): + settings = {} + + settings[self.name + '_open_external'] = gprefs.get(self.name + '_open_external', False) + settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download') + + return settings diff --git a/src/calibre/gui2/store/basic_config_widget.ui b/src/calibre/gui2/store/basic_config_widget.ui new file mode 100644 index 0000000000..cadf7813d6 --- /dev/null +++ b/src/calibre/gui2/store/basic_config_widget.ui @@ -0,0 +1,38 @@ + + + Form + + + + 0 + 0 + 460 + 69 + + + + Form + + + + + + Added Tags: + + + + + + + + + + Open store in external web browswer + + + + + + + + diff --git a/src/calibre/gui2/store/bewrite_plugin.py b/src/calibre/gui2/store/bewrite_plugin.py new file mode 100644 index 0000000000..ffdb3cd4a2 --- /dev/null +++ b/src/calibre/gui2/store/bewrite_plugin.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class BeWriteStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://www.bewrite.net/mm5/merchant.mvc?Screen=SFNT' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(url_slash_cleaner(url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.bewrite.net/mm5/merchant.mvc?Search_Code=B&Screen=SRCH&Search=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@id="content"]//table/tr[position() > 1]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//a/@href')) + if not id: + continue + + heading = ''.join(data.xpath('./td[2]//text()')) + title, q, author = heading.partition('by ') + cover_url = '' + price = '' + + with closing(br.open(id.strip(), timeout=timeout/4)) as nf: + idata = html.fromstring(nf.read()) + price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()')) + price = '$' + price.split('$')[-1] + cover_img = idata.xpath('//div[@id="content"]//img[1]/@src') + if cover_img: + cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url.strip() + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = id.strip() + + yield s diff --git a/src/calibre/gui2/store/bn_plugin.py b/src/calibre/gui2/store/bn_plugin.py new file mode 100644 index 0000000000..4da551fd92 --- /dev/null +++ b/src/calibre/gui2/store/bn_plugin.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class BNStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + + pub_id = '21000000000352219' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + pub_id = '21000000000352583' + + url = 'http://gan.doubleclick.net/gan_click?lid=41000000028437369&pubid=' + pub_id + + if detail_item: + mo = re.search(r'(?<=/)(?P\d+)(?=/|$)', detail_item) + if mo: + isbn = mo.group('isbn') + detail_item = 'http://gan.doubleclick.net/gan_click?lid=41000000012871747&pid=' + isbn + '&adurl=' + detail_item + '&pubid=' + pub_id + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results + url += urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//ul[contains(@class, "wgt-search-results-display")]/li[contains(@class, "search-result-item") and contains(@class, "nook-result-item")]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/@href')) + if not id: + continue + cover_url = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/img/@src')) + + title = ''.join(data.xpath('.//span[@class="product-title"]/a/text()')) + author = ', '.join(data.xpath('.//span[@class="contributers-line"]/a/text()')) + price = ''.join(data.xpath('.//span[contains(@class, "onlinePriceValue2")]/text()')) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = id.strip() + + yield s diff --git a/src/calibre/gui2/store/diesel_ebooks_plugin.py b/src/calibre/gui2/store/diesel_ebooks_plugin.py new file mode 100644 index 0000000000..66c22f847f --- /dev/null +++ b/src/calibre/gui2/store/diesel_ebooks_plugin.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class DieselEbooksStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://www.diesel-ebooks.com/' + + aff_id = '?aid=2049' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + aff_id = '?aid=2053' + + detail_url = None + if detail_item: + detail_url = url + detail_item + aff_id + url = url + aff_id + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.diesel-ebooks.com/index.php?page=seek&id[m]=&id[c]=scope%253Dinventory&id[q]=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="item clearfix"]'): + data = html.fromstring(html.tostring(data)) + if counter <= 0: + break + + id = ''.join(data.xpath('div[@class="cover"]/a/@href')) + if not id or '/item/' not in id: + continue + a, b, id = id.partition('/item/') + + cover_url = ''.join(data.xpath('div[@class="cover"]//img/@src')) + if cover_url.startswith('/'): + cover_url = cover_url[1:] + cover_url = 'http://www.diesel-ebooks.com/' + cover_url + + title = ''.join(data.xpath('.//div[@class="content"]//h2/text()')) + author = ''.join(data.xpath('//div[@class="content"]//div[@class="author"]/a/text()')) + price = '' + price_elem = data.xpath('//td[@class="price"]/text()') + if price_elem: + price = price_elem[0] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/item/' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/ebooks_com_plugin.py b/src/calibre/gui2/store/ebooks_com_plugin.py new file mode 100644 index 0000000000..259e996ebe --- /dev/null +++ b/src/calibre/gui2/store/ebooks_com_plugin.py @@ -0,0 +1,95 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class EbookscomStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + + m_url = 'http://www.dpbolvw.net/' + h_click = 'click-4879827-10364500' + d_click = 'click-4879827-10281551' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + h_click = 'click-4913808-10364500' + d_click = 'click-4913808-10281551' + + url = m_url + h_click + detail_url = None + if detail_item: + detail_url = m_url + d_click + detail_item + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.ebooks.com/SearchApp/SearchResults.net?term=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="book_a" or @class="book_b"]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//a[1]/@href')) + id = id.split('=')[-1] + if not id: + continue + + price = '' + with closing(br.open('http://www.ebooks.com/ebooks/book_display.asp?IID=' + id.strip(), timeout=timeout)) as fp: + pdoc = html.fromstring(fp.read()) + pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()') + if len(pdata) >= 2: + price = pdata[1] + if not price: + continue + + cover_url = ''.join(data.xpath('.//img[1]/@src')) + + title = '' + author = '' + heading_a = data.xpath('.//a[1]/text()') + if heading_a: + title = heading_a[0] + if len(heading_a) >= 2: + author = heading_a[1] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '?url=http://www.ebooks.com/cj.asp?IID=' + id.strip() + '&cjsku=' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/eharlequin_plugin.py b/src/calibre/gui2/store/eharlequin_plugin.py new file mode 100644 index 0000000000..1886671b0a --- /dev/null +++ b/src/calibre/gui2/store/eharlequin_plugin.py @@ -0,0 +1,80 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class EHarlequinStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + + m_url = 'http://www.dpbolvw.net/' + h_click = 'click-4879827-534091' + d_click = 'click-4879827-10375439' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + h_click = 'click-4913808-534091' + d_click = 'click-4913808-10375439' + + url = m_url + h_click + detail_url = None + if detail_item: + detail_url = m_url + d_click + detail_item + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://ebooks.eharlequin.com/BANGSearch.dll?Type=FullText&FullTextField=All&FullTextCriteria=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//table[not(.//@class="sidelink")]/tr[.//ul[@id="details"]]'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//ul[@id="details"]/li[@id="title-results"]/a/@href')) + if not id: + continue + + title = ''.join(data.xpath('.//ul[@id="details"]/li[@id="title-results"]/a/text()')) + author = ''.join(data.xpath('.//ul[@id="details"]/li[@id="author"][1]//a/text()')) + price = ''.join(data.xpath('.//div[@class="ourprice"]/font/text()')) + cover_url = ''.join(data.xpath('.//a[@href="%s"]/img/@src' % id)) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '?url=http://ebooks.eharlequin.com/' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/feedbooks_plugin.py b/src/calibre/gui2/store/feedbooks_plugin.py new file mode 100644 index 0000000000..12873f8bc9 --- /dev/null +++ b/src/calibre/gui2/store/feedbooks_plugin.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class FeedbooksStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://m.feedbooks.com/' + ext_url = 'http://feedbooks.com/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + ext_url = ext_url + detail_item + open_url(QUrl(url_slash_cleaner(ext_url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://m.feedbooks.com/search?query=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//ul[@class="m-list"]//li'): + if counter <= 0: + break + data = html.fromstring(html.tostring(data)) + + id = '' + id_a = data.xpath('//a[@class="buy"]') + if id_a: + id = id_a[0].get('href', None) + id = id.split('/')[-2] + id = '/item/' + id + else: + id_a = data.xpath('//a[@class="download"]') + if id_a: + id = id_a[0].get('href', None) + id = id.split('/')[-1] + id = id.split('.')[0] + id = '/book/' + id + if not id: + continue + + title = ''.join(data.xpath('//h5//a/text()')) + author = ''.join(data.xpath('//h6//a/text()')) + price = ''.join(data.xpath('//a[@class="buy"]/text()')) + if not price: + price = '$0.00' + cover_url = '' + cover_url_img = data.xpath('//img') + if cover_url_img: + cover_url = cover_url_img[0].get('src') + cover_url.split('?')[0] + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.replace(' ', '').strip() + s.detail_item = id.strip() + + yield s diff --git a/src/calibre/gui2/store/gutenberg_plugin.py b/src/calibre/gui2/store/gutenberg_plugin.py new file mode 100644 index 0000000000..8d04b6236d --- /dev/null +++ b/src/calibre/gui2/store/gutenberg_plugin.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class GutenbergStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://m.gutenberg.org/' + ext_url = 'http://gutenberg.org/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + ext_url = ext_url + detail_item + open_url(QUrl(url_slash_cleaner(ext_url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + # Gutenberg's website does not allow searching both author and title. + # Using a google search so we can search on both fields at once. + url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'): + if counter <= 0: + break + + url = '' + url_a = data.xpath('div[@class="jd"]/a') + if url_a: + url_a = url_a[0] + url = url_a.get('href', None) + if url: + url = url.split('u=')[-1].split('&')[0] + if '/ebooks/' not in url: + continue + id = url.split('/')[-1] + + url_a = html.fromstring(html.tostring(url_a)) + heading = ''.join(url_a.xpath('//text()')) + title, _, author = heading.rpartition('by ') + author = author.split('-')[0] + price = '$0.00' + + counter -= 1 + + s = SearchResult() + s.cover_url = '' + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/ebooks/' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/kobo_plugin.py b/src/calibre/gui2/store/kobo_plugin.py new file mode 100644 index 0000000000..d37e806c3f --- /dev/null +++ b/src/calibre/gui2/store/kobo_plugin.py @@ -0,0 +1,84 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class KoboStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + + m_url = 'http://www.dpbolvw.net/' + h_click = 'click-4879827-10762497' + d_click = 'click-4879827-10772898' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + h_click = 'click-4913808-10762497' + d_click = 'click-4913808-10772898' + + url = m_url + h_click + detail_url = None + if detail_item: + detail_url = m_url + d_click + detail_item + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.kobobooks.com/search/search.html?q=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//ul[@class="SCShortCoverList"]/li'): + if counter <= 0: + break + + id = ''.join(data.xpath('.//div[@class="SearchImageContainer"]/a[1]/@href')) + if not id: + continue + + price = ''.join(data.xpath('.//span[@class="SCOurPrice"]/strong/text()')) + if not price: + price = '$0.00' + + cover_url = ''.join(data.xpath('.//div[@class="SearchImageContainer"]//img[1]/@src')) + + title = ''.join(data.xpath('.//div[@class="SCItemHeader"]/h1/a[1]/text()')) + author = ''.join(data.xpath('.//div[@class="SCItemSummary"]/span/a[1]/text()')) + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '?url=http://www.kobobooks.com/' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/manybooks_plugin.py b/src/calibre/gui2/store/manybooks_plugin.py new file mode 100644 index 0000000000..72fe54c427 --- /dev/null +++ b/src/calibre/gui2/store/manybooks_plugin.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class ManyBooksStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://manybooks.net/' + + detail_url = None + if detail_item: + detail_url = url + detail_item + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + # ManyBooks website separates results for title and author. + # It also doesn't do a clear job of references authors and + # secondary titles. Google is also faster. + # Using a google search so we can search on both fields at once. + url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'): + if counter <= 0: + break + + url = '' + url_a = data.xpath('div[@class="jd"]/a') + if url_a: + url_a = url_a[0] + url = url_a.get('href', None) + if url: + url = url.split('u=')[-1][:-2] + if '/titles/' not in url: + continue + id = url.split('/')[-1] + id = id.strip() + + url_a = html.fromstring(html.tostring(url_a)) + heading = ''.join(url_a.xpath('//text()')) + title, _, author = heading.rpartition('by ') + author = author.split('-')[0] + price = '$0.00' + + cover_url = '' + mo = re.match('^\D+', id) + if mo: + cover_name = mo.group() + cover_name = cover_name.replace('etext', '') + cover_id = id.split('.')[0] + cover_url = 'http://manybooks_images.s3.amazonaws.com/original_covers/' + id[0] + '/' + cover_name + '/' + cover_id + '-thumb.jpg' + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/titles/' + id + + yield s diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py new file mode 100644 index 0000000000..49c265d7fe --- /dev/null +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -0,0 +1,304 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import difflib +import heapq +import time +from contextlib import closing +from threading import RLock + +from lxml import html + +from PyQt4.Qt import Qt, QUrl, QDialog, QAbstractItemModel, QModelIndex, QVariant, \ + pyqtSignal + +from calibre import browser +from calibre.gui2 import open_url, NONE +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.mobileread_store_dialog_ui import Ui_Dialog +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog +from calibre.utils.config import DynamicConfig +from calibre.utils.icu import sort_key + +class MobileReadStore(BasicStoreConfig, StorePlugin): + + def genesis(self): + self.config = DynamicConfig('store_' + self.name) + self.rlock = RLock() + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://www.mobileread.com/' + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(detail_item if detail_item else url)) + else: + if detail_item: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + else: + d = MobeReadStoreDialog(self, parent) + d.setWindowTitle(self.name) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + books = self.get_book_list(timeout=timeout) + + query = query.lower() + query_parts = query.split(' ') + matches = [] + s = difflib.SequenceMatcher() + for x in books: + ratio = 0 + t_string = '%s %s' % (x.author.lower(), x.title.lower()) + query_matches = [] + for q in query_parts: + if q in t_string: + query_matches.append(q) + for q in query_matches: + s.set_seq2(q) + for p in t_string.split(' '): + s.set_seq1(p) + ratio += s.ratio() + if ratio > 0: + matches.append((ratio, x)) + + # Move the best scorers to head of list. + matches = heapq.nlargest(max_results, matches) + for score, book in matches: + book.price = '$0.00' + yield book + + def update_book_list(self, timeout=10): + with self.rlock: + url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' + + last_download = self.config.get(self.name + '_last_download', None) + # Don't update the book list if our cache is less than one week old. + if last_download and (time.time() - last_download) < 604800: + return + + # Download the book list HTML file from MobileRead. + br = browser() + raw_data = None + with closing(br.open(url, timeout=timeout)) as f: + raw_data = f.read() + + if not raw_data: + return + + # Turn books listed in the HTML file into BookRef's. + books = [] + try: + data = html.fromstring(raw_data) + for book_data in data.xpath('//ul/li'): + book = BookRef() + book.detail_item = ''.join(book_data.xpath('.//a/@href')) + book.format = ''.join(book_data.xpath('.//i/text()')) + book.format = book.format.strip() + + text = ''.join(book_data.xpath('.//a/text()')) + if ':' in text: + book.author, q, text = text.partition(':') + book.author = book.author.strip() + book.title = text.strip() + books.append(book) + except: + pass + + # Save the book list and it's create time. + if books: + self.config[self.name + '_last_download'] = time.time() + self.config[self.name + '_book_list'] = books + + def get_book_list(self, timeout=10): + self.update_book_list(timeout=timeout) + return self.config.get(self.name + '_book_list', []) + + +class BookRef(SearchResult): + + def __init__(self): + SearchResult.__init__(self) + + self.format = '' + + +class MobeReadStoreDialog(QDialog, Ui_Dialog): + + def __init__(self, plugin, *args): + QDialog.__init__(self, *args) + self.setupUi(self) + + self.plugin = plugin + + self.model = BooksModel() + self.results_view.setModel(self.model) + self.results_view.model().set_books(self.plugin.get_book_list()) + self.total.setText('%s' % self.model.rowCount()) + + self.results_view.activated.connect(self.open_store) + self.search_query.textChanged.connect(self.model.set_filter) + self.results_view.model().total_changed.connect(self.total.setText) + self.finished.connect(self.dialog_closed) + + self.restore_state() + + def open_store(self, index): + result = self.results_view.model().get_book(index) + if result: + self.plugin.open(self, result.detail_item) + + def restore_state(self): + geometry = self.plugin.config['store_mobileread_dialog_geometry'] + if geometry: + self.restoreGeometry(geometry) + + results_cwidth = self.plugin.config['store_mobileread_dialog_results_view_column_width'] + if results_cwidth: + for i, x in enumerate(results_cwidth): + if i >= self.results_view.model().columnCount(): + break + self.results_view.setColumnWidth(i, x) + else: + for i in xrange(self.results_view.model().columnCount()): + self.results_view.resizeColumnToContents(i) + + self.results_view.model().sort_col = self.plugin.config.get('store_mobileread_dialog_sort_col', 0) + self.results_view.model().sort_order = self.plugin.config.get('store_mobileread_dialog_sort_order', Qt.AscendingOrder) + self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) + self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order) + + def save_state(self): + self.plugin.config['store_mobileread_dialog_geometry'] = self.saveGeometry() + self.plugin.config['store_mobileread_dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())] + self.plugin.config['store_mobileread_dialog_sort_col'] = self.results_view.model().sort_col + self.plugin.config['store_mobileread_dialog_sort_order'] = self.results_view.model().sort_order + + def dialog_closed(self, result): + self.save_state() + + +class BooksModel(QAbstractItemModel): + + total_changed = pyqtSignal(unicode) + + HEADERS = [_('Title'), _('Author(s)'), _('Format')] + + def __init__(self): + QAbstractItemModel.__init__(self) + self.books = [] + self.all_books = [] + self.filter = '' + self.sort_col = 0 + self.sort_order = Qt.AscendingOrder + + def set_books(self, books): + self.books = books + self.all_books = books + + self.sort(self.sort_col, self.sort_order) + + def get_book(self, index): + row = index.row() + if row < len(self.books): + return self.books[row] + else: + return None + + def set_filter(self, filter): + #self.layoutAboutToBeChanged.emit() + self.beginResetModel() + + self.filter = unicode(filter) + self.books = [] + if self.filter: + for b in self.all_books: + test = '%s %s %s' % (b.title, b.author, b.format) + test = test.lower() + include = True + for item in self.filter.split(' '): + item = item.lower() + if item not in test: + include = False + break + if include: + self.books.append(b) + else: + self.books = self.all_books + + self.sort(self.sort_col, self.sort_order, reset=False) + self.total_changed.emit('%s' % self.rowCount()) + + self.endResetModel() + #self.layoutChanged.emit() + + def index(self, row, column, parent=QModelIndex()): + return self.createIndex(row, column) + + def parent(self, index): + if not index.isValid() or index.internalId() == 0: + return QModelIndex() + return self.createIndex(0, 0) + + def rowCount(self, *args): + return len(self.books) + + def columnCount(self, *args): + return len(self.HEADERS) + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return NONE + text = '' + if orientation == Qt.Horizontal: + if section < len(self.HEADERS): + text = self.HEADERS[section] + return QVariant(text) + else: + return QVariant(section+1) + + def data(self, index, role): + row, col = index.row(), index.column() + result = self.books[row] + if role == Qt.DisplayRole: + if col == 0: + return QVariant(result.title) + elif col == 1: + return QVariant(result.author) + elif col == 2: + return QVariant(result.format) + return NONE + + def data_as_text(self, result, col): + text = '' + if col == 0: + text = result.title + elif col == 1: + text = result.author + elif col == 2: + text = result.format + return text + + def sort(self, col, order, reset=True): + self.sort_col = col + self.sort_order = order + + if not self.books: + return + descending = order == Qt.DescendingOrder + self.books.sort(None, + lambda x: sort_key(unicode(self.data_as_text(x, col))), + descending) + if reset: + self.reset() + diff --git a/src/calibre/gui2/store/mobileread_store_dialog.ui b/src/calibre/gui2/store/mobileread_store_dialog.ui new file mode 100644 index 0000000000..027d5994f0 --- /dev/null +++ b/src/calibre/gui2/store/mobileread_store_dialog.ui @@ -0,0 +1,112 @@ + + + Dialog + + + + 0 + 0 + 691 + 614 + + + + Dialog + + + + + + + + Search: + + + + + + + + + + + + true + + + false + + + false + + + true + + + false + + + false + + + + + + + + + Books: + + + + + + + 0 + + + + + + + Qt::Horizontal + + + + 308 + 20 + + + + + + + + Close + + + + + + + + + + + close_button + clicked() + Dialog + accept() + + + 440 + 432 + + + 245 + 230 + + + + + diff --git a/src/calibre/gui2/store/open_library_plugin.py b/src/calibre/gui2/store/open_library_plugin.py new file mode 100644 index 0000000000..15b674f262 --- /dev/null +++ b/src/calibre/gui2/store/open_library_plugin.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class OpenLibraryStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://openlibrary.org/' + + if external or settings.get(self.name + '_open_external', False): + if detail_item: + url = url + detail_item + open_url(QUrl(url_slash_cleaner(url))) + else: + detail_url = None + if detail_item: + detail_url = url + detail_item + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://openlibrary.org/search?q=' + urllib2.quote(query) + '&has_fulltext=true' + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@id="searchResults"]/ul[@id="siteSearch"]/li'): + if counter <= 0: + break + + id = ''.join(data.xpath('./span[@class="bookcover"]/a/@href')) + if not id: + continue + cover_url = ''.join(data.xpath('./span[@class="bookcover"]/a/img/@src')) + + title = ''.join(data.xpath('.//h3[@class="booktitle"]/a[@class="results"]/text()')) + author = ''.join(data.xpath('.//span[@class="bookauthor"]/a/text()')) + price = '$0.00' + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price + s.detail_item = id.strip() + + yield s diff --git a/src/calibre/gui2/store/search.py b/src/calibre/gui2/store/search.py new file mode 100644 index 0000000000..970aaf61d2 --- /dev/null +++ b/src/calibre/gui2/store/search.py @@ -0,0 +1,453 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re +import time +from contextlib import closing +from random import shuffle +from threading import Thread +from Queue import Queue + +from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \ + QModelIndex, QPixmap, QSize, QCheckBox, QVBoxLayout, QHBoxLayout, \ + QPushButton, QString, QByteArray + +from calibre import browser +from calibre.gui2 import NONE +from calibre.gui2.progress_indicator import ProgressIndicator +from calibre.gui2.store.search_ui import Ui_Dialog +from calibre.utils.config import DynamicConfig +from calibre.utils.icu import sort_key +from calibre.utils.magick.draw import thumbnail + +HANG_TIME = 75000 # milliseconds seconds +TIMEOUT = 75 # seconds +SEARCH_THREAD_TOTAL = 4 +COVER_DOWNLOAD_THREAD_TOTAL = 2 + +class SearchDialog(QDialog, Ui_Dialog): + + def __init__(self, istores, *args): + QDialog.__init__(self, *args) + self.setupUi(self) + + self.config = DynamicConfig('store_search') + + # We keep a cache of store plugins and reference them by name. + self.store_plugins = istores + self.search_pool = SearchThreadPool(SearchThread, SEARCH_THREAD_TOTAL) + # Check for results and hung threads. + self.checker = QTimer() + self.hang_check = 0 + + self.model = Matches() + self.results_view.setModel(self.model) + + # Add check boxes for each store so the user + # can disable searching specific stores on a + # per search basis. + stores_group_layout = QVBoxLayout() + self.stores_group.setLayout(stores_group_layout) + for x in self.store_plugins: + cbox = QCheckBox(x) + cbox.setChecked(True) + stores_group_layout.addWidget(cbox) + setattr(self, 'store_check_' + x, cbox) + stores_group_layout.addStretch() + + # Create and add the progress indicator + self.pi = ProgressIndicator(self, 24) + self.bottom_layout.insertWidget(0, self.pi) + + self.search.clicked.connect(self.do_search) + self.checker.timeout.connect(self.get_results) + self.results_view.activated.connect(self.open_store) + self.select_all_stores.clicked.connect(self.stores_select_all) + self.select_invert_stores.clicked.connect(self.stores_select_invert) + self.select_none_stores.clicked.connect(self.stores_select_none) + self.finished.connect(self.dialog_closed) + + self.restore_state() + + def resize_columns(self): + total = 600 + # Cover + self.results_view.setColumnWidth(0, 85) + total = total - 85 + # Title + self.results_view.setColumnWidth(1,int(total*.35)) + # Author + self.results_view.setColumnWidth(2,int(total*.35)) + # Price + self.results_view.setColumnWidth(3, int(total*.10)) + # Store + self.results_view.setColumnWidth(4, int(total*.20)) + + def do_search(self, checked=False): + # Stop all running threads. + self.checker.stop() + self.search_pool.abort() + # Clear the visible results. + self.results_view.model().clear_results() + + # Don't start a search if there is nothing to search for. + query = unicode(self.search_edit.text()) + if not query.strip(): + return + + # Plugins are in alphebetic order. Randomize the + # order of plugin names. This way plugins closer + # to a don't have an unfair advantage over + # plugins further from a. + store_names = self.store_plugins.keys() + if not store_names: + return + shuffle(store_names) + # Add plugins that the user has checked to the search pool's work queue. + for n in store_names: + if getattr(self, 'store_check_' + n).isChecked(): + self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT) + if self.search_pool.has_tasks(): + self.hang_check = 0 + self.checker.start(100) + self.search_pool.start_threads() + self.pi.startAnimation() + + def save_state(self): + self.config['store_search_geometry'] = self.saveGeometry() + self.config['store_search_store_splitter_state'] = self.store_splitter.saveState() + self.config['store_search_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())] + + store_check = {} + for n in self.store_plugins: + store_check[n] = getattr(self, 'store_check_' + n).isChecked() + self.config['store_search_store_checked'] = store_check + + def restore_state(self): + geometry = self.config['store_search_geometry'] + if geometry: + self.restoreGeometry(geometry) + + splitter_state = self.config['store_search_store_splitter_state'] + if splitter_state: + self.store_splitter.restoreState(splitter_state) + + results_cwidth = self.config['store_search_results_view_column_width'] + if results_cwidth: + for i, x in enumerate(results_cwidth): + if i >= self.model.columnCount(): + break + self.results_view.setColumnWidth(i, x) + else: + self.resize_columns() + + store_check = self.config['store_search_store_checked'] + if store_check: + for n in store_check: + if hasattr(self, 'store_check_' + n): + getattr(self, 'store_check_' + n).setChecked(store_check[n]) + + def get_results(self): + # We only want the search plugins to run + # a maximum set amount of time before giving up. + self.hang_check += 1 + if self.hang_check >= HANG_TIME: + self.search_pool.abort() + self.checker.stop() + self.pi.stopAnimation() + else: + # Stop the checker if not threads are running. + if not self.search_pool.threads_running() and not self.search_pool.has_tasks(): + self.checker.stop() + self.pi.stopAnimation() + + while self.search_pool.has_results(): + res = self.search_pool.get_result() + if res: + self.results_view.model().add_result(res) + + def open_store(self, index): + result = self.results_view.model().get_result(index) + self.store_plugins[result.store_name].open(self, result.detail_item) + + def get_store_checks(self): + ''' + Returns a list of QCheckBox's for each store. + ''' + checks = [] + for x in self.store_plugins: + check = getattr(self, 'store_check_' + x, None) + if check: + checks.append(check) + return checks + + def stores_select_all(self): + for check in self.get_store_checks(): + check.setChecked(True) + + def stores_select_invert(self): + for check in self.get_store_checks(): + check.setChecked(not check.isChecked()) + + def stores_select_none(self): + for check in self.get_store_checks(): + check.setChecked(False) + + def dialog_closed(self, result): + self.model.closing() + self.search_pool.abort() + self.save_state() + + +class GenericDownloadThreadPool(object): + ''' + add_task must be implemented in a subclass. + ''' + + def __init__(self, thread_type, thread_count): + self.thread_type = thread_type + self.thread_count = thread_count + + self.tasks = Queue() + self.results = Queue() + self.threads = [] + + def add_task(self): + raise NotImplementedError() + + def start_threads(self): + for i in range(self.thread_count): + t = self.thread_type(self.tasks, self.results) + self.threads.append(t) + t.start() + + def abort(self): + self.tasks = Queue() + self.results = Queue() + for t in self.threads: + t.abort() + self.threads = [] + + def has_tasks(self): + return not self.tasks.empty() + + def get_result(self): + return self.results.get() + + def get_result_no_wait(self): + return self.results.get_nowait() + + def result_count(self): + return len(self.results) + + def has_results(self): + return not self.results.empty() + + def threads_running(self): + for t in self.threads: + if t.is_alive(): + return True + return False + + +class SearchThreadPool(GenericDownloadThreadPool): + ''' + Threads will run until there is no work or + abort is called. Create and start new threads + using start_threads(). Reset by calling abort(). + + Example: + sp = SearchThreadPool(SearchThread, 3) + add tasks using add_task(...) + sp.start_threads() + all threads have finished. + sp.abort() + add tasks using add_task(...) + sp.start_threads() + ''' + + def add_task(self, query, store_name, store_plugin, timeout): + self.tasks.put((query, store_name, store_plugin, timeout)) + + +class SearchThread(Thread): + + def __init__(self, tasks, results): + Thread.__init__(self) + self.daemon = True + self.tasks = tasks + self.results = results + self._run = True + + def abort(self): + self._run = False + + def run(self): + while self._run and not self.tasks.empty(): + try: + query, store_name, store_plugin, timeout = self.tasks.get() + for res in store_plugin.search(query, timeout=timeout): + if not self._run: + return + res.store_name = store_name + self.results.put(res) + self.tasks.task_done() + except: + pass + + +class CoverThreadPool(GenericDownloadThreadPool): + ''' + Once started all threads run until abort is called. + ''' + + def add_task(self, search_result, update_callback, timeout=5): + self.tasks.put((search_result, update_callback, timeout)) + + +class CoverThread(Thread): + + def __init__(self, tasks, results): + Thread.__init__(self) + self.daemon = True + self.tasks = tasks + self.results = results + self._run = True + + self.br = browser() + + def abort(self): + self._run = False + + def run(self): + while self._run: + try: + time.sleep(.1) + while not self.tasks.empty(): + if not self._run: + break + result, callback, timeout = self.tasks.get() + if result and result.cover_url: + with closing(self.br.open(result.cover_url, timeout=timeout)) as f: + result.cover_data = f.read() + result.cover_data = thumbnail(result.cover_data, 64, 64)[2] + callback() + self.tasks.task_done() + except: + continue + + +class Matches(QAbstractItemModel): + + HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('Store')] + + def __init__(self): + QAbstractItemModel.__init__(self) + self.matches = [] + self.cover_pool = CoverThreadPool(CoverThread, 2) + self.cover_pool.start_threads() + + def closing(self): + self.cover_pool.abort() + + def clear_results(self): + self.matches = [] + self.cover_pool.abort() + self.cover_pool.start_threads() + self.reset() + + def add_result(self, result): + self.layoutAboutToBeChanged.emit() + self.matches.append(result) + self.cover_pool.add_task(result, self.update_result) + self.layoutChanged.emit() + + def get_result(self, index): + row = index.row() + if row < len(self.matches): + return self.matches[row] + else: + return None + + def update_result(self): + self.layoutAboutToBeChanged.emit() + self.layoutChanged.emit() + + def index(self, row, column, parent=QModelIndex()): + return self.createIndex(row, column) + + def parent(self, index): + if not index.isValid() or index.internalId() == 0: + return QModelIndex() + return self.createIndex(0, 0) + + def rowCount(self, *args): + return len(self.matches) + + def columnCount(self, *args): + return len(self.HEADERS) + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return NONE + text = '' + if orientation == Qt.Horizontal: + if section < len(self.HEADERS): + text = self.HEADERS[section] + return QVariant(text) + else: + return QVariant(section+1) + + def data(self, index, role): + row, col = index.row(), index.column() + result = self.matches[row] + if role == Qt.DisplayRole: + if col == 1: + return QVariant(result.title) + elif col == 2: + return QVariant(result.author) + elif col == 3: + return QVariant(result.price) + elif col == 4: + return QVariant(result.store_name) + return NONE + elif role == Qt.DecorationRole: + if col == 0 and result.cover_data: + p = QPixmap() + p.loadFromData(result.cover_data) + return QVariant(p) + elif role == Qt.SizeHintRole: + return QSize(64, 64) + return NONE + + def data_as_text(self, result, col): + text = '' + if col == 1: + text = result.title + elif col == 2: + text = result.author + elif col == 3: + text = result.price + if len(text) < 3 or text[-3] not in ('.', ','): + text += '00' + text = re.sub(r'\D', '', text) + text = text.rjust(6, '0') + elif col == 4: + text = result.store_name + return text + + def sort(self, col, order, reset=True): + if not self.matches: + return + descending = order == Qt.DescendingOrder + self.matches.sort(None, + lambda x: sort_key(unicode(self.data_as_text(x, col))), + descending) + if reset: + self.reset() + diff --git a/src/calibre/gui2/store/search.ui b/src/calibre/gui2/store/search.ui new file mode 100644 index 0000000000..16fc0c4deb --- /dev/null +++ b/src/calibre/gui2/store/search.ui @@ -0,0 +1,196 @@ + + + Dialog + + + + 0 + 0 + 937 + 669 + + + + calibre Store Search + + + true + + + + + + + + Query: + + + + + + + + + + Search + + + + + + + + + Qt::Horizontal + + + + Stores + + + + + + true + + + + + 0 + 0 + 215 + 116 + + + + + + + + + + + All + + + + + + + Invert + + + + + + + None + + + + + + + + + + + 2 + 0 + + + + Qt::Horizontal + + + + Qt::Horizontal + + + + + 1 + 0 + + + + + 0 + 0 + + + + true + + + + 32 + 32 + + + + false + + + false + + + false + + + true + + + false + + + + + + + + + + + + Qt::Horizontal + + + + 40 + 20 + + + + + + + + Close + + + + + + + + + + + close + clicked() + Dialog + accept() + + + 526 + 525 + + + 307 + 272 + + + + + diff --git a/src/calibre/gui2/store/search_result.py b/src/calibre/gui2/store/search_result.py new file mode 100644 index 0000000000..6e0ed0b572 --- /dev/null +++ b/src/calibre/gui2/store/search_result.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +class SearchResult(object): + + def __init__(self): + self.store_name = '' + self.cover_url = '' + self.cover_data = None + self.title = '' + self.author = '' + self.price = '' + self.detail_item = '' diff --git a/src/calibre/gui2/store/smashwords_plugin.py b/src/calibre/gui2/store/smashwords_plugin.py new file mode 100644 index 0000000000..1806e9f4e1 --- /dev/null +++ b/src/calibre/gui2/store/smashwords_plugin.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import random +import re +import urllib2 +from contextlib import closing + +from lxml import html + +from PyQt4.Qt import QUrl + +from calibre import browser, url_slash_cleaner +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog + +class SmashwordsStore(BasicStoreConfig, StorePlugin): + + def open(self, parent=None, detail_item=None, external=False): + settings = self.get_settings() + url = 'http://www.smashwords.com/' + + aff_id = '?ref=usernone' + # Use Kovid's affiliate id 30% of the time. + if random.randint(1, 10) in (1, 2, 3): + aff_id = '?ref=kovidgoyal' + + detail_url = None + if detail_item: + detail_url = url + detail_item + aff_id + url = url + aff_id + + if external or settings.get(self.name + '_open_external', False): + open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url))) + else: + d = WebStoreDialog(self.gui, url, parent, detail_url) + d.setWindowTitle(self.name) + d.set_tags(settings.get(self.name + '_tags', '')) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + url = 'http://www.smashwords.com/books/search?query=' + urllib2.quote(query) + + br = browser() + + counter = max_results + with closing(br.open(url, timeout=timeout)) as f: + doc = html.fromstring(f.read()) + for data in doc.xpath('//div[@id="pageCenterContent2"]//div[@class="bookCoverImg"]'): + if counter <= 0: + break + data = html.fromstring(html.tostring(data)) + + id = None + id_a = data.xpath('//a[@class="bookTitle"]') + if id_a: + id = id_a[0].get('href', None) + if id: + id = id.split('/')[-1] + if not id: + continue + + cover_url = '' + c_url = data.get('style', None) + if c_url: + mo = re.search(r'http://[^\'"]+', c_url) + if mo: + cover_url = mo.group() + + title = ''.join(data.xpath('//a[@class="bookTitle"]/text()')) + subnote = ''.join(data.xpath('//span[@class="subnote"]/text()')) + author = ''.join(data.xpath('//span[@class="subnote"]/a/text()')) + price = subnote.partition('$')[2] + price = price.split(u'\xa0')[0] + price = '$' + price + + counter -= 1 + + s = SearchResult() + s.cover_url = cover_url + s.title = title.strip() + s.author = author.strip() + s.price = price.strip() + s.detail_item = '/books/view/' + id.strip() + + yield s diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py new file mode 100644 index 0000000000..b7ab75975d --- /dev/null +++ b/src/calibre/gui2/store/web_control.py @@ -0,0 +1,112 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import os +from urlparse import urlparse + +from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString, \ + QFileDialog, QNetworkProxy + +from calibre import USER_AGENT, get_proxies, get_download_filename +from calibre.ebooks import BOOK_EXTENSIONS +from calibre.ptempfile import PersistentTemporaryFile + +class NPWebView(QWebView): + + def __init__(self, *args): + QWebView.__init__(self, *args) + self.gui = None + self.tags = '' + + self.setPage(NPWebPage()) + self.page().networkAccessManager().setCookieJar(QNetworkCookieJar()) + + http_proxy = get_proxies().get('http', None) + if http_proxy: + proxy_parts = urlparse(http_proxy) + proxy = QNetworkProxy() + proxy.setType(QNetworkProxy.HttpProxy) + proxy.setUser(proxy_parts.username) + proxy.setPassword(proxy_parts.password) + proxy.setHostName(proxy_parts.hostname) + proxy.setPort(proxy_parts.port) + self.page().networkAccessManager().setProxy(proxy) + + self.page().setForwardUnsupportedContent(True) + self.page().unsupportedContent.connect(self.start_download) + self.page().downloadRequested.connect(self.start_download) + self.page().networkAccessManager().sslErrors.connect(self.ignore_ssl_errors) + + def createWindow(self, type): + if type == QWebPage.WebBrowserWindow: + return self + else: + return None + + def set_gui(self, gui): + self.gui = gui + + def set_tags(self, tags): + self.tags = tags + + def start_download(self, request): + if not self.gui: + return + + url = unicode(request.url().toString()) + cf = self.get_cookies() + + filename = get_download_filename(url, cf) + ext = os.path.splitext(filename)[1][1:].lower() + if ext not in BOOK_EXTENSIONS: + home = os.path.expanduser('~') + name = QFileDialog.getSaveFileName(self, + _('File is not a supported ebook type. Save to disk?'), + os.path.join(home, filename), + '*.*') + if name: + self.gui.download_ebook(url, cf, name, name, False) + else: + self.gui.download_ebook(url, cf, filename, tags=self.tags) + + def ignore_ssl_errors(self, reply, errors): + reply.ignoreSslErrors(errors) + + def get_cookies(self): + ''' + Writes QNetworkCookies to Mozilla cookie .txt file. + + :return: The file path to the cookie file. + ''' + cf = PersistentTemporaryFile(suffix='.txt') + + cf.write('# Netscape HTTP Cookie File\n\n') + + for c in self.page().networkAccessManager().cookieJar().allCookies(): + cookie = [] + domain = unicode(c.domain()) + + cookie.append(domain) + cookie.append('TRUE' if domain.startswith('.') else 'FALSE') + cookie.append(unicode(c.path())) + cookie.append('TRUE' if c.isSecure() else 'FALSE') + cookie.append(unicode(c.expirationDate().toTime_t())) + cookie.append(unicode(c.name())) + cookie.append(unicode(c.value())) + + cf.write('\t'.join(cookie)) + cf.write('\n') + + cf.close() + return cf.name + + +class NPWebPage(QWebPage): + + def userAgentForUrl(self, url): + return USER_AGENT diff --git a/src/calibre/gui2/store/web_store_dialog.py b/src/calibre/gui2/store/web_store_dialog.py new file mode 100644 index 0000000000..20fb016b6b --- /dev/null +++ b/src/calibre/gui2/store/web_store_dialog.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from PyQt4.Qt import QDialog, QUrl + +from calibre import url_slash_cleaner +from calibre.gui2.store.web_store_dialog_ui import Ui_Dialog + +class WebStoreDialog(QDialog, Ui_Dialog): + + def __init__(self, gui, base_url, parent=None, detail_url=None): + QDialog.__init__(self, parent=parent) + self.setupUi(self) + + self.gui = gui + self.base_url = base_url + + self.view.set_gui(self.gui) + self.view.loadStarted.connect(self.load_started) + self.view.loadProgress.connect(self.load_progress) + self.view.loadFinished.connect(self.load_finished) + self.home.clicked.connect(self.go_home) + self.reload.clicked.connect(self.view.reload) + self.back.clicked.connect(self.view.back) + + self.go_home(detail_url=detail_url) + + def set_tags(self, tags): + self.view.set_tags(tags) + + def load_started(self): + self.progress.setValue(0) + + def load_progress(self, val): + self.progress.setValue(val) + + def load_finished(self, ok=True): + self.progress.setValue(100) + + def go_home(self, checked=False, detail_url=None): + if detail_url: + url = detail_url + else: + url = self.base_url + + # Reduce redundant /'s because some stores + # (Feedbooks) and server frameworks (cherrypy) + # choke on them. + url = url_slash_cleaner(url) + self.view.load(QUrl(url)) diff --git a/src/calibre/gui2/store/web_store_dialog.ui b/src/calibre/gui2/store/web_store_dialog.ui new file mode 100644 index 0000000000..b89b9305be --- /dev/null +++ b/src/calibre/gui2/store/web_store_dialog.ui @@ -0,0 +1,115 @@ + + + Dialog + + + + 0 + 0 + 962 + 656 + + + + + + + true + + + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + 0 + + + + + + about:blank + + + + + + + + + + + Home + + + + + + + Reload + + + + + + + 0 + + + %p% + + + + + + + Back + + + + + + + Close + + + + + + + + QWebView + QWidget +
QtWebKit/QWebView
+ + + NPWebView + QWebView +
web_control.h
+ + + + + + close + clicked() + Dialog + accept() + + + 917 + 635 + + + 480 + 327 + + + + + diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py index 9c791c5b0d..ad295503a0 100644 --- a/src/calibre/gui2/threaded_jobs.py +++ b/src/calibre/gui2/threaded_jobs.py @@ -38,7 +38,7 @@ class ThreadedJob(BaseJob): :func: The function that actually does the work. This function *must* accept at least three keyword arguments: abort, log and notifications. abort is - An Event object. func should periodically check abort.is_set(0 and if + An Event object. func should periodically check abort.is_set() and if it is True, it should stop processing as soon as possible. notifications is a Queue. func should put progress notifications into it in the form of a tuple (frac, msg). frac is a number between 0 and 1 indicating diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index e7853b9491..8d31d9da32 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -23,7 +23,7 @@ from calibre.constants import __appname__, isosx from calibre.utils.config import prefs, dynamic from calibre.utils.ipc.server import Server from calibre.library.database2 import LibraryDatabase2 -from calibre.customize.ui import interface_actions +from calibre.customize.ui import interface_actions, store_plugins from calibre.gui2 import error_dialog, GetMetadata, open_url, \ gprefs, max_available_height, config, info_dialog, Dispatcher, \ question_dialog @@ -34,6 +34,7 @@ from calibre.gui2.main_window import MainWindow from calibre.gui2.layout import MainWindowMixin from calibre.gui2.device import DeviceMixin from calibre.gui2.email import EmailMixin +from calibre.gui2.ebook_download import EbookDownloadMixin from calibre.gui2.jobs import JobManager, JobsDialog, JobsButton from calibre.gui2.init import LibraryViewMixin, LayoutMixin from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin @@ -89,7 +90,8 @@ class SystemTrayIcon(QSystemTrayIcon): # {{{ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin, - SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin + SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin, + EbookDownloadMixin ): 'The main GUI' @@ -100,6 +102,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.device_connected = None self.gui_debug = gui_debug self.iactions = OrderedDict() + # Actions for action in interface_actions(): if opts.ignore_plugins and action.plugin_path is not None: continue @@ -112,11 +115,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ if action.plugin_path is None: raise continue - ac.plugin_path = action.plugin_path ac.interface_action_base_plugin = action - self.add_iaction(ac) + self.load_store_plugins() def init_iaction(self, action): ac = action.load_actual_plugin(self) @@ -133,6 +135,37 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ else: acmap[ac.name] = ac + def load_store_plugins(self): + self.istores = OrderedDict() + for store in store_plugins(): + if self.opts.ignore_plugins and store.plugin_path is not None: + continue + try: + st = self.init_istore(store) + self.add_istore(st) + except: + # Ignore errors in loading user supplied plugins + import traceback + traceback.print_exc() + if store.plugin_path is None: + raise + continue + + def init_istore(self, store): + st = store.load_actual_plugin(self) + st.plugin_path = store.plugin_path + st.base_plugin = store + store.actual_istore_plugin_loaded = True + return st + + def add_istore(self, st): + stmap = self.istores + if st.name in stmap: + if st.priority >= stmap[st.name].priority: + stmap[st.name] = st + else: + stmap[st.name] = st + def initialize(self, library_path, db, listener, actions, show_gui=True): opts = self.opts @@ -154,6 +187,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ for ac in self.iactions.values(): ac.do_genesis() self.donate_action = QAction(QIcon(I('donate.png')), _('&Donate to support calibre'), self) + for st in self.istores.values(): + st.do_genesis() MainWindowMixin.__init__(self, db) # Jobs Button {{{ @@ -165,6 +200,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ LayoutMixin.__init__(self) EmailMixin.__init__(self) + EbookDownloadMixin.__init__(self) DeviceMixin.__init__(self) self.progress_indicator = ProgressIndicator(self)