Initial import of John's store code

src/calibre/__init__.py

@@ -5,7 +5,9 @@ __docformat__ = 'restructuredtext en'
 
 import uuid, sys, os, re, logging, time, random, \
        __builtin__, warnings, multiprocessing
+from contextlib import closing
 from urllib import getproxies
+from urllib2 import unquote as urllib2_unquote
 __builtin__.__dict__['dynamic_property'] = lambda(func): func(None)
 from htmlentitydefs import name2codepoint
 from math import floor
@@ -290,6 +292,9 @@ def get_parsed_proxy(typ='http', debug=True):
                     prints('Using http proxy', str(ans))
                 return ans
 
+USER_AGENT = 'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101210 Gentoo Firefox/3.6.13'
+USER_AGENT_MOBILE = 'Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016'
+
 def random_user_agent():
     choices = [
         'Mozilla/5.0 (Windows NT 5.2; rv:2.0.1) Gecko/20100101 Firefox/4.0.1',
@@ -305,7 +310,6 @@ def random_user_agent():
     #return choices[-1]
     return choices[random.randint(0, len(choices)-1)]
 
-
 def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None):
     '''
     Create a mechanize browser for web scraping. The browser handles cookies,
@@ -319,8 +323,7 @@ def browser(honor_time=True, max_time=2, mobile_browser=False, user_agent=None):
     opener.set_handle_refresh(True, max_time=max_time, honor_time=honor_time)
     opener.set_handle_robots(False)
     if user_agent is None:
-        user_agent = ' Mozilla/5.0 (Windows; U; Windows CE 5.1; rv:1.8.1a3) Gecko/20060610 Minimo/0.016' if mobile_browser else \
+        user_agent = USER_AGENT_MOBILE if mobile_browser else USER_AGENT                          
-                          'Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.13) Gecko/20101210 Gentoo Firefox/3.6.13'
     opener.addheaders = [('User-agent', user_agent)]
     http_proxy = get_proxies().get('http', None)
     if http_proxy:
@@ -537,7 +540,46 @@ def as_unicode(obj, enc=preferred_encoding):
                 obj = repr(obj)
     return force_unicode(obj, enc=enc)
 
+def url_slash_cleaner(url):
+    '''
+    Removes redundant /'s from url's.
+    '''
+    return re.sub(r'(?<!:)/{2,}', '/', url)
 
+def get_download_filename(url, cookie_file=None):
+    filename = ''
+    
+    br = browser()
+    if cookie_file:
+        from mechanize import MozillaCookieJar
+        cj = MozillaCookieJar()
+        cj.load(cookie_file)
+        br.set_cookiejar(cj)
+
+    try:
+        with closing(br.open(url)) as r:
+            disposition = r.info().get('Content-disposition', '')
+            for p in disposition.split(';'):
+                if 'filename' in p:
+                    if '*=' in disposition:
+                        parts = disposition.split('*=')[-1]
+                        filename = parts.split('\'')[-1]
+                    else:
+                        filename = disposition.split('=')[-1]
+                    if filename[0] in ('\'', '"'):
+                        filename = filename[1:]
+                    if filename[-1] in ('\'', '"'):
+                        filename = filename[:-1]
+                    filename = urllib2_unquote(filename)
+                    break
+    except:
+        import traceback
+        traceback.print_exc()
+
+    if not filename:
+        filename = r.geturl().split('/')[-1]
+
+    return filename
 
 def human_readable(size):
     """ Convert a size in bytes into a human readable form """

src/calibre/customize/__init__.py

@@ -602,3 +602,35 @@ class PreferencesPlugin(Plugin): # {{{
 
 # }}}
 
+class StoreBase(Plugin): # {{{
+
+    supported_platforms = ['windows', 'osx', 'linux']
+    author         = 'John Schember'
+    type = _('Store')
+
+    actual_plugin = None
+
+    def load_actual_plugin(self, gui):
+        '''
+        This method must return the actual interface action plugin object.
+        '''
+        mod, cls = self.actual_plugin.split(':')
+        self.actual_plugin_object  = getattr(importlib.import_module(mod), cls)(gui, self.name)
+        return self.actual_plugin_object
+
+    def customization_help(self, gui=False):
+        if getattr(self, 'actual_plugin_object', None) is not None:
+            return self.actual_plugin_object.customization_help(gui)
+        raise NotImplementedError()
+
+    def config_widget(self):
+        if getattr(self, 'actual_plugin_object', None) is not None:
+            return self.actual_plugin_object.config_widget()
+        raise NotImplementedError()
+
+    def save_settings(self, config_widget):
+        if getattr(self, 'actual_plugin_object', None) is not None:
+            return self.actual_plugin_object.save_settings(config_widget)
+        raise NotImplementedError()
+
+# }}}

src/calibre/customize/builtins.py

@@ -5,7 +5,7 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
 import textwrap, os, glob, functools, re
 from calibre import guess_type
 from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \
-    MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase
+    MetadataWriterPlugin, PreferencesPlugin, InterfaceActionBase, StoreBase
 from calibre.constants import numeric_version
 from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata
 from calibre.ebooks.metadata.opf2 import metadata_to_opf
@@ -854,13 +854,18 @@ class ActionNextMatch(InterfaceActionBase):
     name = 'Next Match'
     actual_plugin = 'calibre.gui2.actions.next_match:NextMatchAction'
 
+class ActionStore(InterfaceActionBase):
+    name = 'Store'
+    author = 'John Schember'
+    actual_plugin = 'calibre.gui2.actions.store:StoreAction'
+
 plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
         ActionConvert, ActionDelete, ActionEditMetadata, ActionView,
         ActionFetchNews, ActionSaveToDisk, ActionShowBookDetails,
         ActionRestart, ActionOpenFolder, ActionConnectShare,
         ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks,
         ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary,
-        ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch]
+        ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore]
 
 # }}}
 
@@ -1093,4 +1098,81 @@ if test_eight_code:
 
 #}}}
 
+# Store plugins {{{
+class StoreAmazonKindleStore(StoreBase):
+    name = 'Amazon Kindle'
+    description = _('Kindle books from Amazon')
+    actual_plugin = 'calibre.gui2.store.amazon_plugin:AmazonKindleStore'
 
+class StoreBaenWebScriptionStore(StoreBase):
+    name = 'Baen WebScription'
+    description = _('Ebooks for readers.')
+    actual_plugin = 'calibre.gui2.store.baen_webscription_plugin:BaenWebScriptionStore'
+
+class StoreBNStore(StoreBase):
+    name = 'Barnes and Noble'
+    description = _('Books, Textbooks, eBooks, Toys, Games and More.')
+    actual_plugin = 'calibre.gui2.store.bn_plugin:BNStore'
+
+class StoreBeWriteStore(StoreBase):
+    name = 'BeWrite Books'
+    description = _('Publishers of fine books.')
+    actual_plugin = 'calibre.gui2.store.bewrite_plugin:BeWriteStore'
+
+class StoreDieselEbooksStore(StoreBase):
+    name = 'Diesel eBooks'
+    description = _('World Famous eBook Store.')
+    actual_plugin = 'calibre.gui2.store.diesel_ebooks_plugin:DieselEbooksStore'
+
+class StoreEbookscomStore(StoreBase):
+    name = 'eBooks.com'
+    description = _('The digital bookstore.')
+    actual_plugin = 'calibre.gui2.store.ebooks_com_plugin:EbookscomStore'
+
+class StoreEHarlequinStoretore(StoreBase):
+    name = 'eHarlequin'
+    description = _('entertain, enrich, inspire.')
+    actual_plugin = 'calibre.gui2.store.eharlequin_plugin:EHarlequinStore'
+
+class StoreFeedbooksStore(StoreBase):
+    name = 'Feedbooks'
+    description = _('Read anywhere.')
+    actual_plugin = 'calibre.gui2.store.feedbooks_plugin:FeedbooksStore'
+
+class StoreGutenbergStore(StoreBase):
+    name = 'Project Gutenberg'
+    description = _('The first producer of free ebooks.')
+    actual_plugin = 'calibre.gui2.store.gutenberg_plugin:GutenbergStore'
+
+class StoreKoboStore(StoreBase):
+    name = 'Kobo'
+    description = _('eReading: anytime. anyplace.')
+    actual_plugin = 'calibre.gui2.store.kobo_plugin:KoboStore'
+
+class StoreManyBooksStore(StoreBase):
+    name = 'ManyBooks'
+    description = _('The best ebooks at the best price: free!')
+    actual_plugin = 'calibre.gui2.store.manybooks_plugin:ManyBooksStore'
+
+class StoreMobileReadStore(StoreBase):
+    name = 'MobileRead'
+    description = _('Ebooks handcrafted with the utmost care')
+    actual_plugin = 'calibre.gui2.store.mobileread_plugin:MobileReadStore'
+
+class StoreOpenLibraryStore(StoreBase):
+    name = 'Open Library'
+    description = _('One web page for every book.')
+    actual_plugin = 'calibre.gui2.store.open_library_plugin:OpenLibraryStore'
+
+class StoreSmashwordsStore(StoreBase):
+    name = 'Smashwords'
+    description = _('Your ebook. Your way.')
+    actual_plugin = 'calibre.gui2.store.smashwords_plugin:SmashwordsStore'
+
+plugins += [StoreAmazonKindleStore, StoreBaenWebScriptionStore, StoreBNStore,
+    StoreBeWriteStore, StoreDieselEbooksStore, StoreEbookscomStore,
+    StoreEHarlequinStoretore,
+    StoreFeedbooksStore, StoreGutenbergStore, StoreKoboStore, StoreManyBooksStore,
+    StoreMobileReadStore, StoreOpenLibraryStore, StoreSmashwordsStore]
+
+# }}}

src/calibre/customize/ui.py

@@ -7,7 +7,8 @@ import os, shutil, traceback, functools, sys
 from calibre.customize import (CatalogPlugin, FileTypePlugin, PluginNotFound,
                               MetadataReaderPlugin, MetadataWriterPlugin,
                               InterfaceActionBase as InterfaceAction,
-                              PreferencesPlugin, platform, InvalidPlugin)
+                              PreferencesPlugin, platform, InvalidPlugin,
+                              StoreBase as Store)
 from calibre.customize.conversion import InputFormatPlugin, OutputFormatPlugin
 from calibre.customize.zipplugin import loader
 from calibre.customize.profiles import InputProfile, OutputProfile
@@ -244,6 +245,17 @@ def preferences_plugins():
                 yield plugin
 # }}}
 
+# Store Plugins # {{{
+
+def store_plugins():
+    customization = config['plugin_customization']
+    for plugin in _initialized_plugins:
+        if isinstance(plugin, Store):
+            if not is_disabled(plugin):
+                plugin.site_customization = customization.get(plugin.name, '')
+                yield plugin
+# }}}
+
 # Metadata read/write {{{
 _metadata_readers = {}
 _metadata_writers = {}

src/calibre/gui2/actions/store.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from functools import partial
+
+from PyQt4.Qt import Qt, QMenu, QToolButton, QDialog, QVBoxLayout
+
+from calibre.gui2.actions import InterfaceAction
+
+class StoreAction(InterfaceAction):
+
+    name = 'Store'
+    action_spec = (_('Store'), 'store.png', None, None)
+    
+    def genesis(self):
+        self.qaction.triggered.connect(self.search)
+        self.store_menu = QMenu()
+        self.load_menu()
+        
+    def load_menu(self):
+        self.store_menu.clear()
+        self.store_menu.addAction(_('Search'), self.search)
+        self.store_menu.addSeparator()
+        for n, p in self.gui.istores.items():
+            self.store_menu.addAction(n, partial(self.open_store, p))
+        self.qaction.setMenu(self.store_menu)
+    
+    def search(self):
+        from calibre.gui2.store.search import SearchDialog
+        sd = SearchDialog(self.gui.istores, self.gui)
+        sd.exec_()
+        
+    def open_store(self, store_plugin):
+        store_plugin.open(self.gui)

src/calibre/gui2/ebook_download.py

@@ -0,0 +1,106 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import os
+import shutil
+from contextlib import closing
+from mechanize import MozillaCookieJar
+
+from calibre import browser, get_download_filename
+from calibre.ebooks import BOOK_EXTENSIONS
+from calibre.gui2 import Dispatcher
+from calibre.gui2.threaded_jobs import ThreadedJob
+from calibre.ptempfile import PersistentTemporaryFile
+
+class EbookDownload(object):
+
+    def __call__(self, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[], log=None, abort=None, notifications=None):
+        dfilename = ''
+        try:
+            dfilename = self._download(cookie_file, url, filename, save_loc, add_to_lib)
+            self._add(dfilename, gui, add_to_lib, tags)
+            self._save_as(dfilename, save_loc)
+        except Exception as e:
+            raise e
+        finally:
+            try:
+                if dfilename:
+                    os.remove(dfilename)
+            except:
+                pass
+
+    def _download(self, cookie_file, url, filename, save_loc, add_to_lib):
+        dfilename = ''
+
+        if not url:
+            raise Exception(_('No file specified to download.'))
+        if not save_loc and not add_to_lib:
+            # Nothing to do.
+            return dfilename
+
+        if not filename:
+            filename = get_download_filename(url, cookie_file)
+
+        br = browser()
+        if cookie_file:
+            cj = MozillaCookieJar()
+            cj.load(cookie_file)
+            br.set_cookiejar(cj)
+        with closing(br.open(url)) as r:
+            tf = PersistentTemporaryFile(suffix=filename)
+            tf.write(r.read())
+            dfilename = tf.name
+
+        return dfilename
+
+    def _add(self, filename, gui, add_to_lib, tags):
+        if not add_to_lib or not filename:
+            return
+        ext = os.path.splitext(filename)[1][1:].lower()
+        if ext not in BOOK_EXTENSIONS:
+            raise Exception(_('Not a support ebook format.'))
+
+        from calibre.ebooks.metadata.meta import get_metadata
+        with open(filename) as f:
+            mi = get_metadata(f, ext)
+        mi.tags.extend(tags)
+
+        id = gui.library_view.model().db.create_book_entry(mi)
+        gui.library_view.model().db.add_format_with_hooks(id, ext.upper(), filename, index_is_id=True)
+        gui.library_view.model().books_added(1)
+        gui.library_view.model().count_changed()
+
+    def _save_as(self, dfilename, save_loc):
+        if not save_loc or not dfilename:
+            return
+        shutil.copy(dfilename, save_loc)
+
+
+gui_ebook_download = EbookDownload()
+
+def start_ebook_download(callback, job_manager, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[]):
+    description = _('Downloading %s') % filename if filename else url
+    job = ThreadedJob('ebook_download', description, gui_ebook_download, (gui, cookie_file, url, filename, save_loc, add_to_lib, tags), {}, callback, max_concurrent_count=2, killable=False)
+    job_manager.run_threaded_job(job)
+
+
+class EbookDownloadMixin(object):
+
+    def download_ebook(self, url='', cookie_file=None, filename='', save_loc='', add_to_lib=True, tags=[]):
+        if tags:
+            if isinstance(tags, basestring):
+                tags = tags.split(',')
+        start_ebook_download(Dispatcher(self.downloaded_ebook), self.job_manager, self, cookie_file, url, filename, save_loc, add_to_lib, tags)
+        self.status_bar.show_message(_('Downloading') + ' ' + filename if filename else url, 3000)
+
+    def downloaded_ebook(self, job):
+        if job.failed:
+            self.job_exception(job, dialog_title=_('Failed to download ebook'))
+            return
+
+        self.status_bar.show_message(job.description + ' ' + _('finished'), 5000)

src/calibre/gui2/preferences/plugins.py

@@ -218,6 +218,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
         self.search.search.connect(self.find)
         self.next_button.clicked.connect(self.find_next)
         self.previous_button.clicked.connect(self.find_previous)
+        self.changed_signal.connect(self.reload_store_plugins)
 
     def find(self, query):
         idx = self._plugin_model.find(query)
@@ -344,6 +345,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                          plugin.name + _(' cannot be removed. It is a '
                          'builtin plugin. Try disabling it instead.')).exec_()
 
+    def reload_store_plugins(self):
+        self.gui.load_store_plugins()
+        if self.gui.iactions.has_key('Store'):
+            self.gui.iactions['Store'].load_menu()
+
     def check_for_add_to_toolbars(self, plugin):
         from calibre.gui2.preferences.toolbar import ConfigWidget
         from calibre.customize import InterfaceActionBase
@@ -376,6 +382,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
                 installed_actions.append(plugin_action.name)
                 gprefs['action-layout-'+key] = tuple(installed_actions)
 
+
 if __name__ == '__main__':
     from PyQt4.Qt import QApplication
     app = QApplication([])

src/calibre/gui2/store/__init__.py

@@ -0,0 +1,139 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+class StorePlugin(object): # {{{
+    '''
+    A plugin representing an online ebook repository (store). The store can
+    be a comercial store that sells ebooks or a source of free downloadable
+    ebooks.
+
+    Note that this class is the base class for these plugins, however, to
+    integrate the plugin with calibre's plugin system, you have to make a
+    wrapper class that references the actual plugin. See the
+    :mod:`calibre.customize.builtins` module for examples.
+
+    If two :class:`StorePlugin` objects have the same name, the one with higher
+    priority takes precedence.
+    
+    Sub-classes must implement :meth:`open`, and :meth:`search`.
+    
+    Regarding :meth:`open`. Most stores only make themselves available
+    though a web site thus most store plugins will open using
+    :class:`calibre.gui2.store.web_store_dialog.WebStoreDialog`. This will
+    open a modal window and display the store website in a QWebView.
+    
+    Sub-classes should implement and use the :meth:`genesis` if they require
+    plugin specific initialization. They should not override or otherwise
+    reimplement :meth:`__init__`.
+    
+    Once initialized, this plugin has access to the main calibre GUI via the
+    :attr:`gui` member. You can access other plugins by name, for example::
+
+        self.gui.istores['Amazon Kindle']
+        
+    Plugin authors can use affiliate programs within their plugin. The
+    distribution of money earned from a store plugin is 70/30. 70% going
+    to the pluin author / maintainer and 30% going to the calibre project.
+    
+    The easiest way to handle affiliate money payouts is to randomly select
+    between the author's affiliate id and calibre's affiliate id so that
+    70% of the time the author's id is used.
+    '''
+
+    def __init__(self, gui, name):
+        self.gui = gui
+        self.name = name
+        self.base_plugin = None
+    
+    def open(self, gui, parent=None, detail_item=None, external=False):
+        '''
+        Open the store.
+        
+        :param gui: The main GUI. This will be used to have the job
+        system start downloading an item from the store.
+        
+        :param parent: The parent of the store dialog. This is used
+        to create modal dialogs.
+        
+        :param detail_item: A plugin specific reference to an item
+        in the store that the user should be shown.
+        
+        :param external: When False open an internal dialog with the
+        store. When True open the users default browser to the store's
+        web site. :param:`detail_item` should still be respected when external
+        is True.
+        '''
+        raise NotImplementedError()
+    
+    def search(self, query, max_results=10, timeout=60):
+        '''
+        Searches the store for items matching query. This should
+        return items as a generator.
+        
+        Don't be lazy with the search! Load as much data as possible in the
+        :class:`calibre.gui2.store.search_result.SearchResult` object. If you have to parse
+        multiple pages to get all of the data then do so. However, if data (such as cover_url)
+        isn't available because the store does not display cover images then it's okay to
+        ignore it.
+        
+        Also, by default search results can only include ebooks. A plugin can offer users
+        an option to include physical books in the search results but this must be
+        disabled by default.
+        
+        If a store doesn't provide search on it's own use something like a site specific
+        google search to get search results for this funtion.
+        
+        :param query: The string query search with.
+        :param max_results: The maximum number of results to return.
+        :param timeout: The maximum amount of time in seconds to spend download the search results.
+        
+        :return: :class:`calibre.gui2.store.search_result.SearchResult` objects
+        item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store. 
+        '''
+        raise NotImplementedError()
+    
+    def get_settings(self):
+        '''
+        This is only useful for plugins that implement
+        :attr:`config_widget` that is the only way to save
+        settings. This is used by plugins to get the saved
+        settings and apply when necessary.
+        
+        :return: A dictionary filled with the settings used
+        by this plugin.
+        '''
+        raise NotImplementedError()
+
+    def do_genesis(self):
+        self.genesis()
+
+    def genesis(self):
+        '''
+        Plugin specific initialization.
+        '''
+        pass
+    
+    def config_widget(self):
+        '''
+        See :class:`calibre.customize.Plugin` for details.
+        '''
+        raise NotImplementedError()
+    
+    def save_settings(self, config_widget):
+        '''
+        See :class:`calibre.customize.Plugin` for details.
+        '''
+        raise NotImplementedError()
+    
+    def customization_help(self, gui=False):
+        '''
+        See :class:`calibre.customize.Plugin` for details.
+        '''
+        raise NotImplementedError()
+
+# }}}

src/calibre/gui2/store/amazon_plugin.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import random
+import re
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.search_result import SearchResult
+
+class AmazonKindleStore(StorePlugin):
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        '''
+        Amazon comes with a number of difficulties.
+        
+        QWebView has major issues with Amazon.com. The largest of
+        issues is it simply doesn't work on a number of pages.
+        
+        When connecting to a number parts of Amazon.com (Kindle library
+        for instance) QNetworkAccessManager fails to connect with a
+        NetworkError of 399 - ProtocolFailure. The strange thing is,
+        when I check QNetworkRequest.HttpStatusCodeAttribute when the
+        399 error is returned the status code is 200 (Ok). However, once
+        the QNetworkAccessManager decides there was a NetworkError it
+        does not download the page from Amazon. So I can't even set the
+        HTML in the QWebView myself.
+        
+        There is http://bugreports.qt.nokia.com/browse/QTWEBKIT-259 an
+        open bug about the issue but it is not correct. We can set the
+        useragent (Arora does) to something else and the above issue
+        will persist. This http://developer.qt.nokia.com/forums/viewthread/793
+        gives a bit more information about the issue but as of now (27/Feb/2011)
+        there is no solution or work around.
+        
+        We cannot change the The linkDelegationPolicy to allow us to avoid
+        QNetworkAccessManager because it only works links. Forms aren't
+        included so the same issue persists on any part of the site (login)
+        that use a form to load a new page.
+        
+        Using an aStore was evaluated but I've decided against using it.
+        There are three major issues with an aStore. Because checkout is
+        handled by sending the user to Amazon we can't put it in a QWebView.
+        If we're sending the user to Amazon sending them there directly is
+        nicer. Also, we cannot put the aStore in a QWebView and let it open the
+        redirection the users default browser because the cookies with the
+        shopping cart won't transfer.
+        
+        Another issue with the aStore is how it handles the referral. It only
+        counts the referral for the items in the shopping card / the item
+        that directed the user to Amazon. Kindle books do not use the shopping
+        cart and send the user directly to Amazon for the purchase. In this
+        instance we would only get referral credit for the one book that the
+        aStore directs to Amazon that the user buys. Any other purchases we
+        won't get credit for.
+        
+        The last issue with the aStore is performance. Even though it's an
+        Amazon site it's alow. So much slower than Amazon.com that it makes
+        me not want to browse books using it. The look and feel are lesser
+        issues. So is the fact that it almost seems like the purchase is
+        with calibre. This can cause some support issues because we can't
+        do much for issues with Amazon.com purchase hiccups.
+        
+        Another option that was evaluated was the Product Advertising API.
+        The reasons against this are complexity. It would take a lot of work
+        to basically re-create Amazon.com within calibre. The Product
+        Advertising API is also designed with being run on a server not
+        in an app. The signing keys would have to be made avaliable to ever
+        calibre user which means bad things could be done with our account.
+        
+        The Product Advertising API also assumes the same browser for easy
+        shopping cart transfer to Amazon. With QWebView not working and there
+        not being an easy way to transfer cookies between a QWebView and the
+        users default browser this won't work well.
+        
+        We could create our own website on the calibre server and create an
+        Amazon Product Advertising API store. However, this goes back to the
+        complexity argument. Why spend the time recreating Amazon.com
+        
+        The final and largest issue against using the Product Advertising API
+        is the Efficiency Guidelines:
+        
+        "Each account used to access the Product Advertising API will be allowed
+        an initial usage limit of 2,000 requests per hour. Each account will
+        receive an additional 500 requests per hour (up to a maximum of 25,000
+        requests per hour) for every $1 of shipped item revenue driven per hour
+        in a trailing 30-day period. Usage thresholds are recalculated daily based
+        on revenue performance." 
+        
+        With over two million users a limit of 2,000 request per hour could
+        render our store unusable for no other reason than Amazon rate
+        limiting our traffic.
+        
+        The best (I use the term lightly here) solution is to open Amazon.com
+        in the users default browser and set the affiliate id as part of the url.
+        '''
+        aff_id = {'tag': 'josbl0e-cpb-20'}
+        # Use Kovid's affiliate id 30% of the time.
+        if random.randint(1, 10) in (1, 2, 3):
+            aff_id['tag'] = 'calibrebs-20'
+        store_link = 'http://www.amazon.com/Kindle-eBooks/b/?ie=UTF&node=1286228011&ref_=%(tag)s&ref=%(tag)s&tag=%(tag)s&linkCode=ur2&camp=1789&creative=390957' % aff_id
+        if detail_item:
+            aff_id['asin'] = detail_item
+            store_link = 'http://www.amazon.com/dp/%(asin)s/?tag=%(tag)s' % aff_id
+        open_url(QUrl(store_link))
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query)
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="productData"]'):
+                if counter <= 0:
+                    break
+                
+                # Even though we are searching digital-text only Amazon will still
+                # put in results for non Kindle books (author pages). Se we need
+                # to explicitly check if the item is a Kindle book and ignore it
+                # if it isn't.
+                type = ''.join(data.xpath('//span[@class="format"]/text()'))
+                if 'kindle' not in type.lower():
+                    continue
+                
+                # We must have an asin otherwise we can't easily reference the
+                # book later.
+                asin_href = None
+                asin_a = data.xpath('div[@class="productTitle"]/a[1]')
+                if asin_a:
+                    asin_href = asin_a[0].get('href', '')
+                    m = re.search(r'/dp/(?P<asin>.+?)(/|$)', asin_href)
+                    if m:
+                        asin = m.group('asin')
+                    else:
+                        continue
+                else:
+                    continue
+                
+                cover_url = ''
+                if asin_href:
+                    cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href)
+                    if cover_img:
+                        cover_url = cover_img[0]
+                
+                title = ''.join(data.xpath('div[@class="productTitle"]/a/text()'))
+                author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
+                author = author.split('by')[-1]
+                price = ''.join(data.xpath('div[@class="newPrice"]/span/text()'))
+                    
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = asin.strip()
+                
+                yield s

src/calibre/gui2/store/baen_webscription_plugin.py

@@ -0,0 +1,89 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class BaenWebScriptionStore(BasicStoreConfig, StorePlugin):
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://www.webscription.net/'
+
+        if external or settings.get(self.name + '_open_external', False):
+            if detail_item:
+                url = url + detail_item
+            open_url(QUrl(url_slash_cleaner(url)))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url + detail_item
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.webscription.net/searchadv.aspx?IsSubmit=true&SearchTerm=' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//table/tr/td/img[@src="skins/Skin_1/images/matchingproducts.gif"]/..//tr'):
+                if counter <= 0:
+                    break
+                
+                id = ''.join(data.xpath('./td[1]/a/@href'))
+                if not id:
+                    continue
+                
+                title = ''.join(data.xpath('./td[1]/a/text()'))
+                
+                author = ''
+                cover_url = ''
+                price = ''
+                
+                with closing(br.open('http://www.webscription.net/' + id.strip(), timeout=timeout/4)) as nf:
+                    idata = html.fromstring(nf.read())
+                    author = ''.join(idata.xpath('//span[@class="ProductNameText"]/../b/text()'))
+                    author = author.split('by ')[-1]
+                    price = ''.join(idata.xpath('//span[@class="variantprice"]/text()'))
+                    a, b, price = price.partition('$')
+                    price = b + price
+                    
+                    pnum = ''
+                    mo = re.search(r'p-(?P<num>\d+)-', id.strip())
+                    if mo:
+                        pnum = mo.group('num')
+                    if pnum:
+                        cover_url = 'http://www.webscription.net/' + ''.join(idata.xpath('//img[@id="ProductPic%s"]/@src' % pnum))
+                
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.detail_item = id.strip()
+                
+                yield s

src/calibre/gui2/store/basic_config.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QWidget
+
+from calibre.gui2 import gprefs
+from calibre.gui2.store.basic_config_widget_ui import Ui_Form
+
+def save_settings(config_widget):
+    gprefs[config_widget.store.name + '_open_external'] = config_widget.open_external.isChecked()
+    tags = unicode(config_widget.tags.text())
+    gprefs[config_widget.store.name + '_tags'] = tags
+
+class BasicStoreConfigWidget(QWidget, Ui_Form):
+    
+    def __init__(self, store):
+        QWidget.__init__(self)
+        self.setupUi(self)
+
+        self.store = store
+        
+        self.load_setings()
+
+    def load_setings(self):
+        settings = self.store.get_settings()
+        
+        self.open_external.setChecked(settings.get(self.store.name + '_open_external'))
+        self.tags.setText(settings.get(self.store.name + '_tags', ''))
+
+class BasicStoreConfig(object):
+    
+    def customization_help(self, gui=False):
+        return 'Customize the behavior of this store.'
+
+    def config_widget(self):
+        return BasicStoreConfigWidget(self)
+
+    def save_settings(self, config_widget):
+        save_settings(config_widget)
+
+    def get_settings(self):
+        settings = {}
+        
+        settings[self.name + '_open_external'] = gprefs.get(self.name + '_open_external', False)
+        settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download')
+        
+        return settings

src/calibre/gui2/store/basic_config_widget.ui

@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Form</class>
+ <widget class="QWidget" name="Form">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>460</width>
+    <height>69</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Form</string>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="1" column="0">
+    <widget class="QLabel" name="label">
+     <property name="text">
+      <string>Added Tags:</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="1">
+    <widget class="QLineEdit" name="tags"/>
+   </item>
+   <item row="0" column="0" colspan="2">
+    <widget class="QCheckBox" name="open_external">
+     <property name="text">
+      <string>Open store in external web browswer</string>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections/>
+</ui>

src/calibre/gui2/store/bewrite_plugin.py

@@ -0,0 +1,81 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class BeWriteStore(BasicStoreConfig, StorePlugin):
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://www.bewrite.net/mm5/merchant.mvc?Screen=SFNT'
+
+        if external or settings.get(self.name + '_open_external', False):
+            if detail_item:
+                url = url + detail_item
+            open_url(QUrl(url_slash_cleaner(url)))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url + detail_item
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.bewrite.net/mm5/merchant.mvc?Search_Code=B&Screen=SRCH&Search=' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@id="content"]//table/tr[position() > 1]'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//a/@href'))
+                if not id:
+                    continue
+                
+                heading = ''.join(data.xpath('./td[2]//text()')) 
+                title, q, author = heading.partition('by ')
+                cover_url = ''
+                price = ''
+                
+                with closing(br.open(id.strip(), timeout=timeout/4)) as nf:
+                    idata = html.fromstring(nf.read())
+                    price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()'))
+                    price = '$' + price.split('$')[-1]
+                    cover_img = idata.xpath('//div[@id="content"]//img[1]/@src')
+                    if cover_img:
+                        cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0]
+                
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url.strip()
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = id.strip()
+                
+                yield s

src/calibre/gui2/store/bn_plugin.py

@@ -0,0 +1,82 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import random
+import re
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class BNStore(BasicStoreConfig, StorePlugin):
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+
+        pub_id = '21000000000352219'
+        # Use Kovid's affiliate id 30% of the time.
+        if random.randint(1, 10) in (1, 2, 3):
+            pub_id = '21000000000352583'
+        
+        url = 'http://gan.doubleclick.net/gan_click?lid=41000000028437369&pubid=' + pub_id
+
+        if detail_item:
+            mo = re.search(r'(?<=/)(?P<isbn>\d+)(?=/|$)', detail_item)
+            if mo:
+                isbn = mo.group('isbn')
+                detail_item = 'http://gan.doubleclick.net/gan_click?lid=41000000012871747&pid=' + isbn + '&adurl=' + detail_item + '&pubid=' + pub_id
+
+        if external or settings.get(self.name + '_open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_item if detail_item else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_item)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://productsearch.barnesandnoble.com/search/results.aspx?STORE=EBOOK&SZE=%s&WRD=' % max_results
+        url += urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//ul[contains(@class, "wgt-search-results-display")]/li[contains(@class, "search-result-item") and contains(@class, "nook-result-item")]'):
+                if counter <= 0:
+                    break
+                
+                id = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/@href'))
+                if not id:
+                    continue
+                cover_url = ''.join(data.xpath('.//div[contains(@class, "wgt-product-image-module")]/a/img/@src'))
+                
+                title = ''.join(data.xpath('.//span[@class="product-title"]/a/text()'))
+                author = ', '.join(data.xpath('.//span[@class="contributers-line"]/a/text()'))
+                price = ''.join(data.xpath('.//span[contains(@class, "onlinePriceValue2")]/text()'))
+                
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.detail_item = id.strip()
+                
+                yield s

src/calibre/gui2/store/diesel_ebooks_plugin.py

@@ -0,0 +1,87 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import random
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class DieselEbooksStore(BasicStoreConfig, StorePlugin):
+        
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://www.diesel-ebooks.com/'
+
+        aff_id = '?aid=2049'
+        # Use Kovid's affiliate id 30% of the time.
+        if random.randint(1, 10) in (1, 2, 3):
+            aff_id = '?aid=2053'
+
+        detail_url = None
+        if detail_item:
+            detail_url = url + detail_item + aff_id
+        url = url + aff_id
+
+        if external or settings.get(self.name + '_open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.diesel-ebooks.com/index.php?page=seek&id[m]=&id[c]=scope%253Dinventory&id[q]=' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="item clearfix"]'):
+                data = html.fromstring(html.tostring(data))
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('div[@class="cover"]/a/@href'))
+                if not id or '/item/' not in id:
+                    continue
+                a, b, id = id.partition('/item/')
+
+                cover_url = ''.join(data.xpath('div[@class="cover"]//img/@src'))
+                if cover_url.startswith('/'):
+                    cover_url = cover_url[1:]
+                cover_url = 'http://www.diesel-ebooks.com/' + cover_url
+
+                title = ''.join(data.xpath('.//div[@class="content"]//h2/text()'))
+                author = ''.join(data.xpath('//div[@class="content"]//div[@class="author"]/a/text()'))
+                price = ''
+                price_elem = data.xpath('//td[@class="price"]/text()')
+                if price_elem:
+                    price = price_elem[0]
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = '/item/' + id.strip()
+                
+                yield s

src/calibre/gui2/store/ebooks_com_plugin.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import random
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class EbookscomStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+
+        m_url = 'http://www.dpbolvw.net/'
+        h_click = 'click-4879827-10364500'
+        d_click = 'click-4879827-10281551'
+        # Use Kovid's affiliate id 30% of the time.
+        if random.randint(1, 10) in (1, 2, 3):
+            h_click = 'click-4913808-10364500'
+            d_click = 'click-4913808-10281551'
+        
+        url = m_url + h_click
+        detail_url = None
+        if detail_item:
+            detail_url = m_url + d_click + detail_item
+
+        if external or settings.get(self.name + '_open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.ebooks.com/SearchApp/SearchResults.net?term=' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="book_a" or @class="book_b"]'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//a[1]/@href'))
+                id = id.split('=')[-1]
+                if not id:
+                    continue
+
+                price = ''
+                with closing(br.open('http://www.ebooks.com/ebooks/book_display.asp?IID=' + id.strip(), timeout=timeout)) as fp:
+                    pdoc = html.fromstring(fp.read())
+                    pdata = pdoc.xpath('//table[@class="price"]/tr/td/text()')
+                    if len(pdata) >= 2:
+                        price = pdata[1]
+                if not price:
+                    continue
+                
+                cover_url = ''.join(data.xpath('.//img[1]/@src'))
+                
+                title = ''
+                author = ''
+                heading_a = data.xpath('.//a[1]/text()')
+                if heading_a:
+                    title = heading_a[0]
+                if len(heading_a) >= 2:
+                    author = heading_a[1]
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = '?url=http://www.ebooks.com/cj.asp?IID=' + id.strip() + '&cjsku=' + id.strip()
+                
+                yield s

src/calibre/gui2/store/eharlequin_plugin.py

@@ -0,0 +1,80 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import random
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class EHarlequinStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+
+        m_url = 'http://www.dpbolvw.net/'
+        h_click = 'click-4879827-534091'
+        d_click = 'click-4879827-10375439'
+        # Use Kovid's affiliate id 30% of the time.
+        if random.randint(1, 10) in (1, 2, 3):
+            h_click = 'click-4913808-534091'
+            d_click = 'click-4913808-10375439'
+
+        url = m_url + h_click
+        detail_url = None
+        if detail_item:
+            detail_url = m_url + d_click + detail_item
+
+        if external or settings.get(self.name + '_open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://ebooks.eharlequin.com/BANGSearch.dll?Type=FullText&FullTextField=All&FullTextCriteria=' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//table[not(.//@class="sidelink")]/tr[.//ul[@id="details"]]'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//ul[@id="details"]/li[@id="title-results"]/a/@href'))
+                if not id:
+                    continue
+
+                title = ''.join(data.xpath('.//ul[@id="details"]/li[@id="title-results"]/a/text()'))
+                author = ''.join(data.xpath('.//ul[@id="details"]/li[@id="author"][1]//a/text()'))
+                price = ''.join(data.xpath('.//div[@class="ourprice"]/font/text()'))
+                cover_url = ''.join(data.xpath('.//a[@href="%s"]/img/@src' % id))
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = '?url=http://ebooks.eharlequin.com/' + id.strip()
+                
+                yield s

src/calibre/gui2/store/feedbooks_plugin.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class FeedbooksStore(BasicStoreConfig, StorePlugin):
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://m.feedbooks.com/'
+        ext_url = 'http://feedbooks.com/'
+
+        if external or settings.get(self.name + '_open_external', False):
+            if detail_item:
+                ext_url = ext_url + detail_item
+            open_url(QUrl(url_slash_cleaner(ext_url)))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url + detail_item
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://m.feedbooks.com/search?query=' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//ul[@class="m-list"]//li'):
+                if counter <= 0:
+                    break
+                data = html.fromstring(html.tostring(data))
+                
+                id = ''
+                id_a = data.xpath('//a[@class="buy"]')
+                if id_a:
+                    id = id_a[0].get('href', None)
+                    id = id.split('/')[-2]
+                    id = '/item/' + id
+                else:
+                    id_a = data.xpath('//a[@class="download"]')
+                    if id_a:
+                        id = id_a[0].get('href', None)
+                        id = id.split('/')[-1]
+                        id = id.split('.')[0]
+                        id = '/book/' + id
+                if not id:
+                    continue
+                
+                title = ''.join(data.xpath('//h5//a/text()'))
+                author = ''.join(data.xpath('//h6//a/text()'))
+                price = ''.join(data.xpath('//a[@class="buy"]/text()'))
+                if not price:
+                    price = '$0.00'
+                cover_url = ''
+                cover_url_img =  data.xpath('//img')
+                if cover_url_img:
+                    cover_url = cover_url_img[0].get('src')
+                    cover_url.split('?')[0]
+                
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.replace(' ', '').strip()
+                s.detail_item = id.strip()
+                
+                yield s

src/calibre/gui2/store/gutenberg_plugin.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class GutenbergStore(BasicStoreConfig, StorePlugin):
+        
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://m.gutenberg.org/'
+        ext_url = 'http://gutenberg.org/'
+
+        if external or settings.get(self.name + '_open_external', False):
+            if detail_item:
+                ext_url = ext_url + detail_item
+            open_url(QUrl(url_slash_cleaner(ext_url)))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url + detail_item
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        # Gutenberg's website does not allow searching both author and title.
+        # Using a google search so we can search on both fields at once.
+        url = 'http://www.google.com/xhtml?q=site:gutenberg.org+' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'):
+                if counter <= 0:
+                    break
+                
+                url = ''
+                url_a = data.xpath('div[@class="jd"]/a')
+                if url_a:
+                    url_a = url_a[0]
+                    url = url_a.get('href', None)
+                if url:
+                    url = url.split('u=')[-1].split('&')[0]
+                if '/ebooks/' not in url:
+                    continue
+                id = url.split('/')[-1]
+                
+                url_a = html.fromstring(html.tostring(url_a))
+                heading = ''.join(url_a.xpath('//text()'))
+                title, _, author = heading.rpartition('by ')
+                author = author.split('-')[0]
+                price = '$0.00'
+                
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = ''
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = '/ebooks/' + id.strip()
+                
+                yield s

src/calibre/gui2/store/kobo_plugin.py

@@ -0,0 +1,84 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import random
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class KoboStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+
+        m_url = 'http://www.dpbolvw.net/'
+        h_click = 'click-4879827-10762497'
+        d_click = 'click-4879827-10772898'
+        # Use Kovid's affiliate id 30% of the time.
+        if random.randint(1, 10) in (1, 2, 3):
+            h_click = 'click-4913808-10762497'
+            d_click = 'click-4913808-10772898'
+        
+        url = m_url + h_click
+        detail_url = None
+        if detail_item:
+            detail_url = m_url + d_click + detail_item
+
+        if external or settings.get(self.name + '_open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.kobobooks.com/search/search.html?q=' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//ul[@class="SCShortCoverList"]/li'):
+                if counter <= 0:
+                    break
+
+                id = ''.join(data.xpath('.//div[@class="SearchImageContainer"]/a[1]/@href'))
+                if not id:
+                    continue
+
+                price = ''.join(data.xpath('.//span[@class="SCOurPrice"]/strong/text()'))
+                if not price:
+                    price = '$0.00'
+                
+                cover_url = ''.join(data.xpath('.//div[@class="SearchImageContainer"]//img[1]/@src'))
+                
+                title = ''.join(data.xpath('.//div[@class="SCItemHeader"]/h1/a[1]/text()'))
+                author = ''.join(data.xpath('.//div[@class="SCItemSummary"]/span/a[1]/text()'))
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = '?url=http://www.kobobooks.com/' + id.strip()
+                
+                yield s

src/calibre/gui2/store/manybooks_plugin.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+        
+class ManyBooksStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://manybooks.net/'
+
+        detail_url = None
+        if detail_item:
+            detail_url = url + detail_item
+        
+        if external or settings.get(self.name + '_open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        # ManyBooks website separates results for title and author.
+        # It also doesn't do a clear job of references authors and
+        # secondary titles. Google is also faster.
+        # Using a google search so we can search on both fields at once.
+        url = 'http://www.google.com/xhtml?q=site:manybooks.net+' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@class="edewpi"]//div[@class="r ld"]'):
+                if counter <= 0:
+                    break
+                
+                url = ''
+                url_a = data.xpath('div[@class="jd"]/a')
+                if url_a:
+                    url_a = url_a[0]
+                    url = url_a.get('href', None)
+                if url:
+                    url = url.split('u=')[-1][:-2]
+                if '/titles/' not in url:
+                    continue
+                id = url.split('/')[-1]
+                id = id.strip()
+                
+                url_a = html.fromstring(html.tostring(url_a))
+                heading = ''.join(url_a.xpath('//text()'))
+                title, _, author = heading.rpartition('by ')
+                author = author.split('-')[0]
+                price = '$0.00'
+                
+                cover_url = ''
+                mo = re.match('^\D+', id)
+                if mo:
+                    cover_name = mo.group()
+                    cover_name = cover_name.replace('etext', '')
+                    cover_id = id.split('.')[0]
+                    cover_url = 'http://manybooks_images.s3.amazonaws.com/original_covers/' + id[0] + '/' + cover_name + '/' + cover_id + '-thumb.jpg' 
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = '/titles/' + id
+                
+                yield s

src/calibre/gui2/store/mobileread_plugin.py

@@ -0,0 +1,304 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import difflib
+import heapq
+import time
+from contextlib import closing
+from threading import RLock
+
+from lxml import html
+
+from PyQt4.Qt import Qt, QUrl, QDialog, QAbstractItemModel, QModelIndex, QVariant, \
+    pyqtSignal
+
+from calibre import browser
+from calibre.gui2 import open_url, NONE
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.mobileread_store_dialog_ui import Ui_Dialog
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+from calibre.utils.config import DynamicConfig
+from calibre.utils.icu import sort_key
+
+class MobileReadStore(BasicStoreConfig, StorePlugin):
+    
+    def genesis(self):
+        self.config = DynamicConfig('store_' + self.name)
+        self.rlock = RLock()
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://www.mobileread.com/'
+        
+        if external or settings.get(self.name + '_open_external', False):
+            open_url(QUrl(detail_item if detail_item else url))
+        else:
+            if detail_item:
+                d = WebStoreDialog(self.gui, url, parent, detail_item)
+                d.setWindowTitle(self.name)
+                d.set_tags(settings.get(self.name + '_tags', ''))
+                d.exec_()
+            else:
+                d = MobeReadStoreDialog(self, parent)
+                d.setWindowTitle(self.name)
+                d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        books = self.get_book_list(timeout=timeout)
+
+        query = query.lower()
+        query_parts = query.split(' ')
+        matches = []
+        s = difflib.SequenceMatcher()
+        for x in books:
+            ratio = 0
+            t_string = '%s %s' % (x.author.lower(), x.title.lower())
+            query_matches = []
+            for q in query_parts:
+                if q in t_string:
+                    query_matches.append(q)
+            for q in query_matches:
+                s.set_seq2(q)
+                for p in t_string.split(' '):
+                    s.set_seq1(p)
+                    ratio += s.ratio()
+            if ratio > 0:
+                matches.append((ratio, x))
+        
+        # Move the best scorers to head of list.
+        matches = heapq.nlargest(max_results, matches)
+        for score, book in matches:
+            book.price = '$0.00'
+            yield book
+
+    def update_book_list(self, timeout=10):
+        with self.rlock:
+            url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html'
+            
+            last_download = self.config.get(self.name + '_last_download', None)
+            # Don't update the book list if our cache is less than one week old.
+            if last_download and (time.time() - last_download) < 604800:
+                return
+            
+            # Download the book list HTML file from MobileRead.
+            br = browser()
+            raw_data = None
+            with closing(br.open(url, timeout=timeout)) as f:
+                raw_data = f.read()
+            
+            if not raw_data:
+                return
+            
+            # Turn books listed in the HTML file into BookRef's.
+            books = []
+            try:
+                data = html.fromstring(raw_data)
+                for book_data in data.xpath('//ul/li'):
+                    book = BookRef()
+                    book.detail_item = ''.join(book_data.xpath('.//a/@href'))
+                    book.format = ''.join(book_data.xpath('.//i/text()'))
+                    book.format = book.format.strip()
+        
+                    text = ''.join(book_data.xpath('.//a/text()'))
+                    if ':' in text:
+                        book.author, q, text = text.partition(':')
+                    book.author = book.author.strip()
+                    book.title = text.strip()
+                    books.append(book)
+            except:
+                pass
+    
+            # Save the book list and it's create time.
+            if books:
+                self.config[self.name + '_last_download'] = time.time()
+                self.config[self.name + '_book_list'] = books
+
+    def get_book_list(self, timeout=10):
+        self.update_book_list(timeout=timeout)
+        return self.config.get(self.name + '_book_list', [])
+
+
+class BookRef(SearchResult):
+    
+    def __init__(self):
+        SearchResult.__init__(self)
+        
+        self.format = ''
+
+
+class MobeReadStoreDialog(QDialog, Ui_Dialog):
+    
+    def __init__(self, plugin, *args):
+        QDialog.__init__(self, *args)
+        self.setupUi(self)
+
+        self.plugin = plugin
+        
+        self.model = BooksModel()
+        self.results_view.setModel(self.model)
+        self.results_view.model().set_books(self.plugin.get_book_list())
+        self.total.setText('%s' % self.model.rowCount())
+
+        self.results_view.activated.connect(self.open_store)
+        self.search_query.textChanged.connect(self.model.set_filter)
+        self.results_view.model().total_changed.connect(self.total.setText)
+        self.finished.connect(self.dialog_closed)
+        
+        self.restore_state()
+        
+    def open_store(self, index):
+        result = self.results_view.model().get_book(index)
+        if result:
+            self.plugin.open(self, result.detail_item)
+    
+    def restore_state(self):
+        geometry = self.plugin.config['store_mobileread_dialog_geometry']
+        if geometry:
+            self.restoreGeometry(geometry)
+
+        results_cwidth = self.plugin.config['store_mobileread_dialog_results_view_column_width']
+        if results_cwidth:
+            for i, x in enumerate(results_cwidth):
+                if i >= self.results_view.model().columnCount():
+                    break
+                self.results_view.setColumnWidth(i, x)
+        else:
+            for i in xrange(self.results_view.model().columnCount()):
+                self.results_view.resizeColumnToContents(i)
+                
+        self.results_view.model().sort_col = self.plugin.config.get('store_mobileread_dialog_sort_col', 0)
+        self.results_view.model().sort_order = self.plugin.config.get('store_mobileread_dialog_sort_order', Qt.AscendingOrder)
+        self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order)
+        self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order)
+
+    def save_state(self):
+        self.plugin.config['store_mobileread_dialog_geometry'] = self.saveGeometry()
+        self.plugin.config['store_mobileread_dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())]
+        self.plugin.config['store_mobileread_dialog_sort_col'] = self.results_view.model().sort_col
+        self.plugin.config['store_mobileread_dialog_sort_order'] = self.results_view.model().sort_order
+
+    def dialog_closed(self, result):
+        self.save_state()
+
+
+class BooksModel(QAbstractItemModel):
+    
+    total_changed = pyqtSignal(unicode)
+
+    HEADERS = [_('Title'), _('Author(s)'), _('Format')]
+
+    def __init__(self):
+        QAbstractItemModel.__init__(self)
+        self.books = []
+        self.all_books = []
+        self.filter = ''
+        self.sort_col = 0
+        self.sort_order = Qt.AscendingOrder
+
+    def set_books(self, books):
+        self.books = books
+        self.all_books = books
+        
+        self.sort(self.sort_col, self.sort_order)
+
+    def get_book(self, index):
+        row = index.row()
+        if row < len(self.books):
+            return self.books[row]
+        else:
+            return None
+    
+    def set_filter(self, filter):
+        #self.layoutAboutToBeChanged.emit()
+        self.beginResetModel()
+        
+        self.filter = unicode(filter)
+        self.books = []
+        if self.filter:
+            for b in self.all_books:
+                test = '%s %s %s' % (b.title, b.author, b.format)
+                test = test.lower()
+                include = True
+                for item in self.filter.split(' '):
+                    item = item.lower()
+                    if item not in test:
+                        include = False
+                        break
+                if include:
+                    self.books.append(b)
+        else:
+            self.books = self.all_books
+
+        self.sort(self.sort_col, self.sort_order, reset=False)
+        self.total_changed.emit('%s' % self.rowCount())
+
+        self.endResetModel()
+        #self.layoutChanged.emit()
+    
+    def index(self, row, column, parent=QModelIndex()):
+        return self.createIndex(row, column)
+
+    def parent(self, index):
+        if not index.isValid() or index.internalId() == 0:
+            return QModelIndex()
+        return self.createIndex(0, 0)
+
+    def rowCount(self, *args):
+        return len(self.books)
+
+    def columnCount(self, *args):
+        return len(self.HEADERS)
+    
+    def headerData(self, section, orientation, role):
+        if role != Qt.DisplayRole:
+            return NONE
+        text = ''
+        if orientation == Qt.Horizontal:
+            if section < len(self.HEADERS):
+                text = self.HEADERS[section]
+            return QVariant(text)
+        else:
+            return QVariant(section+1)
+
+    def data(self, index, role):
+        row, col = index.row(), index.column()
+        result = self.books[row]
+        if role == Qt.DisplayRole:
+            if col == 0:
+                return QVariant(result.title)
+            elif col == 1:
+                return QVariant(result.author)
+            elif col == 2:
+                return QVariant(result.format)
+        return NONE
+
+    def data_as_text(self, result, col):
+        text = ''
+        if col == 0:
+            text = result.title
+        elif col == 1:
+            text = result.author
+        elif col == 2:
+            text = result.format
+        return text
+
+    def sort(self, col, order, reset=True):
+        self.sort_col = col
+        self.sort_order = order
+
+        if not self.books:
+            return
+        descending = order == Qt.DescendingOrder       
+        self.books.sort(None,
+            lambda x: sort_key(unicode(self.data_as_text(x, col))),
+            descending)
+        if reset:
+            self.reset()
+

src/calibre/gui2/store/mobileread_store_dialog.ui

@@ -0,0 +1,112 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>691</width>
+    <height>614</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Dialog</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <layout class="QHBoxLayout" name="horizontalLayout_2">
+     <item>
+      <widget class="QLabel" name="label">
+       <property name="text">
+        <string>Search:</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QLineEdit" name="search_query"/>
+     </item>
+    </layout>
+   </item>
+   <item>
+    <widget class="QTreeView" name="results_view">
+     <property name="alternatingRowColors">
+      <bool>true</bool>
+     </property>
+     <property name="rootIsDecorated">
+      <bool>false</bool>
+     </property>
+     <property name="itemsExpandable">
+      <bool>false</bool>
+     </property>
+     <property name="sortingEnabled">
+      <bool>true</bool>
+     </property>
+     <property name="expandsOnDoubleClick">
+      <bool>false</bool>
+     </property>
+     <attribute name="headerCascadingSectionResizes">
+      <bool>false</bool>
+     </attribute>
+    </widget>
+   </item>
+   <item>
+    <layout class="QHBoxLayout" name="horizontalLayout">
+     <item>
+      <widget class="QLabel" name="label_2">
+       <property name="text">
+        <string>Books:</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QLabel" name="total">
+       <property name="text">
+        <string>0</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <spacer name="horizontalSpacer">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>308</width>
+         <height>20</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <widget class="QPushButton" name="close_button">
+       <property name="text">
+        <string>Close</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>close_button</sender>
+   <signal>clicked()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>440</x>
+     <y>432</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>245</x>
+     <y>230</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/store/open_library_plugin.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class OpenLibraryStore(BasicStoreConfig, StorePlugin):
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://openlibrary.org/'
+
+        if external or settings.get(self.name + '_open_external', False):
+            if detail_item:
+                url = url + detail_item
+            open_url(QUrl(url_slash_cleaner(url)))
+        else:
+            detail_url = None
+            if detail_item:
+                detail_url = url + detail_item
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://openlibrary.org/search?q=' + urllib2.quote(query) + '&has_fulltext=true'
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@id="searchResults"]/ul[@id="siteSearch"]/li'):
+                if counter <= 0:
+                    break
+                
+                id = ''.join(data.xpath('./span[@class="bookcover"]/a/@href'))
+                if not id:
+                    continue
+                cover_url = ''.join(data.xpath('./span[@class="bookcover"]/a/img/@src'))
+                
+                title = ''.join(data.xpath('.//h3[@class="booktitle"]/a[@class="results"]/text()'))
+                author = ''.join(data.xpath('.//span[@class="bookauthor"]/a/text()'))
+                price = '$0.00'
+                
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price
+                s.detail_item = id.strip()
+                
+                yield s

src/calibre/gui2/store/search.py

@@ -0,0 +1,453 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+import time
+from contextlib import closing
+from random import shuffle
+from threading import Thread
+from Queue import Queue
+
+from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \
+    QModelIndex, QPixmap, QSize, QCheckBox, QVBoxLayout, QHBoxLayout, \
+    QPushButton, QString, QByteArray
+
+from calibre import browser
+from calibre.gui2 import NONE
+from calibre.gui2.progress_indicator import ProgressIndicator
+from calibre.gui2.store.search_ui import Ui_Dialog
+from calibre.utils.config import DynamicConfig
+from calibre.utils.icu import sort_key
+from calibre.utils.magick.draw import thumbnail
+
+HANG_TIME = 75000 # milliseconds seconds
+TIMEOUT = 75 # seconds
+SEARCH_THREAD_TOTAL = 4
+COVER_DOWNLOAD_THREAD_TOTAL = 2
+
+class SearchDialog(QDialog, Ui_Dialog):
+
+    def __init__(self, istores, *args):
+        QDialog.__init__(self, *args)
+        self.setupUi(self)
+        
+        self.config = DynamicConfig('store_search')
+
+        # We keep a cache of store plugins and reference them by name.
+        self.store_plugins = istores
+        self.search_pool = SearchThreadPool(SearchThread, SEARCH_THREAD_TOTAL)
+        # Check for results and hung threads.
+        self.checker = QTimer()
+        self.hang_check = 0
+        
+        self.model = Matches()
+        self.results_view.setModel(self.model)
+
+        # Add check boxes for each store so the user
+        # can disable searching specific stores on a
+        # per search basis.
+        stores_group_layout = QVBoxLayout()
+        self.stores_group.setLayout(stores_group_layout)
+        for x in self.store_plugins:
+            cbox = QCheckBox(x)
+            cbox.setChecked(True)
+            stores_group_layout.addWidget(cbox)
+            setattr(self, 'store_check_' + x, cbox)
+        stores_group_layout.addStretch()
+        
+        # Create and add the progress indicator
+        self.pi = ProgressIndicator(self, 24)
+        self.bottom_layout.insertWidget(0, self.pi)
+
+        self.search.clicked.connect(self.do_search)
+        self.checker.timeout.connect(self.get_results)
+        self.results_view.activated.connect(self.open_store)
+        self.select_all_stores.clicked.connect(self.stores_select_all)
+        self.select_invert_stores.clicked.connect(self.stores_select_invert)
+        self.select_none_stores.clicked.connect(self.stores_select_none)
+        self.finished.connect(self.dialog_closed)
+        
+        self.restore_state()
+        
+    def resize_columns(self):
+        total = 600
+        # Cover
+        self.results_view.setColumnWidth(0, 85)
+        total = total - 85
+        # Title
+        self.results_view.setColumnWidth(1,int(total*.35))
+        # Author
+        self.results_view.setColumnWidth(2,int(total*.35))
+        # Price
+        self.results_view.setColumnWidth(3, int(total*.10))
+        # Store
+        self.results_view.setColumnWidth(4, int(total*.20))
+        
+    def do_search(self, checked=False):
+        # Stop all running threads.
+        self.checker.stop()
+        self.search_pool.abort()
+        # Clear the visible results.
+        self.results_view.model().clear_results()
+        
+        # Don't start a search if there is nothing to search for.
+        query = unicode(self.search_edit.text())
+        if not query.strip():
+            return
+        
+        # Plugins are in alphebetic order. Randomize the
+        # order of plugin names. This way plugins closer
+        # to a don't have an unfair advantage over
+        # plugins further from a.
+        store_names = self.store_plugins.keys()
+        if not store_names:
+            return
+        shuffle(store_names)
+        # Add plugins that the user has checked to the search pool's work queue.
+        for n in store_names:
+            if getattr(self, 'store_check_' + n).isChecked():
+                self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT)
+        if self.search_pool.has_tasks():
+            self.hang_check = 0
+            self.checker.start(100)
+            self.search_pool.start_threads()
+            self.pi.startAnimation()
+    
+    def save_state(self):
+        self.config['store_search_geometry'] = self.saveGeometry()
+        self.config['store_search_store_splitter_state'] = self.store_splitter.saveState()
+        self.config['store_search_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())]
+        
+        store_check = {}
+        for n in self.store_plugins:
+            store_check[n] = getattr(self, 'store_check_' + n).isChecked()
+        self.config['store_search_store_checked'] = store_check
+
+    def restore_state(self):
+        geometry = self.config['store_search_geometry']
+        if geometry:
+            self.restoreGeometry(geometry)
+            
+        splitter_state = self.config['store_search_store_splitter_state']
+        if splitter_state:
+            self.store_splitter.restoreState(splitter_state)
+            
+        results_cwidth = self.config['store_search_results_view_column_width']
+        if results_cwidth:
+            for i, x in enumerate(results_cwidth):
+                if i >= self.model.columnCount():
+                    break
+                self.results_view.setColumnWidth(i, x)
+        else:
+            self.resize_columns()
+            
+        store_check = self.config['store_search_store_checked']
+        if store_check:
+            for n in store_check:
+                if hasattr(self, 'store_check_' + n):
+                    getattr(self, 'store_check_' + n).setChecked(store_check[n])
+
+    def get_results(self):
+        # We only want the search plugins to run
+        # a maximum set amount of time before giving up.
+        self.hang_check += 1
+        if self.hang_check >= HANG_TIME:
+            self.search_pool.abort()
+            self.checker.stop()
+            self.pi.stopAnimation()
+        else:
+            # Stop the checker if not threads are running.
+            if not self.search_pool.threads_running() and not self.search_pool.has_tasks():
+                self.checker.stop()
+                self.pi.stopAnimation()
+        
+        while self.search_pool.has_results():
+            res = self.search_pool.get_result()
+            if res:
+                self.results_view.model().add_result(res)
+
+    def open_store(self, index):
+        result = self.results_view.model().get_result(index)
+        self.store_plugins[result.store_name].open(self, result.detail_item)
+
+    def get_store_checks(self):
+        '''
+        Returns a list of QCheckBox's for each store.
+        '''
+        checks = []
+        for x in self.store_plugins:
+            check = getattr(self, 'store_check_' + x, None)
+            if check:
+                checks.append(check)
+        return checks
+
+    def stores_select_all(self):
+        for check in self.get_store_checks():
+            check.setChecked(True)
+    
+    def stores_select_invert(self):
+        for check in self.get_store_checks():
+            check.setChecked(not check.isChecked())
+    
+    def stores_select_none(self):
+        for check in self.get_store_checks():
+            check.setChecked(False)
+    
+    def dialog_closed(self, result):
+        self.model.closing()
+        self.search_pool.abort()
+        self.save_state()
+
+
+class GenericDownloadThreadPool(object):
+    '''
+    add_task must be implemented in a subclass.
+    '''
+    
+    def __init__(self, thread_type, thread_count):
+        self.thread_type = thread_type
+        self.thread_count = thread_count
+        
+        self.tasks = Queue()
+        self.results = Queue()
+        self.threads = []
+    
+    def add_task(self):
+        raise NotImplementedError()
+        
+    def start_threads(self):
+        for i in range(self.thread_count):
+            t = self.thread_type(self.tasks, self.results)
+            self.threads.append(t)
+            t.start()
+        
+    def abort(self):
+        self.tasks = Queue()
+        self.results = Queue()
+        for t in self.threads:
+            t.abort()
+        self.threads = []
+    
+    def has_tasks(self):
+        return not self.tasks.empty()
+    
+    def get_result(self):
+        return self.results.get()
+    
+    def get_result_no_wait(self):
+        return self.results.get_nowait()
+    
+    def result_count(self):
+        return len(self.results)
+    
+    def has_results(self):
+        return not self.results.empty()
+    
+    def threads_running(self):
+        for t in self.threads:
+            if t.is_alive():
+                return True
+        return False
+
+
+class SearchThreadPool(GenericDownloadThreadPool):
+    '''
+    Threads will run until there is no work or
+    abort is called. Create and start new threads
+    using start_threads(). Reset by calling abort().
+    
+    Example:
+    sp = SearchThreadPool(SearchThread, 3)
+    add tasks using add_task(...)
+    sp.start_threads()
+    all threads have finished.
+    sp.abort()
+    add tasks using add_task(...)
+    sp.start_threads()
+    '''
+        
+    def add_task(self, query, store_name, store_plugin, timeout):
+        self.tasks.put((query, store_name, store_plugin, timeout))
+
+
+class SearchThread(Thread):
+    
+    def __init__(self, tasks, results):
+        Thread.__init__(self)
+        self.daemon = True
+        self.tasks = tasks
+        self.results = results
+        self._run = True
+
+    def abort(self):
+        self._run = False
+    
+    def run(self):
+        while self._run and not self.tasks.empty():
+            try:
+                query, store_name, store_plugin, timeout = self.tasks.get()
+                for res in store_plugin.search(query, timeout=timeout):
+                    if not self._run:
+                        return
+                    res.store_name = store_name
+                    self.results.put(res)
+                self.tasks.task_done()
+            except:
+                pass
+
+
+class CoverThreadPool(GenericDownloadThreadPool):
+    '''
+    Once started all threads run until abort is called.
+    '''
+    
+    def add_task(self, search_result, update_callback, timeout=5):
+        self.tasks.put((search_result, update_callback, timeout))
+
+
+class CoverThread(Thread):
+
+    def __init__(self, tasks, results):
+        Thread.__init__(self)
+        self.daemon = True
+        self.tasks = tasks
+        self.results = results
+        self._run = True
+        
+        self.br = browser()
+
+    def abort(self):
+        self._run = False
+        
+    def run(self):
+        while self._run:
+            try:
+                time.sleep(.1)
+                while not self.tasks.empty():
+                    if not self._run:
+                        break
+                    result, callback, timeout = self.tasks.get()
+                    if result and result.cover_url:
+                        with closing(self.br.open(result.cover_url, timeout=timeout)) as f:
+                            result.cover_data = f.read()
+                        result.cover_data = thumbnail(result.cover_data, 64, 64)[2]
+                        callback()
+                    self.tasks.task_done()
+            except:
+                continue
+
+
+class Matches(QAbstractItemModel):
+
+    HEADERS = [_('Cover'), _('Title'), _('Author(s)'), _('Price'), _('Store')]
+
+    def __init__(self):
+        QAbstractItemModel.__init__(self)
+        self.matches = []
+        self.cover_pool = CoverThreadPool(CoverThread, 2)
+        self.cover_pool.start_threads()
+
+    def closing(self):
+        self.cover_pool.abort()
+        
+    def clear_results(self):
+        self.matches = []
+        self.cover_pool.abort()
+        self.cover_pool.start_threads()
+        self.reset()
+    
+    def add_result(self, result):
+        self.layoutAboutToBeChanged.emit()
+        self.matches.append(result)
+        self.cover_pool.add_task(result, self.update_result)
+        self.layoutChanged.emit()
+
+    def get_result(self, index):
+        row = index.row()
+        if row < len(self.matches):
+            return self.matches[row]
+        else:
+            return None
+
+    def update_result(self):
+        self.layoutAboutToBeChanged.emit()
+        self.layoutChanged.emit()
+
+    def index(self, row, column, parent=QModelIndex()):
+        return self.createIndex(row, column)
+
+    def parent(self, index):
+        if not index.isValid() or index.internalId() == 0:
+            return QModelIndex()
+        return self.createIndex(0, 0)
+
+    def rowCount(self, *args):
+        return len(self.matches)
+
+    def columnCount(self, *args):
+        return len(self.HEADERS)
+    
+    def headerData(self, section, orientation, role):
+        if role != Qt.DisplayRole:
+            return NONE
+        text = ''
+        if orientation == Qt.Horizontal:
+            if section < len(self.HEADERS):
+                text = self.HEADERS[section]
+            return QVariant(text)
+        else:
+            return QVariant(section+1)
+
+    def data(self, index, role):
+        row, col = index.row(), index.column()
+        result = self.matches[row]
+        if role == Qt.DisplayRole:
+            if col == 1:
+                return QVariant(result.title)
+            elif col == 2:
+                return QVariant(result.author)
+            elif col == 3:
+                return QVariant(result.price)
+            elif col == 4:
+                return QVariant(result.store_name)
+            return NONE
+        elif role == Qt.DecorationRole:
+            if col == 0 and result.cover_data:
+                p = QPixmap()
+                p.loadFromData(result.cover_data)
+                return QVariant(p)
+        elif role == Qt.SizeHintRole:
+            return QSize(64, 64)
+        return NONE
+
+    def data_as_text(self, result, col):
+        text = ''
+        if col == 1:
+            text = result.title
+        elif col == 2:
+            text = result.author
+        elif col == 3:
+            text = result.price
+            if len(text) < 3 or text[-3] not in ('.', ','):
+                text += '00' 
+            text = re.sub(r'\D', '', text)
+            text = text.rjust(6, '0')
+        elif col == 4:
+            text = result.store_name
+        return text
+
+    def sort(self, col, order, reset=True):
+        if not self.matches:
+            return
+        descending = order == Qt.DescendingOrder       
+        self.matches.sort(None,
+            lambda x: sort_key(unicode(self.data_as_text(x, col))),
+            descending)
+        if reset:
+            self.reset()
+

src/calibre/gui2/store/search.ui

@@ -0,0 +1,196 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>937</width>
+    <height>669</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>calibre Store Search</string>
+  </property>
+  <property name="sizeGripEnabled">
+   <bool>true</bool>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <layout class="QHBoxLayout" name="horizontalLayout_2">
+     <item>
+      <widget class="QLabel" name="label">
+       <property name="text">
+        <string>Query:</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QLineEdit" name="search_edit"/>
+     </item>
+     <item>
+      <widget class="QPushButton" name="search">
+       <property name="text">
+        <string>Search</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item>
+    <widget class="QSplitter" name="store_splitter">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <widget class="QGroupBox" name="groupBox">
+      <property name="title">
+       <string>Stores</string>
+      </property>
+      <layout class="QVBoxLayout" name="verticalLayout_2">
+       <item>
+        <widget class="QScrollArea" name="stores_group">
+         <property name="widgetResizable">
+          <bool>true</bool>
+         </property>
+         <widget class="QWidget" name="scrollAreaWidgetContents">
+          <property name="geometry">
+           <rect>
+            <x>0</x>
+            <y>0</y>
+            <width>215</width>
+            <height>116</height>
+           </rect>
+          </property>
+         </widget>
+        </widget>
+       </item>
+       <item>
+        <layout class="QHBoxLayout" name="horizontalLayout_3">
+         <item>
+          <widget class="QPushButton" name="select_all_stores">
+           <property name="text">
+            <string>All</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="select_invert_stores">
+           <property name="text">
+            <string>Invert</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QPushButton" name="select_none_stores">
+           <property name="text">
+            <string>None</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </widget>
+     <widget class="QSplitter" name="splitter_2">
+      <property name="sizePolicy">
+       <sizepolicy hsizetype="Expanding" vsizetype="Preferred">
+        <horstretch>2</horstretch>
+        <verstretch>0</verstretch>
+       </sizepolicy>
+      </property>
+      <property name="orientation">
+       <enum>Qt::Horizontal</enum>
+      </property>
+      <widget class="QSplitter" name="splitter">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <widget class="QTreeView" name="results_view">
+        <property name="sizePolicy">
+         <sizepolicy hsizetype="Expanding" vsizetype="Expanding">
+          <horstretch>1</horstretch>
+          <verstretch>0</verstretch>
+         </sizepolicy>
+        </property>
+        <property name="minimumSize">
+         <size>
+          <width>0</width>
+          <height>0</height>
+         </size>
+        </property>
+        <property name="alternatingRowColors">
+         <bool>true</bool>
+        </property>
+        <property name="iconSize">
+         <size>
+          <width>32</width>
+          <height>32</height>
+         </size>
+        </property>
+        <property name="rootIsDecorated">
+         <bool>false</bool>
+        </property>
+        <property name="uniformRowHeights">
+         <bool>false</bool>
+        </property>
+        <property name="itemsExpandable">
+         <bool>false</bool>
+        </property>
+        <property name="sortingEnabled">
+         <bool>true</bool>
+        </property>
+        <property name="expandsOnDoubleClick">
+         <bool>false</bool>
+        </property>
+       </widget>
+      </widget>
+     </widget>
+    </widget>
+   </item>
+   <item>
+    <layout class="QHBoxLayout" name="bottom_layout">
+     <item>
+      <spacer name="horizontalSpacer">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+       <property name="sizeHint" stdset="0">
+        <size>
+         <width>40</width>
+         <height>20</height>
+        </size>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <widget class="QPushButton" name="close">
+       <property name="text">
+        <string>Close</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>close</sender>
+   <signal>clicked()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>526</x>
+     <y>525</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>307</x>
+     <y>272</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/store/search_result.py

@@ -0,0 +1,18 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+class SearchResult(object):
+    
+    def __init__(self):
+        self.store_name = ''
+        self.cover_url = ''
+        self.cover_data = None
+        self.title = ''
+        self.author = ''
+        self.price = ''
+        self.detail_item = ''

src/calibre/gui2/store/smashwords_plugin.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import random
+import re
+import urllib2
+from contextlib import closing
+
+from lxml import html
+
+from PyQt4.Qt import QUrl
+
+from calibre import browser, url_slash_cleaner
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+
+class SmashwordsStore(BasicStoreConfig, StorePlugin):
+
+    def open(self, parent=None, detail_item=None, external=False):
+        settings = self.get_settings()
+        url = 'http://www.smashwords.com/'
+
+        aff_id = '?ref=usernone'
+        # Use Kovid's affiliate id 30% of the time.
+        if random.randint(1, 10) in (1, 2, 3):
+            aff_id = '?ref=kovidgoyal'
+
+        detail_url = None
+        if detail_item:
+            detail_url = url + detail_item + aff_id
+        url = url + aff_id
+            
+        if external or settings.get(self.name + '_open_external', False):
+            open_url(QUrl(url_slash_cleaner(detail_url if detail_url else url)))
+        else:
+            d = WebStoreDialog(self.gui, url, parent, detail_url)
+            d.setWindowTitle(self.name)
+            d.set_tags(settings.get(self.name + '_tags', ''))
+            d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        url = 'http://www.smashwords.com/books/search?query=' + urllib2.quote(query)
+        
+        br = browser()
+        
+        counter = max_results
+        with closing(br.open(url, timeout=timeout)) as f:
+            doc = html.fromstring(f.read())
+            for data in doc.xpath('//div[@id="pageCenterContent2"]//div[@class="bookCoverImg"]'):
+                if counter <= 0:
+                    break
+                data = html.fromstring(html.tostring(data))
+                
+                id = None
+                id_a = data.xpath('//a[@class="bookTitle"]')
+                if id_a:
+                    id = id_a[0].get('href', None)
+                    if id:
+                        id = id.split('/')[-1]
+                if not id:
+                    continue
+                
+                cover_url = ''
+                c_url = data.get('style', None)
+                if c_url:
+                    mo = re.search(r'http://[^\'"]+', c_url)
+                    if mo:
+                        cover_url = mo.group()
+                
+                title = ''.join(data.xpath('//a[@class="bookTitle"]/text()'))
+                subnote = ''.join(data.xpath('//span[@class="subnote"]/text()'))
+                author = ''.join(data.xpath('//span[@class="subnote"]/a/text()'))
+                price = subnote.partition('$')[2]
+                price = price.split(u'\xa0')[0]
+                price = '$' + price
+
+                counter -= 1
+                
+                s = SearchResult()
+                s.cover_url = cover_url
+                s.title = title.strip()
+                s.author = author.strip()
+                s.price = price.strip()
+                s.detail_item = '/books/view/' + id.strip()
+                
+                yield s

src/calibre/gui2/store/web_control.py

@@ -0,0 +1,112 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import os
+from urlparse import urlparse
+
+from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString, \
+    QFileDialog, QNetworkProxy
+
+from calibre import USER_AGENT, get_proxies, get_download_filename
+from calibre.ebooks import BOOK_EXTENSIONS
+from calibre.ptempfile import PersistentTemporaryFile
+
+class NPWebView(QWebView):
+
+    def __init__(self, *args):
+        QWebView.__init__(self, *args)
+        self.gui = None
+        self.tags = ''
+
+        self.setPage(NPWebPage())
+        self.page().networkAccessManager().setCookieJar(QNetworkCookieJar())
+
+        http_proxy = get_proxies().get('http', None)
+        if http_proxy:
+            proxy_parts = urlparse(http_proxy)
+            proxy = QNetworkProxy()
+            proxy.setType(QNetworkProxy.HttpProxy)
+            proxy.setUser(proxy_parts.username)
+            proxy.setPassword(proxy_parts.password)
+            proxy.setHostName(proxy_parts.hostname)
+            proxy.setPort(proxy_parts.port)
+            self.page().networkAccessManager().setProxy(proxy)            
+        
+        self.page().setForwardUnsupportedContent(True)
+        self.page().unsupportedContent.connect(self.start_download)
+        self.page().downloadRequested.connect(self.start_download)
+        self.page().networkAccessManager().sslErrors.connect(self.ignore_ssl_errors)
+    
+    def createWindow(self, type):
+        if type == QWebPage.WebBrowserWindow:
+            return self
+        else:
+            return None
+
+    def set_gui(self, gui):
+        self.gui = gui
+        
+    def set_tags(self, tags):
+        self.tags = tags
+        
+    def start_download(self, request):
+        if not self.gui:
+            return
+        
+        url = unicode(request.url().toString())
+        cf = self.get_cookies()
+        
+        filename = get_download_filename(url, cf)
+        ext = os.path.splitext(filename)[1][1:].lower()
+        if ext not in BOOK_EXTENSIONS:
+            home = os.path.expanduser('~')
+            name = QFileDialog.getSaveFileName(self,
+                _('File is not a supported ebook type. Save to disk?'),
+                os.path.join(home, filename),
+                '*.*')
+            if name:
+                self.gui.download_ebook(url, cf, name, name, False)
+        else:
+            self.gui.download_ebook(url, cf, filename, tags=self.tags)
+
+    def ignore_ssl_errors(self, reply, errors):
+        reply.ignoreSslErrors(errors)
+    
+    def get_cookies(self):
+        '''
+        Writes QNetworkCookies to Mozilla cookie .txt file.
+        
+        :return: The file path to the cookie file.
+        '''
+        cf = PersistentTemporaryFile(suffix='.txt')
+        
+        cf.write('# Netscape HTTP Cookie File\n\n')
+        
+        for c in self.page().networkAccessManager().cookieJar().allCookies():
+            cookie = []
+            domain = unicode(c.domain())
+            
+            cookie.append(domain)
+            cookie.append('TRUE' if domain.startswith('.') else 'FALSE')
+            cookie.append(unicode(c.path()))
+            cookie.append('TRUE' if c.isSecure() else 'FALSE')
+            cookie.append(unicode(c.expirationDate().toTime_t()))
+            cookie.append(unicode(c.name()))
+            cookie.append(unicode(c.value()))
+            
+            cf.write('\t'.join(cookie))
+            cf.write('\n')
+        
+        cf.close()
+        return cf.name
+
+
+class NPWebPage(QWebPage):
+    
+    def userAgentForUrl(self, url):
+        return USER_AGENT

src/calibre/gui2/store/web_store_dialog.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QDialog, QUrl
+
+from calibre import url_slash_cleaner
+from calibre.gui2.store.web_store_dialog_ui import Ui_Dialog
+
+class WebStoreDialog(QDialog, Ui_Dialog):
+
+    def __init__(self, gui, base_url, parent=None, detail_url=None):
+        QDialog.__init__(self, parent=parent)
+        self.setupUi(self)
+        
+        self.gui = gui
+        self.base_url = base_url
+        
+        self.view.set_gui(self.gui)
+        self.view.loadStarted.connect(self.load_started)
+        self.view.loadProgress.connect(self.load_progress)
+        self.view.loadFinished.connect(self.load_finished)
+        self.home.clicked.connect(self.go_home)
+        self.reload.clicked.connect(self.view.reload)
+        self.back.clicked.connect(self.view.back)
+        
+        self.go_home(detail_url=detail_url)
+
+    def set_tags(self, tags):
+        self.view.set_tags(tags)
+
+    def load_started(self):
+        self.progress.setValue(0)
+    
+    def load_progress(self, val):
+        self.progress.setValue(val)
+        
+    def load_finished(self, ok=True):
+        self.progress.setValue(100)
+    
+    def go_home(self, checked=False, detail_url=None):
+        if detail_url:
+            url = detail_url
+        else:
+            url = self.base_url
+            
+        # Reduce redundant /'s because some stores
+        # (Feedbooks) and server frameworks (cherrypy)
+        # choke on them. 
+        url = url_slash_cleaner(url)
+        self.view.load(QUrl(url))

src/calibre/gui2/store/web_store_dialog.ui

@@ -0,0 +1,115 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>962</width>
+    <height>656</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string/>
+  </property>
+  <property name="sizeGripEnabled">
+   <bool>true</bool>
+  </property>
+  <layout class="QGridLayout" name="gridLayout">
+   <item row="0" column="0" colspan="5">
+    <widget class="QFrame" name="frame">
+     <property name="frameShape">
+      <enum>QFrame::StyledPanel</enum>
+     </property>
+     <property name="frameShadow">
+      <enum>QFrame::Raised</enum>
+     </property>
+     <layout class="QVBoxLayout" name="verticalLayout">
+      <property name="margin">
+       <number>0</number>
+      </property>
+      <item>
+       <widget class="NPWebView" name="view">
+        <property name="url">
+         <url>
+          <string>about:blank</string>
+         </url>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="1" column="0">
+    <widget class="QPushButton" name="home">
+     <property name="text">
+      <string>Home</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="1">
+    <widget class="QPushButton" name="reload">
+     <property name="text">
+      <string>Reload</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="3">
+    <widget class="QProgressBar" name="progress">
+     <property name="value">
+      <number>0</number>
+     </property>
+     <property name="format">
+      <string>%p%</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="2">
+    <widget class="QPushButton" name="back">
+     <property name="text">
+      <string>Back</string>
+     </property>
+    </widget>
+   </item>
+   <item row="1" column="4">
+    <widget class="QPushButton" name="close">
+     <property name="text">
+      <string>Close</string>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>QWebView</class>
+   <extends>QWidget</extends>
+   <header>QtWebKit/QWebView</header>
+  </customwidget>
+  <customwidget>
+   <class>NPWebView</class>
+   <extends>QWebView</extends>
+   <header>web_control.h</header>
+  </customwidget>
+ </customwidgets>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>close</sender>
+   <signal>clicked()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>917</x>
+     <y>635</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>480</x>
+     <y>327</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/threaded_jobs.py

@@ -38,7 +38,7 @@ class ThreadedJob(BaseJob):
 
         :func: The function that actually does the work. This function *must*
         accept at least three keyword arguments: abort, log and notifications. abort is
-        An Event object. func should periodically check abort.is_set(0 and if
+        An Event object. func should periodically check abort.is_set() and if
         it is True, it should stop processing as soon as possible. notifications
         is a Queue. func should put progress notifications into it in the form
         of a tuple (frac, msg). frac is a number between 0 and 1 indicating

src/calibre/gui2/ui.py

@@ -23,7 +23,7 @@ from calibre.constants import __appname__, isosx
 from calibre.utils.config import prefs, dynamic
 from calibre.utils.ipc.server import Server
 from calibre.library.database2 import LibraryDatabase2
-from calibre.customize.ui import interface_actions
+from calibre.customize.ui import interface_actions, store_plugins
 from calibre.gui2 import error_dialog, GetMetadata, open_url, \
         gprefs, max_available_height, config, info_dialog, Dispatcher, \
         question_dialog
@@ -34,6 +34,7 @@ from calibre.gui2.main_window import MainWindow
 from calibre.gui2.layout import MainWindowMixin
 from calibre.gui2.device import DeviceMixin
 from calibre.gui2.email import EmailMixin
+from calibre.gui2.ebook_download import EbookDownloadMixin
 from calibre.gui2.jobs import JobManager, JobsDialog, JobsButton
 from calibre.gui2.init import LibraryViewMixin, LayoutMixin
 from calibre.gui2.search_box import SearchBoxMixin, SavedSearchBoxMixin
@@ -89,7 +90,8 @@ class SystemTrayIcon(QSystemTrayIcon): # {{{
 
 class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         TagBrowserMixin, CoverFlowMixin, LibraryViewMixin, SearchBoxMixin,
-        SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin
+        SavedSearchBoxMixin, SearchRestrictionMixin, LayoutMixin, UpdateMixin,
+        EbookDownloadMixin
         ):
     'The main GUI'
 
@@ -100,6 +102,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         self.device_connected = None
         self.gui_debug = gui_debug
         self.iactions = OrderedDict()
+        # Actions
         for action in interface_actions():
             if opts.ignore_plugins and action.plugin_path is not None:
                 continue
@@ -112,11 +115,10 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
                 if action.plugin_path is None:
                     raise
                 continue
-
             ac.plugin_path = action.plugin_path
             ac.interface_action_base_plugin = action
-
             self.add_iaction(ac)
+        self.load_store_plugins()
 
     def init_iaction(self, action):
         ac = action.load_actual_plugin(self)
@@ -133,6 +135,37 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         else:
             acmap[ac.name] = ac
 
+    def load_store_plugins(self):
+        self.istores = OrderedDict()
+        for store in store_plugins():
+            if self.opts.ignore_plugins and store.plugin_path is not None:
+                continue
+            try:
+                st = self.init_istore(store)
+                self.add_istore(st)
+            except:
+                # Ignore errors in loading user supplied plugins
+                import traceback
+                traceback.print_exc()
+                if store.plugin_path is None:
+                    raise
+                continue
+
+    def init_istore(self, store):
+        st = store.load_actual_plugin(self)
+        st.plugin_path = store.plugin_path
+        st.base_plugin = store
+        store.actual_istore_plugin_loaded = True
+        return st
+
+    def add_istore(self, st):
+        stmap = self.istores
+        if st.name in stmap:
+            if st.priority >= stmap[st.name].priority:
+                stmap[st.name] = st
+        else:
+            stmap[st.name] = st
+
 
     def initialize(self, library_path, db, listener, actions, show_gui=True):
         opts = self.opts
@@ -154,6 +187,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
         for ac in self.iactions.values():
             ac.do_genesis()
         self.donate_action = QAction(QIcon(I('donate.png')), _('&Donate to support calibre'), self)
+        for st in self.istores.values():
+            st.do_genesis()
         MainWindowMixin.__init__(self, db)
 
         # Jobs Button {{{
@@ -165,6 +200,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{
 
         LayoutMixin.__init__(self)
         EmailMixin.__init__(self)
+        EbookDownloadMixin.__init__(self)
         DeviceMixin.__init__(self)
 
         self.progress_indicator = ProgressIndicator(self)