Get Books: Allow stores to customize the browser instance used for downloading book files

src/calibre/gui2/ebook_download.py

@@ -12,7 +12,6 @@ from contextlib import closing
 from mechanize import MozillaCookieJar
 
 from calibre import browser
-from calibre.constants import __appname__, __version__
 from calibre.ebooks import BOOK_EXTENSIONS
 from calibre.gui2 import Dispatcher, gprefs
 from calibre.gui2.dialogs.message_box import MessageBox
@@ -52,15 +51,13 @@ def get_download_filename(response):
     filename = ascii_filename(filename)
     return filename
 
-def download_file(url, cookie_file=None, filename=None):
+def download_file(url, cookie_file=None, filename=None, create_browser=None):
-    user_agent = None
     if url.startswith('//'):
         url = 'http:' + url
-    if url.startswith('http://www.gutenberg.org'):
+    try:
-        # Project Gutenberg returns an HTML page if the user agent is a normal
+        br = browser() if create_browser is None else create_browser()
-        # browser user agent
+    except NotImplementedError:
-        user_agent = '%s/%s' % (__appname__, __version__)
+        br = browser()
-    br = browser(user_agent=user_agent)
     if cookie_file:
         cj = MozillaCookieJar()
         cj.load(cookie_file)
@@ -78,10 +75,11 @@ def download_file(url, cookie_file=None, filename=None):
 
 class EbookDownload(object):
 
-    def __call__(self, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[], log=None, abort=None, notifications=None):
+    def __call__(self, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[], create_browser=None,
+                 log=None, abort=None, notifications=None):
         dfilename = ''
         try:
-            dfilename = self._download(cookie_file, url, filename, save_loc, add_to_lib)
+            dfilename = self._download(cookie_file, url, filename, save_loc, add_to_lib, create_browser)
             self._add(dfilename, gui, add_to_lib, tags)
             self._save_as(dfilename, save_loc)
         finally:
@@ -91,13 +89,13 @@ class EbookDownload(object):
             except:
                 pass
 
-    def _download(self, cookie_file, url, filename, save_loc, add_to_lib):
+    def _download(self, cookie_file, url, filename, save_loc, add_to_lib, create_browser):
         if not url:
             raise Exception(_('No file specified to download.'))
         if not save_loc and not add_to_lib:
             # Nothing to do.
             return ''
-        return download_file(url, cookie_file, filename)
+        return download_file(url, cookie_file, filename, create_browser=create_browser)
 
     def _add(self, filename, gui, add_to_lib, tags):
         if not add_to_lib or not filename:
@@ -124,10 +122,10 @@ class EbookDownload(object):
 
 gui_ebook_download = EbookDownload()
 
-def start_ebook_download(callback, job_manager, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[]):
+def start_ebook_download(callback, job_manager, gui, cookie_file=None, url='', filename='', save_loc='', add_to_lib=True, tags=[], create_browser=None):
     description = _('Downloading %s') % filename.decode('utf-8', 'ignore') if filename else url.decode('utf-8', 'ignore')
     job = ThreadedJob('ebook_download', description, gui_ebook_download, (
-        gui, cookie_file, url, filename, save_loc, add_to_lib, tags), {},
+        gui, cookie_file, url, filename, save_loc, add_to_lib, tags, create_browser), {},
                       callback, max_concurrent_count=2, killable=False)
     job_manager.run_threaded_job(job)
 
@@ -137,11 +135,11 @@ class EbookDownloadMixin(object):
     def __init__(self, *args, **kwargs):
         pass
 
-    def download_ebook(self, url='', cookie_file=None, filename='', save_loc='', add_to_lib=True, tags=[]):
+    def download_ebook(self, url='', cookie_file=None, filename='', save_loc='', add_to_lib=True, tags=[], create_browser=None):
         if tags:
             if isinstance(tags, basestring):
                 tags = tags.split(',')
-        start_ebook_download(Dispatcher(self.downloaded_ebook), self.job_manager, self, cookie_file, url, filename, save_loc, add_to_lib, tags)
+        start_ebook_download(Dispatcher(self.downloaded_ebook), self.job_manager, self, cookie_file, url, filename, save_loc, add_to_lib, tags, create_browser)
         self.status_bar.show_message(_('Downloading') + ' ' + filename.decode('utf-8', 'ignore') if filename else url.decode('utf-8', 'ignore'), 3000)
 
     def downloaded_ebook(self, job):

src/calibre/gui2/store/__init__.py

@@ -8,7 +8,8 @@ __docformat__ = 'restructuredtext en'
 
 from calibre.utils.filenames import ascii_filename
 
-class StorePlugin(object): # {{{
+class StorePlugin(object):  # {{{
+
     '''
     A plugin representing an online ebook repository (store). The store can
     be a commercial store that sells ebooks or a source of free downloadable
@@ -60,6 +61,18 @@ class StorePlugin(object): # {{{
             config = JSONConfig('store/stores/' + ascii_filename(self.name))
         self.config = config
 
+    def create_browser(self):
+        '''
+        If the server requires special headers, such as a particular user agent
+        or a referrer, then implement this method in you plugin to return a
+        customized browser instance. See the Gutenberg plugin for an example.
+
+        Note that if you implement the open() method in your plugin and use the
+        WebStoreDialog class, remember to pass self.createbrowser in the
+        constructor of WebStoreDialog.
+        '''
+        raise NotImplementedError()
+
     def open(self, gui, parent=None, detail_item=None, external=False):
         '''
         Open the store.

src/calibre/gui2/store/opensearch_store.py

@@ -32,7 +32,7 @@ class OpenSearchOPDSStore(StorePlugin):
         if external or self.config.get('open_external', False):
             open_url(QUrl(detail_item if detail_item else self.web_url))
         else:
-            d = WebStoreDialog(self.gui, self.web_url, parent, detail_item)
+            d = WebStoreDialog(self.gui, self.web_url, parent, detail_item, create_browser=self.create_browser)
             d.setWindowTitle(self.name)
             d.set_tags(self.config.get('tags', ''))
             d.exec_()
@@ -97,5 +97,4 @@ class OpenSearchOPDSStore(StorePlugin):
                     s.price = currency_code + ' ' + price
                     s.price = s.price.strip()
 
-
                 yield s

src/calibre/gui2/store/search/download_thread.py

@@ -122,6 +122,7 @@ class SearchThread(Thread):
                     res.store_name = store_name
                     res.affiliate = store_plugin.base_plugin.affiliate
                     res.plugin_author = store_plugin.base_plugin.author
+                    res.create_browser = store_plugin.create_browser
                     self.results.put((res, store_plugin))
                 self.tasks.task_done()
             except:

src/calibre/gui2/store/search/search.py

@@ -395,7 +395,7 @@ class SearchDialog(QDialog, Ui_Dialog):
             fname = result.title[:60] + '.' + ext.lower()
             fname = ascii_filename(fname)
             show_download_info(result.title, parent=self)
-            self.gui.download_ebook(result.downloads[ext], filename=fname)
+            self.gui.download_ebook(result.downloads[ext], filename=fname, create_browser=result.create_browser)
 
     def open_store(self, result):
         self.gui.istores[result.store_name].open(self, result.detail_item, self.open_external.isChecked())

src/calibre/gui2/store/search_result.py

@@ -27,6 +27,7 @@ class SearchResult(object):
         self.downloads = {}
         self.affiliate = False
         self.plugin_author = ''
+        self.create_browser = None
 
     def __eq__(self, other):
         return self.title == other.title and self.author == other.author and self.store_name == other.store_name and self.formats == other.formats

src/calibre/gui2/store/stores/gutenberg_plugin.py

@@ -16,7 +16,7 @@ from contextlib import closing
 from lxml import etree
 
 from calibre import browser, url_slash_cleaner
-from calibre.constants import __version__
+from calibre.constants import __appname__, __version__
 from calibre.gui2.store.basic_config import BasicStoreConfig
 from calibre.gui2.store.opensearch_store import OpenSearchOPDSStore
 from calibre.gui2.store.search_result import SearchResult
@@ -93,6 +93,11 @@ class GutenbergStore(BasicStoreConfig, OpenSearchOPDSStore):
     open_search_url = 'http://www.gutenberg.org/catalog/osd-books.xml'
     web_url = web_url
 
+    def create_browser(self):
+        from calibre import browser
+        user_agent = '%s/%s' % (__appname__, __version__)
+        return browser(user_agent=user_agent)
+
     def search(self, query, max_results=10, timeout=60):
         '''
         Gutenberg's ODPS feed is poorly implmented and has a number of issues

src/calibre/gui2/store/web_control.py

@@ -26,6 +26,7 @@ class NPWebView(QWebView):
         QWebView.__init__(self, *args)
         self.gui = None
         self.tags = ''
+        self.create_browser = None
 
         self._page = NPWebPage()
         self.setPage(self._page)
@@ -90,10 +91,10 @@ class NPWebView(QWebView):
                     return
             name = choose_save_file(self, 'web-store-download-unknown', _('File is not a supported ebook type. Save to disk?'), initial_filename=filename)
             if name:
-                self.gui.download_ebook(url, cf, name, name, False)
+                self.gui.download_ebook(url, cf, name, name, False, create_browser=self.create_browser)
         else:
             show_download_info(filename, self)
-            self.gui.download_ebook(url, cf, filename, tags=self.tags)
+            self.gui.download_ebook(url, cf, filename, tags=self.tags, create_browser=self.create_browser)
 
     def ignore_ssl_errors(self, reply, errors):
         reply.ignoreSslErrors(errors)

src/calibre/gui2/store/web_store_dialog.py

@@ -13,21 +13,22 @@ from calibre.gui2.store.web_store_dialog_ui import Ui_Dialog
 
 class WebStoreDialog(QDialog, Ui_Dialog):
 
-    def __init__(self, gui, base_url, parent=None, detail_url=None):
+    def __init__(self, gui, base_url, parent=None, detail_url=None, create_browser=None):
         QDialog.__init__(self, parent=parent)
         self.setupUi(self)
-        
+
         self.gui = gui
         self.base_url = base_url
-        
+
         self.view.set_gui(self.gui)
+        self.view.create_browser = create_browser
         self.view.loadStarted.connect(self.load_started)
         self.view.loadProgress.connect(self.load_progress)
         self.view.loadFinished.connect(self.load_finished)
         self.home.clicked.connect(self.go_home)
         self.reload.clicked.connect(self.view.reload)
         self.back.clicked.connect(self.view.back)
-        
+
         self.go_home(detail_url=detail_url)
 
     def set_tags(self, tags):
@@ -35,21 +36,21 @@ class WebStoreDialog(QDialog, Ui_Dialog):
 
     def load_started(self):
         self.progress.setValue(0)
-    
+
     def load_progress(self, val):
         self.progress.setValue(val)
-        
+
     def load_finished(self, ok=True):
         self.progress.setValue(100)
-    
+
     def go_home(self, checked=False, detail_url=None):
         if detail_url:
             url = detail_url
         else:
             url = self.base_url
-            
+
         # Reduce redundant /'s because some stores
         # (Feedbooks) and server frameworks (cherrypy)
-        # choke on them. 
+        # choke on them.
         url = url_slash_cleaner(url)
         self.view.load(QUrl(url))