Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

...

Browse Source
This commit is contained in:
Kovid Goyal 2011-04-17 08:40:10 -06:00
parent 03cb146ad0
commit 94c4b74e53
6 changed files with 122 additions and 124 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/calibre/gui2/actions/store.py
View File

@ -8,20 +8,20 @@ __docformat__ = 'restructuredtext en'
from functools import partial
from PyQt4.Qt import Qt, QMenu, QToolButton, QDialog, QVBoxLayout
from PyQt4.Qt import QMenu
from calibre.gui2.actions import InterfaceAction
class StoreAction(InterfaceAction):
name = 'Store'
action_spec = (_('Store'), 'store.png', None, None)
action_spec = (_('Get books'), 'store.png', None, None)
def genesis(self):
self.qaction.triggered.connect(self.search)
self.store_menu = QMenu()
self.load_menu()
def load_menu(self):
self.store_menu.clear()
self.store_menu.addAction(_('Search'), self.search)
@ -29,11 +29,11 @@ class StoreAction(InterfaceAction):
for n, p in self.gui.istores.items():
self.store_menu.addAction(n, partial(self.open_store, p))
self.qaction.setMenu(self.store_menu)
def search(self):
from calibre.gui2.store.search import SearchDialog
sd = SearchDialog(self.gui.istores, self.gui)
sd.exec_()
def open_store(self, store_plugin):
store_plugin.open(self.gui)

48
src/calibre/gui2/store/__init__.py
View File

@ -19,27 +19,27 @@ class StorePlugin(object): # {{{
If two :class:`StorePlugin` objects have the same name, the one with higher
priority takes precedence.
Sub-classes must implement :meth:`open`, and :meth:`search`.
Regarding :meth:`open`. Most stores only make themselves available
though a web site thus most store plugins will open using
:class:`calibre.gui2.store.web_store_dialog.WebStoreDialog`. This will
open a modal window and display the store website in a QWebView.
Sub-classes should implement and use the :meth:`genesis` if they require
plugin specific initialization. They should not override or otherwise
reimplement :meth:`__init__`.
Once initialized, this plugin has access to the main calibre GUI via the
:attr:`gui` member. You can access other plugins by name, for example::
self.gui.istores['Amazon Kindle']
Plugin authors can use affiliate programs within their plugin. The
distribution of money earned from a store plugin is 70/30. 70% going
to the pluin author / maintainer and 30% going to the calibre project.
The easiest way to handle affiliate money payouts is to randomly select
between the author's affiliate id and calibre's affiliate id so that
70% of the time the author's id is used.
@ -49,61 +49,61 @@ class StorePlugin(object): # {{{
self.gui = gui
self.name = name
self.base_plugin = None
def open(self, gui, parent=None, detail_item=None, external=False):
'''
Open the store.
:param gui: The main GUI. This will be used to have the job
system start downloading an item from the store.
:param parent: The parent of the store dialog. This is used
to create modal dialogs.
:param detail_item: A plugin specific reference to an item
in the store that the user should be shown.
:param external: When False open an internal dialog with the
store. When True open the users default browser to the store's
web site. :param:`detail_item` should still be respected when external
is True.
'''
raise NotImplementedError()
def search(self, query, max_results=10, timeout=60):
'''
Searches the store for items matching query. This should
return items as a generator.
Don't be lazy with the search! Load as much data as possible in the
:class:`calibre.gui2.store.search_result.SearchResult` object. If you have to parse
multiple pages to get all of the data then do so. However, if data (such as cover_url)
isn't available because the store does not display cover images then it's okay to
ignore it.
Also, by default search results can only include ebooks. A plugin can offer users
an option to include physical books in the search results but this must be
disabled by default.
If a store doesn't provide search on it's own use something like a site specific
google search to get search results for this funtion.
:param query: The string query search with.
:param max_results: The maximum number of results to return.
:param timeout: The maximum amount of time in seconds to spend download the search results.
:return: :class:`calibre.gui2.store.search_result.SearchResult` objects
item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store.
item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store.
'''
raise NotImplementedError()
def get_settings(self):
'''
This is only useful for plugins that implement
:attr:`config_widget` that is the only way to save
settings. This is used by plugins to get the saved
settings and apply when necessary.
:return: A dictionary filled with the settings used
by this plugin.
'''
@ -117,23 +117,23 @@ class StorePlugin(object): # {{{
Plugin specific initialization.
'''
pass
def config_widget(self):
'''
See :class:`calibre.customize.Plugin` for details.
'''
raise NotImplementedError()
def save_settings(self, config_widget):
'''
See :class:`calibre.customize.Plugin` for details.
'''
raise NotImplementedError()
def customization_help(self, gui=False):
'''
See :class:`calibre.customize.Plugin` for details.
'''
raise NotImplementedError()
# }}}
# }}}

48
src/calibre/gui2/store/amazon_plugin.py
View File

@ -21,14 +21,14 @@ from calibre.gui2.store import StorePlugin
from calibre.gui2.store.search_result import SearchResult
class AmazonKindleStore(StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
'''
Amazon comes with a number of difficulties.
QWebView has major issues with Amazon.com. The largest of
issues is it simply doesn't work on a number of pages.
When connecting to a number parts of Amazon.com (Kindle library
for instance) QNetworkAccessManager fails to connect with a
NetworkError of 399 - ProtocolFailure. The strange thing is,
@ -37,19 +37,19 @@ class AmazonKindleStore(StorePlugin):
the QNetworkAccessManager decides there was a NetworkError it
does not download the page from Amazon. So I can't even set the
HTML in the QWebView myself.
There is http://bugreports.qt.nokia.com/browse/QTWEBKIT-259 an
open bug about the issue but it is not correct. We can set the
useragent (Arora does) to something else and the above issue
will persist. This http://developer.qt.nokia.com/forums/viewthread/793
gives a bit more information about the issue but as of now (27/Feb/2011)
there is no solution or work around.
We cannot change the The linkDelegationPolicy to allow us to avoid
QNetworkAccessManager because it only works links. Forms aren't
included so the same issue persists on any part of the site (login)
that use a form to load a new page.
Using an aStore was evaluated but I've decided against using it.
There are three major issues with an aStore. Because checkout is
handled by sending the user to Amazon we can't put it in a QWebView.
@ -57,7 +57,7 @@ class AmazonKindleStore(StorePlugin):
nicer. Also, we cannot put the aStore in a QWebView and let it open the
redirection the users default browser because the cookies with the
shopping cart won't transfer.
Another issue with the aStore is how it handles the referral. It only
counts the referral for the items in the shopping card / the item
that directed the user to Amazon. Kindle books do not use the shopping
@ -65,44 +65,44 @@ class AmazonKindleStore(StorePlugin):
instance we would only get referral credit for the one book that the
aStore directs to Amazon that the user buys. Any other purchases we
won't get credit for.
The last issue with the aStore is performance. Even though it's an
Amazon site it's alow. So much slower than Amazon.com that it makes
me not want to browse books using it. The look and feel are lesser
issues. So is the fact that it almost seems like the purchase is
with calibre. This can cause some support issues because we can't
do much for issues with Amazon.com purchase hiccups.
Another option that was evaluated was the Product Advertising API.
The reasons against this are complexity. It would take a lot of work
to basically re-create Amazon.com within calibre. The Product
Advertising API is also designed with being run on a server not
in an app. The signing keys would have to be made avaliable to ever
calibre user which means bad things could be done with our account.
The Product Advertising API also assumes the same browser for easy
shopping cart transfer to Amazon. With QWebView not working and there
not being an easy way to transfer cookies between a QWebView and the
users default browser this won't work well.
We could create our own website on the calibre server and create an
Amazon Product Advertising API store. However, this goes back to the
complexity argument. Why spend the time recreating Amazon.com
The final and largest issue against using the Product Advertising API
is the Efficiency Guidelines:
"Each account used to access the Product Advertising API will be allowed
an initial usage limit of 2,000 requests per hour. Each account will
receive an additional 500 requests per hour (up to a maximum of 25,000
requests per hour) for every $1 of shipped item revenue driven per hour
in a trailing 30-day period. Usage thresholds are recalculated daily based
on revenue performance."
on revenue performance."
With over two million users a limit of 2,000 request per hour could
render our store unusable for no other reason than Amazon rate
limiting our traffic.
The best (I use the term lightly here) solution is to open Amazon.com
in the users default browser and set the affiliate id as part of the url.
'''
@ -119,14 +119,14 @@ class AmazonKindleStore(StorePlugin):
def search(self, query, max_results=10, timeout=60):
url = 'http://www.amazon.com/s/url=search-alias%3Ddigital-text&field-keywords=' + urllib2.quote(query)
br = browser()
counter = max_results
with closing(br.open(url, timeout=timeout)) as f:
doc = html.fromstring(f.read())
for data in doc.xpath('//div[@class="productData"]'):
if counter <= 0:
break
# Even though we are searching digital-text only Amazon will still
# put in results for non Kindle books (author pages). Se we need
# to explicitly check if the item is a Kindle book and ignore it
@ -134,7 +134,7 @@ class AmazonKindleStore(StorePlugin):
type = ''.join(data.xpath('//span[@class="format"]/text()'))
if 'kindle' not in type.lower():
continue
# We must have an asin otherwise we can't easily reference the
# book later.
asin_href = None
@ -148,25 +148,25 @@ class AmazonKindleStore(StorePlugin):
continue
else:
continue
cover_url = ''
if asin_href:
cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href)
if cover_img:
cover_url = cover_img[0]
title = ''.join(data.xpath('div[@class="productTitle"]/a/text()'))
author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()'))
author = author.split('by')[-1]
price = ''.join(data.xpath('div[@class="newPrice"]/span/text()'))
counter -= 1
s = SearchResult()
s.cover_url = cover_url
s.title = title.strip()
s.author = author.strip()
s.price = price.strip()
s.detail_item = asin.strip()
yield s

19
src/calibre/gui2/store/bewrite_plugin.py
View File

@ -6,7 +6,6 @@ __license__ = 'GPL 3'
__copyright__ = '2011, John Schember <john@nachtimwald.com>'
__docformat__ = 'restructuredtext en'
import re
import urllib2
from contextlib import closing
@ -22,7 +21,7 @@ from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
class BeWriteStore(BasicStoreConfig, StorePlugin):
def open(self, parent=None, detail_item=None, external=False):
settings = self.get_settings()
url = 'http://www.bewrite.net/mm5/merchant.mvc?Screen=SFNT'
@ -42,9 +41,9 @@ class BeWriteStore(BasicStoreConfig, StorePlugin):
def search(self, query, max_results=10, timeout=60):
url = 'http://www.bewrite.net/mm5/merchant.mvc?Search_Code=B&Screen=SRCH&Search=' + urllib2.quote(query)
br = browser()
counter = max_results
with closing(br.open(url, timeout=timeout)) as f:
doc = html.fromstring(f.read())
@ -55,12 +54,12 @@ class BeWriteStore(BasicStoreConfig, StorePlugin):
id = ''.join(data.xpath('.//a/@href'))
if not id:
continue
heading = ''.join(data.xpath('./td[2]//text()'))
heading = ''.join(data.xpath('./td[2]//text()'))
title, q, author = heading.partition('by ')
cover_url = ''
price = ''
with closing(br.open(id.strip(), timeout=timeout/4)) as nf:
idata = html.fromstring(nf.read())
price = ''.join(idata.xpath('//div[@id="content"]//td[contains(text(), "ePub")]/text()'))
@ -68,14 +67,14 @@ class BeWriteStore(BasicStoreConfig, StorePlugin):
cover_img = idata.xpath('//div[@id="content"]//img[1]/@src')
if cover_img:
cover_url = 'http://www.bewrite.net/mm5/' + cover_img[0]
counter -= 1
s = SearchResult()
s.cover_url = cover_url.strip()
s.title = title.strip()
s.author = author.strip()
s.price = price.strip()
s.detail_item = id.strip()
yield s

85
src/calibre/gui2/store/search.py
View File

@ -13,9 +13,8 @@ from random import shuffle
from threading import Thread
from Queue import Queue
from PyQt4.Qt import Qt, QAbstractItemModel, QDialog, QTimer, QVariant, \
QModelIndex, QPixmap, QSize, QCheckBox, QVBoxLayout, QHBoxLayout, \
QPushButton, QString, QByteArray
from PyQt4.Qt import (Qt, QAbstractItemModel, QDialog, QTimer, QVariant,
QModelIndex, QPixmap, QSize, QCheckBox, QVBoxLayout)
from calibre import browser
from calibre.gui2 import NONE
@ -35,7 +34,7 @@ class SearchDialog(QDialog, Ui_Dialog):
def __init__(self, istores, *args):
QDialog.__init__(self, *args)
self.setupUi(self)
self.config = DynamicConfig('store_search')
# We keep a cache of store plugins and reference them by name.
@ -44,7 +43,7 @@ class SearchDialog(QDialog, Ui_Dialog):
# Check for results and hung threads.
self.checker = QTimer()
self.hang_check = 0
self.model = Matches()
self.results_view.setModel(self.model)
@ -59,7 +58,7 @@ class SearchDialog(QDialog, Ui_Dialog):
stores_group_layout.addWidget(cbox)
setattr(self, 'store_check_' + x, cbox)
stores_group_layout.addStretch()
# Create and add the progress indicator
self.pi = ProgressIndicator(self, 24)
self.bottom_layout.insertWidget(0, self.pi)
@ -71,9 +70,9 @@ class SearchDialog(QDialog, Ui_Dialog):
self.select_invert_stores.clicked.connect(self.stores_select_invert)
self.select_none_stores.clicked.connect(self.stores_select_none)
self.finished.connect(self.dialog_closed)
self.restore_state()
def resize_columns(self):
total = 600
# Cover
@ -87,19 +86,19 @@ class SearchDialog(QDialog, Ui_Dialog):
self.results_view.setColumnWidth(3, int(total*.10))
# Store
self.results_view.setColumnWidth(4, int(total*.20))
def do_search(self, checked=False):
# Stop all running threads.
self.checker.stop()
self.search_pool.abort()
# Clear the visible results.
self.results_view.model().clear_results()
# Don't start a search if there is nothing to search for.
query = unicode(self.search_edit.text())
if not query.strip():
return
# Plugins are in alphebetic order. Randomize the
# order of plugin names. This way plugins closer
# to a don't have an unfair advantage over
@ -117,12 +116,12 @@ class SearchDialog(QDialog, Ui_Dialog):
self.checker.start(100)
self.search_pool.start_threads()
self.pi.startAnimation()
def save_state(self):
self.config['store_search_geometry'] = self.saveGeometry()
self.config['store_search_store_splitter_state'] = self.store_splitter.saveState()
self.config['store_search_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())]
store_check = {}
for n in self.store_plugins:
store_check[n] = getattr(self, 'store_check_' + n).isChecked()
@ -132,11 +131,11 @@ class SearchDialog(QDialog, Ui_Dialog):
geometry = self.config['store_search_geometry']
if geometry:
self.restoreGeometry(geometry)
splitter_state = self.config['store_search_store_splitter_state']
if splitter_state:
self.store_splitter.restoreState(splitter_state)
results_cwidth = self.config['store_search_results_view_column_width']
if results_cwidth:
for i, x in enumerate(results_cwidth):
@ -145,7 +144,7 @@ class SearchDialog(QDialog, Ui_Dialog):
self.results_view.setColumnWidth(i, x)
else:
self.resize_columns()
store_check = self.config['store_search_store_checked']
if store_check:
for n in store_check:
@ -165,7 +164,7 @@ class SearchDialog(QDialog, Ui_Dialog):
if not self.search_pool.threads_running() and not self.search_pool.has_tasks():
self.checker.stop()
self.pi.stopAnimation()
while self.search_pool.has_results():
res = self.search_pool.get_result()
if res:
@ -189,15 +188,15 @@ class SearchDialog(QDialog, Ui_Dialog):
def stores_select_all(self):
for check in self.get_store_checks():
check.setChecked(True)
def stores_select_invert(self):
for check in self.get_store_checks():
check.setChecked(not check.isChecked())
def stores_select_none(self):
for check in self.get_store_checks():
check.setChecked(False)
def dialog_closed(self, result):
self.model.closing()
self.search_pool.abort()
@ -208,46 +207,46 @@ class GenericDownloadThreadPool(object):
'''
add_task must be implemented in a subclass.
'''
def __init__(self, thread_type, thread_count):
self.thread_type = thread_type
self.thread_count = thread_count
self.tasks = Queue()
self.results = Queue()
self.threads = []
def add_task(self):
raise NotImplementedError()
def start_threads(self):
for i in range(self.thread_count):
t = self.thread_type(self.tasks, self.results)
self.threads.append(t)
t.start()
def abort(self):
self.tasks = Queue()
self.results = Queue()
for t in self.threads:
t.abort()
self.threads = []
def has_tasks(self):
return not self.tasks.empty()
def get_result(self):
return self.results.get()
def get_result_no_wait(self):
return self.results.get_nowait()
def result_count(self):
return len(self.results)
def has_results(self):
return not self.results.empty()
def threads_running(self):
for t in self.threads:
if t.is_alive():
@ -260,7 +259,7 @@ class SearchThreadPool(GenericDownloadThreadPool):
Threads will run until there is no work or
abort is called. Create and start new threads
using start_threads(). Reset by calling abort().
Example:
sp = SearchThreadPool(SearchThread, 3)
add tasks using add_task(...)
@ -270,13 +269,13 @@ class SearchThreadPool(GenericDownloadThreadPool):
add tasks using add_task(...)
sp.start_threads()
'''
def add_task(self, query, store_name, store_plugin, timeout):
self.tasks.put((query, store_name, store_plugin, timeout))
class SearchThread(Thread):
def __init__(self, tasks, results):
Thread.__init__(self)
self.daemon = True
@ -286,7 +285,7 @@ class SearchThread(Thread):
def abort(self):
self._run = False
def run(self):
while self._run and not self.tasks.empty():
try:
@ -305,7 +304,7 @@ class CoverThreadPool(GenericDownloadThreadPool):
'''
Once started all threads run until abort is called.
'''
def add_task(self, search_result, update_callback, timeout=5):
self.tasks.put((search_result, update_callback, timeout))
@ -318,12 +317,12 @@ class CoverThread(Thread):
self.tasks = tasks
self.results = results
self._run = True
self.br = browser()
def abort(self):
self._run = False
def run(self):
while self._run:
try:
@ -354,13 +353,13 @@ class Matches(QAbstractItemModel):
def closing(self):
self.cover_pool.abort()
def clear_results(self):
self.matches = []
self.cover_pool.abort()
self.cover_pool.start_threads()
self.reset()
def add_result(self, result):
self.layoutAboutToBeChanged.emit()
self.matches.append(result)
@ -391,7 +390,7 @@ class Matches(QAbstractItemModel):
def columnCount(self, *args):
return len(self.HEADERS)
def headerData(self, section, orientation, role):
if role != Qt.DisplayRole:
return NONE
@ -434,7 +433,7 @@ class Matches(QAbstractItemModel):
elif col == 3:
text = result.price
if len(text) < 3 or text[-3] not in ('.', ','):
text += '00'
text += '00'
text = re.sub(r'\D', '', text)
text = text.rjust(6, '0')
elif col == 4:
@ -444,7 +443,7 @@ class Matches(QAbstractItemModel):
def sort(self, col, order, reset=True):
if not self.matches:
return
descending = order == Qt.DescendingOrder
descending = order == Qt.DescendingOrder
self.matches.sort(None,
lambda x: sort_key(unicode(self.data_as_text(x, col))),
descending)

34
src/calibre/gui2/store/web_control.py
View File

@ -9,8 +9,8 @@ __docformat__ = 'restructuredtext en'
import os
from urlparse import urlparse
from PyQt4.Qt import QWebView, QWebPage, QNetworkCookieJar, QNetworkRequest, QString, \
QFileDialog, QNetworkProxy
from PyQt4.Qt import (QWebView, QWebPage, QNetworkCookieJar,
QFileDialog, QNetworkProxy)
from calibre import USER_AGENT, get_proxies, get_download_filename
from calibre.ebooks import BOOK_EXTENSIONS
@ -35,13 +35,13 @@ class NPWebView(QWebView):
proxy.setPassword(proxy_parts.password)
proxy.setHostName(proxy_parts.hostname)
proxy.setPort(proxy_parts.port)
self.page().networkAccessManager().setProxy(proxy)
self.page().networkAccessManager().setProxy(proxy)
self.page().setForwardUnsupportedContent(True)
self.page().unsupportedContent.connect(self.start_download)
self.page().downloadRequested.connect(self.start_download)
self.page().networkAccessManager().sslErrors.connect(self.ignore_ssl_errors)
def createWindow(self, type):
if type == QWebPage.WebBrowserWindow:
return self
@ -50,17 +50,17 @@ class NPWebView(QWebView):
def set_gui(self, gui):
self.gui = gui
def set_tags(self, tags):
self.tags = tags
def start_download(self, request):
if not self.gui:
return
url = unicode(request.url().toString())
cf = self.get_cookies()
filename = get_download_filename(url, cf)
ext = os.path.splitext(filename)[1][1:].lower()
if ext not in BOOK_EXTENSIONS:
@ -76,21 +76,21 @@ class NPWebView(QWebView):
def ignore_ssl_errors(self, reply, errors):
reply.ignoreSslErrors(errors)
def get_cookies(self):
'''
Writes QNetworkCookies to Mozilla cookie .txt file.
:return: The file path to the cookie file.
'''
cf = PersistentTemporaryFile(suffix='.txt')
cf.write('# Netscape HTTP Cookie File\n\n')
for c in self.page().networkAccessManager().cookieJar().allCookies():
cookie = []
domain = unicode(c.domain())
cookie.append(domain)
cookie.append('TRUE' if domain.startswith('.') else 'FALSE')
cookie.append(unicode(c.path()))
@ -98,15 +98,15 @@ class NPWebView(QWebView):
cookie.append(unicode(c.expirationDate().toTime_t()))
cookie.append(unicode(c.name()))
cookie.append(unicode(c.value()))
cf.write('\t'.join(cookie))
cf.write('\n')
cf.close()
return cf.name
class NPWebPage(QWebPage):
def userAgentForUrl(self, url):
return USER_AGENT