From d4de706fc8df27dc6feee8ae6f108a664539bf30 Mon Sep 17 00:00:00 2001 From: John Schember Date: Sat, 23 Apr 2011 23:33:38 -0400 Subject: [PATCH] Store: Break MobileRead into more manageable pieces. API for updating store caches. --- src/calibre/customize/builtins.py | 2 +- src/calibre/gui2/store/__init__.py | 34 ++ src/calibre/gui2/store/mobileread/__init__.py | 0 .../store/mobileread/cache_progress_dialog.py | 62 +++ .../store/mobileread/cache_progress_dialog.ui | 104 +++++ .../store/mobileread/cache_update_thread.py | 94 +++++ .../store/mobileread/mobileread_plugin.py | 105 +++++ src/calibre/gui2/store/mobileread/models.py | 190 +++++++++ .../gui2/store/mobileread/store_dialog.py | 83 ++++ .../store_dialog.ui} | 0 src/calibre/gui2/store/mobileread_plugin.py | 376 ------------------ .../gui2/store/search/download_thread.py | 30 ++ src/calibre/gui2/store/search/search.py | 22 +- 13 files changed, 716 insertions(+), 386 deletions(-) create mode 100644 src/calibre/gui2/store/mobileread/__init__.py create mode 100644 src/calibre/gui2/store/mobileread/cache_progress_dialog.py create mode 100644 src/calibre/gui2/store/mobileread/cache_progress_dialog.ui create mode 100644 src/calibre/gui2/store/mobileread/cache_update_thread.py create mode 100644 src/calibre/gui2/store/mobileread/mobileread_plugin.py create mode 100644 src/calibre/gui2/store/mobileread/models.py create mode 100644 src/calibre/gui2/store/mobileread/store_dialog.py rename src/calibre/gui2/store/{mobileread_store_dialog.ui => mobileread/store_dialog.ui} (100%) delete mode 100644 src/calibre/gui2/store/mobileread_plugin.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 00af4e5117..c27fa2a57b 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1162,7 +1162,7 @@ class StoreManyBooksStore(StoreBase): class StoreMobileReadStore(StoreBase): name = 'MobileRead' description = _('Ebooks handcrafted with the utmost care') - actual_plugin = 'calibre.gui2.store.mobileread_plugin:MobileReadStore' + actual_plugin = 'calibre.gui2.store.mobileread.mobileread_plugin:MobileReadStore' class StoreOpenLibraryStore(StoreBase): name = 'Open Library' diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 16e9f5689d..c95d794975 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -127,6 +127,40 @@ class StorePlugin(object): # {{{ ''' return False + def update_cache(self, parent=None, timeout=60, force=False, suppress_progress=False): + ''' + Some plugins need to keep an local cache of available books. This function + is called to update the caches. It is recommended to call this function + from :meth:`open`. Especially if :meth:`open` does anything other than + open a web page. + + This function can be called at any time. It is up to the plugin to determine + if the cache really does need updating. Unless :param:`force` is True, then + the plugin must update the cache. The only time force should be True is if + this function is called by the plugin's configuration dialog. + + if :param:`suppress_progress` is False it is safe to assume that this function + is being called from the main GUI thread so it is safe and recommended to use + a QProgressDialog to display what is happening and allow the user to cancel + the operation. if :param:`suppress_progress` is True then run the update + silently. In this case there is no guarantee what thread is calling this + function so no Qt related functionality that requires being run in the main + GUI thread should be run. E.G. Open a QProgressDialog. + + :param parent: The parent object to be used by an GUI dialogs. + + :param timeout: The maximum amount of time that should be spent in + any given network connection. + + :param force: Force updating the cache even if the plugin has determined + it is not necessary. + + :param suppress_progress: Should a progress indicator be shown. + + :return: True if the cache was updated, False otherwise. + ''' + return False + def get_settings(self): ''' This is only useful for plugins that implement diff --git a/src/calibre/gui2/store/mobileread/__init__.py b/src/calibre/gui2/store/mobileread/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/calibre/gui2/store/mobileread/cache_progress_dialog.py b/src/calibre/gui2/store/mobileread/cache_progress_dialog.py new file mode 100644 index 0000000000..3c372144c4 --- /dev/null +++ b/src/calibre/gui2/store/mobileread/cache_progress_dialog.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from PyQt4.Qt import (QCoreApplication, QDialog, QTimer) + +from calibre.gui2.store.mobileread.cache_progress_dialog_ui import Ui_Dialog + +class CacheProgressDialog(QDialog, Ui_Dialog): + + def __init__(self, parent=None, total=None): + QDialog.__init__(self, parent) + self.setupUi(self) + + self.completed = 0 + self.canceled = False + + self.progress.setValue(0) + self.progress.setMinimum(0) + self.progress.setMaximum(total if total else 0) + + def exec_(self): + self.completed = 0 + self.canceled = False + QDialog.exec_(self) + + def open(self): + self.completed = 0 + self.canceled = False + QDialog.open(self) + + def reject(self): + self.canceled = True + QDialog.reject(self) + + def update_progress(self): + ''' + completed is an int from 0 to total representing the number + records that have bee completed. + ''' + self.set_progress(self.completed + 1) + + def set_message(self, msg): + self.message.setText(msg) + + def set_details(self, msg): + self.details.setText(msg) + + def set_progress(self, completed): + ''' + completed is an int from 0 to total representing the number + records that have bee completed. + ''' + self.completed = completed + self.progress.setValue(self.completed) + + def set_total(self, total): + self.progress.setMaximum(total) diff --git a/src/calibre/gui2/store/mobileread/cache_progress_dialog.ui b/src/calibre/gui2/store/mobileread/cache_progress_dialog.ui new file mode 100644 index 0000000000..4690f14e7f --- /dev/null +++ b/src/calibre/gui2/store/mobileread/cache_progress_dialog.ui @@ -0,0 +1,104 @@ + + + Dialog + + + + 0 + 0 + 402 + 138 + + + + Dialog + + + + + + Updating book cache + + + Qt::AlignCenter + + + + + + + 24 + + + + + + + + + + Qt::AlignCenter + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel + + + + + + + + + buttonBox + accepted() + Dialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + Dialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + diff --git a/src/calibre/gui2/store/mobileread/cache_update_thread.py b/src/calibre/gui2/store/mobileread/cache_update_thread.py new file mode 100644 index 0000000000..f81e7951d4 --- /dev/null +++ b/src/calibre/gui2/store/mobileread/cache_update_thread.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import time +from contextlib import closing +from threading import Thread + +from lxml import html + +from PyQt4.Qt import (pyqtSignal, QObject) + +from calibre import browser +from calibre.gui2.store.search_result import SearchResult + +class CacheUpdateThread(Thread, QObject): + + total_changed = pyqtSignal(int) + update_progress = pyqtSignal(int) + update_details = pyqtSignal(unicode) + + def __init__(self, config, seralize_books_function, timeout): + Thread.__init__(self) + QObject.__init__(self) + + self.daemon = True + self.config = config + self.seralize_books = seralize_books_function + self.timeout = timeout + self._run = True + + def abort(self): + self._run = False + + def run(self): + url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' + + self.update_details.emit(_('Checking last download date.')) + last_download = self.config.get('last_download', None) + # Don't update the book list if our cache is less than one week old. + if last_download and (time.time() - last_download) < 604800: + return + + self.update_details.emit(_('Downloading book list from MobileRead.')) + # Download the book list HTML file from MobileRead. + br = browser() + raw_data = None + try: + with closing(br.open(url, timeout=self.timeout)) as f: + raw_data = f.read() + except: + return + + if not raw_data or not self._run: + return + + self.update_details.emit(_('Processing books.')) + # Turn books listed in the HTML file into SearchResults's. + books = [] + try: + data = html.fromstring(raw_data) + raw_books = data.xpath('//ul/li') + self.total_changed.emit(len(raw_books)) + + for i, book_data in enumerate(raw_books): + self.update_details.emit(_('%s of %s books processed.') % (i, len(raw_books))) + book = SearchResult() + book.detail_item = ''.join(book_data.xpath('.//a/@href')) + book.formats = ''.join(book_data.xpath('.//i/text()')) + book.formats = book.formats.strip() + + text = ''.join(book_data.xpath('.//a/text()')) + if ':' in text: + book.author, q, text = text.partition(':') + book.author = book.author.strip() + book.title = text.strip() + books.append(book) + + if not self._run: + books = [] + break + else: + self.update_progress.emit(i) + except: + pass + + # Save the book list and it's create time. + if books: + self.config['book_list'] = self.seralize_books(books) + self.config['last_download'] = time.time() diff --git a/src/calibre/gui2/store/mobileread/mobileread_plugin.py b/src/calibre/gui2/store/mobileread/mobileread_plugin.py new file mode 100644 index 0000000000..54242ce0b2 --- /dev/null +++ b/src/calibre/gui2/store/mobileread/mobileread_plugin.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from threading import Lock + +from PyQt4.Qt import (QUrl, QCoreApplication) + +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog +from calibre.gui2.store.mobileread.models import SearchFilter +from calibre.gui2.store.mobileread.cache_progress_dialog import CacheProgressDialog +from calibre.gui2.store.mobileread.cache_update_thread import CacheUpdateThread +from calibre.gui2.store.mobileread.store_dialog import MobeReadStoreDialog + +class MobileReadStore(BasicStoreConfig, StorePlugin): + + def genesis(self): + self.lock = Lock() + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://www.mobileread.com/' + + if external or self.config.get('open_external', False): + open_url(QUrl(detail_item if detail_item else url)) + else: + if detail_item: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + else: + if self.update_cache(parent, 30): + d = MobeReadStoreDialog(self, parent) + d.setWindowTitle(self.name) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + books = self.get_book_list() + + sf = SearchFilter(books) + matches = sf.parse(query) + + for book in matches: + book.price = '$0.00' + book.drm = SearchResult.DRM_UNLOCKED + yield book + + def update_cache(self, parent=None, timeout=10, force=False, suppress_progress=False): + if self.lock.acquire(False): + try: + update_thread = CacheUpdateThread(self.config, self.seralize_books, timeout) + if not suppress_progress: + progress = CacheProgressDialog(parent) + progress.set_message(_('Updating MobileRead book cache...')) + + update_thread.total_changed.connect(progress.set_total) + update_thread.update_progress.connect(progress.set_progress) + update_thread.update_details.connect(progress.set_details) + progress.rejected.connect(update_thread.abort) + + progress.open() + update_thread.start() + while update_thread.is_alive() and not progress.canceled: + QCoreApplication.processEvents() + + if progress.isVisible(): + progress.accept() + return not progress.canceled + else: + update_thread.start() + finally: + self.lock.release() + + def get_book_list(self): + return self.deseralize_books(self.config.get('book_list', [])) + + def seralize_books(self, books): + sbooks = [] + for b in books: + data = {} + data['author'] = b.author + data['title'] = b.title + data['detail_item'] = b.detail_item + data['formats'] = b.formats + sbooks.append(data) + return sbooks + + def deseralize_books(self, sbooks): + books = [] + for s in sbooks: + b = SearchResult() + b.author = s.get('author', '') + b.title = s.get('title', '') + b.detail_item = s.get('detail_item', '') + b.formats = s.get('formats', '') + books.append(b) + return books diff --git a/src/calibre/gui2/store/mobileread/models.py b/src/calibre/gui2/store/mobileread/models.py new file mode 100644 index 0000000000..a080affb51 --- /dev/null +++ b/src/calibre/gui2/store/mobileread/models.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from operator import attrgetter + +from PyQt4.Qt import (Qt, QAbstractItemModel, QModelIndex, QVariant, pyqtSignal) + +from calibre.gui2 import NONE +from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ + REGEXP_MATCH +from calibre.utils.icu import sort_key +from calibre.utils.search_query_parser import SearchQueryParser + +class BooksModel(QAbstractItemModel): + + total_changed = pyqtSignal(int) + + HEADERS = [_('Title'), _('Author(s)'), _('Format')] + + def __init__(self, all_books): + QAbstractItemModel.__init__(self) + self.books = all_books + self.all_books = all_books + self.filter = '' + self.search_filter = SearchFilter(all_books) + self.sort_col = 0 + self.sort_order = Qt.AscendingOrder + + def get_book(self, index): + row = index.row() + if row < len(self.books): + return self.books[row] + else: + return None + + def search(self, filter): + self.filter = filter.strip() + if not self.filter: + self.books = self.all_books + else: + try: + self.books = list(self.search_filter.parse(self.filter)) + except: + self.books = self.all_books + self.sort(self.sort_col, self.sort_order) + self.total_changed.emit(self.rowCount()) + + def index(self, row, column, parent=QModelIndex()): + return self.createIndex(row, column) + + def parent(self, index): + if not index.isValid() or index.internalId() == 0: + return QModelIndex() + return self.createIndex(0, 0) + + def rowCount(self, *args): + return len(self.books) + + def columnCount(self, *args): + return len(self.HEADERS) + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return NONE + text = '' + if orientation == Qt.Horizontal: + if section < len(self.HEADERS): + text = self.HEADERS[section] + return QVariant(text) + else: + return QVariant(section+1) + + def data(self, index, role): + row, col = index.row(), index.column() + result = self.books[row] + if role == Qt.DisplayRole: + if col == 0: + return QVariant(result.title) + elif col == 1: + return QVariant(result.author) + elif col == 2: + return QVariant(result.formats) + return NONE + + def data_as_text(self, result, col): + text = '' + if col == 0: + text = result.title + elif col == 1: + text = result.author + elif col == 2: + text = result.formats + return text + + def sort(self, col, order, reset=True): + self.sort_col = col + self.sort_order = order + if not self.books: + return + descending = order == Qt.DescendingOrder + self.books.sort(None, + lambda x: sort_key(unicode(self.data_as_text(x, col))), + descending) + if reset: + self.reset() + + +class SearchFilter(SearchQueryParser): + + USABLE_LOCATIONS = [ + 'all', + 'author', + 'authors', + 'format', + 'formats', + 'title', + ] + + def __init__(self, all_books=[]): + SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS) + self.srs = set(all_books) + + def universal_set(self): + return self.srs + + def get_matches(self, location, query): + location = location.lower().strip() + if location == 'authors': + location = 'author' + elif location == 'formats': + location = 'format' + + matchkind = CONTAINS_MATCH + if len(query) > 1: + if query.startswith('\\'): + query = query[1:] + elif query.startswith('='): + matchkind = EQUALS_MATCH + query = query[1:] + elif query.startswith('~'): + matchkind = REGEXP_MATCH + query = query[1:] + if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D + query = query.lower() + + if location not in self.USABLE_LOCATIONS: + return set([]) + matches = set([]) + all_locs = set(self.USABLE_LOCATIONS) - set(['all']) + locations = all_locs if location == 'all' else [location] + q = { + 'author': lambda x: x.author.lower(), + 'format': attrgetter('formats'), + 'title': lambda x: x.title.lower(), + } + for x in ('author', 'format'): + q[x+'s'] = q[x] + for sr in self.srs: + for locvalue in locations: + accessor = q[locvalue] + if query == 'true': + if accessor(sr) is not None: + matches.add(sr) + continue + if query == 'false': + if accessor(sr) is None: + matches.add(sr) + continue + try: + ### Can't separate authors because comma is used for name sep and author sep + ### Exact match might not get what you want. For that reason, turn author + ### exactmatch searches into contains searches. + if locvalue == 'author' and matchkind == EQUALS_MATCH: + m = CONTAINS_MATCH + else: + m = matchkind + + vals = [accessor(sr)] + if _match(query, vals, m): + matches.add(sr) + break + except ValueError: # Unicode errors + import traceback + traceback.print_exc() + return matches diff --git a/src/calibre/gui2/store/mobileread/store_dialog.py b/src/calibre/gui2/store/mobileread/store_dialog.py new file mode 100644 index 0000000000..af300565aa --- /dev/null +++ b/src/calibre/gui2/store/mobileread/store_dialog.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import (Qt, QDialog, QIcon) + +from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog +from calibre.gui2.store.mobileread.models import BooksModel +from calibre.gui2.store.mobileread.store_dialog_ui import Ui_Dialog + +class MobeReadStoreDialog(QDialog, Ui_Dialog): + + def __init__(self, plugin, *args): + QDialog.__init__(self, *args) + self.setupUi(self) + + self.plugin = plugin + + self.adv_search_button.setIcon(QIcon(I('search.png'))) + + self._model = BooksModel(self.plugin.get_book_list()) + self.results_view.setModel(self._model) + self.total.setText('%s' % self.results_view.model().rowCount()) + + self.search_button.clicked.connect(self.do_search) + self.adv_search_button.clicked.connect(self.build_adv_search) + self.results_view.activated.connect(self.open_store) + self.results_view.model().total_changed.connect(self.update_book_total) + self.finished.connect(self.dialog_closed) + + self.restore_state() + + def do_search(self): + self.results_view.model().search(unicode(self.search_query.text())) + + def open_store(self, index): + result = self.results_view.model().get_book(index) + if result: + self.plugin.open(self, result.detail_item) + + def update_book_total(self, total): + self.total.setText('%s' % total) + + def build_adv_search(self): + adv = AdvSearchBuilderDialog(self) + adv.price_label.hide() + adv.price_box.hide() + if adv.exec_() == QDialog.Accepted: + self.search_query.setText(adv.search_string()) + + def restore_state(self): + geometry = self.plugin.config.get('dialog_geometry', None) + if geometry: + self.restoreGeometry(geometry) + + results_cwidth = self.plugin.config.get('dialog_results_view_column_width') + if results_cwidth: + for i, x in enumerate(results_cwidth): + if i >= self.results_view.model().columnCount(): + break + self.results_view.setColumnWidth(i, x) + else: + for i in xrange(self.results_view.model().columnCount()): + self.results_view.resizeColumnToContents(i) + + self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0) + self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder) + self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) + self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order) + + def save_state(self): + self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry()) + self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.results_view.model().columnCount())] + self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col + self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order + + def dialog_closed(self, result): + self.save_state() diff --git a/src/calibre/gui2/store/mobileread_store_dialog.ui b/src/calibre/gui2/store/mobileread/store_dialog.ui similarity index 100% rename from src/calibre/gui2/store/mobileread_store_dialog.ui rename to src/calibre/gui2/store/mobileread/store_dialog.ui diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py deleted file mode 100644 index 3547eb555c..0000000000 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ /dev/null @@ -1,376 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import (unicode_literals, division, absolute_import, print_function) - -__license__ = 'GPL 3' -__copyright__ = '2011, John Schember ' -__docformat__ = 'restructuredtext en' - -import difflib -import heapq -import time -from contextlib import closing -from operator import attrgetter -from threading import RLock - -from lxml import html - -from PyQt4.Qt import Qt, QUrl, QDialog, QAbstractItemModel, QModelIndex, QVariant, \ - pyqtSignal, QIcon - -from calibre import browser -from calibre.gui2 import open_url, NONE -from calibre.gui2.store import StorePlugin -from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.mobileread_store_dialog_ui import Ui_Dialog -from calibre.gui2.store.search_result import SearchResult -from calibre.gui2.store.web_store_dialog import WebStoreDialog -from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog -from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ - REGEXP_MATCH -from calibre.utils.icu import sort_key -from calibre.utils.search_query_parser import SearchQueryParser - -class MobileReadStore(BasicStoreConfig, StorePlugin): - - def genesis(self): - self.rlock = RLock() - - def open(self, parent=None, detail_item=None, external=False): - url = 'http://www.mobileread.com/' - - if external or self.config.get('open_external', False): - open_url(QUrl(detail_item if detail_item else url)) - else: - if detail_item: - d = WebStoreDialog(self.gui, url, parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(self.config.get('tags', '')) - d.exec_() - else: - d = MobeReadStoreDialog(self, parent) - d.setWindowTitle(self.name) - d.exec_() - - def search(self, query, max_results=10, timeout=60): - books = self.get_book_list(timeout=timeout) - - sf = SearchFilter(books) - matches = sf.parse(query) - - for book in matches: - book.price = '$0.00' - book.drm = SearchResult.DRM_UNLOCKED - yield book - - def update_book_list(self, timeout=10): - with self.rlock: - url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' - - last_download = self.config.get('last_download', None) - # Don't update the book list if our cache is less than one week old. - if last_download and (time.time() - last_download) < 604800: - return - - # Download the book list HTML file from MobileRead. - br = browser() - raw_data = None - with closing(br.open(url, timeout=timeout)) as f: - raw_data = f.read() - - if not raw_data: - return - - # Turn books listed in the HTML file into SearchResults's. - books = [] - try: - data = html.fromstring(raw_data) - for book_data in data.xpath('//ul/li'): - book = SearchResult() - book.detail_item = ''.join(book_data.xpath('.//a/@href')) - book.formats = ''.join(book_data.xpath('.//i/text()')) - book.formats = book.formats.strip() - - text = ''.join(book_data.xpath('.//a/text()')) - if ':' in text: - book.author, q, text = text.partition(':') - book.author = book.author.strip() - book.title = text.strip() - books.append(book) - except: - pass - - # Save the book list and it's create time. - if books: - self.config['last_download'] = time.time() - self.config['book_list'] = self.seralize_books(books) - - def get_book_list(self, timeout=10): - self.update_book_list(timeout=timeout) - return self.deseralize_books(self.config.get('book_list', [])) - - def seralize_books(self, books): - sbooks = [] - for b in books: - data = {} - data['author'] = b.author - data['title'] = b.title - data['detail_item'] = b.detail_item - data['formats'] = b.formats - sbooks.append(data) - return sbooks - - def deseralize_books(self, sbooks): - books = [] - for s in sbooks: - b = SearchResult() - b.author = s.get('author', '') - b.title = s.get('title', '') - b.detail_item = s.get('detail_item', '') - b.formats = s.get('formats', '') - books.append(b) - return books - - -class MobeReadStoreDialog(QDialog, Ui_Dialog): - - def __init__(self, plugin, *args): - QDialog.__init__(self, *args) - self.setupUi(self) - - self.plugin = plugin - - self.adv_search_button.setIcon(QIcon(I('search.png'))) - - self._model = BooksModel(self.plugin.get_book_list()) - self.results_view.setModel(self._model) - self.total.setText('%s' % self.results_view.model().rowCount()) - - self.search_button.clicked.connect(self.do_search) - self.adv_search_button.clicked.connect(self.build_adv_search) - self.results_view.activated.connect(self.open_store) - self.results_view.model().total_changed.connect(self.update_book_total) - self.finished.connect(self.dialog_closed) - - self.restore_state() - - def do_search(self): - self.results_view.model().search(unicode(self.search_query.text())) - - def open_store(self, index): - result = self.results_view.model().get_book(index) - if result: - self.plugin.open(self, result.detail_item) - - def update_book_total(self, total): - self.total.setText('%s' % total) - - def build_adv_search(self): - adv = AdvSearchBuilderDialog(self) - adv.price_label.hide() - adv.price_box.hide() - if adv.exec_() == QDialog.Accepted: - self.search_query.setText(adv.search_string()) - - def restore_state(self): - geometry = self.plugin.config.get('dialog_geometry', None) - if geometry: - self.restoreGeometry(geometry) - - results_cwidth = self.plugin.config.get('dialog_results_view_column_width') - if results_cwidth: - for i, x in enumerate(results_cwidth): - if i >= self.results_view.model().columnCount(): - break - self.results_view.setColumnWidth(i, x) - else: - for i in xrange(self.results_view.model().columnCount()): - self.results_view.resizeColumnToContents(i) - - self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0) - self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder) - self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) - self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order) - - def save_state(self): - self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry()) - self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.results_view.model().columnCount())] - self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col - self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order - - def dialog_closed(self, result): - self.save_state() - - -class BooksModel(QAbstractItemModel): - - total_changed = pyqtSignal(int) - - HEADERS = [_('Title'), _('Author(s)'), _('Format')] - - def __init__(self, all_books): - QAbstractItemModel.__init__(self) - self.books = all_books - self.all_books = all_books - self.filter = '' - self.search_filter = SearchFilter(all_books) - self.sort_col = 0 - self.sort_order = Qt.AscendingOrder - - def get_book(self, index): - row = index.row() - if row < len(self.books): - return self.books[row] - else: - return None - - def search(self, filter): - self.filter = filter.strip() - if not self.filter: - self.books = self.all_books - else: - try: - self.books = list(self.search_filter.parse(self.filter)) - except: - self.books = self.all_books - self.sort(self.sort_col, self.sort_order) - self.total_changed.emit(self.rowCount()) - - def index(self, row, column, parent=QModelIndex()): - return self.createIndex(row, column) - - def parent(self, index): - if not index.isValid() or index.internalId() == 0: - return QModelIndex() - return self.createIndex(0, 0) - - def rowCount(self, *args): - return len(self.books) - - def columnCount(self, *args): - return len(self.HEADERS) - - def headerData(self, section, orientation, role): - if role != Qt.DisplayRole: - return NONE - text = '' - if orientation == Qt.Horizontal: - if section < len(self.HEADERS): - text = self.HEADERS[section] - return QVariant(text) - else: - return QVariant(section+1) - - def data(self, index, role): - row, col = index.row(), index.column() - result = self.books[row] - if role == Qt.DisplayRole: - if col == 0: - return QVariant(result.title) - elif col == 1: - return QVariant(result.author) - elif col == 2: - return QVariant(result.formats) - return NONE - - def data_as_text(self, result, col): - text = '' - if col == 0: - text = result.title - elif col == 1: - text = result.author - elif col == 2: - text = result.formats - return text - - def sort(self, col, order, reset=True): - self.sort_col = col - self.sort_order = order - if not self.books: - return - descending = order == Qt.DescendingOrder - self.books.sort(None, - lambda x: sort_key(unicode(self.data_as_text(x, col))), - descending) - if reset: - self.reset() - - -class SearchFilter(SearchQueryParser): - - USABLE_LOCATIONS = [ - 'all', - 'author', - 'authors', - 'format', - 'formats', - 'title', - ] - - def __init__(self, all_books=[]): - SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS) - self.srs = set(all_books) - - def universal_set(self): - return self.srs - - def get_matches(self, location, query): - location = location.lower().strip() - if location == 'authors': - location = 'author' - elif location == 'formats': - location = 'format' - - matchkind = CONTAINS_MATCH - if len(query) > 1: - if query.startswith('\\'): - query = query[1:] - elif query.startswith('='): - matchkind = EQUALS_MATCH - query = query[1:] - elif query.startswith('~'): - matchkind = REGEXP_MATCH - query = query[1:] - if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D - query = query.lower() - - if location not in self.USABLE_LOCATIONS: - return set([]) - matches = set([]) - all_locs = set(self.USABLE_LOCATIONS) - set(['all']) - locations = all_locs if location == 'all' else [location] - q = { - 'author': lambda x: x.author.lower(), - 'format': attrgetter('formats'), - 'title': lambda x: x.title.lower(), - } - for x in ('author', 'format'): - q[x+'s'] = q[x] - for sr in self.srs: - for locvalue in locations: - accessor = q[locvalue] - if query == 'true': - if accessor(sr) is not None: - matches.add(sr) - continue - if query == 'false': - if accessor(sr) is None: - matches.add(sr) - continue - try: - ### Can't separate authors because comma is used for name sep and author sep - ### Exact match might not get what you want. For that reason, turn author - ### exactmatch searches into contains searches. - if locvalue == 'author' and matchkind == EQUALS_MATCH: - m = CONTAINS_MATCH - else: - m = matchkind - - vals = [accessor(sr)] - if _match(query, vals, m): - matches.add(sr) - break - except ValueError: # Unicode errors - import traceback - traceback.print_exc() - return matches diff --git a/src/calibre/gui2/store/search/download_thread.py b/src/calibre/gui2/store/search/download_thread.py index 4dd3c4a59b..6dd59cc5a7 100644 --- a/src/calibre/gui2/store/search/download_thread.py +++ b/src/calibre/gui2/store/search/download_thread.py @@ -192,3 +192,33 @@ class DetailsThread(Thread): self.tasks.task_done() except: continue + + +class CacheUpdateThreadPool(GenericDownloadThreadPool): + + def __init__(self, thread_count): + GenericDownloadThreadPool.__init__(self, CacheUpdateThread, thread_count) + + def add_task(self, store_plugin, timeout=10): + self.tasks.put((store_plugin, timeout)) + GenericDownloadThreadPool.add_task(self) + + +class CacheUpdateThread(Thread): + + def __init__(self, tasks, results): + Thread.__init__(self) + self.daemon = True + self.tasks = tasks + self._run = True + + def abort(self): + self._run = False + + def run(self): + while self._run and not self.tasks.empty(): + try: + store_plugin, timeout = self.tasks.get() + store_plugin.update_cache(timeout=timeout, suppress_progress=True) + except: + traceback.print_exc() diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 7e92621932..70e92d1756 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -14,7 +14,8 @@ from PyQt4.Qt import (Qt, QDialog, QTimer, QCheckBox, QVBoxLayout, QIcon) from calibre.gui2 import JSONConfig, info_dialog from calibre.gui2.progress_indicator import ProgressIndicator from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog -from calibre.gui2.store.search.download_thread import SearchThreadPool +from calibre.gui2.store.search.download_thread import SearchThreadPool, \ + CacheUpdateThreadPool from calibre.gui2.store.search.search_ui import Ui_Dialog HANG_TIME = 75000 # milliseconds seconds @@ -31,10 +32,15 @@ class SearchDialog(QDialog, Ui_Dialog): # We keep a cache of store plugins and reference them by name. self.store_plugins = istores self.search_pool = SearchThreadPool(4) + self.cache_pool = CacheUpdateThreadPool(2) # Check for results and hung threads. self.checker = QTimer() self.progress_checker = QTimer() self.hang_check = 0 + + # Update store caches silently. + for p in self.store_plugins.values(): + self.cache_pool.add_task(p, 30) # Add check boxes for each store so the user # can disable searching specific stores on a @@ -116,10 +122,9 @@ class SearchDialog(QDialog, Ui_Dialog): for n in store_names: if getattr(self, 'store_check_' + n).isChecked(): self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT) - if self.search_pool.has_tasks() or self.search_pool.threads_running(): - self.hang_check = 0 - self.checker.start(100) - self.pi.startAnimation() + self.hang_check = 0 + self.checker.start(100) + self.pi.startAnimation() def clean_query(self, query): query = query.lower() @@ -200,10 +205,9 @@ class SearchDialog(QDialog, Ui_Dialog): res, store_plugin = self.search_pool.get_result() if res: self.results_view.model().add_result(res, store_plugin) - - if not self.checker.isActive(): - if not self.results_view.model().has_results(): - info_dialog(self, _('No matches'), _('Couldn\'t find any books matching your query.'), show=True, show_copy_button=False) + + if not self.results_view.model().has_results(): + info_dialog(self, _('No matches'), _('Couldn\'t find any books matching your query.'), show=True, show_copy_button=False) def open_store(self, index):