diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 00af4e5117..c27fa2a57b 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -1162,7 +1162,7 @@ class StoreManyBooksStore(StoreBase): class StoreMobileReadStore(StoreBase): name = 'MobileRead' description = _('Ebooks handcrafted with the utmost care') - actual_plugin = 'calibre.gui2.store.mobileread_plugin:MobileReadStore' + actual_plugin = 'calibre.gui2.store.mobileread.mobileread_plugin:MobileReadStore' class StoreOpenLibraryStore(StoreBase): name = 'Open Library' diff --git a/src/calibre/gui2/store/__init__.py b/src/calibre/gui2/store/__init__.py index 16e9f5689d..c95d794975 100644 --- a/src/calibre/gui2/store/__init__.py +++ b/src/calibre/gui2/store/__init__.py @@ -127,6 +127,40 @@ class StorePlugin(object): # {{{ ''' return False + def update_cache(self, parent=None, timeout=60, force=False, suppress_progress=False): + ''' + Some plugins need to keep an local cache of available books. This function + is called to update the caches. It is recommended to call this function + from :meth:`open`. Especially if :meth:`open` does anything other than + open a web page. + + This function can be called at any time. It is up to the plugin to determine + if the cache really does need updating. Unless :param:`force` is True, then + the plugin must update the cache. The only time force should be True is if + this function is called by the plugin's configuration dialog. + + if :param:`suppress_progress` is False it is safe to assume that this function + is being called from the main GUI thread so it is safe and recommended to use + a QProgressDialog to display what is happening and allow the user to cancel + the operation. if :param:`suppress_progress` is True then run the update + silently. In this case there is no guarantee what thread is calling this + function so no Qt related functionality that requires being run in the main + GUI thread should be run. E.G. Open a QProgressDialog. + + :param parent: The parent object to be used by an GUI dialogs. + + :param timeout: The maximum amount of time that should be spent in + any given network connection. + + :param force: Force updating the cache even if the plugin has determined + it is not necessary. + + :param suppress_progress: Should a progress indicator be shown. + + :return: True if the cache was updated, False otherwise. + ''' + return False + def get_settings(self): ''' This is only useful for plugins that implement diff --git a/src/calibre/gui2/store/mobileread/__init__.py b/src/calibre/gui2/store/mobileread/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/calibre/gui2/store/mobileread/cache_progress_dialog.py b/src/calibre/gui2/store/mobileread/cache_progress_dialog.py new file mode 100644 index 0000000000..71416d8680 --- /dev/null +++ b/src/calibre/gui2/store/mobileread/cache_progress_dialog.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from PyQt4.Qt import QDialog + +from calibre.gui2.store.mobileread.cache_progress_dialog_ui import Ui_Dialog + +class CacheProgressDialog(QDialog, Ui_Dialog): + + def __init__(self, parent=None, total=None): + QDialog.__init__(self, parent) + self.setupUi(self) + + self.completed = 0 + self.canceled = False + + self.progress.setValue(0) + self.progress.setMinimum(0) + self.progress.setMaximum(total if total else 0) + + def exec_(self): + self.completed = 0 + self.canceled = False + QDialog.exec_(self) + + def open(self): + self.completed = 0 + self.canceled = False + QDialog.open(self) + + def reject(self): + self.canceled = True + QDialog.reject(self) + + def update_progress(self): + ''' + completed is an int from 0 to total representing the number + records that have bee completed. + ''' + self.set_progress(self.completed + 1) + + def set_message(self, msg): + self.message.setText(msg) + + def set_details(self, msg): + self.details.setText(msg) + + def set_progress(self, completed): + ''' + completed is an int from 0 to total representing the number + records that have bee completed. + ''' + self.completed = completed + self.progress.setValue(self.completed) + + def set_total(self, total): + self.progress.setMaximum(total) diff --git a/src/calibre/gui2/store/mobileread/cache_progress_dialog.ui b/src/calibre/gui2/store/mobileread/cache_progress_dialog.ui new file mode 100644 index 0000000000..4690f14e7f --- /dev/null +++ b/src/calibre/gui2/store/mobileread/cache_progress_dialog.ui @@ -0,0 +1,104 @@ + + + Dialog + + + + 0 + 0 + 402 + 138 + + + + Dialog + + + + + + Updating book cache + + + Qt::AlignCenter + + + + + + + 24 + + + + + + + + + + Qt::AlignCenter + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel + + + + + + + + + buttonBox + accepted() + Dialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + Dialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + diff --git a/src/calibre/gui2/store/mobileread/cache_update_thread.py b/src/calibre/gui2/store/mobileread/cache_update_thread.py new file mode 100644 index 0000000000..f81e7951d4 --- /dev/null +++ b/src/calibre/gui2/store/mobileread/cache_update_thread.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import time +from contextlib import closing +from threading import Thread + +from lxml import html + +from PyQt4.Qt import (pyqtSignal, QObject) + +from calibre import browser +from calibre.gui2.store.search_result import SearchResult + +class CacheUpdateThread(Thread, QObject): + + total_changed = pyqtSignal(int) + update_progress = pyqtSignal(int) + update_details = pyqtSignal(unicode) + + def __init__(self, config, seralize_books_function, timeout): + Thread.__init__(self) + QObject.__init__(self) + + self.daemon = True + self.config = config + self.seralize_books = seralize_books_function + self.timeout = timeout + self._run = True + + def abort(self): + self._run = False + + def run(self): + url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' + + self.update_details.emit(_('Checking last download date.')) + last_download = self.config.get('last_download', None) + # Don't update the book list if our cache is less than one week old. + if last_download and (time.time() - last_download) < 604800: + return + + self.update_details.emit(_('Downloading book list from MobileRead.')) + # Download the book list HTML file from MobileRead. + br = browser() + raw_data = None + try: + with closing(br.open(url, timeout=self.timeout)) as f: + raw_data = f.read() + except: + return + + if not raw_data or not self._run: + return + + self.update_details.emit(_('Processing books.')) + # Turn books listed in the HTML file into SearchResults's. + books = [] + try: + data = html.fromstring(raw_data) + raw_books = data.xpath('//ul/li') + self.total_changed.emit(len(raw_books)) + + for i, book_data in enumerate(raw_books): + self.update_details.emit(_('%s of %s books processed.') % (i, len(raw_books))) + book = SearchResult() + book.detail_item = ''.join(book_data.xpath('.//a/@href')) + book.formats = ''.join(book_data.xpath('.//i/text()')) + book.formats = book.formats.strip() + + text = ''.join(book_data.xpath('.//a/text()')) + if ':' in text: + book.author, q, text = text.partition(':') + book.author = book.author.strip() + book.title = text.strip() + books.append(book) + + if not self._run: + books = [] + break + else: + self.update_progress.emit(i) + except: + pass + + # Save the book list and it's create time. + if books: + self.config['book_list'] = self.seralize_books(books) + self.config['last_download'] = time.time() diff --git a/src/calibre/gui2/store/mobileread/mobileread_plugin.py b/src/calibre/gui2/store/mobileread/mobileread_plugin.py new file mode 100644 index 0000000000..54242ce0b2 --- /dev/null +++ b/src/calibre/gui2/store/mobileread/mobileread_plugin.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from threading import Lock + +from PyQt4.Qt import (QUrl, QCoreApplication) + +from calibre.gui2 import open_url +from calibre.gui2.store import StorePlugin +from calibre.gui2.store.basic_config import BasicStoreConfig +from calibre.gui2.store.search_result import SearchResult +from calibre.gui2.store.web_store_dialog import WebStoreDialog +from calibre.gui2.store.mobileread.models import SearchFilter +from calibre.gui2.store.mobileread.cache_progress_dialog import CacheProgressDialog +from calibre.gui2.store.mobileread.cache_update_thread import CacheUpdateThread +from calibre.gui2.store.mobileread.store_dialog import MobeReadStoreDialog + +class MobileReadStore(BasicStoreConfig, StorePlugin): + + def genesis(self): + self.lock = Lock() + + def open(self, parent=None, detail_item=None, external=False): + url = 'http://www.mobileread.com/' + + if external or self.config.get('open_external', False): + open_url(QUrl(detail_item if detail_item else url)) + else: + if detail_item: + d = WebStoreDialog(self.gui, url, parent, detail_item) + d.setWindowTitle(self.name) + d.set_tags(self.config.get('tags', '')) + d.exec_() + else: + if self.update_cache(parent, 30): + d = MobeReadStoreDialog(self, parent) + d.setWindowTitle(self.name) + d.exec_() + + def search(self, query, max_results=10, timeout=60): + books = self.get_book_list() + + sf = SearchFilter(books) + matches = sf.parse(query) + + for book in matches: + book.price = '$0.00' + book.drm = SearchResult.DRM_UNLOCKED + yield book + + def update_cache(self, parent=None, timeout=10, force=False, suppress_progress=False): + if self.lock.acquire(False): + try: + update_thread = CacheUpdateThread(self.config, self.seralize_books, timeout) + if not suppress_progress: + progress = CacheProgressDialog(parent) + progress.set_message(_('Updating MobileRead book cache...')) + + update_thread.total_changed.connect(progress.set_total) + update_thread.update_progress.connect(progress.set_progress) + update_thread.update_details.connect(progress.set_details) + progress.rejected.connect(update_thread.abort) + + progress.open() + update_thread.start() + while update_thread.is_alive() and not progress.canceled: + QCoreApplication.processEvents() + + if progress.isVisible(): + progress.accept() + return not progress.canceled + else: + update_thread.start() + finally: + self.lock.release() + + def get_book_list(self): + return self.deseralize_books(self.config.get('book_list', [])) + + def seralize_books(self, books): + sbooks = [] + for b in books: + data = {} + data['author'] = b.author + data['title'] = b.title + data['detail_item'] = b.detail_item + data['formats'] = b.formats + sbooks.append(data) + return sbooks + + def deseralize_books(self, sbooks): + books = [] + for s in sbooks: + b = SearchResult() + b.author = s.get('author', '') + b.title = s.get('title', '') + b.detail_item = s.get('detail_item', '') + b.formats = s.get('formats', '') + books.append(b) + return books diff --git a/src/calibre/gui2/store/mobileread/models.py b/src/calibre/gui2/store/mobileread/models.py new file mode 100644 index 0000000000..a080affb51 --- /dev/null +++ b/src/calibre/gui2/store/mobileread/models.py @@ -0,0 +1,190 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +from operator import attrgetter + +from PyQt4.Qt import (Qt, QAbstractItemModel, QModelIndex, QVariant, pyqtSignal) + +from calibre.gui2 import NONE +from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ + REGEXP_MATCH +from calibre.utils.icu import sort_key +from calibre.utils.search_query_parser import SearchQueryParser + +class BooksModel(QAbstractItemModel): + + total_changed = pyqtSignal(int) + + HEADERS = [_('Title'), _('Author(s)'), _('Format')] + + def __init__(self, all_books): + QAbstractItemModel.__init__(self) + self.books = all_books + self.all_books = all_books + self.filter = '' + self.search_filter = SearchFilter(all_books) + self.sort_col = 0 + self.sort_order = Qt.AscendingOrder + + def get_book(self, index): + row = index.row() + if row < len(self.books): + return self.books[row] + else: + return None + + def search(self, filter): + self.filter = filter.strip() + if not self.filter: + self.books = self.all_books + else: + try: + self.books = list(self.search_filter.parse(self.filter)) + except: + self.books = self.all_books + self.sort(self.sort_col, self.sort_order) + self.total_changed.emit(self.rowCount()) + + def index(self, row, column, parent=QModelIndex()): + return self.createIndex(row, column) + + def parent(self, index): + if not index.isValid() or index.internalId() == 0: + return QModelIndex() + return self.createIndex(0, 0) + + def rowCount(self, *args): + return len(self.books) + + def columnCount(self, *args): + return len(self.HEADERS) + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return NONE + text = '' + if orientation == Qt.Horizontal: + if section < len(self.HEADERS): + text = self.HEADERS[section] + return QVariant(text) + else: + return QVariant(section+1) + + def data(self, index, role): + row, col = index.row(), index.column() + result = self.books[row] + if role == Qt.DisplayRole: + if col == 0: + return QVariant(result.title) + elif col == 1: + return QVariant(result.author) + elif col == 2: + return QVariant(result.formats) + return NONE + + def data_as_text(self, result, col): + text = '' + if col == 0: + text = result.title + elif col == 1: + text = result.author + elif col == 2: + text = result.formats + return text + + def sort(self, col, order, reset=True): + self.sort_col = col + self.sort_order = order + if not self.books: + return + descending = order == Qt.DescendingOrder + self.books.sort(None, + lambda x: sort_key(unicode(self.data_as_text(x, col))), + descending) + if reset: + self.reset() + + +class SearchFilter(SearchQueryParser): + + USABLE_LOCATIONS = [ + 'all', + 'author', + 'authors', + 'format', + 'formats', + 'title', + ] + + def __init__(self, all_books=[]): + SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS) + self.srs = set(all_books) + + def universal_set(self): + return self.srs + + def get_matches(self, location, query): + location = location.lower().strip() + if location == 'authors': + location = 'author' + elif location == 'formats': + location = 'format' + + matchkind = CONTAINS_MATCH + if len(query) > 1: + if query.startswith('\\'): + query = query[1:] + elif query.startswith('='): + matchkind = EQUALS_MATCH + query = query[1:] + elif query.startswith('~'): + matchkind = REGEXP_MATCH + query = query[1:] + if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D + query = query.lower() + + if location not in self.USABLE_LOCATIONS: + return set([]) + matches = set([]) + all_locs = set(self.USABLE_LOCATIONS) - set(['all']) + locations = all_locs if location == 'all' else [location] + q = { + 'author': lambda x: x.author.lower(), + 'format': attrgetter('formats'), + 'title': lambda x: x.title.lower(), + } + for x in ('author', 'format'): + q[x+'s'] = q[x] + for sr in self.srs: + for locvalue in locations: + accessor = q[locvalue] + if query == 'true': + if accessor(sr) is not None: + matches.add(sr) + continue + if query == 'false': + if accessor(sr) is None: + matches.add(sr) + continue + try: + ### Can't separate authors because comma is used for name sep and author sep + ### Exact match might not get what you want. For that reason, turn author + ### exactmatch searches into contains searches. + if locvalue == 'author' and matchkind == EQUALS_MATCH: + m = CONTAINS_MATCH + else: + m = matchkind + + vals = [accessor(sr)] + if _match(query, vals, m): + matches.add(sr) + break + except ValueError: # Unicode errors + import traceback + traceback.print_exc() + return matches diff --git a/src/calibre/gui2/store/mobileread/store_dialog.py b/src/calibre/gui2/store/mobileread/store_dialog.py new file mode 100644 index 0000000000..af300565aa --- /dev/null +++ b/src/calibre/gui2/store/mobileread/store_dialog.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import (Qt, QDialog, QIcon) + +from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog +from calibre.gui2.store.mobileread.models import BooksModel +from calibre.gui2.store.mobileread.store_dialog_ui import Ui_Dialog + +class MobeReadStoreDialog(QDialog, Ui_Dialog): + + def __init__(self, plugin, *args): + QDialog.__init__(self, *args) + self.setupUi(self) + + self.plugin = plugin + + self.adv_search_button.setIcon(QIcon(I('search.png'))) + + self._model = BooksModel(self.plugin.get_book_list()) + self.results_view.setModel(self._model) + self.total.setText('%s' % self.results_view.model().rowCount()) + + self.search_button.clicked.connect(self.do_search) + self.adv_search_button.clicked.connect(self.build_adv_search) + self.results_view.activated.connect(self.open_store) + self.results_view.model().total_changed.connect(self.update_book_total) + self.finished.connect(self.dialog_closed) + + self.restore_state() + + def do_search(self): + self.results_view.model().search(unicode(self.search_query.text())) + + def open_store(self, index): + result = self.results_view.model().get_book(index) + if result: + self.plugin.open(self, result.detail_item) + + def update_book_total(self, total): + self.total.setText('%s' % total) + + def build_adv_search(self): + adv = AdvSearchBuilderDialog(self) + adv.price_label.hide() + adv.price_box.hide() + if adv.exec_() == QDialog.Accepted: + self.search_query.setText(adv.search_string()) + + def restore_state(self): + geometry = self.plugin.config.get('dialog_geometry', None) + if geometry: + self.restoreGeometry(geometry) + + results_cwidth = self.plugin.config.get('dialog_results_view_column_width') + if results_cwidth: + for i, x in enumerate(results_cwidth): + if i >= self.results_view.model().columnCount(): + break + self.results_view.setColumnWidth(i, x) + else: + for i in xrange(self.results_view.model().columnCount()): + self.results_view.resizeColumnToContents(i) + + self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0) + self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder) + self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) + self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order) + + def save_state(self): + self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry()) + self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.results_view.model().columnCount())] + self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col + self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order + + def dialog_closed(self, result): + self.save_state() diff --git a/src/calibre/gui2/store/mobileread_store_dialog.ui b/src/calibre/gui2/store/mobileread/store_dialog.ui similarity index 84% rename from src/calibre/gui2/store/mobileread_store_dialog.ui rename to src/calibre/gui2/store/mobileread/store_dialog.ui index 027d5994f0..6d31efab6d 100644 --- a/src/calibre/gui2/store/mobileread_store_dialog.ui +++ b/src/calibre/gui2/store/mobileread/store_dialog.ui @@ -19,13 +19,30 @@ - Search: + &Query: + + + search_query + + + + + + + ... + + + + Search + + + diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py deleted file mode 100644 index 86cdb77e4e..0000000000 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ /dev/null @@ -1,316 +0,0 @@ -# -*- coding: utf-8 -*- - -from __future__ import (unicode_literals, division, absolute_import, print_function) - -__license__ = 'GPL 3' -__copyright__ = '2011, John Schember ' -__docformat__ = 'restructuredtext en' - -import difflib -import heapq -import time -from contextlib import closing -from threading import RLock - -from lxml import html - -from PyQt4.Qt import Qt, QUrl, QDialog, QAbstractItemModel, QModelIndex, QVariant, \ - pyqtSignal - -from calibre import browser -from calibre.gui2 import open_url, NONE -from calibre.gui2.store import StorePlugin -from calibre.gui2.store.basic_config import BasicStoreConfig -from calibre.gui2.store.mobileread_store_dialog_ui import Ui_Dialog -from calibre.gui2.store.search_result import SearchResult -from calibre.gui2.store.web_store_dialog import WebStoreDialog -from calibre.utils.icu import sort_key - -class MobileReadStore(BasicStoreConfig, StorePlugin): - - def genesis(self): - self.rlock = RLock() - - def open(self, parent=None, detail_item=None, external=False): - url = 'http://www.mobileread.com/' - - if external or self.config.get('open_external', False): - open_url(QUrl(detail_item if detail_item else url)) - else: - if detail_item: - d = WebStoreDialog(self.gui, url, parent, detail_item) - d.setWindowTitle(self.name) - d.set_tags(self.config.get('tags', '')) - d.exec_() - else: - d = MobeReadStoreDialog(self, parent) - d.setWindowTitle(self.name) - d.exec_() - - def search(self, query, max_results=10, timeout=60): - books = self.get_book_list(timeout=timeout) - - query = query.lower() - query_parts = query.split(' ') - matches = [] - s = difflib.SequenceMatcher() - for x in books: - ratio = 0 - t_string = '%s %s' % (x.author.lower(), x.title.lower()) - query_matches = [] - for q in query_parts: - if q in t_string: - query_matches.append(q) - for q in query_matches: - s.set_seq2(q) - for p in t_string.split(' '): - s.set_seq1(p) - ratio += s.ratio() - if ratio > 0: - matches.append((ratio, x)) - - # Move the best scorers to head of list. - matches = heapq.nlargest(max_results, matches) - for score, book in matches: - book.price = '$0.00' - book.drm = SearchResult.DRM_UNLOCKED - yield book - - def update_book_list(self, timeout=10): - with self.rlock: - url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' - - last_download = self.config.get('last_download', None) - # Don't update the book list if our cache is less than one week old. - if last_download and (time.time() - last_download) < 604800: - return - - # Download the book list HTML file from MobileRead. - br = browser() - raw_data = None - with closing(br.open(url, timeout=timeout)) as f: - raw_data = f.read() - - if not raw_data: - return - - # Turn books listed in the HTML file into SearchResults's. - books = [] - try: - data = html.fromstring(raw_data) - for book_data in data.xpath('//ul/li'): - book = SearchResult() - book.detail_item = ''.join(book_data.xpath('.//a/@href')) - book.formats = ''.join(book_data.xpath('.//i/text()')) - book.formats = book.formats.strip() - - text = ''.join(book_data.xpath('.//a/text()')) - if ':' in text: - book.author, q, text = text.partition(':') - book.author = book.author.strip() - book.title = text.strip() - books.append(book) - except: - pass - - # Save the book list and it's create time. - if books: - self.config['last_download'] = time.time() - self.config['book_list'] = self.seralize_books(books) - - def get_book_list(self, timeout=10): - self.update_book_list(timeout=timeout) - return self.deseralize_books(self.config.get('book_list', [])) - - def seralize_books(self, books): - sbooks = [] - for b in books: - data = {} - data['author'] = b.author - data['title'] = b.title - data['detail_item'] = b.detail_item - data['formats'] = b.formats - sbooks.append(data) - return sbooks - - def deseralize_books(self, sbooks): - books = [] - for s in sbooks: - b = SearchResult() - b.author = s.get('author', '') - b.title = s.get('title', '') - b.detail_item = s.get('detail_item', '') - b.formats = s.get('formats', '') - books.append(b) - return books - - -class MobeReadStoreDialog(QDialog, Ui_Dialog): - - def __init__(self, plugin, *args): - QDialog.__init__(self, *args) - self.setupUi(self) - - self.plugin = plugin - - self.model = BooksModel() - self.results_view.setModel(self.model) - self.results_view.model().set_books(self.plugin.get_book_list()) - self.total.setText('%s' % self.model.rowCount()) - - self.results_view.activated.connect(self.open_store) - self.search_query.textChanged.connect(self.model.set_filter) - self.results_view.model().total_changed.connect(self.total.setText) - self.finished.connect(self.dialog_closed) - - self.restore_state() - - def open_store(self, index): - result = self.results_view.model().get_book(index) - if result: - self.plugin.open(self, result.detail_item) - - def restore_state(self): - geometry = self.plugin.config.get('dialog_geometry', None) - if geometry: - self.restoreGeometry(geometry) - - results_cwidth = self.plugin.config.get('dialog_results_view_column_width') - if results_cwidth: - for i, x in enumerate(results_cwidth): - if i >= self.results_view.model().columnCount(): - break - self.results_view.setColumnWidth(i, x) - else: - for i in xrange(self.results_view.model().columnCount()): - self.results_view.resizeColumnToContents(i) - - self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0) - self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder) - self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) - self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order) - - def save_state(self): - self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry()) - self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())] - self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col - self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order - - def dialog_closed(self, result): - self.save_state() - - -class BooksModel(QAbstractItemModel): - - total_changed = pyqtSignal(unicode) - - HEADERS = [_('Title'), _('Author(s)'), _('Format')] - - def __init__(self): - QAbstractItemModel.__init__(self) - self.books = [] - self.all_books = [] - self.filter = '' - self.sort_col = 0 - self.sort_order = Qt.AscendingOrder - - def set_books(self, books): - self.books = books - self.all_books = books - - self.sort(self.sort_col, self.sort_order) - - def get_book(self, index): - row = index.row() - if row < len(self.books): - return self.books[row] - else: - return None - - def set_filter(self, filter): - #self.layoutAboutToBeChanged.emit() - self.beginResetModel() - - self.filter = unicode(filter) - self.books = [] - if self.filter: - for b in self.all_books: - test = '%s %s %s' % (b.title, b.author, b.formats) - test = test.lower() - include = True - for item in self.filter.split(' '): - item = item.lower() - if item not in test: - include = False - break - if include: - self.books.append(b) - else: - self.books = self.all_books - - self.sort(self.sort_col, self.sort_order, reset=False) - self.total_changed.emit('%s' % self.rowCount()) - - self.endResetModel() - #self.layoutChanged.emit() - - def index(self, row, column, parent=QModelIndex()): - return self.createIndex(row, column) - - def parent(self, index): - if not index.isValid() or index.internalId() == 0: - return QModelIndex() - return self.createIndex(0, 0) - - def rowCount(self, *args): - return len(self.books) - - def columnCount(self, *args): - return len(self.HEADERS) - - def headerData(self, section, orientation, role): - if role != Qt.DisplayRole: - return NONE - text = '' - if orientation == Qt.Horizontal: - if section < len(self.HEADERS): - text = self.HEADERS[section] - return QVariant(text) - else: - return QVariant(section+1) - - def data(self, index, role): - row, col = index.row(), index.column() - result = self.books[row] - if role == Qt.DisplayRole: - if col == 0: - return QVariant(result.title) - elif col == 1: - return QVariant(result.author) - elif col == 2: - return QVariant(result.formats) - return NONE - - def data_as_text(self, result, col): - text = '' - if col == 0: - text = result.title - elif col == 1: - text = result.author - elif col == 2: - text = result.formats - return text - - def sort(self, col, order, reset=True): - self.sort_col = col - self.sort_order = order - - if not self.books: - return - descending = order == Qt.DescendingOrder - self.books.sort(None, - lambda x: sort_key(unicode(self.data_as_text(x, col))), - descending) - if reset: - self.reset() - diff --git a/src/calibre/gui2/store/open_library_plugin.py b/src/calibre/gui2/store/open_library_plugin.py index 93b9c02dcf..b95f1bf930 100644 --- a/src/calibre/gui2/store/open_library_plugin.py +++ b/src/calibre/gui2/store/open_library_plugin.py @@ -50,6 +50,9 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin): if counter <= 0: break + # Don't include books that don't have downloadable files. + if not data.xpath('boolean(./span[@class="actions"]//span[@class="label" and contains(text(), "Read")])'): + continue id = ''.join(data.xpath('./span[@class="bookcover"]/a/@href')) if not id: continue @@ -67,7 +70,7 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin): s.author = author.strip() s.price = price s.detail_item = id.strip() - s.drm = SearchResult.DRM_UNKNOWN + s.drm = SearchResult.DRM_UNLOCKED yield s diff --git a/src/calibre/gui2/store/search/adv_search_builder.py b/src/calibre/gui2/store/search/adv_search_builder.py new file mode 100644 index 0000000000..50d4d3f3f4 --- /dev/null +++ b/src/calibre/gui2/store/search/adv_search_builder.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- + +from __future__ import (unicode_literals, division, absolute_import, print_function) + +__license__ = 'GPL 3' +__copyright__ = '2011, John Schember ' +__docformat__ = 'restructuredtext en' + +import re + +from PyQt4.Qt import (QDialog, QDialogButtonBox) + +from calibre.gui2.store.search.adv_search_builder_ui import Ui_Dialog +from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH + +class AdvSearchBuilderDialog(QDialog, Ui_Dialog): + + def __init__(self, parent): + QDialog.__init__(self, parent) + self.setupUi(self) + + self.buttonBox.accepted.connect(self.advanced_search_button_pushed) + self.tab_2_button_box.accepted.connect(self.accept) + self.tab_2_button_box.rejected.connect(self.reject) + self.clear_button.clicked.connect(self.clear_button_pushed) + self.adv_search_used = False + self.mc = '' + + self.tabWidget.setCurrentIndex(0) + self.tabWidget.currentChanged[int].connect(self.tab_changed) + self.tab_changed(0) + + def tab_changed(self, idx): + if idx == 1: + self.tab_2_button_box.button(QDialogButtonBox.Ok).setDefault(True) + else: + self.buttonBox.button(QDialogButtonBox.Ok).setDefault(True) + + def advanced_search_button_pushed(self): + self.adv_search_used = True + self.accept() + + def clear_button_pushed(self): + self.title_box.setText('') + self.author_box.setText('') + self.price_box.setText('') + self.format_box.setText('') + + def tokens(self, raw): + phrases = re.findall(r'\s*".*?"\s*', raw) + for f in phrases: + raw = raw.replace(f, ' ') + phrases = [t.strip('" ') for t in phrases] + return ['"' + self.mc + t + '"' for t in phrases + [r.strip() for r in raw.split()]] + + def search_string(self): + if self.adv_search_used: + return self.adv_search_string() + else: + return self.box_search_string() + + def adv_search_string(self): + mk = self.matchkind.currentIndex() + if mk == CONTAINS_MATCH: + self.mc = '' + elif mk == EQUALS_MATCH: + self.mc = '=' + else: + self.mc = '~' + all, any, phrase, none = map(lambda x: unicode(x.text()), + (self.all, self.any, self.phrase, self.none)) + all, any, none = map(self.tokens, (all, any, none)) + phrase = phrase.strip() + all = ' and '.join(all) + any = ' or '.join(any) + none = ' and not '.join(none) + ans = '' + if phrase: + ans += '"%s"'%phrase + if all: + ans += (' and ' if ans else '') + all + if none: + ans += (' and not ' if ans else 'not ') + none + if any: + ans += (' or ' if ans else '') + any + return ans + + def token(self): + txt = unicode(self.text.text()).strip() + if txt: + if self.negate.isChecked(): + txt = '!'+txt + tok = self.FIELDS[unicode(self.field.currentText())]+txt + if re.search(r'\s', tok): + tok = '"%s"'%tok + return tok + + def box_search_string(self): + mk = self.matchkind.currentIndex() + if mk == CONTAINS_MATCH: + self.mc = '' + elif mk == EQUALS_MATCH: + self.mc = '=' + else: + self.mc = '~' + + ans = [] + self.box_last_values = {} + title = unicode(self.title_box.text()).strip() + if title: + ans.append('title:"' + self.mc + title + '"') + author = unicode(self.author_box.text()).strip() + if author: + ans.append('author:"' + self.mc + author + '"') + price = unicode(self.price_box.text()).strip() + if price: + ans.append('price:"' + self.mc + price + '"') + format = unicode(self.format_box.text()).strip() + if author: + ans.append('format:"' + self.mc + format + '"') + if ans: + return ' and '.join(ans) + return '' diff --git a/src/calibre/gui2/store/search/adv_search_builder.ui b/src/calibre/gui2/store/search/adv_search_builder.ui new file mode 100644 index 0000000000..a758057311 --- /dev/null +++ b/src/calibre/gui2/store/search/adv_search_builder.ui @@ -0,0 +1,364 @@ + + + Dialog + + + + 0 + 0 + 752 + 472 + + + + Advanced Search + + + + :/images/search.png:/images/search.png + + + + + + &What kind of match to use: + + + matchkind + + + + + + + + Contains: the word or phrase matches anywhere in the metadata field + + + + + Equals: the word or phrase must match the entire metadata field + + + + + Regular expression: the expression must match anywhere in the metadata field + + + + + + + + 0 + + + + A&dvanced Search + + + + + + Find entries that have... + + + + + + + + &All these words: + + + all + + + + + + + + + + + + + + This exact &phrase: + + + all + + + + + + + + + + + + + + &One or more of these words: + + + all + + + + + + + + + + + + + + + But dont show entries that have... + + + + + + + + Any of these &unwanted words: + + + all + + + + + + + + + + + + + 16777215 + 30 + + + + See the <a href="http://calibre-ebook.com/user_manual/gui.html#the-search-interface">User Manual</a> for more help + + + true + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Qt::Horizontal + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + + Titl&e/Author/Price ... + + + + + + &Title: + + + title_box + + + + + + + Enter the title. + + + + + + + &Author: + + + author_box + + + + + + + &Price: + + + price_box + + + + + + + + + &Clear + + + + + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + Search only in specific fields: + + + + + + + + + + + + + &Format: + + + format_box + + + + + + + + + + + + + + Qt::Vertical + + + + 20 + 40 + + + + + + + + + EnLineEdit + QLineEdit +
widgets.h
+ + + + all + phrase + any + none + buttonBox + title_box + author_box + price_box + format_box + clear_button + tab_2_button_box + tabWidget + matchkind + + + + + + + buttonBox + accepted() + Dialog + accept() + + + 248 + 254 + + + 157 + 274 + + + + + buttonBox + rejected() + Dialog + reject() + + + 316 + 260 + + + 286 + 274 + + + + + diff --git a/src/calibre/gui2/store/search/download_thread.py b/src/calibre/gui2/store/search/download_thread.py index a6f92011f6..6dd59cc5a7 100644 --- a/src/calibre/gui2/store/search/download_thread.py +++ b/src/calibre/gui2/store/search/download_thread.py @@ -6,7 +6,6 @@ __license__ = 'GPL 3' __copyright__ = '2011, John Schember ' __docformat__ = 'restructuredtext en' -import time import traceback from contextlib import closing from threading import Thread @@ -17,7 +16,9 @@ from calibre.utils.magick.draw import thumbnail class GenericDownloadThreadPool(object): ''' - add_task must be implemented in a subclass. + add_task must be implemented in a subclass and must + GenericDownloadThreadPool.add_task must be called + at the end of the function. ''' def __init__(self, thread_type, thread_count): @@ -29,10 +30,16 @@ class GenericDownloadThreadPool(object): self.threads = [] def add_task(self): - raise NotImplementedError() - - def start_threads(self): - for i in range(self.thread_count): + ''' + This must be implemented in a sub class and this function + must be called at the end of the add_task function in + the sub class. + + The implementation of this function (in this base class) + starts any threads necessary to fill the pool if it is + not already full. + ''' + for i in xrange(self.thread_count - self.running_threads_count()): t = self.thread_type(self.tasks, self.results) self.threads.append(t) t.start() @@ -60,10 +67,14 @@ class GenericDownloadThreadPool(object): return not self.results.empty() def threads_running(self): + return self.running_threads_count() > 0 + + def running_threads_count(self): + count = 0 for t in self.threads: if t.is_alive(): - return True - return False + count += 1 + return count class SearchThreadPool(GenericDownloadThreadPool): @@ -73,17 +84,16 @@ class SearchThreadPool(GenericDownloadThreadPool): using start_threads(). Reset by calling abort(). Example: - sp = SearchThreadPool(SearchThread, 3) - add tasks using add_task(...) - sp.start_threads() - all threads have finished. - sp.abort() - add tasks using add_task(...) - sp.start_threads() + sp = SearchThreadPool(3) + sp.add_task(...) ''' + + def __init__(self, thread_count): + GenericDownloadThreadPool.__init__(self, SearchThread, thread_count) def add_task(self, query, store_name, store_plugin, timeout): self.tasks.put((query, store_name, store_plugin, timeout)) + GenericDownloadThreadPool.add_task(self) class SearchThread(Thread): @@ -113,12 +123,13 @@ class SearchThread(Thread): class CoverThreadPool(GenericDownloadThreadPool): - ''' - Once started all threads run until abort is called. - ''' + + def __init__(self, thread_count): + GenericDownloadThreadPool.__init__(self, CoverThread, thread_count) def add_task(self, search_result, update_callback, timeout=5): self.tasks.put((search_result, update_callback, timeout)) + GenericDownloadThreadPool.add_task(self) class CoverThread(Thread): @@ -136,30 +147,27 @@ class CoverThread(Thread): self._run = False def run(self): - while self._run: + while self._run and not self.tasks.empty(): try: - time.sleep(.1) - while not self.tasks.empty(): - if not self._run: - break - result, callback, timeout = self.tasks.get() - if result and result.cover_url: - with closing(self.br.open(result.cover_url, timeout=timeout)) as f: - result.cover_data = f.read() - result.cover_data = thumbnail(result.cover_data, 64, 64)[2] - callback() - self.tasks.task_done() + result, callback, timeout = self.tasks.get() + if result and result.cover_url: + with closing(self.br.open(result.cover_url, timeout=timeout)) as f: + result.cover_data = f.read() + result.cover_data = thumbnail(result.cover_data, 64, 64)[2] + callback() + self.tasks.task_done() except: continue class DetailsThreadPool(GenericDownloadThreadPool): - ''' - Once started all threads run until abort is called. - ''' + + def __init__(self, thread_count): + GenericDownloadThreadPool.__init__(self, DetailsThread, thread_count) def add_task(self, search_result, store_plugin, update_callback, timeout=10): self.tasks.put((search_result, store_plugin, update_callback, timeout)) + GenericDownloadThreadPool.add_task(self) class DetailsThread(Thread): @@ -175,16 +183,42 @@ class DetailsThread(Thread): self._run = False def run(self): - while self._run: + while self._run and not self.tasks.empty(): try: - time.sleep(.1) - while not self.tasks.empty(): - if not self._run: - break - result, store_plugin, callback, timeout = self.tasks.get() - if result: - store_plugin.get_details(result, timeout) - callback(result) - self.tasks.task_done() + result, store_plugin, callback, timeout = self.tasks.get() + if result: + store_plugin.get_details(result, timeout) + callback(result) + self.tasks.task_done() except: continue + + +class CacheUpdateThreadPool(GenericDownloadThreadPool): + + def __init__(self, thread_count): + GenericDownloadThreadPool.__init__(self, CacheUpdateThread, thread_count) + + def add_task(self, store_plugin, timeout=10): + self.tasks.put((store_plugin, timeout)) + GenericDownloadThreadPool.add_task(self) + + +class CacheUpdateThread(Thread): + + def __init__(self, tasks, results): + Thread.__init__(self) + self.daemon = True + self.tasks = tasks + self._run = True + + def abort(self): + self._run = False + + def run(self): + while self._run and not self.tasks.empty(): + try: + store_plugin, timeout = self.tasks.get() + store_plugin.update_cache(timeout=timeout, suppress_progress=True) + except: + traceback.print_exc() diff --git a/src/calibre/gui2/store/search/models.py b/src/calibre/gui2/store/search/models.py index 73b7bcc90a..adc90e3b14 100644 --- a/src/calibre/gui2/store/search/models.py +++ b/src/calibre/gui2/store/search/models.py @@ -14,7 +14,7 @@ from PyQt4.Qt import (Qt, QAbstractItemModel, QVariant, QPixmap, QModelIndex, QS from calibre.gui2 import NONE from calibre.gui2.store.search_result import SearchResult from calibre.gui2.store.search.download_thread import DetailsThreadPool, \ - DetailsThread, CoverThreadPool, CoverThread + CoverThreadPool from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ REGEXP_MATCH from calibre.utils.icu import sort_key @@ -51,10 +51,8 @@ class Matches(QAbstractItemModel): self.matches = [] self.query = '' self.search_filter = SearchFilter() - self.cover_pool = CoverThreadPool(CoverThread, 2) - self.cover_pool.start_threads() - self.details_pool = DetailsThreadPool(DetailsThread, 4) - self.details_pool.start_threads() + self.cover_pool = CoverThreadPool(2) + self.details_pool = DetailsThreadPool(4) self.sort_col = 2 self.sort_order = Qt.AscendingOrder @@ -70,9 +68,7 @@ class Matches(QAbstractItemModel): self.search_filter.clear_search_results() self.query = '' self.cover_pool.abort() - self.cover_pool.start_threads() self.details_pool.abort() - self.details_pool.start_threads() self.reset() def add_result(self, result, store_plugin): diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 5c4b1cee00..70e92d1756 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -9,17 +9,17 @@ __docformat__ = 'restructuredtext en' import re from random import shuffle -from PyQt4.Qt import (Qt, QDialog, QTimer, QCheckBox, QVBoxLayout) +from PyQt4.Qt import (Qt, QDialog, QTimer, QCheckBox, QVBoxLayout, QIcon) from calibre.gui2 import JSONConfig, info_dialog from calibre.gui2.progress_indicator import ProgressIndicator -from calibre.gui2.store.search.download_thread import SearchThreadPool, SearchThread +from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog +from calibre.gui2.store.search.download_thread import SearchThreadPool, \ + CacheUpdateThreadPool from calibre.gui2.store.search.search_ui import Ui_Dialog HANG_TIME = 75000 # milliseconds seconds TIMEOUT = 75 # seconds -SEARCH_THREAD_TOTAL = 4 -COVER_DOWNLOAD_THREAD_TOTAL = 2 class SearchDialog(QDialog, Ui_Dialog): @@ -31,10 +31,16 @@ class SearchDialog(QDialog, Ui_Dialog): # We keep a cache of store plugins and reference them by name. self.store_plugins = istores - self.search_pool = SearchThreadPool(SearchThread, SEARCH_THREAD_TOTAL) + self.search_pool = SearchThreadPool(4) + self.cache_pool = CacheUpdateThreadPool(2) # Check for results and hung threads. self.checker = QTimer() + self.progress_checker = QTimer() self.hang_check = 0 + + # Update store caches silently. + for p in self.store_plugins.values(): + self.cache_pool.add_task(p, 30) # Add check boxes for each store so the user # can disable searching specific stores on a @@ -51,17 +57,28 @@ class SearchDialog(QDialog, Ui_Dialog): # Create and add the progress indicator self.pi = ProgressIndicator(self, 24) self.top_layout.addWidget(self.pi) + + self.adv_search_button.setIcon(QIcon(I('search.png'))) + self.adv_search_button.clicked.connect(self.build_adv_search) self.search.clicked.connect(self.do_search) self.checker.timeout.connect(self.get_results) + self.progress_checker.timeout.connect(self.check_progress) self.results_view.activated.connect(self.open_store) self.select_all_stores.clicked.connect(self.stores_select_all) self.select_invert_stores.clicked.connect(self.stores_select_invert) self.select_none_stores.clicked.connect(self.stores_select_none) self.finished.connect(self.dialog_closed) + self.progress_checker.start(100) + self.restore_state() + def build_adv_search(self): + adv = AdvSearchBuilderDialog(self) + if adv.exec_() == QDialog.Accepted: + self.search_edit.setText(adv.search_string()) + def resize_columns(self): total = 600 # Cover @@ -105,11 +122,9 @@ class SearchDialog(QDialog, Ui_Dialog): for n in store_names: if getattr(self, 'store_check_' + n).isChecked(): self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT) - if self.search_pool.has_tasks(): - self.hang_check = 0 - self.checker.start(100) - self.search_pool.start_threads() - self.pi.startAnimation() + self.hang_check = 0 + self.checker.start(100) + self.pi.startAnimation() def clean_query(self, query): query = query.lower() @@ -181,27 +196,31 @@ class SearchDialog(QDialog, Ui_Dialog): if self.hang_check >= HANG_TIME: self.search_pool.abort() self.checker.stop() - self.pi.stopAnimation() else: # Stop the checker if not threads are running. if not self.search_pool.threads_running() and not self.search_pool.has_tasks(): self.checker.stop() - self.pi.stopAnimation() while self.search_pool.has_results(): res, store_plugin = self.search_pool.get_result() if res: self.results_view.model().add_result(res, store_plugin) - - if not self.checker.isActive(): - if not self.results_view.model().has_results(): - info_dialog(self, _('No matches'), _('Couldn\'t find any books matching your query.'), show=True, show_copy_button=False) + + if not self.results_view.model().has_results(): + info_dialog(self, _('No matches'), _('Couldn\'t find any books matching your query.'), show=True, show_copy_button=False) def open_store(self, index): result = self.results_view.model().get_result(index) self.store_plugins[result.store_name].open(self, result.detail_item) + def check_progress(self): + if not self.search_pool.threads_running() and not self.results_view.model().cover_pool.threads_running() and not self.results_view.model().details_pool.threads_running(): + self.pi.stopAnimation() + else: + if not self.pi.isAnimated(): + self.pi.startAnimation() + def get_store_checks(self): ''' Returns a list of QCheckBox's for each store. diff --git a/src/calibre/gui2/store/search/search.ui b/src/calibre/gui2/store/search/search.ui index bdf875113e..0d39a70a29 100644 --- a/src/calibre/gui2/store/search/search.ui +++ b/src/calibre/gui2/store/search/search.ui @@ -30,6 +30,13 @@ + + + + ... + + +