Store: Break MobileRead into more manageable pieces. API for updating store caches.

src/calibre/customize/builtins.py

@@ -1162,7 +1162,7 @@ class StoreManyBooksStore(StoreBase):
 class StoreMobileReadStore(StoreBase):
     name = 'MobileRead'
     description = _('Ebooks handcrafted with the utmost care')
-    actual_plugin = 'calibre.gui2.store.mobileread_plugin:MobileReadStore'
+    actual_plugin = 'calibre.gui2.store.mobileread.mobileread_plugin:MobileReadStore'
 
 class StoreOpenLibraryStore(StoreBase):
     name = 'Open Library'

src/calibre/gui2/store/__init__.py

@@ -127,6 +127,40 @@ class StorePlugin(object): # {{{
         '''
         return False
 
+    def update_cache(self, parent=None, timeout=60, force=False, suppress_progress=False):
+        '''
+        Some plugins need to keep an local cache of available books. This function
+        is called to update the caches. It is recommended to call this function
+        from :meth:`open`. Especially if :meth:`open` does anything other than
+        open a web page.
+        
+        This function can be called at any time. It is up to the plugin to determine
+        if the cache really does need updating. Unless :param:`force` is True, then
+        the plugin must update the cache. The only time force should be True is if
+        this function is called by the plugin's configuration dialog.
+        
+        if :param:`suppress_progress` is False it is safe to assume that this function
+        is being called from the main GUI thread so it is safe and recommended to use
+        a QProgressDialog to display what is happening and allow the user to cancel
+        the operation. if :param:`suppress_progress` is True then run the update
+        silently. In this case there is no guarantee what thread is calling this
+        function so no Qt related functionality that requires being run in the main
+        GUI thread should be run. E.G. Open a QProgressDialog. 
+        
+        :param parent: The parent object to be used by an GUI dialogs.
+        
+        :param timeout: The maximum amount of time that should be spent in
+        any given network connection.
+        
+        :param force: Force updating the cache even if the plugin has determined
+        it is not necessary.
+        
+        :param suppress_progress: Should a progress indicator be shown.
+        
+        :return: True if the cache was updated, False otherwise.
+        '''
+        return False
+
     def get_settings(self):
         '''
         This is only useful for plugins that implement

src/calibre/gui2/store/mobileread/cache_progress_dialog.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import (QCoreApplication, QDialog, QTimer)
+
+from calibre.gui2.store.mobileread.cache_progress_dialog_ui import Ui_Dialog
+
+class CacheProgressDialog(QDialog, Ui_Dialog):
+    
+    def __init__(self, parent=None, total=None):
+        QDialog.__init__(self, parent)
+        self.setupUi(self)
+        
+        self.completed = 0
+        self.canceled = False        
+        
+        self.progress.setValue(0)
+        self.progress.setMinimum(0)
+        self.progress.setMaximum(total if total else 0)
+
+    def exec_(self):
+        self.completed = 0
+        self.canceled = False
+        QDialog.exec_(self)
+
+    def open(self):
+        self.completed = 0
+        self.canceled = False
+        QDialog.open(self)
+        
+    def reject(self):
+        self.canceled = True
+        QDialog.reject(self)
+
+    def update_progress(self):
+        '''
+        completed is an int from 0 to total representing the number
+        records that have bee completed.
+        '''
+        self.set_progress(self.completed + 1)
+    
+    def set_message(self, msg):
+        self.message.setText(msg)
+    
+    def set_details(self, msg):
+        self.details.setText(msg)
+    
+    def set_progress(self, completed):
+        '''
+        completed is an int from 0 to total representing the number
+        records that have bee completed.
+        '''
+        self.completed = completed
+        self.progress.setValue(self.completed)
+    
+    def set_total(self, total):
+        self.progress.setMaximum(total)

src/calibre/gui2/store/mobileread/cache_progress_dialog.ui

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>402</width>
+    <height>138</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Dialog</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <widget class="QLabel" name="message">
+     <property name="text">
+      <string>Updating book cache</string>
+     </property>
+     <property name="alignment">
+      <set>Qt::AlignCenter</set>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QProgressBar" name="progress">
+     <property name="value">
+      <number>24</number>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QLabel" name="details">
+     <property name="text">
+      <string/>
+     </property>
+     <property name="alignment">
+      <set>Qt::AlignCenter</set>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>40</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+   <item>
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Cancel</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/store/mobileread/cache_update_thread.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import time
+from contextlib import closing
+from threading import Thread
+
+from lxml import html
+
+from PyQt4.Qt import (pyqtSignal, QObject)
+
+from calibre import browser
+from calibre.gui2.store.search_result import SearchResult
+
+class CacheUpdateThread(Thread, QObject):
+    
+    total_changed = pyqtSignal(int)
+    update_progress = pyqtSignal(int)
+    update_details = pyqtSignal(unicode)
+    
+    def __init__(self, config, seralize_books_function, timeout):
+        Thread.__init__(self)
+        QObject.__init__(self)
+
+        self.daemon = True
+        self.config = config
+        self.seralize_books = seralize_books_function
+        self.timeout = timeout
+        self._run = True
+        
+    def abort(self):
+        self._run = False
+        
+    def run(self):
+        url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html'
+        
+        self.update_details.emit(_('Checking last download date.'))
+        last_download = self.config.get('last_download', None)
+        # Don't update the book list if our cache is less than one week old.
+        if last_download and (time.time() - last_download) < 604800:
+            return
+        
+        self.update_details.emit(_('Downloading book list from MobileRead.'))
+        # Download the book list HTML file from MobileRead.
+        br = browser()
+        raw_data = None
+        try:
+            with closing(br.open(url, timeout=self.timeout)) as f:
+                raw_data = f.read()
+        except:
+            return
+        
+        if not raw_data or not self._run:
+            return
+        
+        self.update_details.emit(_('Processing books.'))
+        # Turn books listed in the HTML file into SearchResults's.
+        books = []
+        try:
+            data = html.fromstring(raw_data)
+            raw_books = data.xpath('//ul/li')
+            self.total_changed.emit(len(raw_books))
+            
+            for i, book_data in enumerate(raw_books):
+                self.update_details.emit(_('%s of %s books processed.') % (i, len(raw_books)))
+                book = SearchResult()
+                book.detail_item = ''.join(book_data.xpath('.//a/@href'))
+                book.formats = ''.join(book_data.xpath('.//i/text()'))
+                book.formats = book.formats.strip()
+    
+                text = ''.join(book_data.xpath('.//a/text()'))
+                if ':' in text:
+                    book.author, q, text = text.partition(':')
+                book.author = book.author.strip()
+                book.title = text.strip()
+                books.append(book)
+                
+                if not self._run:
+                    books = []
+                    break
+                else:
+                    self.update_progress.emit(i)
+        except:
+            pass
+
+        # Save the book list and it's create time.
+        if books:
+            self.config['book_list'] = self.seralize_books(books)
+            self.config['last_download'] = time.time()

src/calibre/gui2/store/mobileread/mobileread_plugin.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from threading import Lock
+
+from PyQt4.Qt import (QUrl, QCoreApplication)
+
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+from calibre.gui2.store.mobileread.models import SearchFilter
+from calibre.gui2.store.mobileread.cache_progress_dialog import CacheProgressDialog
+from calibre.gui2.store.mobileread.cache_update_thread import CacheUpdateThread
+from calibre.gui2.store.mobileread.store_dialog import MobeReadStoreDialog
+
+class MobileReadStore(BasicStoreConfig, StorePlugin):
+    
+    def genesis(self):
+        self.lock = Lock()
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://www.mobileread.com/'
+        
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(detail_item if detail_item else url))
+        else:
+            if detail_item:
+                d = WebStoreDialog(self.gui, url, parent, detail_item)
+                d.setWindowTitle(self.name)
+                d.set_tags(self.config.get('tags', ''))
+                d.exec_()
+            else:
+                if self.update_cache(parent, 30):
+                    d = MobeReadStoreDialog(self, parent)
+                    d.setWindowTitle(self.name)
+                    d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        books = self.get_book_list()
+
+        sf = SearchFilter(books)
+        matches = sf.parse(query)
+
+        for book in matches:
+            book.price = '$0.00'
+            book.drm = SearchResult.DRM_UNLOCKED
+            yield book
+
+    def update_cache(self, parent=None, timeout=10, force=False, suppress_progress=False):
+        if self.lock.acquire(False):
+            try:
+                update_thread = CacheUpdateThread(self.config, self.seralize_books, timeout)
+                if not suppress_progress:
+                    progress = CacheProgressDialog(parent)
+                    progress.set_message(_('Updating MobileRead book cache...'))
+            
+                    update_thread.total_changed.connect(progress.set_total)
+                    update_thread.update_progress.connect(progress.set_progress)
+                    update_thread.update_details.connect(progress.set_details)
+                    progress.rejected.connect(update_thread.abort)
+            
+                    progress.open()
+                    update_thread.start()
+                    while update_thread.is_alive() and not progress.canceled:
+                        QCoreApplication.processEvents()
+            
+                    if progress.isVisible():
+                        progress.accept()
+                    return not progress.canceled
+                else:
+                    update_thread.start()
+            finally:
+                self.lock.release()
+
+    def get_book_list(self):
+        return self.deseralize_books(self.config.get('book_list', []))
+    
+    def seralize_books(self, books):
+        sbooks = []
+        for b in books:
+            data = {}
+            data['author'] = b.author
+            data['title'] = b.title
+            data['detail_item'] = b.detail_item
+            data['formats'] = b.formats
+            sbooks.append(data)
+        return sbooks
+    
+    def deseralize_books(self, sbooks):
+        books = []
+        for s in sbooks:
+            b = SearchResult()
+            b.author = s.get('author', '')
+            b.title = s.get('title', '')
+            b.detail_item = s.get('detail_item', '')
+            b.formats = s.get('formats', '')
+            books.append(b)
+        return books

src/calibre/gui2/store/mobileread/models.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from operator import attrgetter
+
+from PyQt4.Qt import (Qt, QAbstractItemModel, QModelIndex, QVariant, pyqtSignal)
+
+from calibre.gui2 import NONE
+from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
+    REGEXP_MATCH
+from calibre.utils.icu import sort_key
+from calibre.utils.search_query_parser import SearchQueryParser
+
+class BooksModel(QAbstractItemModel):
+    
+    total_changed = pyqtSignal(int)
+
+    HEADERS = [_('Title'), _('Author(s)'), _('Format')]
+
+    def __init__(self, all_books):
+        QAbstractItemModel.__init__(self)
+        self.books = all_books
+        self.all_books = all_books
+        self.filter = ''
+        self.search_filter = SearchFilter(all_books)
+        self.sort_col = 0
+        self.sort_order = Qt.AscendingOrder
+
+    def get_book(self, index):
+        row = index.row()
+        if row < len(self.books):
+            return self.books[row]
+        else:
+            return None
+    
+    def search(self, filter):        
+        self.filter = filter.strip()
+        if not self.filter:
+            self.books = self.all_books
+        else:
+            try:
+                self.books = list(self.search_filter.parse(self.filter))
+            except:
+                self.books = self.all_books
+        self.sort(self.sort_col, self.sort_order)
+        self.total_changed.emit(self.rowCount())
+    
+    def index(self, row, column, parent=QModelIndex()):
+        return self.createIndex(row, column)
+
+    def parent(self, index):
+        if not index.isValid() or index.internalId() == 0:
+            return QModelIndex()
+        return self.createIndex(0, 0)
+
+    def rowCount(self, *args):
+        return len(self.books)
+
+    def columnCount(self, *args):
+        return len(self.HEADERS)
+    
+    def headerData(self, section, orientation, role):
+        if role != Qt.DisplayRole:
+            return NONE
+        text = ''
+        if orientation == Qt.Horizontal:
+            if section < len(self.HEADERS):
+                text = self.HEADERS[section]
+            return QVariant(text)
+        else:
+            return QVariant(section+1)
+
+    def data(self, index, role):
+        row, col = index.row(), index.column()
+        result = self.books[row]
+        if role == Qt.DisplayRole:
+            if col == 0:
+                return QVariant(result.title)
+            elif col == 1:
+                return QVariant(result.author)
+            elif col == 2:
+                return QVariant(result.formats)
+        return NONE
+
+    def data_as_text(self, result, col):
+        text = ''
+        if col == 0:
+            text = result.title
+        elif col == 1:
+            text = result.author
+        elif col == 2:
+            text = result.formats
+        return text
+
+    def sort(self, col, order, reset=True):
+        self.sort_col = col
+        self.sort_order = order
+        if not self.books:
+            return
+        descending = order == Qt.DescendingOrder
+        self.books.sort(None,
+            lambda x: sort_key(unicode(self.data_as_text(x, col))),
+            descending)
+        if reset:
+            self.reset()
+
+
+class SearchFilter(SearchQueryParser):
+    
+    USABLE_LOCATIONS = [
+        'all',
+        'author',
+        'authors',
+        'format',
+        'formats',
+        'title',
+    ]
+
+    def __init__(self, all_books=[]):
+        SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS)
+        self.srs = set(all_books)
+
+    def universal_set(self):
+        return self.srs
+
+    def get_matches(self, location, query):
+        location = location.lower().strip()
+        if location == 'authors':
+            location = 'author'
+        elif location == 'formats':
+            location = 'format'
+
+        matchkind = CONTAINS_MATCH
+        if len(query) > 1:
+            if query.startswith('\\'):
+                query = query[1:]
+            elif query.startswith('='):
+                matchkind = EQUALS_MATCH
+                query = query[1:]
+            elif query.startswith('~'):
+                matchkind = REGEXP_MATCH
+                query = query[1:]
+        if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D
+            query = query.lower()
+
+        if location not in self.USABLE_LOCATIONS:
+            return set([])
+        matches = set([])
+        all_locs = set(self.USABLE_LOCATIONS) - set(['all'])
+        locations = all_locs if location == 'all' else [location]
+        q = {
+             'author': lambda x: x.author.lower(),
+             'format': attrgetter('formats'),
+             'title': lambda x: x.title.lower(),
+        }
+        for x in ('author', 'format'):
+            q[x+'s'] = q[x]
+        for sr in self.srs:
+            for locvalue in locations:
+                accessor = q[locvalue]
+                if query == 'true':
+                    if accessor(sr) is not None:
+                        matches.add(sr)
+                    continue
+                if query == 'false':
+                    if accessor(sr) is None:
+                        matches.add(sr)
+                    continue
+                try:
+                    ### Can't separate authors because comma is used for name sep and author sep
+                    ### Exact match might not get what you want. For that reason, turn author
+                    ### exactmatch searches into contains searches.
+                    if locvalue == 'author' and matchkind == EQUALS_MATCH:
+                        m = CONTAINS_MATCH
+                    else:
+                        m = matchkind
+
+                    vals = [accessor(sr)]
+                    if _match(query, vals, m):
+                        matches.add(sr)
+                        break
+                except ValueError: # Unicode errors
+                    import traceback
+                    traceback.print_exc()
+        return matches

src/calibre/gui2/store/mobileread/store_dialog.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import (Qt, QDialog, QIcon)
+
+from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog
+from calibre.gui2.store.mobileread.models import BooksModel
+from calibre.gui2.store.mobileread.store_dialog_ui import Ui_Dialog
+
+class MobeReadStoreDialog(QDialog, Ui_Dialog):
+    
+    def __init__(self, plugin, *args):
+        QDialog.__init__(self, *args)
+        self.setupUi(self)
+
+        self.plugin = plugin
+        
+        self.adv_search_button.setIcon(QIcon(I('search.png')))
+        
+        self._model = BooksModel(self.plugin.get_book_list())
+        self.results_view.setModel(self._model)
+        self.total.setText('%s' % self.results_view.model().rowCount())
+
+        self.search_button.clicked.connect(self.do_search)
+        self.adv_search_button.clicked.connect(self.build_adv_search)
+        self.results_view.activated.connect(self.open_store)
+        self.results_view.model().total_changed.connect(self.update_book_total)
+        self.finished.connect(self.dialog_closed)
+        
+        self.restore_state()
+        
+    def do_search(self):
+        self.results_view.model().search(unicode(self.search_query.text()))
+        
+    def open_store(self, index):
+        result = self.results_view.model().get_book(index)
+        if result:
+            self.plugin.open(self, result.detail_item)
+    
+    def update_book_total(self, total):
+        self.total.setText('%s' % total)
+    
+    def build_adv_search(self):
+        adv = AdvSearchBuilderDialog(self)
+        adv.price_label.hide()
+        adv.price_box.hide()
+        if adv.exec_() == QDialog.Accepted:
+            self.search_query.setText(adv.search_string())
+    
+    def restore_state(self):
+        geometry = self.plugin.config.get('dialog_geometry', None)
+        if geometry:
+            self.restoreGeometry(geometry)
+
+        results_cwidth = self.plugin.config.get('dialog_results_view_column_width')
+        if results_cwidth:
+            for i, x in enumerate(results_cwidth):
+                if i >= self.results_view.model().columnCount():
+                    break
+                self.results_view.setColumnWidth(i, x)
+        else:
+            for i in xrange(self.results_view.model().columnCount()):
+                self.results_view.resizeColumnToContents(i)
+                
+        self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0)
+        self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder)
+        self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order)
+        self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order)
+
+    def save_state(self):
+        self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry())
+        self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.results_view.model().columnCount())]
+        self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col
+        self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order
+
+    def dialog_closed(self, result):
+        self.save_state()

src/calibre/gui2/store/mobileread_plugin.py

@@ -1,376 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import (unicode_literals, division, absolute_import, print_function)
-
-__license__ = 'GPL 3'
-__copyright__ = '2011, John Schember <john@nachtimwald.com>'
-__docformat__ = 'restructuredtext en'
-
-import difflib
-import heapq
-import time
-from contextlib import closing
-from operator import attrgetter
-from threading import RLock
-
-from lxml import html
-
-from PyQt4.Qt import Qt, QUrl, QDialog, QAbstractItemModel, QModelIndex, QVariant, \
-    pyqtSignal, QIcon
-
-from calibre import browser
-from calibre.gui2 import open_url, NONE
-from calibre.gui2.store import StorePlugin
-from calibre.gui2.store.basic_config import BasicStoreConfig
-from calibre.gui2.store.mobileread_store_dialog_ui import Ui_Dialog
-from calibre.gui2.store.search_result import SearchResult
-from calibre.gui2.store.web_store_dialog import WebStoreDialog
-from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog
-from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
-    REGEXP_MATCH
-from calibre.utils.icu import sort_key
-from calibre.utils.search_query_parser import SearchQueryParser
-
-class MobileReadStore(BasicStoreConfig, StorePlugin):
-    
-    def genesis(self):
-        self.rlock = RLock()
-    
-    def open(self, parent=None, detail_item=None, external=False):
-        url = 'http://www.mobileread.com/'
-        
-        if external or self.config.get('open_external', False):
-            open_url(QUrl(detail_item if detail_item else url))
-        else:
-            if detail_item:
-                d = WebStoreDialog(self.gui, url, parent, detail_item)
-                d.setWindowTitle(self.name)
-                d.set_tags(self.config.get('tags', ''))
-                d.exec_()
-            else:
-                d = MobeReadStoreDialog(self, parent)
-                d.setWindowTitle(self.name)
-                d.exec_()
-
-    def search(self, query, max_results=10, timeout=60):
-        books = self.get_book_list(timeout=timeout)
-
-        sf = SearchFilter(books)
-        matches = sf.parse(query)
-
-        for book in matches:
-            book.price = '$0.00'
-            book.drm = SearchResult.DRM_UNLOCKED
-            yield book
-
-    def update_book_list(self, timeout=10):
-        with self.rlock:
-            url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html'
-            
-            last_download = self.config.get('last_download', None)
-            # Don't update the book list if our cache is less than one week old.
-            if last_download and (time.time() - last_download) < 604800:
-                return
-            
-            # Download the book list HTML file from MobileRead.
-            br = browser()
-            raw_data = None
-            with closing(br.open(url, timeout=timeout)) as f:
-                raw_data = f.read()
-            
-            if not raw_data:
-                return
-            
-            # Turn books listed in the HTML file into SearchResults's.
-            books = []
-            try:
-                data = html.fromstring(raw_data)
-                for book_data in data.xpath('//ul/li'):
-                    book = SearchResult()
-                    book.detail_item = ''.join(book_data.xpath('.//a/@href'))
-                    book.formats = ''.join(book_data.xpath('.//i/text()'))
-                    book.formats = book.formats.strip()
-        
-                    text = ''.join(book_data.xpath('.//a/text()'))
-                    if ':' in text:
-                        book.author, q, text = text.partition(':')
-                    book.author = book.author.strip()
-                    book.title = text.strip()
-                    books.append(book)
-            except:
-                pass
-    
-            # Save the book list and it's create time.
-            if books:
-                self.config['last_download'] = time.time()
-                self.config['book_list'] = self.seralize_books(books)
-
-    def get_book_list(self, timeout=10):
-        self.update_book_list(timeout=timeout)
-        return self.deseralize_books(self.config.get('book_list', []))
-    
-    def seralize_books(self, books):
-        sbooks = []
-        for b in books:
-            data = {}
-            data['author'] = b.author
-            data['title'] = b.title
-            data['detail_item'] = b.detail_item
-            data['formats'] = b.formats
-            sbooks.append(data)
-        return sbooks
-    
-    def deseralize_books(self, sbooks):
-        books = []
-        for s in sbooks:
-            b = SearchResult()
-            b.author = s.get('author', '')
-            b.title = s.get('title', '')
-            b.detail_item = s.get('detail_item', '')
-            b.formats = s.get('formats', '')
-            books.append(b)
-        return books
-
-
-class MobeReadStoreDialog(QDialog, Ui_Dialog):
-    
-    def __init__(self, plugin, *args):
-        QDialog.__init__(self, *args)
-        self.setupUi(self)
-
-        self.plugin = plugin
-        
-        self.adv_search_button.setIcon(QIcon(I('search.png')))
-        
-        self._model = BooksModel(self.plugin.get_book_list())
-        self.results_view.setModel(self._model)
-        self.total.setText('%s' % self.results_view.model().rowCount())
-
-        self.search_button.clicked.connect(self.do_search)
-        self.adv_search_button.clicked.connect(self.build_adv_search)
-        self.results_view.activated.connect(self.open_store)
-        self.results_view.model().total_changed.connect(self.update_book_total)
-        self.finished.connect(self.dialog_closed)
-        
-        self.restore_state()
-        
-    def do_search(self):
-        self.results_view.model().search(unicode(self.search_query.text()))
-        
-    def open_store(self, index):
-        result = self.results_view.model().get_book(index)
-        if result:
-            self.plugin.open(self, result.detail_item)
-    
-    def update_book_total(self, total):
-        self.total.setText('%s' % total)
-    
-    def build_adv_search(self):
-        adv = AdvSearchBuilderDialog(self)
-        adv.price_label.hide()
-        adv.price_box.hide()
-        if adv.exec_() == QDialog.Accepted:
-            self.search_query.setText(adv.search_string())
-    
-    def restore_state(self):
-        geometry = self.plugin.config.get('dialog_geometry', None)
-        if geometry:
-            self.restoreGeometry(geometry)
-
-        results_cwidth = self.plugin.config.get('dialog_results_view_column_width')
-        if results_cwidth:
-            for i, x in enumerate(results_cwidth):
-                if i >= self.results_view.model().columnCount():
-                    break
-                self.results_view.setColumnWidth(i, x)
-        else:
-            for i in xrange(self.results_view.model().columnCount()):
-                self.results_view.resizeColumnToContents(i)
-                
-        self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0)
-        self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder)
-        self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order)
-        self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order)
-
-    def save_state(self):
-        self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry())
-        self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.results_view.model().columnCount())]
-        self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col
-        self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order
-
-    def dialog_closed(self, result):
-        self.save_state()
-
-
-class BooksModel(QAbstractItemModel):
-    
-    total_changed = pyqtSignal(int)
-
-    HEADERS = [_('Title'), _('Author(s)'), _('Format')]
-
-    def __init__(self, all_books):
-        QAbstractItemModel.__init__(self)
-        self.books = all_books
-        self.all_books = all_books
-        self.filter = ''
-        self.search_filter = SearchFilter(all_books)
-        self.sort_col = 0
-        self.sort_order = Qt.AscendingOrder
-
-    def get_book(self, index):
-        row = index.row()
-        if row < len(self.books):
-            return self.books[row]
-        else:
-            return None
-    
-    def search(self, filter):        
-        self.filter = filter.strip()
-        if not self.filter:
-            self.books = self.all_books
-        else:
-            try:
-                self.books = list(self.search_filter.parse(self.filter))
-            except:
-                self.books = self.all_books
-        self.sort(self.sort_col, self.sort_order)
-        self.total_changed.emit(self.rowCount())
-    
-    def index(self, row, column, parent=QModelIndex()):
-        return self.createIndex(row, column)
-
-    def parent(self, index):
-        if not index.isValid() or index.internalId() == 0:
-            return QModelIndex()
-        return self.createIndex(0, 0)
-
-    def rowCount(self, *args):
-        return len(self.books)
-
-    def columnCount(self, *args):
-        return len(self.HEADERS)
-    
-    def headerData(self, section, orientation, role):
-        if role != Qt.DisplayRole:
-            return NONE
-        text = ''
-        if orientation == Qt.Horizontal:
-            if section < len(self.HEADERS):
-                text = self.HEADERS[section]
-            return QVariant(text)
-        else:
-            return QVariant(section+1)
-
-    def data(self, index, role):
-        row, col = index.row(), index.column()
-        result = self.books[row]
-        if role == Qt.DisplayRole:
-            if col == 0:
-                return QVariant(result.title)
-            elif col == 1:
-                return QVariant(result.author)
-            elif col == 2:
-                return QVariant(result.formats)
-        return NONE
-
-    def data_as_text(self, result, col):
-        text = ''
-        if col == 0:
-            text = result.title
-        elif col == 1:
-            text = result.author
-        elif col == 2:
-            text = result.formats
-        return text
-
-    def sort(self, col, order, reset=True):
-        self.sort_col = col
-        self.sort_order = order
-        if not self.books:
-            return
-        descending = order == Qt.DescendingOrder
-        self.books.sort(None,
-            lambda x: sort_key(unicode(self.data_as_text(x, col))),
-            descending)
-        if reset:
-            self.reset()
-
-
-class SearchFilter(SearchQueryParser):
-    
-    USABLE_LOCATIONS = [
-        'all',
-        'author',
-        'authors',
-        'format',
-        'formats',
-        'title',
-    ]
-
-    def __init__(self, all_books=[]):
-        SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS)
-        self.srs = set(all_books)
-
-    def universal_set(self):
-        return self.srs
-
-    def get_matches(self, location, query):
-        location = location.lower().strip()
-        if location == 'authors':
-            location = 'author'
-        elif location == 'formats':
-            location = 'format'
-
-        matchkind = CONTAINS_MATCH
-        if len(query) > 1:
-            if query.startswith('\\'):
-                query = query[1:]
-            elif query.startswith('='):
-                matchkind = EQUALS_MATCH
-                query = query[1:]
-            elif query.startswith('~'):
-                matchkind = REGEXP_MATCH
-                query = query[1:]
-        if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D
-            query = query.lower()
-
-        if location not in self.USABLE_LOCATIONS:
-            return set([])
-        matches = set([])
-        all_locs = set(self.USABLE_LOCATIONS) - set(['all'])
-        locations = all_locs if location == 'all' else [location]
-        q = {
-             'author': lambda x: x.author.lower(),
-             'format': attrgetter('formats'),
-             'title': lambda x: x.title.lower(),
-        }
-        for x in ('author', 'format'):
-            q[x+'s'] = q[x]
-        for sr in self.srs:
-            for locvalue in locations:
-                accessor = q[locvalue]
-                if query == 'true':
-                    if accessor(sr) is not None:
-                        matches.add(sr)
-                    continue
-                if query == 'false':
-                    if accessor(sr) is None:
-                        matches.add(sr)
-                    continue
-                try:
-                    ### Can't separate authors because comma is used for name sep and author sep
-                    ### Exact match might not get what you want. For that reason, turn author
-                    ### exactmatch searches into contains searches.
-                    if locvalue == 'author' and matchkind == EQUALS_MATCH:
-                        m = CONTAINS_MATCH
-                    else:
-                        m = matchkind
-
-                    vals = [accessor(sr)]
-                    if _match(query, vals, m):
-                        matches.add(sr)
-                        break
-                except ValueError: # Unicode errors
-                    import traceback
-                    traceback.print_exc()
-        return matches

src/calibre/gui2/store/search/download_thread.py

@@ -192,3 +192,33 @@ class DetailsThread(Thread):
                 self.tasks.task_done()
             except:
                 continue
+
+
+class CacheUpdateThreadPool(GenericDownloadThreadPool):
+
+    def __init__(self, thread_count):
+        GenericDownloadThreadPool.__init__(self, CacheUpdateThread, thread_count)
+
+    def add_task(self, store_plugin, timeout=10):
+        self.tasks.put((store_plugin, timeout))
+        GenericDownloadThreadPool.add_task(self)
+
+
+class CacheUpdateThread(Thread):
+
+    def __init__(self, tasks, results):
+        Thread.__init__(self)
+        self.daemon = True
+        self.tasks = tasks
+        self._run = True
+
+    def abort(self):
+        self._run = False
+
+    def run(self):
+        while self._run and not self.tasks.empty():
+            try:
+                store_plugin, timeout = self.tasks.get()
+                store_plugin.update_cache(timeout=timeout, suppress_progress=True)
+            except:
+                traceback.print_exc()

src/calibre/gui2/store/search/search.py

@@ -14,7 +14,8 @@ from PyQt4.Qt import (Qt, QDialog, QTimer, QCheckBox, QVBoxLayout, QIcon)
 from calibre.gui2 import JSONConfig, info_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
 from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog
-from calibre.gui2.store.search.download_thread import SearchThreadPool
+from calibre.gui2.store.search.download_thread import SearchThreadPool, \
+    CacheUpdateThreadPool
 from calibre.gui2.store.search.search_ui import Ui_Dialog
 
 HANG_TIME = 75000 # milliseconds seconds
@@ -31,10 +32,15 @@ class SearchDialog(QDialog, Ui_Dialog):
         # We keep a cache of store plugins and reference them by name.
         self.store_plugins = istores
         self.search_pool = SearchThreadPool(4)
+        self.cache_pool = CacheUpdateThreadPool(2)
         # Check for results and hung threads.
         self.checker = QTimer()
         self.progress_checker = QTimer()
         self.hang_check = 0
+        
+        # Update store caches silently.
+        for p in self.store_plugins.values():
+            self.cache_pool.add_task(p, 30)
 
         # Add check boxes for each store so the user
         # can disable searching specific stores on a
@@ -116,10 +122,9 @@ class SearchDialog(QDialog, Ui_Dialog):
         for n in store_names:
             if getattr(self, 'store_check_' + n).isChecked():
                 self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT)
-        if self.search_pool.has_tasks() or self.search_pool.threads_running():
+        self.hang_check = 0
-            self.hang_check = 0
+        self.checker.start(100)
-            self.checker.start(100)
+        self.pi.startAnimation()
-            self.pi.startAnimation()
 
     def clean_query(self, query):
         query = query.lower()
@@ -200,10 +205,9 @@ class SearchDialog(QDialog, Ui_Dialog):
             res, store_plugin = self.search_pool.get_result()
             if res:
                 self.results_view.model().add_result(res, store_plugin)
-                
+
-        if not self.checker.isActive():
+        if not self.results_view.model().has_results():
-            if not self.results_view.model().has_results():
+            info_dialog(self, _('No matches'), _('Couldn\'t find any books matching your query.'), show=True, show_copy_button=False)
-                info_dialog(self, _('No matches'), _('Couldn\'t find any books matching your query.'), show=True, show_copy_button=False)
 
 
     def open_store(self, index):