Stores stuff

src/calibre/customize/builtins.py

@@ -1162,7 +1162,7 @@ class StoreManyBooksStore(StoreBase):
 class StoreMobileReadStore(StoreBase):
     name = 'MobileRead'
     description = _('Ebooks handcrafted with the utmost care')
-    actual_plugin = 'calibre.gui2.store.mobileread_plugin:MobileReadStore'
+    actual_plugin = 'calibre.gui2.store.mobileread.mobileread_plugin:MobileReadStore'
 
 class StoreOpenLibraryStore(StoreBase):
     name = 'Open Library'

src/calibre/gui2/store/__init__.py

@@ -127,6 +127,40 @@ class StorePlugin(object): # {{{
         '''
         return False
 
+    def update_cache(self, parent=None, timeout=60, force=False, suppress_progress=False):
+        '''
+        Some plugins need to keep an local cache of available books. This function
+        is called to update the caches. It is recommended to call this function
+        from :meth:`open`. Especially if :meth:`open` does anything other than
+        open a web page.
+        
+        This function can be called at any time. It is up to the plugin to determine
+        if the cache really does need updating. Unless :param:`force` is True, then
+        the plugin must update the cache. The only time force should be True is if
+        this function is called by the plugin's configuration dialog.
+        
+        if :param:`suppress_progress` is False it is safe to assume that this function
+        is being called from the main GUI thread so it is safe and recommended to use
+        a QProgressDialog to display what is happening and allow the user to cancel
+        the operation. if :param:`suppress_progress` is True then run the update
+        silently. In this case there is no guarantee what thread is calling this
+        function so no Qt related functionality that requires being run in the main
+        GUI thread should be run. E.G. Open a QProgressDialog. 
+        
+        :param parent: The parent object to be used by an GUI dialogs.
+        
+        :param timeout: The maximum amount of time that should be spent in
+        any given network connection.
+        
+        :param force: Force updating the cache even if the plugin has determined
+        it is not necessary.
+        
+        :param suppress_progress: Should a progress indicator be shown.
+        
+        :return: True if the cache was updated, False otherwise.
+        '''
+        return False
+
     def get_settings(self):
         '''
         This is only useful for plugins that implement

src/calibre/gui2/store/mobileread/cache_progress_dialog.py

@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from PyQt4.Qt import QDialog
+
+from calibre.gui2.store.mobileread.cache_progress_dialog_ui import Ui_Dialog
+
+class CacheProgressDialog(QDialog, Ui_Dialog):
+
+    def __init__(self, parent=None, total=None):
+        QDialog.__init__(self, parent)
+        self.setupUi(self)
+
+        self.completed = 0
+        self.canceled = False
+
+        self.progress.setValue(0)
+        self.progress.setMinimum(0)
+        self.progress.setMaximum(total if total else 0)
+
+    def exec_(self):
+        self.completed = 0
+        self.canceled = False
+        QDialog.exec_(self)
+
+    def open(self):
+        self.completed = 0
+        self.canceled = False
+        QDialog.open(self)
+
+    def reject(self):
+        self.canceled = True
+        QDialog.reject(self)
+
+    def update_progress(self):
+        '''
+        completed is an int from 0 to total representing the number
+        records that have bee completed.
+        '''
+        self.set_progress(self.completed + 1)
+
+    def set_message(self, msg):
+        self.message.setText(msg)
+
+    def set_details(self, msg):
+        self.details.setText(msg)
+
+    def set_progress(self, completed):
+        '''
+        completed is an int from 0 to total representing the number
+        records that have bee completed.
+        '''
+        self.completed = completed
+        self.progress.setValue(self.completed)
+
+    def set_total(self, total):
+        self.progress.setMaximum(total)

src/calibre/gui2/store/mobileread/cache_progress_dialog.ui

@@ -0,0 +1,104 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>402</width>
+    <height>138</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Dialog</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout">
+   <item>
+    <widget class="QLabel" name="message">
+     <property name="text">
+      <string>Updating book cache</string>
+     </property>
+     <property name="alignment">
+      <set>Qt::AlignCenter</set>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QProgressBar" name="progress">
+     <property name="value">
+      <number>24</number>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <widget class="QLabel" name="details">
+     <property name="text">
+      <string/>
+     </property>
+     <property name="alignment">
+      <set>Qt::AlignCenter</set>
+     </property>
+    </widget>
+   </item>
+   <item>
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>40</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+   <item>
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Cancel</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/store/mobileread/cache_update_thread.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import time
+from contextlib import closing
+from threading import Thread
+
+from lxml import html
+
+from PyQt4.Qt import (pyqtSignal, QObject)
+
+from calibre import browser
+from calibre.gui2.store.search_result import SearchResult
+
+class CacheUpdateThread(Thread, QObject):
+    
+    total_changed = pyqtSignal(int)
+    update_progress = pyqtSignal(int)
+    update_details = pyqtSignal(unicode)
+    
+    def __init__(self, config, seralize_books_function, timeout):
+        Thread.__init__(self)
+        QObject.__init__(self)
+
+        self.daemon = True
+        self.config = config
+        self.seralize_books = seralize_books_function
+        self.timeout = timeout
+        self._run = True
+        
+    def abort(self):
+        self._run = False
+        
+    def run(self):
+        url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html'
+        
+        self.update_details.emit(_('Checking last download date.'))
+        last_download = self.config.get('last_download', None)
+        # Don't update the book list if our cache is less than one week old.
+        if last_download and (time.time() - last_download) < 604800:
+            return
+        
+        self.update_details.emit(_('Downloading book list from MobileRead.'))
+        # Download the book list HTML file from MobileRead.
+        br = browser()
+        raw_data = None
+        try:
+            with closing(br.open(url, timeout=self.timeout)) as f:
+                raw_data = f.read()
+        except:
+            return
+        
+        if not raw_data or not self._run:
+            return
+        
+        self.update_details.emit(_('Processing books.'))
+        # Turn books listed in the HTML file into SearchResults's.
+        books = []
+        try:
+            data = html.fromstring(raw_data)
+            raw_books = data.xpath('//ul/li')
+            self.total_changed.emit(len(raw_books))
+            
+            for i, book_data in enumerate(raw_books):
+                self.update_details.emit(_('%s of %s books processed.') % (i, len(raw_books)))
+                book = SearchResult()
+                book.detail_item = ''.join(book_data.xpath('.//a/@href'))
+                book.formats = ''.join(book_data.xpath('.//i/text()'))
+                book.formats = book.formats.strip()
+    
+                text = ''.join(book_data.xpath('.//a/text()'))
+                if ':' in text:
+                    book.author, q, text = text.partition(':')
+                book.author = book.author.strip()
+                book.title = text.strip()
+                books.append(book)
+                
+                if not self._run:
+                    books = []
+                    break
+                else:
+                    self.update_progress.emit(i)
+        except:
+            pass
+
+        # Save the book list and it's create time.
+        if books:
+            self.config['book_list'] = self.seralize_books(books)
+            self.config['last_download'] = time.time()

src/calibre/gui2/store/mobileread/mobileread_plugin.py

@@ -0,0 +1,105 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from threading import Lock
+
+from PyQt4.Qt import (QUrl, QCoreApplication)
+
+from calibre.gui2 import open_url
+from calibre.gui2.store import StorePlugin
+from calibre.gui2.store.basic_config import BasicStoreConfig
+from calibre.gui2.store.search_result import SearchResult
+from calibre.gui2.store.web_store_dialog import WebStoreDialog
+from calibre.gui2.store.mobileread.models import SearchFilter
+from calibre.gui2.store.mobileread.cache_progress_dialog import CacheProgressDialog
+from calibre.gui2.store.mobileread.cache_update_thread import CacheUpdateThread
+from calibre.gui2.store.mobileread.store_dialog import MobeReadStoreDialog
+
+class MobileReadStore(BasicStoreConfig, StorePlugin):
+    
+    def genesis(self):
+        self.lock = Lock()
+    
+    def open(self, parent=None, detail_item=None, external=False):
+        url = 'http://www.mobileread.com/'
+        
+        if external or self.config.get('open_external', False):
+            open_url(QUrl(detail_item if detail_item else url))
+        else:
+            if detail_item:
+                d = WebStoreDialog(self.gui, url, parent, detail_item)
+                d.setWindowTitle(self.name)
+                d.set_tags(self.config.get('tags', ''))
+                d.exec_()
+            else:
+                if self.update_cache(parent, 30):
+                    d = MobeReadStoreDialog(self, parent)
+                    d.setWindowTitle(self.name)
+                    d.exec_()
+
+    def search(self, query, max_results=10, timeout=60):
+        books = self.get_book_list()
+
+        sf = SearchFilter(books)
+        matches = sf.parse(query)
+
+        for book in matches:
+            book.price = '$0.00'
+            book.drm = SearchResult.DRM_UNLOCKED
+            yield book
+
+    def update_cache(self, parent=None, timeout=10, force=False, suppress_progress=False):
+        if self.lock.acquire(False):
+            try:
+                update_thread = CacheUpdateThread(self.config, self.seralize_books, timeout)
+                if not suppress_progress:
+                    progress = CacheProgressDialog(parent)
+                    progress.set_message(_('Updating MobileRead book cache...'))
+            
+                    update_thread.total_changed.connect(progress.set_total)
+                    update_thread.update_progress.connect(progress.set_progress)
+                    update_thread.update_details.connect(progress.set_details)
+                    progress.rejected.connect(update_thread.abort)
+            
+                    progress.open()
+                    update_thread.start()
+                    while update_thread.is_alive() and not progress.canceled:
+                        QCoreApplication.processEvents()
+            
+                    if progress.isVisible():
+                        progress.accept()
+                    return not progress.canceled
+                else:
+                    update_thread.start()
+            finally:
+                self.lock.release()
+
+    def get_book_list(self):
+        return self.deseralize_books(self.config.get('book_list', []))
+    
+    def seralize_books(self, books):
+        sbooks = []
+        for b in books:
+            data = {}
+            data['author'] = b.author
+            data['title'] = b.title
+            data['detail_item'] = b.detail_item
+            data['formats'] = b.formats
+            sbooks.append(data)
+        return sbooks
+    
+    def deseralize_books(self, sbooks):
+        books = []
+        for s in sbooks:
+            b = SearchResult()
+            b.author = s.get('author', '')
+            b.title = s.get('title', '')
+            b.detail_item = s.get('detail_item', '')
+            b.formats = s.get('formats', '')
+            books.append(b)
+        return books

src/calibre/gui2/store/mobileread/models.py

@@ -0,0 +1,190 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+from operator import attrgetter
+
+from PyQt4.Qt import (Qt, QAbstractItemModel, QModelIndex, QVariant, pyqtSignal)
+
+from calibre.gui2 import NONE
+from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
+    REGEXP_MATCH
+from calibre.utils.icu import sort_key
+from calibre.utils.search_query_parser import SearchQueryParser
+
+class BooksModel(QAbstractItemModel):
+    
+    total_changed = pyqtSignal(int)
+
+    HEADERS = [_('Title'), _('Author(s)'), _('Format')]
+
+    def __init__(self, all_books):
+        QAbstractItemModel.__init__(self)
+        self.books = all_books
+        self.all_books = all_books
+        self.filter = ''
+        self.search_filter = SearchFilter(all_books)
+        self.sort_col = 0
+        self.sort_order = Qt.AscendingOrder
+
+    def get_book(self, index):
+        row = index.row()
+        if row < len(self.books):
+            return self.books[row]
+        else:
+            return None
+    
+    def search(self, filter):        
+        self.filter = filter.strip()
+        if not self.filter:
+            self.books = self.all_books
+        else:
+            try:
+                self.books = list(self.search_filter.parse(self.filter))
+            except:
+                self.books = self.all_books
+        self.sort(self.sort_col, self.sort_order)
+        self.total_changed.emit(self.rowCount())
+    
+    def index(self, row, column, parent=QModelIndex()):
+        return self.createIndex(row, column)
+
+    def parent(self, index):
+        if not index.isValid() or index.internalId() == 0:
+            return QModelIndex()
+        return self.createIndex(0, 0)
+
+    def rowCount(self, *args):
+        return len(self.books)
+
+    def columnCount(self, *args):
+        return len(self.HEADERS)
+    
+    def headerData(self, section, orientation, role):
+        if role != Qt.DisplayRole:
+            return NONE
+        text = ''
+        if orientation == Qt.Horizontal:
+            if section < len(self.HEADERS):
+                text = self.HEADERS[section]
+            return QVariant(text)
+        else:
+            return QVariant(section+1)
+
+    def data(self, index, role):
+        row, col = index.row(), index.column()
+        result = self.books[row]
+        if role == Qt.DisplayRole:
+            if col == 0:
+                return QVariant(result.title)
+            elif col == 1:
+                return QVariant(result.author)
+            elif col == 2:
+                return QVariant(result.formats)
+        return NONE
+
+    def data_as_text(self, result, col):
+        text = ''
+        if col == 0:
+            text = result.title
+        elif col == 1:
+            text = result.author
+        elif col == 2:
+            text = result.formats
+        return text
+
+    def sort(self, col, order, reset=True):
+        self.sort_col = col
+        self.sort_order = order
+        if not self.books:
+            return
+        descending = order == Qt.DescendingOrder
+        self.books.sort(None,
+            lambda x: sort_key(unicode(self.data_as_text(x, col))),
+            descending)
+        if reset:
+            self.reset()
+
+
+class SearchFilter(SearchQueryParser):
+    
+    USABLE_LOCATIONS = [
+        'all',
+        'author',
+        'authors',
+        'format',
+        'formats',
+        'title',
+    ]
+
+    def __init__(self, all_books=[]):
+        SearchQueryParser.__init__(self, locations=self.USABLE_LOCATIONS)
+        self.srs = set(all_books)
+
+    def universal_set(self):
+        return self.srs
+
+    def get_matches(self, location, query):
+        location = location.lower().strip()
+        if location == 'authors':
+            location = 'author'
+        elif location == 'formats':
+            location = 'format'
+
+        matchkind = CONTAINS_MATCH
+        if len(query) > 1:
+            if query.startswith('\\'):
+                query = query[1:]
+            elif query.startswith('='):
+                matchkind = EQUALS_MATCH
+                query = query[1:]
+            elif query.startswith('~'):
+                matchkind = REGEXP_MATCH
+                query = query[1:]
+        if matchkind != REGEXP_MATCH: ### leave case in regexps because it can be significant e.g. \S \W \D
+            query = query.lower()
+
+        if location not in self.USABLE_LOCATIONS:
+            return set([])
+        matches = set([])
+        all_locs = set(self.USABLE_LOCATIONS) - set(['all'])
+        locations = all_locs if location == 'all' else [location]
+        q = {
+             'author': lambda x: x.author.lower(),
+             'format': attrgetter('formats'),
+             'title': lambda x: x.title.lower(),
+        }
+        for x in ('author', 'format'):
+            q[x+'s'] = q[x]
+        for sr in self.srs:
+            for locvalue in locations:
+                accessor = q[locvalue]
+                if query == 'true':
+                    if accessor(sr) is not None:
+                        matches.add(sr)
+                    continue
+                if query == 'false':
+                    if accessor(sr) is None:
+                        matches.add(sr)
+                    continue
+                try:
+                    ### Can't separate authors because comma is used for name sep and author sep
+                    ### Exact match might not get what you want. For that reason, turn author
+                    ### exactmatch searches into contains searches.
+                    if locvalue == 'author' and matchkind == EQUALS_MATCH:
+                        m = CONTAINS_MATCH
+                    else:
+                        m = matchkind
+
+                    vals = [accessor(sr)]
+                    if _match(query, vals, m):
+                        matches.add(sr)
+                        break
+                except ValueError: # Unicode errors
+                    import traceback
+                    traceback.print_exc()
+        return matches

src/calibre/gui2/store/mobileread/store_dialog.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+
+from PyQt4.Qt import (Qt, QDialog, QIcon)
+
+from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog
+from calibre.gui2.store.mobileread.models import BooksModel
+from calibre.gui2.store.mobileread.store_dialog_ui import Ui_Dialog
+
+class MobeReadStoreDialog(QDialog, Ui_Dialog):
+    
+    def __init__(self, plugin, *args):
+        QDialog.__init__(self, *args)
+        self.setupUi(self)
+
+        self.plugin = plugin
+        
+        self.adv_search_button.setIcon(QIcon(I('search.png')))
+        
+        self._model = BooksModel(self.plugin.get_book_list())
+        self.results_view.setModel(self._model)
+        self.total.setText('%s' % self.results_view.model().rowCount())
+
+        self.search_button.clicked.connect(self.do_search)
+        self.adv_search_button.clicked.connect(self.build_adv_search)
+        self.results_view.activated.connect(self.open_store)
+        self.results_view.model().total_changed.connect(self.update_book_total)
+        self.finished.connect(self.dialog_closed)
+        
+        self.restore_state()
+        
+    def do_search(self):
+        self.results_view.model().search(unicode(self.search_query.text()))
+        
+    def open_store(self, index):
+        result = self.results_view.model().get_book(index)
+        if result:
+            self.plugin.open(self, result.detail_item)
+    
+    def update_book_total(self, total):
+        self.total.setText('%s' % total)
+    
+    def build_adv_search(self):
+        adv = AdvSearchBuilderDialog(self)
+        adv.price_label.hide()
+        adv.price_box.hide()
+        if adv.exec_() == QDialog.Accepted:
+            self.search_query.setText(adv.search_string())
+    
+    def restore_state(self):
+        geometry = self.plugin.config.get('dialog_geometry', None)
+        if geometry:
+            self.restoreGeometry(geometry)
+
+        results_cwidth = self.plugin.config.get('dialog_results_view_column_width')
+        if results_cwidth:
+            for i, x in enumerate(results_cwidth):
+                if i >= self.results_view.model().columnCount():
+                    break
+                self.results_view.setColumnWidth(i, x)
+        else:
+            for i in xrange(self.results_view.model().columnCount()):
+                self.results_view.resizeColumnToContents(i)
+                
+        self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0)
+        self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder)
+        self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order)
+        self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order)
+
+    def save_state(self):
+        self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry())
+        self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.results_view.model().columnCount())]
+        self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col
+        self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order
+
+    def dialog_closed(self, result):
+        self.save_state()

src/calibre/gui2/store/mobileread/store_dialog.ui

@@ -19,13 +19,30 @@
      <item>
       <widget class="QLabel" name="label">
        <property name="text">
-        <string>Search:</string>
+        <string>&amp;Query:</string>
+       </property>
+       <property name="buddy">
+        <cstring>search_query</cstring>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QToolButton" name="adv_search_button">
+       <property name="text">
+        <string>...</string>
        </property>
       </widget>
      </item>
      <item>
       <widget class="QLineEdit" name="search_query"/>
      </item>
+     <item>
+      <widget class="QPushButton" name="search_button">
+       <property name="text">
+        <string>Search</string>
+       </property>
+      </widget>
+     </item>
     </layout>
    </item>
    <item>

src/calibre/gui2/store/mobileread_plugin.py

@@ -1,316 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from __future__ import (unicode_literals, division, absolute_import, print_function)
-
-__license__ = 'GPL 3'
-__copyright__ = '2011, John Schember <john@nachtimwald.com>'
-__docformat__ = 'restructuredtext en'
-
-import difflib
-import heapq
-import time
-from contextlib import closing
-from threading import RLock
-
-from lxml import html
-
-from PyQt4.Qt import Qt, QUrl, QDialog, QAbstractItemModel, QModelIndex, QVariant, \
-    pyqtSignal
-
-from calibre import browser
-from calibre.gui2 import open_url, NONE
-from calibre.gui2.store import StorePlugin
-from calibre.gui2.store.basic_config import BasicStoreConfig
-from calibre.gui2.store.mobileread_store_dialog_ui import Ui_Dialog
-from calibre.gui2.store.search_result import SearchResult
-from calibre.gui2.store.web_store_dialog import WebStoreDialog
-from calibre.utils.icu import sort_key
-
-class MobileReadStore(BasicStoreConfig, StorePlugin):
-
-    def genesis(self):
-        self.rlock = RLock()
-
-    def open(self, parent=None, detail_item=None, external=False):
-        url = 'http://www.mobileread.com/'
-
-        if external or self.config.get('open_external', False):
-            open_url(QUrl(detail_item if detail_item else url))
-        else:
-            if detail_item:
-                d = WebStoreDialog(self.gui, url, parent, detail_item)
-                d.setWindowTitle(self.name)
-                d.set_tags(self.config.get('tags', ''))
-                d.exec_()
-            else:
-                d = MobeReadStoreDialog(self, parent)
-                d.setWindowTitle(self.name)
-                d.exec_()
-
-    def search(self, query, max_results=10, timeout=60):
-        books = self.get_book_list(timeout=timeout)
-
-        query = query.lower()
-        query_parts = query.split(' ')
-        matches = []
-        s = difflib.SequenceMatcher()
-        for x in books:
-            ratio = 0
-            t_string = '%s %s' % (x.author.lower(), x.title.lower())
-            query_matches = []
-            for q in query_parts:
-                if q in t_string:
-                    query_matches.append(q)
-            for q in query_matches:
-                s.set_seq2(q)
-                for p in t_string.split(' '):
-                    s.set_seq1(p)
-                    ratio += s.ratio()
-            if ratio > 0:
-                matches.append((ratio, x))
-
-        # Move the best scorers to head of list.
-        matches = heapq.nlargest(max_results, matches)
-        for score, book in matches:
-            book.price = '$0.00'
-            book.drm = SearchResult.DRM_UNLOCKED
-            yield book
-
-    def update_book_list(self, timeout=10):
-        with self.rlock:
-            url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html'
-
-            last_download = self.config.get('last_download', None)
-            # Don't update the book list if our cache is less than one week old.
-            if last_download and (time.time() - last_download) < 604800:
-                return
-
-            # Download the book list HTML file from MobileRead.
-            br = browser()
-            raw_data = None
-            with closing(br.open(url, timeout=timeout)) as f:
-                raw_data = f.read()
-
-            if not raw_data:
-                return
-
-            # Turn books listed in the HTML file into SearchResults's.
-            books = []
-            try:
-                data = html.fromstring(raw_data)
-                for book_data in data.xpath('//ul/li'):
-                    book = SearchResult()
-                    book.detail_item = ''.join(book_data.xpath('.//a/@href'))
-                    book.formats = ''.join(book_data.xpath('.//i/text()'))
-                    book.formats = book.formats.strip()
-
-                    text = ''.join(book_data.xpath('.//a/text()'))
-                    if ':' in text:
-                        book.author, q, text = text.partition(':')
-                    book.author = book.author.strip()
-                    book.title = text.strip()
-                    books.append(book)
-            except:
-                pass
-
-            # Save the book list and it's create time.
-            if books:
-                self.config['last_download'] = time.time()
-                self.config['book_list'] = self.seralize_books(books)
-
-    def get_book_list(self, timeout=10):
-        self.update_book_list(timeout=timeout)
-        return self.deseralize_books(self.config.get('book_list', []))
-
-    def seralize_books(self, books):
-        sbooks = []
-        for b in books:
-            data = {}
-            data['author'] = b.author
-            data['title'] = b.title
-            data['detail_item'] = b.detail_item
-            data['formats'] = b.formats
-            sbooks.append(data)
-        return sbooks
-
-    def deseralize_books(self, sbooks):
-        books = []
-        for s in sbooks:
-            b = SearchResult()
-            b.author = s.get('author', '')
-            b.title = s.get('title', '')
-            b.detail_item = s.get('detail_item', '')
-            b.formats = s.get('formats', '')
-            books.append(b)
-        return books
-
-
-class MobeReadStoreDialog(QDialog, Ui_Dialog):
-
-    def __init__(self, plugin, *args):
-        QDialog.__init__(self, *args)
-        self.setupUi(self)
-
-        self.plugin = plugin
-
-        self.model = BooksModel()
-        self.results_view.setModel(self.model)
-        self.results_view.model().set_books(self.plugin.get_book_list())
-        self.total.setText('%s' % self.model.rowCount())
-
-        self.results_view.activated.connect(self.open_store)
-        self.search_query.textChanged.connect(self.model.set_filter)
-        self.results_view.model().total_changed.connect(self.total.setText)
-        self.finished.connect(self.dialog_closed)
-
-        self.restore_state()
-
-    def open_store(self, index):
-        result = self.results_view.model().get_book(index)
-        if result:
-            self.plugin.open(self, result.detail_item)
-
-    def restore_state(self):
-        geometry = self.plugin.config.get('dialog_geometry', None)
-        if geometry:
-            self.restoreGeometry(geometry)
-
-        results_cwidth = self.plugin.config.get('dialog_results_view_column_width')
-        if results_cwidth:
-            for i, x in enumerate(results_cwidth):
-                if i >= self.results_view.model().columnCount():
-                    break
-                self.results_view.setColumnWidth(i, x)
-        else:
-            for i in xrange(self.results_view.model().columnCount()):
-                self.results_view.resizeColumnToContents(i)
-
-        self.results_view.model().sort_col = self.plugin.config.get('dialog_sort_col', 0)
-        self.results_view.model().sort_order = self.plugin.config.get('dialog_sort_order', Qt.AscendingOrder)
-        self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order)
-        self.results_view.header().setSortIndicator(self.results_view.model().sort_col, self.results_view.model().sort_order)
-
-    def save_state(self):
-        self.plugin.config['dialog_geometry'] = bytearray(self.saveGeometry())
-        self.plugin.config['dialog_results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.model.columnCount())]
-        self.plugin.config['dialog_sort_col'] = self.results_view.model().sort_col
-        self.plugin.config['dialog_sort_order'] = self.results_view.model().sort_order
-
-    def dialog_closed(self, result):
-        self.save_state()
-
-
-class BooksModel(QAbstractItemModel):
-
-    total_changed = pyqtSignal(unicode)
-
-    HEADERS = [_('Title'), _('Author(s)'), _('Format')]
-
-    def __init__(self):
-        QAbstractItemModel.__init__(self)
-        self.books = []
-        self.all_books = []
-        self.filter = ''
-        self.sort_col = 0
-        self.sort_order = Qt.AscendingOrder
-
-    def set_books(self, books):
-        self.books = books
-        self.all_books = books
-
-        self.sort(self.sort_col, self.sort_order)
-
-    def get_book(self, index):
-        row = index.row()
-        if row < len(self.books):
-            return self.books[row]
-        else:
-            return None
-
-    def set_filter(self, filter):
-        #self.layoutAboutToBeChanged.emit()
-        self.beginResetModel()
-
-        self.filter = unicode(filter)
-        self.books = []
-        if self.filter:
-            for b in self.all_books:
-                test = '%s %s %s' % (b.title, b.author, b.formats)
-                test = test.lower()
-                include = True
-                for item in self.filter.split(' '):
-                    item = item.lower()
-                    if item not in test:
-                        include = False
-                        break
-                if include:
-                    self.books.append(b)
-        else:
-            self.books = self.all_books
-
-        self.sort(self.sort_col, self.sort_order, reset=False)
-        self.total_changed.emit('%s' % self.rowCount())
-
-        self.endResetModel()
-        #self.layoutChanged.emit()
-
-    def index(self, row, column, parent=QModelIndex()):
-        return self.createIndex(row, column)
-
-    def parent(self, index):
-        if not index.isValid() or index.internalId() == 0:
-            return QModelIndex()
-        return self.createIndex(0, 0)
-
-    def rowCount(self, *args):
-        return len(self.books)
-
-    def columnCount(self, *args):
-        return len(self.HEADERS)
-
-    def headerData(self, section, orientation, role):
-        if role != Qt.DisplayRole:
-            return NONE
-        text = ''
-        if orientation == Qt.Horizontal:
-            if section < len(self.HEADERS):
-                text = self.HEADERS[section]
-            return QVariant(text)
-        else:
-            return QVariant(section+1)
-
-    def data(self, index, role):
-        row, col = index.row(), index.column()
-        result = self.books[row]
-        if role == Qt.DisplayRole:
-            if col == 0:
-                return QVariant(result.title)
-            elif col == 1:
-                return QVariant(result.author)
-            elif col == 2:
-                return QVariant(result.formats)
-        return NONE
-
-    def data_as_text(self, result, col):
-        text = ''
-        if col == 0:
-            text = result.title
-        elif col == 1:
-            text = result.author
-        elif col == 2:
-            text = result.formats
-        return text
-
-    def sort(self, col, order, reset=True):
-        self.sort_col = col
-        self.sort_order = order
-
-        if not self.books:
-            return
-        descending = order == Qt.DescendingOrder
-        self.books.sort(None,
-            lambda x: sort_key(unicode(self.data_as_text(x, col))),
-            descending)
-        if reset:
-            self.reset()
-

src/calibre/gui2/store/open_library_plugin.py

@@ -50,6 +50,9 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin):
                 if counter <= 0:
                     break
                 
+                # Don't include books that don't have downloadable files.
+                if not data.xpath('boolean(./span[@class="actions"]//span[@class="label" and contains(text(), "Read")])'):
+                    continue
                 id = ''.join(data.xpath('./span[@class="bookcover"]/a/@href'))
                 if not id:
                     continue
@@ -67,7 +70,7 @@ class OpenLibraryStore(BasicStoreConfig, StorePlugin):
                 s.author = author.strip()
                 s.price = price
                 s.detail_item = id.strip()
-                s.drm = SearchResult.DRM_UNKNOWN
+                s.drm = SearchResult.DRM_UNLOCKED
                 
                 yield s
 

src/calibre/gui2/store/search/adv_search_builder.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import (unicode_literals, division, absolute_import, print_function)
+
+__license__ = 'GPL 3'
+__copyright__ = '2011, John Schember <john@nachtimwald.com>'
+__docformat__ = 'restructuredtext en'
+
+import re
+
+from PyQt4.Qt import (QDialog, QDialogButtonBox)
+
+from calibre.gui2.store.search.adv_search_builder_ui import Ui_Dialog
+from calibre.library.caches import CONTAINS_MATCH, EQUALS_MATCH
+
+class AdvSearchBuilderDialog(QDialog, Ui_Dialog):
+
+    def __init__(self, parent):
+        QDialog.__init__(self, parent)
+        self.setupUi(self)
+        
+        self.buttonBox.accepted.connect(self.advanced_search_button_pushed)
+        self.tab_2_button_box.accepted.connect(self.accept)
+        self.tab_2_button_box.rejected.connect(self.reject)
+        self.clear_button.clicked.connect(self.clear_button_pushed)
+        self.adv_search_used = False
+        self.mc = ''
+        
+        self.tabWidget.setCurrentIndex(0)
+        self.tabWidget.currentChanged[int].connect(self.tab_changed)
+        self.tab_changed(0)
+        
+    def tab_changed(self, idx):
+        if idx == 1:
+            self.tab_2_button_box.button(QDialogButtonBox.Ok).setDefault(True)
+        else:
+            self.buttonBox.button(QDialogButtonBox.Ok).setDefault(True)
+
+    def advanced_search_button_pushed(self):
+        self.adv_search_used = True
+        self.accept()
+
+    def clear_button_pushed(self):
+        self.title_box.setText('')
+        self.author_box.setText('')
+        self.price_box.setText('')
+        self.format_box.setText('')
+
+    def tokens(self, raw):
+        phrases = re.findall(r'\s*".*?"\s*', raw)
+        for f in phrases:
+            raw = raw.replace(f, ' ')
+        phrases = [t.strip('" ') for t in phrases]
+        return ['"' + self.mc + t + '"' for t in phrases + [r.strip() for r in raw.split()]]
+
+    def search_string(self):
+        if self.adv_search_used:
+            return self.adv_search_string()
+        else:
+            return self.box_search_string()
+
+    def adv_search_string(self):
+        mk = self.matchkind.currentIndex()
+        if mk == CONTAINS_MATCH:
+            self.mc = ''
+        elif mk == EQUALS_MATCH:
+            self.mc = '='
+        else:
+            self.mc = '~'
+        all, any, phrase, none = map(lambda x: unicode(x.text()),
+                (self.all, self.any, self.phrase, self.none))
+        all, any, none = map(self.tokens, (all, any, none))
+        phrase = phrase.strip()
+        all = ' and '.join(all)
+        any = ' or '.join(any)
+        none = ' and not '.join(none)
+        ans = ''
+        if phrase:
+            ans += '"%s"'%phrase
+        if all:
+            ans += (' and ' if ans else '') + all
+        if none:
+            ans += (' and not ' if ans else 'not ') + none
+        if any:
+            ans += (' or ' if ans else '') + any
+        return ans
+
+    def token(self):
+        txt = unicode(self.text.text()).strip()
+        if txt:
+            if self.negate.isChecked():
+                txt = '!'+txt
+            tok = self.FIELDS[unicode(self.field.currentText())]+txt
+            if re.search(r'\s', tok):
+                tok = '"%s"'%tok
+            return tok
+
+    def box_search_string(self):
+        mk = self.matchkind.currentIndex()
+        if mk == CONTAINS_MATCH:
+            self.mc = ''
+        elif mk == EQUALS_MATCH:
+            self.mc = '='
+        else:
+            self.mc = '~'
+
+        ans = []
+        self.box_last_values = {}
+        title = unicode(self.title_box.text()).strip()
+        if title:
+            ans.append('title:"' + self.mc + title + '"')
+        author = unicode(self.author_box.text()).strip()
+        if author:
+            ans.append('author:"' + self.mc + author + '"')
+        price = unicode(self.price_box.text()).strip()
+        if price:
+            ans.append('price:"' + self.mc + price + '"')
+        format = unicode(self.format_box.text()).strip()
+        if author:
+            ans.append('format:"' + self.mc + format + '"')            
+        if ans:
+            return ' and '.join(ans)
+        return ''

src/calibre/gui2/store/search/adv_search_builder.ui

@@ -0,0 +1,364 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>Dialog</class>
+ <widget class="QDialog" name="Dialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>752</width>
+    <height>472</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string>Advanced Search</string>
+  </property>
+  <property name="windowIcon">
+   <iconset>
+    <normaloff>:/images/search.png</normaloff>:/images/search.png</iconset>
+  </property>
+  <layout class="QGridLayout" name="gridLayout_2">
+   <item row="0" column="0">
+    <widget class="QLabel" name="label_5">
+     <property name="text">
+      <string>&amp;What kind of match to use:</string>
+     </property>
+     <property name="buddy">
+      <cstring>matchkind</cstring>
+     </property>
+    </widget>
+   </item>
+   <item row="0" column="1">
+    <widget class="QComboBox" name="matchkind">
+     <item>
+      <property name="text">
+       <string>Contains: the word or phrase matches anywhere in the metadata field</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>Equals: the word or phrase must match the entire metadata field</string>
+      </property>
+     </item>
+     <item>
+      <property name="text">
+       <string>Regular expression: the expression must match anywhere in the metadata field</string>
+      </property>
+     </item>
+    </widget>
+   </item>
+   <item row="2" column="0" colspan="2">
+    <widget class="QTabWidget" name="tabWidget">
+     <property name="currentIndex">
+      <number>0</number>
+     </property>
+     <widget class="QWidget" name="tab">
+      <attribute name="title">
+       <string>A&amp;dvanced Search</string>
+      </attribute>
+      <layout class="QGridLayout" name="gridLayout_3">
+       <item row="0" column="0">
+        <widget class="QGroupBox" name="groupBox">
+         <property name="title">
+          <string>Find entries that have...</string>
+         </property>
+         <layout class="QVBoxLayout" name="verticalLayout">
+          <item>
+           <layout class="QHBoxLayout" name="horizontalLayout">
+            <item>
+             <widget class="QLabel" name="label">
+              <property name="text">
+               <string>&amp;All these words:</string>
+              </property>
+              <property name="buddy">
+               <cstring>all</cstring>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QLineEdit" name="all"/>
+            </item>
+           </layout>
+          </item>
+          <item>
+           <layout class="QHBoxLayout" name="horizontalLayout_2">
+            <item>
+             <widget class="QLabel" name="label_2">
+              <property name="text">
+               <string>This exact &amp;phrase:</string>
+              </property>
+              <property name="buddy">
+               <cstring>all</cstring>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QLineEdit" name="phrase"/>
+            </item>
+           </layout>
+          </item>
+          <item>
+           <layout class="QHBoxLayout" name="horizontalLayout_3">
+            <item>
+             <widget class="QLabel" name="label_3">
+              <property name="text">
+               <string>&amp;One or more of these words:</string>
+              </property>
+              <property name="buddy">
+               <cstring>all</cstring>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QLineEdit" name="any"/>
+            </item>
+           </layout>
+          </item>
+         </layout>
+        </widget>
+       </item>
+       <item row="1" column="0">
+        <widget class="QGroupBox" name="groupBox_2">
+         <property name="title">
+          <string>But dont show entries that have...</string>
+         </property>
+         <layout class="QVBoxLayout" name="verticalLayout_2">
+          <item>
+           <layout class="QHBoxLayout" name="horizontalLayout_4">
+            <item>
+             <widget class="QLabel" name="label_4">
+              <property name="text">
+               <string>Any of these &amp;unwanted words:</string>
+              </property>
+              <property name="buddy">
+               <cstring>all</cstring>
+              </property>
+             </widget>
+            </item>
+            <item>
+             <widget class="QLineEdit" name="none"/>
+            </item>
+           </layout>
+          </item>
+          <item>
+           <widget class="QLabel" name="label_6">
+            <property name="maximumSize">
+             <size>
+              <width>16777215</width>
+              <height>30</height>
+             </size>
+            </property>
+            <property name="text">
+             <string>See the &lt;a href=&quot;http://calibre-ebook.com/user_manual/gui.html#the-search-interface&quot;&gt;User Manual&lt;/a&gt; for more help</string>
+            </property>
+            <property name="openExternalLinks">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
+         </layout>
+        </widget>
+       </item>
+       <item row="2" column="0">
+        <spacer name="verticalSpacer_2">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>40</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item row="3" column="0">
+        <widget class="QDialogButtonBox" name="buttonBox">
+         <property name="orientation">
+          <enum>Qt::Horizontal</enum>
+         </property>
+         <property name="standardButtons">
+          <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+         </property>
+        </widget>
+       </item>
+      </layout>
+     </widget>
+     <widget class="QWidget" name="tab_2">
+      <attribute name="title">
+       <string>Titl&amp;e/Author/Price ...</string>
+      </attribute>
+      <layout class="QGridLayout" name="gridLayout">
+       <item row="1" column="0">
+        <widget class="QLabel" name="label_7">
+         <property name="text">
+          <string>&amp;Title:</string>
+         </property>
+         <property name="buddy">
+          <cstring>title_box</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="1" column="1">
+        <widget class="EnLineEdit" name="title_box">
+         <property name="toolTip">
+          <string>Enter the title.</string>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="0">
+        <widget class="QLabel" name="label_8">
+         <property name="text">
+          <string>&amp;Author:</string>
+         </property>
+         <property name="buddy">
+          <cstring>author_box</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="0">
+        <widget class="QLabel" name="price_label">
+         <property name="text">
+          <string>&amp;Price:</string>
+         </property>
+         <property name="buddy">
+          <cstring>price_box</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="6" column="0" colspan="2">
+        <layout class="QHBoxLayout" name="horizontalLayout_6">
+         <item>
+          <widget class="QPushButton" name="clear_button">
+           <property name="text">
+            <string>&amp;Clear</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QDialogButtonBox" name="tab_2_button_box">
+           <property name="standardButtons">
+            <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+       <item row="5" column="1">
+        <spacer name="verticalSpacer">
+         <property name="orientation">
+          <enum>Qt::Vertical</enum>
+         </property>
+         <property name="sizeHint" stdset="0">
+          <size>
+           <width>20</width>
+           <height>40</height>
+          </size>
+         </property>
+        </spacer>
+       </item>
+       <item row="0" column="0" colspan="2">
+        <widget class="QLabel" name="label_11">
+         <property name="text">
+          <string>Search only in specific fields:</string>
+         </property>
+        </widget>
+       </item>
+       <item row="2" column="1">
+        <widget class="EnLineEdit" name="author_box"/>
+       </item>
+       <item row="4" column="1">
+        <widget class="QLineEdit" name="format_box"/>
+       </item>
+       <item row="4" column="0">
+        <widget class="QLabel" name="label_10">
+         <property name="text">
+          <string>&amp;Format:</string>
+         </property>
+         <property name="buddy">
+          <cstring>format_box</cstring>
+         </property>
+        </widget>
+       </item>
+       <item row="3" column="1">
+        <widget class="EnLineEdit" name="price_box"/>
+       </item>
+      </layout>
+     </widget>
+    </widget>
+   </item>
+   <item row="1" column="1">
+    <spacer name="verticalSpacer_3">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+     <property name="sizeHint" stdset="0">
+      <size>
+       <width>20</width>
+       <height>40</height>
+      </size>
+     </property>
+    </spacer>
+   </item>
+  </layout>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>EnLineEdit</class>
+   <extends>QLineEdit</extends>
+   <header>widgets.h</header>
+  </customwidget>
+ </customwidgets>
+ <tabstops>
+  <tabstop>all</tabstop>
+  <tabstop>phrase</tabstop>
+  <tabstop>any</tabstop>
+  <tabstop>none</tabstop>
+  <tabstop>buttonBox</tabstop>
+  <tabstop>title_box</tabstop>
+  <tabstop>author_box</tabstop>
+  <tabstop>price_box</tabstop>
+  <tabstop>format_box</tabstop>
+  <tabstop>clear_button</tabstop>
+  <tabstop>tab_2_button_box</tabstop>
+  <tabstop>tabWidget</tabstop>
+  <tabstop>matchkind</tabstop>
+ </tabstops>
+ <resources>
+  <include location="../../../../resources/images.qrc"/>
+ </resources>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>Dialog</receiver>
+   <slot>accept()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>248</x>
+     <y>254</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>157</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>Dialog</receiver>
+   <slot>reject()</slot>
+   <hints>
+    <hint type="sourcelabel">
+     <x>316</x>
+     <y>260</y>
+    </hint>
+    <hint type="destinationlabel">
+     <x>286</x>
+     <y>274</y>
+    </hint>
+   </hints>
+  </connection>
+ </connections>
+</ui>

src/calibre/gui2/store/search/download_thread.py

@@ -6,7 +6,6 @@ __license__ = 'GPL 3'
 __copyright__ = '2011, John Schember <john@nachtimwald.com>'
 __docformat__ = 'restructuredtext en'
 
-import time
 import traceback
 from contextlib import closing
 from threading import Thread
@@ -17,7 +16,9 @@ from calibre.utils.magick.draw import thumbnail
 
 class GenericDownloadThreadPool(object):
     '''
-    add_task must be implemented in a subclass.
+    add_task must be implemented in a subclass and must
+    GenericDownloadThreadPool.add_task must be called
+    at the end of the function.
     '''
 
     def __init__(self, thread_type, thread_count):
@@ -29,10 +30,16 @@ class GenericDownloadThreadPool(object):
         self.threads = []
 
     def add_task(self):
-        raise NotImplementedError()
+        '''
+        This must be implemented in a sub class and this function
+        must be called at the end of the add_task function in
+        the sub class.
         
-    def start_threads(self):
+        The implementation of this function (in this base class)
-        for i in range(self.thread_count):
+        starts any threads necessary to fill the pool if it is
+        not already full.
+        '''
+        for i in xrange(self.thread_count - self.running_threads_count()):
             t = self.thread_type(self.tasks, self.results)
             self.threads.append(t)
             t.start()
@@ -60,10 +67,14 @@ class GenericDownloadThreadPool(object):
         return not self.results.empty()
 
     def threads_running(self):
+        return self.running_threads_count() > 0
+
+    def running_threads_count(self):
+        count = 0
         for t in self.threads:
             if t.is_alive():
-                return True
+                count += 1
-        return False
+        return count
 
 
 class SearchThreadPool(GenericDownloadThreadPool):
@@ -73,17 +84,16 @@ class SearchThreadPool(GenericDownloadThreadPool):
     using start_threads(). Reset by calling abort().
 
     Example:
-    sp = SearchThreadPool(SearchThread, 3)
+    sp = SearchThreadPool(3)
-    add tasks using add_task(...)
+    sp.add_task(...)
-    sp.start_threads()
-    all threads have finished.
-    sp.abort()
-    add tasks using add_task(...)
-    sp.start_threads()
     '''
     
+    def __init__(self, thread_count):
+        GenericDownloadThreadPool.__init__(self, SearchThread, thread_count)
+
     def add_task(self, query, store_name, store_plugin, timeout):
         self.tasks.put((query, store_name, store_plugin, timeout))
+        GenericDownloadThreadPool.add_task(self)
 
 
 class SearchThread(Thread):
@@ -113,12 +123,13 @@ class SearchThread(Thread):
 
 
 class CoverThreadPool(GenericDownloadThreadPool):
-    '''
+
-    Once started all threads run until abort is called.
+    def __init__(self, thread_count):
-    '''
+        GenericDownloadThreadPool.__init__(self, CoverThread, thread_count)
 
     def add_task(self, search_result, update_callback, timeout=5):
         self.tasks.put((search_result, update_callback, timeout))
+        GenericDownloadThreadPool.add_task(self)
 
 
 class CoverThread(Thread):
@@ -136,12 +147,8 @@ class CoverThread(Thread):
         self._run = False
 
     def run(self):
-        while self._run:
+        while self._run and not self.tasks.empty():
             try:
-                time.sleep(.1)
-                while not self.tasks.empty():
-                    if not self._run:
-                        break
                 result, callback, timeout = self.tasks.get()
                 if result and result.cover_url:
                     with closing(self.br.open(result.cover_url, timeout=timeout)) as f:
@@ -154,12 +161,13 @@ class CoverThread(Thread):
 
 
 class DetailsThreadPool(GenericDownloadThreadPool):
-    '''
+    
-    Once started all threads run until abort is called.
+    def __init__(self, thread_count):
-    '''
+        GenericDownloadThreadPool.__init__(self, DetailsThread, thread_count)
 
     def add_task(self, search_result, store_plugin, update_callback, timeout=10):
         self.tasks.put((search_result, store_plugin, update_callback, timeout))
+        GenericDownloadThreadPool.add_task(self)
 
 
 class DetailsThread(Thread):
@@ -175,12 +183,8 @@ class DetailsThread(Thread):
         self._run = False
 
     def run(self):
-        while self._run:
+        while self._run and not self.tasks.empty():
             try:
-                time.sleep(.1)
-                while not self.tasks.empty():
-                    if not self._run:
-                        break
                 result, store_plugin, callback, timeout = self.tasks.get()
                 if result:
                     store_plugin.get_details(result, timeout)
@@ -188,3 +192,33 @@ class DetailsThread(Thread):
                 self.tasks.task_done()
             except:
                 continue
+
+
+class CacheUpdateThreadPool(GenericDownloadThreadPool):
+
+    def __init__(self, thread_count):
+        GenericDownloadThreadPool.__init__(self, CacheUpdateThread, thread_count)
+
+    def add_task(self, store_plugin, timeout=10):
+        self.tasks.put((store_plugin, timeout))
+        GenericDownloadThreadPool.add_task(self)
+
+
+class CacheUpdateThread(Thread):
+
+    def __init__(self, tasks, results):
+        Thread.__init__(self)
+        self.daemon = True
+        self.tasks = tasks
+        self._run = True
+
+    def abort(self):
+        self._run = False
+
+    def run(self):
+        while self._run and not self.tasks.empty():
+            try:
+                store_plugin, timeout = self.tasks.get()
+                store_plugin.update_cache(timeout=timeout, suppress_progress=True)
+            except:
+                traceback.print_exc()

src/calibre/gui2/store/search/models.py

@@ -14,7 +14,7 @@ from PyQt4.Qt import (Qt, QAbstractItemModel, QVariant, QPixmap, QModelIndex, QS
 from calibre.gui2 import NONE
 from calibre.gui2.store.search_result import SearchResult
 from calibre.gui2.store.search.download_thread import DetailsThreadPool, \
-    DetailsThread, CoverThreadPool, CoverThread
+    CoverThreadPool
 from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \
     REGEXP_MATCH
 from calibre.utils.icu import sort_key
@@ -51,10 +51,8 @@ class Matches(QAbstractItemModel):
         self.matches = []
         self.query = ''
         self.search_filter = SearchFilter()
-        self.cover_pool = CoverThreadPool(CoverThread, 2)
+        self.cover_pool = CoverThreadPool(2)
-        self.cover_pool.start_threads()
+        self.details_pool = DetailsThreadPool(4)
-        self.details_pool = DetailsThreadPool(DetailsThread, 4)
-        self.details_pool.start_threads()
 
         self.sort_col = 2
         self.sort_order = Qt.AscendingOrder
@@ -70,9 +68,7 @@ class Matches(QAbstractItemModel):
         self.search_filter.clear_search_results()
         self.query = ''
         self.cover_pool.abort()
-        self.cover_pool.start_threads()
         self.details_pool.abort()
-        self.details_pool.start_threads()
         self.reset()
 
     def add_result(self, result, store_plugin):

src/calibre/gui2/store/search/search.py

@@ -9,17 +9,17 @@ __docformat__ = 'restructuredtext en'
 import re
 from random import shuffle
 
-from PyQt4.Qt import (Qt, QDialog, QTimer, QCheckBox, QVBoxLayout) 
+from PyQt4.Qt import (Qt, QDialog, QTimer, QCheckBox, QVBoxLayout, QIcon) 
 
 from calibre.gui2 import JSONConfig, info_dialog
 from calibre.gui2.progress_indicator import ProgressIndicator
-from calibre.gui2.store.search.download_thread import SearchThreadPool, SearchThread
+from calibre.gui2.store.search.adv_search_builder import AdvSearchBuilderDialog
+from calibre.gui2.store.search.download_thread import SearchThreadPool, \
+    CacheUpdateThreadPool
 from calibre.gui2.store.search.search_ui import Ui_Dialog
 
 HANG_TIME = 75000 # milliseconds seconds
 TIMEOUT = 75 # seconds
-SEARCH_THREAD_TOTAL = 4
-COVER_DOWNLOAD_THREAD_TOTAL = 2
 
 class SearchDialog(QDialog, Ui_Dialog):
 
@@ -31,11 +31,17 @@ class SearchDialog(QDialog, Ui_Dialog):
 
         # We keep a cache of store plugins and reference them by name.
         self.store_plugins = istores
-        self.search_pool = SearchThreadPool(SearchThread, SEARCH_THREAD_TOTAL)
+        self.search_pool = SearchThreadPool(4)
+        self.cache_pool = CacheUpdateThreadPool(2)
         # Check for results and hung threads.
         self.checker = QTimer()
+        self.progress_checker = QTimer()
         self.hang_check = 0
         
+        # Update store caches silently.
+        for p in self.store_plugins.values():
+            self.cache_pool.add_task(p, 30)
+
         # Add check boxes for each store so the user
         # can disable searching specific stores on a
         # per search basis.
@@ -52,16 +58,27 @@ class SearchDialog(QDialog, Ui_Dialog):
         self.pi = ProgressIndicator(self, 24)
         self.top_layout.addWidget(self.pi)
         
+        self.adv_search_button.setIcon(QIcon(I('search.png')))
+
+        self.adv_search_button.clicked.connect(self.build_adv_search)
         self.search.clicked.connect(self.do_search)
         self.checker.timeout.connect(self.get_results)
+        self.progress_checker.timeout.connect(self.check_progress)
         self.results_view.activated.connect(self.open_store)
         self.select_all_stores.clicked.connect(self.stores_select_all)
         self.select_invert_stores.clicked.connect(self.stores_select_invert)
         self.select_none_stores.clicked.connect(self.stores_select_none)
         self.finished.connect(self.dialog_closed)
 
+        self.progress_checker.start(100)
+
         self.restore_state()
 
+    def build_adv_search(self):
+        adv = AdvSearchBuilderDialog(self)
+        if adv.exec_() == QDialog.Accepted:
+            self.search_edit.setText(adv.search_string())
+
     def resize_columns(self):
         total = 600
         # Cover
@@ -105,10 +122,8 @@ class SearchDialog(QDialog, Ui_Dialog):
         for n in store_names:
             if getattr(self, 'store_check_' + n).isChecked():
                 self.search_pool.add_task(query, n, self.store_plugins[n], TIMEOUT)
-        if self.search_pool.has_tasks():
         self.hang_check = 0
         self.checker.start(100)
-            self.search_pool.start_threads()
         self.pi.startAnimation()
 
     def clean_query(self, query):
@@ -181,19 +196,16 @@ class SearchDialog(QDialog, Ui_Dialog):
         if self.hang_check >= HANG_TIME:
             self.search_pool.abort()
             self.checker.stop()
-            self.pi.stopAnimation()
         else:
             # Stop the checker if not threads are running.
             if not self.search_pool.threads_running() and not self.search_pool.has_tasks():
                 self.checker.stop()
-                self.pi.stopAnimation()
 
         while self.search_pool.has_results():
             res, store_plugin = self.search_pool.get_result()
             if res:
                 self.results_view.model().add_result(res, store_plugin)
 
-        if not self.checker.isActive():
         if not self.results_view.model().has_results():
             info_dialog(self, _('No matches'), _('Couldn\'t find any books matching your query.'), show=True, show_copy_button=False)
 
@@ -202,6 +214,13 @@ class SearchDialog(QDialog, Ui_Dialog):
         result = self.results_view.model().get_result(index)
         self.store_plugins[result.store_name].open(self, result.detail_item)
 
+    def check_progress(self):
+        if not self.search_pool.threads_running() and not self.results_view.model().cover_pool.threads_running() and not self.results_view.model().details_pool.threads_running(): 
+            self.pi.stopAnimation()
+        else:
+            if not self.pi.isAnimated():
+                self.pi.startAnimation()
+
     def get_store_checks(self):
         '''
         Returns a list of QCheckBox's for each store.

src/calibre/gui2/store/search/search.ui

@@ -30,6 +30,13 @@
        </property>
       </widget>
      </item>
+     <item>
+      <widget class="QToolButton" name="adv_search_button">
+       <property name="text">
+        <string>...</string>
+       </property>
+      </widget>
+     </item>
      <item>
       <widget class="QLineEdit" name="search_edit"/>
      </item>