mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Use 4 thread threadpool for downloading search results.
This commit is contained in:
parent
588adb70e4
commit
98a0211463
@ -586,7 +586,9 @@ class StorePlugin(Plugin): # {{{
|
|||||||
|
|
||||||
supported_platforms = ['windows', 'osx', 'linux']
|
supported_platforms = ['windows', 'osx', 'linux']
|
||||||
author = 'John Schember'
|
author = 'John Schember'
|
||||||
type = _('Stores')
|
type = _('Store')
|
||||||
|
# This needs to be changed to (0, 8, 0)
|
||||||
|
minimum_calibre_version = (0, 4, 118)
|
||||||
|
|
||||||
def open(self, gui, parent=None, detail_item=None):
|
def open(self, gui, parent=None, detail_item=None):
|
||||||
'''
|
'''
|
||||||
@ -609,6 +611,9 @@ class StorePlugin(Plugin): # {{{
|
|||||||
item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store.
|
item_data is plugin specific and is used in :meth:`open` to open to a specifc place in the store.
|
||||||
'''
|
'''
|
||||||
raise NotImplementedError()
|
raise NotImplementedError()
|
||||||
|
|
||||||
|
def get_settings(self):
|
||||||
|
raise NotImplementedError()
|
||||||
|
|
||||||
# }}}
|
# }}}
|
||||||
|
|
||||||
|
@ -806,6 +806,7 @@ class ActionNextMatch(InterfaceActionBase):
|
|||||||
|
|
||||||
class ActionStore(InterfaceActionBase):
|
class ActionStore(InterfaceActionBase):
|
||||||
name = 'Store'
|
name = 'Store'
|
||||||
|
author = 'John Schember'
|
||||||
actual_plugin = 'calibre.gui2.actions.store:StoreAction'
|
actual_plugin = 'calibre.gui2.actions.store:StoreAction'
|
||||||
|
|
||||||
plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
|
plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
|
||||||
|
@ -76,3 +76,22 @@ class FeedbooksStore(StorePlugin):
|
|||||||
s.detail_item = id.strip()
|
s.detail_item = id.strip()
|
||||||
|
|
||||||
yield s
|
yield s
|
||||||
|
|
||||||
|
def customization_help(self, gui=False):
|
||||||
|
return 'Customize the behavior of this store.'
|
||||||
|
|
||||||
|
def config_widget(self):
|
||||||
|
from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget
|
||||||
|
return BasicStoreConfigWidget(self)
|
||||||
|
|
||||||
|
def save_settings(self, config_widget):
|
||||||
|
from calibre.gui2.store.basic_config_widget import save_settings
|
||||||
|
save_settings(config_widget)
|
||||||
|
|
||||||
|
def get_settings(self):
|
||||||
|
from calibre.gui2 import gprefs
|
||||||
|
settings = {}
|
||||||
|
|
||||||
|
settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download')
|
||||||
|
|
||||||
|
return settings
|
||||||
|
@ -17,13 +17,13 @@ class GutenbergStore(StorePlugin):
|
|||||||
|
|
||||||
name = 'Project Gutenberg'
|
name = 'Project Gutenberg'
|
||||||
description = _('The first producer of free ebooks.')
|
description = _('The first producer of free ebooks.')
|
||||||
|
|
||||||
|
|
||||||
def open(self, gui, parent=None, detail_item=None):
|
def open(self, gui, parent=None, detail_item=None):
|
||||||
|
settings = self.get_settings()
|
||||||
from calibre.gui2.store.web_store_dialog import WebStoreDialog
|
from calibre.gui2.store.web_store_dialog import WebStoreDialog
|
||||||
d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, detail_item)
|
d = WebStoreDialog(gui, 'http://m.gutenberg.org/', parent, detail_item)
|
||||||
d.setWindowTitle(self.name)
|
d.setWindowTitle(self.name)
|
||||||
d.set_tags(self.name + ',' + _('store'))
|
d.set_tags(settings.get(self.name + '_tags', ''))
|
||||||
d = d.exec_()
|
d = d.exec_()
|
||||||
|
|
||||||
def search(self, query, max_results=10, timeout=60):
|
def search(self, query, max_results=10, timeout=60):
|
||||||
@ -66,3 +66,22 @@ class GutenbergStore(StorePlugin):
|
|||||||
s.detail_item = '/ebooks/' + id.strip()
|
s.detail_item = '/ebooks/' + id.strip()
|
||||||
|
|
||||||
yield s
|
yield s
|
||||||
|
|
||||||
|
def customization_help(self, gui=False):
|
||||||
|
return 'Customize the behavior of this store.'
|
||||||
|
|
||||||
|
def config_widget(self):
|
||||||
|
from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget
|
||||||
|
return BasicStoreConfigWidget(self)
|
||||||
|
|
||||||
|
def save_settings(self, config_widget):
|
||||||
|
from calibre.gui2.store.basic_config_widget import save_settings
|
||||||
|
save_settings(config_widget)
|
||||||
|
|
||||||
|
def get_settings(self):
|
||||||
|
from calibre.gui2 import gprefs
|
||||||
|
settings = {}
|
||||||
|
|
||||||
|
settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download')
|
||||||
|
|
||||||
|
return settings
|
||||||
|
@ -78,3 +78,22 @@ class ManyBooksStore(StorePlugin):
|
|||||||
s.detail_item = '/titles/' + id
|
s.detail_item = '/titles/' + id
|
||||||
|
|
||||||
yield s
|
yield s
|
||||||
|
|
||||||
|
def customization_help(self, gui=False):
|
||||||
|
return 'Customize the behavior of this store.'
|
||||||
|
|
||||||
|
def config_widget(self):
|
||||||
|
from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget
|
||||||
|
return BasicStoreConfigWidget(self)
|
||||||
|
|
||||||
|
def save_settings(self, config_widget):
|
||||||
|
from calibre.gui2.store.basic_config_widget import save_settings
|
||||||
|
save_settings(config_widget)
|
||||||
|
|
||||||
|
def get_settings(self):
|
||||||
|
from calibre.gui2 import gprefs
|
||||||
|
settings = {}
|
||||||
|
|
||||||
|
settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download')
|
||||||
|
|
||||||
|
return settings
|
||||||
|
@ -25,6 +25,8 @@ class SearchDialog(QDialog, Ui_Dialog):
|
|||||||
|
|
||||||
HANG_TIME = 75000 # milliseconds seconds
|
HANG_TIME = 75000 # milliseconds seconds
|
||||||
TIMEOUT = 75 # seconds
|
TIMEOUT = 75 # seconds
|
||||||
|
SEARCH_THREAD_TOTAL = 4
|
||||||
|
COVER_DOWNLOAD_THREAD_TOTAL = 2
|
||||||
|
|
||||||
def __init__(self, gui, *args):
|
def __init__(self, gui, *args):
|
||||||
QDialog.__init__(self, *args)
|
QDialog.__init__(self, *args)
|
||||||
@ -33,9 +35,7 @@ class SearchDialog(QDialog, Ui_Dialog):
|
|||||||
self.gui = gui
|
self.gui = gui
|
||||||
|
|
||||||
self.store_plugins = {}
|
self.store_plugins = {}
|
||||||
self.running_threads = []
|
self.search_thread_pool = SearchThreadPool(self.SEARCH_THREAD_TOTAL)
|
||||||
self.results = Queue()
|
|
||||||
self.abort = Event()
|
|
||||||
self.checker = QTimer()
|
self.checker = QTimer()
|
||||||
self.hang_check = 0
|
self.hang_check = 0
|
||||||
|
|
||||||
@ -78,10 +78,7 @@ class SearchDialog(QDialog, Ui_Dialog):
|
|||||||
def do_search(self, checked=False):
|
def do_search(self, checked=False):
|
||||||
# Stop all running threads.
|
# Stop all running threads.
|
||||||
self.checker.stop()
|
self.checker.stop()
|
||||||
self.abort.set()
|
self.search_thread_pool.abort()
|
||||||
self.running_threads = []
|
|
||||||
self.results = Queue()
|
|
||||||
self.abort = Event()
|
|
||||||
# Clear the visible results.
|
# Clear the visible results.
|
||||||
self.results_view.model().clear_results()
|
self.results_view.model().clear_results()
|
||||||
|
|
||||||
@ -92,40 +89,32 @@ class SearchDialog(QDialog, Ui_Dialog):
|
|||||||
|
|
||||||
for n in self.store_plugins:
|
for n in self.store_plugins:
|
||||||
if getattr(self, 'store_check_' + n).isChecked():
|
if getattr(self, 'store_check_' + n).isChecked():
|
||||||
t = SearchThread(query, (n, self.store_plugins[n]), self.results, self.abort, self.TIMEOUT)
|
self.search_thread_pool.add_task(query, n, self.store_plugins[n], self.TIMEOUT)
|
||||||
self.running_threads.append(t)
|
if self.search_thread_pool.has_tasks():
|
||||||
t.start()
|
|
||||||
if self.running_threads:
|
|
||||||
self.hang_check = 0
|
self.hang_check = 0
|
||||||
self.checker.start(100)
|
self.checker.start(100)
|
||||||
|
self.search_thread_pool.start_threads()
|
||||||
|
|
||||||
def get_results(self):
|
def get_results(self):
|
||||||
# We only want the search plugins to run
|
# We only want the search plugins to run
|
||||||
# a maximum set amount of time before giving up.
|
# a maximum set amount of time before giving up.
|
||||||
self.hang_check += 1
|
self.hang_check += 1
|
||||||
if self.hang_check >= self.HANG_TIME:
|
if self.hang_check >= self.HANG_TIME:
|
||||||
self.abort.set()
|
self.search_thread_pool.abort()
|
||||||
self.checker.stop()
|
self.checker.stop()
|
||||||
else:
|
else:
|
||||||
# Stop the checker if not threads are running.
|
# Stop the checker if not threads are running.
|
||||||
running = False
|
if not self.search_thread_pool.threads_running():
|
||||||
for t in self.running_threads:
|
|
||||||
if t.is_alive():
|
|
||||||
running = True
|
|
||||||
if not running:
|
|
||||||
self.checker.stop()
|
self.checker.stop()
|
||||||
|
|
||||||
while not self.results.empty():
|
while self.search_thread_pool.has_results():
|
||||||
res = self.results.get_nowait()
|
res = self.search_thread_pool.get_result_no_wait()
|
||||||
if res:
|
if res:
|
||||||
result = res[1]
|
self.results_view.model().add_result(res)
|
||||||
result.store = res[0]
|
|
||||||
|
|
||||||
self.results_view.model().add_result(result)
|
|
||||||
|
|
||||||
def open_store(self, index):
|
def open_store(self, index):
|
||||||
result = self.results_view.model().get_result(index)
|
result = self.results_view.model().get_result(index)
|
||||||
self.store_plugins[result.store].open(self.gui, self, result.detail_item)
|
self.store_plugins[result.store_name].open(self.gui, self, result.detail_item)
|
||||||
|
|
||||||
def get_store_checks(self):
|
def get_store_checks(self):
|
||||||
checks = []
|
checks = []
|
||||||
@ -147,27 +136,77 @@ class SearchDialog(QDialog, Ui_Dialog):
|
|||||||
for check in self.get_store_checks():
|
for check in self.get_store_checks():
|
||||||
check.setChecked(False)
|
check.setChecked(False)
|
||||||
|
|
||||||
|
|
||||||
|
class SearchThreadPool(object):
|
||||||
|
|
||||||
|
def __init__(self, thread_count):
|
||||||
|
self.tasks = Queue()
|
||||||
|
self.results = Queue()
|
||||||
|
self.threads = []
|
||||||
|
self.thread_count = thread_count
|
||||||
|
|
||||||
|
def start_threads(self):
|
||||||
|
for i in range(self.thread_count):
|
||||||
|
t = SearchThread(self.tasks, self.results)
|
||||||
|
self.threads.append(t)
|
||||||
|
t.start()
|
||||||
|
|
||||||
|
def abort(self):
|
||||||
|
self.tasks = Queue()
|
||||||
|
self.results = Queue()
|
||||||
|
self.threads = []
|
||||||
|
for t in self.threads:
|
||||||
|
t.abort()
|
||||||
|
|
||||||
|
def add_task(self, query, store_name, store_plugin, timeout):
|
||||||
|
self.tasks.put((query, store_name, store_plugin, timeout))
|
||||||
|
|
||||||
|
def has_tasks(self):
|
||||||
|
return not self.tasks.empty()
|
||||||
|
|
||||||
|
def get_result(self):
|
||||||
|
return self.results.get()
|
||||||
|
|
||||||
|
def get_result_no_wait(self):
|
||||||
|
return self.results.get_nowait()
|
||||||
|
|
||||||
|
def result_count(self):
|
||||||
|
return len(self.results)
|
||||||
|
|
||||||
|
def has_results(self):
|
||||||
|
return not self.results.empty()
|
||||||
|
|
||||||
|
def threads_running(self):
|
||||||
|
for t in self.threads:
|
||||||
|
if t.is_alive():
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
class SearchThread(Thread):
|
class SearchThread(Thread):
|
||||||
|
|
||||||
def __init__(self, query, store, results, abort, timeout):
|
def __init__(self, tasks, results):
|
||||||
Thread.__init__(self)
|
Thread.__init__(self)
|
||||||
self.daemon = True
|
self.daemon = True
|
||||||
self.query = query
|
self.tasks = tasks
|
||||||
self.store_name = store[0]
|
|
||||||
self.store_plugin = store[1]
|
|
||||||
self.results = results
|
self.results = results
|
||||||
self.abort = abort
|
self._run = True
|
||||||
self.timeout = timeout
|
|
||||||
self.br = browser()
|
def abort(self):
|
||||||
|
self._run = False
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
try:
|
while self._run and not self.tasks.empty():
|
||||||
for res in self.store_plugin.search(self.query, timeout=self.timeout):
|
try:
|
||||||
if self.abort.is_set():
|
query, store_name, store_plugin, timeout = self.tasks.get()
|
||||||
return
|
for res in store_plugin.search(query, timeout=timeout):
|
||||||
self.results.put((self.store_name, res))
|
if not self._run:
|
||||||
except:
|
return
|
||||||
pass
|
res.store_name = store_name
|
||||||
|
self.results.put(res)
|
||||||
|
self.tasks.task_done()
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class CoverDownloadThread(Thread):
|
class CoverDownloadThread(Thread):
|
||||||
@ -278,7 +317,7 @@ class Matches(QAbstractItemModel):
|
|||||||
elif col == 3:
|
elif col == 3:
|
||||||
return QVariant(result.price)
|
return QVariant(result.price)
|
||||||
elif col == 4:
|
elif col == 4:
|
||||||
return QVariant(result.store)
|
return QVariant(result.store_name)
|
||||||
return NONE
|
return NONE
|
||||||
elif role == Qt.DecorationRole:
|
elif role == Qt.DecorationRole:
|
||||||
if col == 0 and result.cover_data:
|
if col == 0 and result.cover_data:
|
||||||
@ -302,7 +341,7 @@ class Matches(QAbstractItemModel):
|
|||||||
text = re.sub(r'\D', '', text)
|
text = re.sub(r'\D', '', text)
|
||||||
text = text.rjust(6, '0')
|
text = text.rjust(6, '0')
|
||||||
elif col == 4:
|
elif col == 4:
|
||||||
text = result.store
|
text = result.store_name
|
||||||
return text
|
return text
|
||||||
|
|
||||||
def sort(self, col, order, reset=True):
|
def sort(self, col, order, reset=True):
|
||||||
|
@ -7,11 +7,10 @@ __docformat__ = 'restructuredtext en'
|
|||||||
class SearchResult(object):
|
class SearchResult(object):
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
self.store = ''
|
self.store_name = ''
|
||||||
self.cover_url = ''
|
self.cover_url = ''
|
||||||
self.cover_data = None
|
self.cover_data = None
|
||||||
self.title = ''
|
self.title = ''
|
||||||
self.author = ''
|
self.author = ''
|
||||||
self.price = ''
|
self.price = ''
|
||||||
self.store = ''
|
|
||||||
self.detail_item = ''
|
self.detail_item = ''
|
||||||
|
@ -78,3 +78,22 @@ class SmashwordsStore(StorePlugin):
|
|||||||
s.detail_item = '/books/view/' + id.strip()
|
s.detail_item = '/books/view/' + id.strip()
|
||||||
|
|
||||||
yield s
|
yield s
|
||||||
|
|
||||||
|
def customization_help(self, gui=False):
|
||||||
|
return 'Customize the behavior of this store.'
|
||||||
|
|
||||||
|
def config_widget(self):
|
||||||
|
from calibre.gui2.store.basic_config_widget import BasicStoreConfigWidget
|
||||||
|
return BasicStoreConfigWidget(self)
|
||||||
|
|
||||||
|
def save_settings(self, config_widget):
|
||||||
|
from calibre.gui2.store.basic_config_widget import save_settings
|
||||||
|
save_settings(config_widget)
|
||||||
|
|
||||||
|
def get_settings(self):
|
||||||
|
from calibre.gui2 import gprefs
|
||||||
|
settings = {}
|
||||||
|
|
||||||
|
settings[self.name + '_tags'] = gprefs.get(self.name + '_tags', self.name + ', store, download')
|
||||||
|
|
||||||
|
return settings
|
||||||
|
Loading…
x
Reference in New Issue
Block a user