Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Run get_cached_url backends in parallel

Browse Source
This commit is contained in:
Kovid Goyal 2024-10-18 14:13:05 +05:30
parent 761e2b5c6f
commit 758725699a
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 26 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
src/calibre/ebooks/metadata/sources/search_engines.py
View File

@ -31,7 +31,7 @@ from calibre.ebooks.chardet import xml_to_unicode
from calibre.utils.lock import ExclusiveFile
from calibre.utils.random_ua import accept_header_for_ua
current_version = (1, 2, 10)
current_version = (1, 2, 11)
minimum_calibre_version = (2, 80, 0)
webcache = {}
webcache_lock = Lock()
@ -443,7 +443,31 @@ def google_develop(search_terms='1423146786', raw_from=''):
def get_cached_url(url, br=None, log=prints, timeout=60):
return bing_cached_url(url, br, log, timeout) or wayback_machine_cached_url(url, br, log, timeout)
from threading import Lock, Thread
from polyglot.queue import Queue
print_lock = Lock()
q = Queue()
def safe_print(*a):
with print_lock:
log(*a)
def doit(func):
try:
q.put(func(url, br, safe_print, timeout))
except Exception as e:
safe_print(e)
q.put(None)
threads = []
threads.append(Thread(target=doit, args=(wayback_machine_cached_url,), daemon=True).start())
threads.append(Thread(target=doit, args=(bing_cached_url,), daemon=True).start())
while threads:
x = q.get()
if x is not None:
return x
threads.pop()
def get_data_for_cached_url(url):