Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

DRYer

Browse Source
This commit is contained in:
Kovid Goyal 2022-04-02 13:50:14 +05:30
parent 7a4a6d0d19
commit dc92c8f9bb
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 17 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/calibre/gui2/store/stores/kobo_plugin.py
View File

@ -15,7 +15,6 @@ except ImportError:
from lxml import etree, html
from calibre import url_slash_cleaner
from calibre.ebooks.chardet import strip_encoding_declarations
from calibre.ebooks.metadata import authors_to_string
from calibre.gui2 import open_url
from calibre.gui2.store import StorePlugin
@ -23,17 +22,15 @@ from calibre.gui2.store.basic_config import BasicStoreConfig
from calibre.gui2.store.search_result import SearchResult
from calibre.gui2.store.web_store_dialog import WebStoreDialog
scraper = None
def read_url(url):
# Kobo uses Akamai which has some bot detection that uses network/tls
# protocol data. So use the Chromium network stack to make the request
global scraper
if scraper is None:
from calibre.scraper.simple import Overseer
scraper = Overseer()
return strip_encoding_declarations(scraper.fetch_url(url))
from calibre.scraper.simple import read_url as ru
return ru(read_url.storage, url)
read_url.storage = []
def search_kobo(query, max_results=10, timeout=60, write_html_to=None):

12
src/calibre/scraper/simple.py
View File

@ -114,6 +114,18 @@ def cleanup_overseers():
return join_all
read_url_lock = Lock()
def read_url(storage, url):
with read_url_lock:
if not storage:
storage.append(Overseer())
scraper = storage[0]
from calibre.ebooks.chardet import strip_encoding_declarations
return strip_encoding_declarations(scraper.fetch_url(url))
def find_tests():
import re
import unittest