Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature-recipe-get-delay' of https://github.com/ping/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2023-07-24 11:18:20 +05:30
commit 1f5810159f
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/calibre/web/feeds/news.py
View File

@ -474,6 +474,15 @@ class BasicNewsRecipe(Recipe):
return self.feeds[:self.test[0]]
return self.feeds
def get_delay(self, url=None):
'''
Return the delay in seconds before downloading `url`. If you want to programmatically
determine the delay for the specified url, override this method in your subclass.
When overriding, you should cater for when `url` may be `None`.
You may return a floating point number to indicate a more precise time.
'''
return getattr(self, 'delay', 0)
@classmethod
def print_version(cls, url):
'''
@ -934,7 +943,7 @@ class BasicNewsRecipe(Recipe):
web2disk_cmdline = ['web2disk',
'--timeout', str(self.timeout),
'--max-recursions', str(self.recursions),
'--delay', str(self.delay),
'--delay', str(self.get_delay()),
]
if self.verbose:
@ -966,8 +975,9 @@ class BasicNewsRecipe(Recipe):
self.web2disk_options.preprocess_image = self.preprocess_image
self.web2disk_options.encoding = self.encoding
self.web2disk_options.preprocess_raw_html = self.preprocess_raw_html_
self.web2disk_options.get_delay = self.get_delay
if self.delay > 0:
if self.get_delay() > 0:
self.simultaneous_downloads = 1
self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \
@ -1711,8 +1721,9 @@ class BasicNewsRecipe(Recipe):
feed.description = as_unicode(err)
parsed_feeds.append(feed)
self.log.exception(msg)
if self.delay > 0:
time.sleep(self.delay)
delay = self.get_delay(url)
if delay > 0:
time.sleep(delay)
remove = [fl for fl in parsed_feeds if len(fl) == 0 and self.remove_empty_feeds]
for f in remove:

6
src/calibre/web/fetch/simple.py
View File

@ -180,6 +180,7 @@ class RecursiveFetcher:
self.compress_news_images = getattr(options, 'compress_news_images', False)
self.compress_news_images_auto_size = getattr(options, 'compress_news_images_auto_size', 16)
self.scale_news_images = getattr(options, 'scale_news_images', None)
self.get_delay = getattr(options, 'get_delay', lambda url=None: self.delay)
self.download_stylesheets = not options.no_stylesheets
self.show_progress = True
self.failed_links = []
@ -268,8 +269,9 @@ class RecursiveFetcher:
return data
delta = time.monotonic() - self.last_fetch_at
if delta < self.delay:
time.sleep(self.delay - delta)
delay = self.get_delay(url)
if delta < delay:
time.sleep(delay - delta)
url = canonicalize_url(url)
open_func = getattr(self.browser, 'open_novisit', self.browser.open)
try: