Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup previous PR

Browse Source
This commit is contained in:
Kovid Goyal 2023-07-24 11:19:18 +05:30
parent 1f5810159f
commit ce457c61f2
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/calibre/web/feeds/news.py
View File

@ -103,8 +103,9 @@ class BasicNewsRecipe(Recipe):
#: Number of levels of links to follow on article webpages
recursions = 0
#: Delay between consecutive downloads in seconds. The argument may be a
#: floating point number to indicate a more precise time.
#: The default delay between consecutive downloads in seconds. The argument may be a
#: floating point number to indicate a more precise time. See :meth:`get_url_specific_delay`
#: to implement per URL delays.
delay = 0
#: Publication type
@ -474,14 +475,15 @@ class BasicNewsRecipe(Recipe):
return self.feeds[:self.test[0]]
return self.feeds
def get_delay(self, url=None):
def get_url_specific_delay(self, url):
'''
Return the delay in seconds before downloading `url`. If you want to programmatically
determine the delay for the specified url, override this method in your subclass.
When overriding, you should cater for when `url` may be `None`.
You may return a floating point number to indicate a more precise time.
Return the delay in seconds before downloading this URL. If you want to programmatically
determine the delay for the specified URL, override this method in your subclass, returning
self.delay by default for URLs you do not want to affect.
:return: A floating point number, the delay in seconds.
'''
return getattr(self, 'delay', 0)
return self.delay
@classmethod
def print_version(cls, url):
@ -943,7 +945,7 @@ class BasicNewsRecipe(Recipe):
web2disk_cmdline = ['web2disk',
'--timeout', str(self.timeout),
'--max-recursions', str(self.recursions),
'--delay', str(self.get_delay()),
'--delay', str(self.delay),
]
if self.verbose:
@ -975,9 +977,9 @@ class BasicNewsRecipe(Recipe):
self.web2disk_options.preprocess_image = self.preprocess_image
self.web2disk_options.encoding = self.encoding
self.web2disk_options.preprocess_raw_html = self.preprocess_raw_html_
self.web2disk_options.get_delay = self.get_delay
self.web2disk_options.get_delay = self.get_url_specific_delay
if self.get_delay() > 0:
if self.delay > 0:
self.simultaneous_downloads = 1
self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \
@ -1721,7 +1723,7 @@ class BasicNewsRecipe(Recipe):
feed.description = as_unicode(err)
parsed_feeds.append(feed)
self.log.exception(msg)
delay = self.get_delay(url)
delay = self.get_url_specific_delay(url)
if delay > 0:
time.sleep(delay)

2
src/calibre/web/fetch/simple.py
View File

@ -180,7 +180,7 @@ class RecursiveFetcher:
self.compress_news_images = getattr(options, 'compress_news_images', False)
self.compress_news_images_auto_size = getattr(options, 'compress_news_images_auto_size', 16)
self.scale_news_images = getattr(options, 'scale_news_images', None)
self.get_delay = getattr(options, 'get_delay', lambda url=None: self.delay)
self.get_delay = getattr(options, 'get_delay', lambda url: self.delay)
self.download_stylesheets = not options.no_stylesheets
self.show_progress = True
self.failed_links = []