Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup previous PR

Browse Source
This commit is contained in:
Kovid Goyal 2023-07-24 11:19:18 +05:30
parent 1f5810159f
commit ce457c61f2
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 15 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/calibre/web/feeds/news.py
View File

@ -103,8 +103,9 @@ class BasicNewsRecipe(Recipe):
#: Number of levels of links to follow on article webpages #: Number of levels of links to follow on article webpages
recursions = 0 recursions = 0
#: Delay between consecutive downloads in seconds. The argument may be a #: The default delay between consecutive downloads in seconds. The argument may be a
#: floating point number to indicate a more precise time. #: floating point number to indicate a more precise time. See :meth:`get_url_specific_delay`
#: to implement per URL delays.
delay = 0 delay = 0
#: Publication type #: Publication type
@ -474,14 +475,15 @@ class BasicNewsRecipe(Recipe):
return self.feeds[:self.test[0]] return self.feeds[:self.test[0]]
return self.feeds return self.feeds
def get_delay(self, url=None): def get_url_specific_delay(self, url):
''' '''
Return the delay in seconds before downloading `url`. If you want to programmatically Return the delay in seconds before downloading this URL. If you want to programmatically
determine the delay for the specified url, override this method in your subclass. determine the delay for the specified URL, override this method in your subclass, returning
When overriding, you should cater for when `url` may be `None`. self.delay by default for URLs you do not want to affect.
You may return a floating point number to indicate a more precise time.
:return: A floating point number, the delay in seconds.
''' '''
return getattr(self, 'delay', 0) return self.delay
@classmethod @classmethod
def print_version(cls, url): def print_version(cls, url):
@ -943,7 +945,7 @@ class BasicNewsRecipe(Recipe):
web2disk_cmdline = ['web2disk', web2disk_cmdline = ['web2disk',
'--timeout', str(self.timeout), '--timeout', str(self.timeout),
'--max-recursions', str(self.recursions), '--max-recursions', str(self.recursions),
'--delay', str(self.get_delay()), '--delay', str(self.delay),
] ]
if self.verbose: if self.verbose:
@ -975,9 +977,9 @@ class BasicNewsRecipe(Recipe):
self.web2disk_options.preprocess_image = self.preprocess_image self.web2disk_options.preprocess_image = self.preprocess_image
self.web2disk_options.encoding = self.encoding self.web2disk_options.encoding = self.encoding
self.web2disk_options.preprocess_raw_html = self.preprocess_raw_html_ self.web2disk_options.preprocess_raw_html = self.preprocess_raw_html_
self.web2disk_options.get_delay = self.get_delay self.web2disk_options.get_delay = self.get_url_specific_delay
if self.get_delay() > 0: if self.delay > 0:
self.simultaneous_downloads = 1 self.simultaneous_downloads = 1
self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \ self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \
@ -1721,7 +1723,7 @@ class BasicNewsRecipe(Recipe):
feed.description = as_unicode(err) feed.description = as_unicode(err)
parsed_feeds.append(feed) parsed_feeds.append(feed)
self.log.exception(msg) self.log.exception(msg)
delay = self.get_delay(url) delay = self.get_url_specific_delay(url)
if delay > 0: if delay > 0:
time.sleep(delay) time.sleep(delay)

2
src/calibre/web/fetch/simple.py
View File

@ -180,7 +180,7 @@ class RecursiveFetcher:
self.compress_news_images = getattr(options, 'compress_news_images', False) self.compress_news_images = getattr(options, 'compress_news_images', False)
self.compress_news_images_auto_size = getattr(options, 'compress_news_images_auto_size', 16) self.compress_news_images_auto_size = getattr(options, 'compress_news_images_auto_size', 16)
self.scale_news_images = getattr(options, 'scale_news_images', None) self.scale_news_images = getattr(options, 'scale_news_images', None)
self.get_delay = getattr(options, 'get_delay', lambda url=None: self.delay) self.get_delay = getattr(options, 'get_delay', lambda url: self.delay)
self.download_stylesheets = not options.no_stylesheets self.download_stylesheets = not options.no_stylesheets
self.show_progress = True self.show_progress = True
self.failed_links = [] self.failed_links = []