Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'feature-recipe-get-delay' of https://github.com/ping/calibre

Browse Source
This commit is contained in:
Kovid Goyal 2023-07-24 11:18:20 +05:30
commit 1f5810159f
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 19 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
src/calibre/web/feeds/news.py
View File

@ -474,6 +474,15 @@ class BasicNewsRecipe(Recipe):
return self.feeds[:self.test[0]] return self.feeds[:self.test[0]]
return self.feeds return self.feeds
def get_delay(self, url=None):
'''
Return the delay in seconds before downloading `url`. If you want to programmatically
determine the delay for the specified url, override this method in your subclass.
When overriding, you should cater for when `url` may be `None`.
You may return a floating point number to indicate a more precise time.
'''
return getattr(self, 'delay', 0)
@classmethod @classmethod
def print_version(cls, url): def print_version(cls, url):
''' '''
@ -934,7 +943,7 @@ class BasicNewsRecipe(Recipe):
web2disk_cmdline = ['web2disk', web2disk_cmdline = ['web2disk',
'--timeout', str(self.timeout), '--timeout', str(self.timeout),
'--max-recursions', str(self.recursions), '--max-recursions', str(self.recursions),
'--delay', str(self.delay), '--delay', str(self.get_delay()),
] ]
if self.verbose: if self.verbose:
@ -966,8 +975,9 @@ class BasicNewsRecipe(Recipe):
self.web2disk_options.preprocess_image = self.preprocess_image self.web2disk_options.preprocess_image = self.preprocess_image
self.web2disk_options.encoding = self.encoding self.web2disk_options.encoding = self.encoding
self.web2disk_options.preprocess_raw_html = self.preprocess_raw_html_ self.web2disk_options.preprocess_raw_html = self.preprocess_raw_html_
self.web2disk_options.get_delay = self.get_delay
if self.delay > 0: if self.get_delay() > 0:
self.simultaneous_downloads = 1 self.simultaneous_downloads = 1
self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \ self.navbar = templates.TouchscreenNavBarTemplate() if self.touchscreen else \
@ -1711,8 +1721,9 @@ class BasicNewsRecipe(Recipe):
feed.description = as_unicode(err) feed.description = as_unicode(err)
parsed_feeds.append(feed) parsed_feeds.append(feed)
self.log.exception(msg) self.log.exception(msg)
if self.delay > 0: delay = self.get_delay(url)
time.sleep(self.delay) if delay > 0:
time.sleep(delay)
remove = [fl for fl in parsed_feeds if len(fl) == 0 and self.remove_empty_feeds] remove = [fl for fl in parsed_feeds if len(fl) == 0 and self.remove_empty_feeds]
for f in remove: for f in remove:

6
src/calibre/web/fetch/simple.py
View File

@ -180,6 +180,7 @@ class RecursiveFetcher:
self.compress_news_images = getattr(options, 'compress_news_images', False) self.compress_news_images = getattr(options, 'compress_news_images', False)
self.compress_news_images_auto_size = getattr(options, 'compress_news_images_auto_size', 16) self.compress_news_images_auto_size = getattr(options, 'compress_news_images_auto_size', 16)
self.scale_news_images = getattr(options, 'scale_news_images', None) self.scale_news_images = getattr(options, 'scale_news_images', None)
self.get_delay = getattr(options, 'get_delay', lambda url=None: self.delay)
self.download_stylesheets = not options.no_stylesheets self.download_stylesheets = not options.no_stylesheets
self.show_progress = True self.show_progress = True
self.failed_links = [] self.failed_links = []
@ -268,8 +269,9 @@ class RecursiveFetcher:
return data return data
delta = time.monotonic() - self.last_fetch_at delta = time.monotonic() - self.last_fetch_at
if delta < self.delay: delay = self.get_delay(url)
time.sleep(self.delay - delta) if delta < delay:
time.sleep(delay - delta)
url = canonicalize_url(url) url = canonicalize_url(url)
open_func = getattr(self.browser, 'open_novisit', self.browser.open) open_func = getattr(self.browser, 'open_novisit', self.browser.open)
try: try: