mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-06-23 15:30:45 -04:00
NYTimes has apparently tweaked their bot detection algorithms again
Sigh. Add a delay to avoid triggering the bot detection.
This commit is contained in:
parent
2c8afa639c
commit
9e58974ec9
@ -91,6 +91,7 @@ class NewYorkTimes(BasicNewsRecipe):
|
|||||||
compress_news_images = True
|
compress_news_images = True
|
||||||
compress_news_images_auto_size = 5
|
compress_news_images_auto_size = 5
|
||||||
conversion_options = {'flow_size': 0}
|
conversion_options = {'flow_size': 0}
|
||||||
|
delay = 0 if use_wayback_machine else 1
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def nyt_parser(self):
|
def nyt_parser(self):
|
||||||
|
@ -32,6 +32,8 @@ class NYTimesSports(BasicNewsRecipe):
|
|||||||
use_embedded_content = False
|
use_embedded_content = False
|
||||||
no_stylesheets = True
|
no_stylesheets = True
|
||||||
language = 'en'
|
language = 'en'
|
||||||
|
delay = 1
|
||||||
|
|
||||||
feeds = [
|
feeds = [
|
||||||
(u'The Fifth Down', u'https://fifthdown.blogs.nytimes.com/feed/'),
|
(u'The Fifth Down', u'https://fifthdown.blogs.nytimes.com/feed/'),
|
||||||
(u'The Quad', u'https://thequad.blogs.nytimes.com/feed/'),
|
(u'The Quad', u'https://thequad.blogs.nytimes.com/feed/'),
|
||||||
|
@ -91,6 +91,7 @@ class NewYorkTimes(BasicNewsRecipe):
|
|||||||
compress_news_images = True
|
compress_news_images = True
|
||||||
compress_news_images_auto_size = 5
|
compress_news_images_auto_size = 5
|
||||||
conversion_options = {'flow_size': 0}
|
conversion_options = {'flow_size': 0}
|
||||||
|
delay = 0 if use_wayback_machine else 1
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def nyt_parser(self):
|
def nyt_parser(self):
|
||||||
|
@ -23,6 +23,7 @@ class NewYorkTimesBookReview(BasicNewsRecipe):
|
|||||||
no_javascript = True
|
no_javascript = True
|
||||||
ignore_duplicate_articles = {'title', 'url'}
|
ignore_duplicate_articles = {'title', 'url'}
|
||||||
encoding = 'utf-8'
|
encoding = 'utf-8'
|
||||||
|
delay = 0 if use_wayback_machine else 1
|
||||||
|
|
||||||
articles_are_obfuscated = use_wayback_machine
|
articles_are_obfuscated = use_wayback_machine
|
||||||
|
|
||||||
|
Loading…
x
Reference in New Issue
Block a user