NYTimes has apparently tweaked their bot detection algorithms again

Sigh. Add a delay to avoid triggering the bot detection.
This commit is contained in:
Kovid Goyal 2022-11-14 10:41:52 +05:30
parent 2c8afa639c
commit 9e58974ec9
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
4 changed files with 5 additions and 0 deletions

View File

@ -91,6 +91,7 @@ class NewYorkTimes(BasicNewsRecipe):
compress_news_images = True
compress_news_images_auto_size = 5
conversion_options = {'flow_size': 0}
delay = 0 if use_wayback_machine else 1
@property
def nyt_parser(self):

View File

@ -32,6 +32,8 @@ class NYTimesSports(BasicNewsRecipe):
use_embedded_content = False
no_stylesheets = True
language = 'en'
delay = 1
feeds = [
(u'The Fifth Down', u'https://fifthdown.blogs.nytimes.com/feed/'),
(u'The Quad', u'https://thequad.blogs.nytimes.com/feed/'),

View File

@ -91,6 +91,7 @@ class NewYorkTimes(BasicNewsRecipe):
compress_news_images = True
compress_news_images_auto_size = 5
conversion_options = {'flow_size': 0}
delay = 0 if use_wayback_machine else 1
@property
def nyt_parser(self):

View File

@ -23,6 +23,7 @@ class NewYorkTimesBookReview(BasicNewsRecipe):
no_javascript = True
ignore_duplicate_articles = {'title', 'url'}
encoding = 'utf-8'
delay = 0 if use_wayback_machine else 1
articles_are_obfuscated = use_wayback_machine