Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-07 10:14:46 -04:00
Code Issues Packages Projects Releases Wiki Activity

Dont use the wayback machine for nytimes book review

Browse Source 
Its API returns unavailable for a lot of pages even though they are
actually available.
This commit is contained in:
Kovid Goyal 2022-11-27 09:27:09 +05:30
parent 6edce6fef3
commit 5b5b7eeab1
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
1 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
recipes/nytimesbook.recipe
View File

@ -4,7 +4,7 @@
from calibre.web.feeds.news import BasicNewsRecipe from calibre.web.feeds.news import BasicNewsRecipe
use_wayback_machine = True use_wayback_machine = False
def absolutize(url): def absolutize(url):
@ -42,8 +42,8 @@ class NewYorkTimesBookReview(BasicNewsRecipe):
self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes') self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes')
return ans return ans
def get_nyt_page(self, url): def get_nyt_page(self, url, skip_wayback=False):
if use_wayback_machine: if use_wayback_machine and not skip_wayback:
from calibre import browser from calibre import browser
return self.nyt_parser.download_url(url, browser()) return self.nyt_parser.download_url(url, browser())
return self.browser.open_novisit(url).read() return self.browser.open_novisit(url).read()
@ -55,7 +55,7 @@ class NewYorkTimesBookReview(BasicNewsRecipe):
def parse_index(self): def parse_index(self):
# return [('Articles', [{'url': 'https://www.nytimes.com/2022/09/08/books/review/karen-armstrong-by-the-book-interview.html', 'title':'test'}])] # return [('Articles', [{'url': 'https://www.nytimes.com/2022/09/08/books/review/karen-armstrong-by-the-book-interview.html', 'title':'test'}])]
soup = self.index_to_soup( soup = self.index_to_soup(
self.get_nyt_page('https://www.nytimes.com/pages/books/review/index.html')) self.get_nyt_page('https://www.nytimes.com/pages/books/review/index.html', skip_wayback=True))
# Find TOC # Find TOC
toc = soup.find('section', id='collection-book-review').find('section').find('ol') toc = soup.find('section', id='collection-book-review').find('section').find('ol')