mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
Fix #5317 (New York Review of Books: Recipe broken, needs updating)
This commit is contained in:
Kovid Goyal
parent
20ac15a4a8
commit
7def5d376a
@ -1,3 +1,4 @@
|
|||||||
|
|
||||||
#!/usr/bin/env python
|
#!/usr/bin/env python
|
||||||
__license__ = 'GPL v3'
|
__license__ = 'GPL v3'
|
||||||
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
|
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
|
||||||
@ -6,51 +7,81 @@ __docformat__ = 'restructuredtext en'
|
|||||||
'''
|
'''
|
||||||
nybooks.com
|
nybooks.com
|
||||||
'''
|
'''
|
||||||
|
import re
|
||||||
|
|
||||||
from calibre.web.feeds.news import BasicNewsRecipe
|
from calibre.web.feeds.news import BasicNewsRecipe
|
||||||
from lxml import html
|
|
||||||
from calibre.constants import preferred_encoding
|
|
||||||
|
|
||||||
class NewYorkReviewOfBooks(BasicNewsRecipe):
|
class NewYorkReviewOfBooks(BasicNewsRecipe):
|
||||||
|
|
||||||
title = u'New York Review of Books'
|
title = u'New York Review of Books (no subscription)'
|
||||||
description = u'Book reviews'
|
description = u'Book reviews'
|
||||||
language = 'en'
|
language = 'en'
|
||||||
|
|
||||||
__author__ = 'Kovid Goyal'
|
__author__ = 'Kovid Goyal'
|
||||||
|
|
||||||
|
no_stylesheets = True
|
||||||
|
no_javascript = True
|
||||||
needs_subscription = True
|
needs_subscription = True
|
||||||
remove_tags_before = {'id':'container'}
|
|
||||||
remove_tags = [{'class':['noprint', 'ad', 'footer']}, {'id':'right-content'}]
|
keep_only_tags = [dict(id='article-body')]
|
||||||
|
remove_tags = [dict(attrs={'class':['article-tools', 'article-links',
|
||||||
|
'center advertisement']})]
|
||||||
|
|
||||||
|
preprocess_regexps = [(re.compile(r'<head>.*?</head>', re.DOTALL), lambda
|
||||||
|
m:'<head></head>')]
|
||||||
|
|
||||||
def get_browser(self):
|
def get_browser(self):
|
||||||
br = BasicNewsRecipe.get_browser()
|
br = BasicNewsRecipe.get_browser()
|
||||||
if self.username is not None and self.password is not None:
|
br.open('http://www.nybooks.com/account/signin/')
|
||||||
br.open('http://www.nybooks.com/register/')
|
br.select_form(nr = 1)
|
||||||
br.select_form(name='login')
|
br['username'] = self.username
|
||||||
br['email'] = self.username
|
br['password'] = self.password
|
||||||
br['password'] = self.password
|
br.submit()
|
||||||
br.submit()
|
|
||||||
return br
|
return br
|
||||||
|
|
||||||
|
def print_version(self, url):
|
||||||
|
return url+'?pagination=false'
|
||||||
|
|
||||||
def parse_index(self):
|
def parse_index(self):
|
||||||
root = html.fromstring(self.browser.open('http://www.nybooks.com/current-issue').read())
|
soup = self.index_to_soup('http://www.nybooks.com/current-issue')
|
||||||
date = root.xpath('//h4[@class = "date"]')[0]
|
|
||||||
self.timefmt = ' ['+date.text.encode(preferred_encoding)+']'
|
# Find cover
|
||||||
|
sidebar = soup.find(id='sidebar')
|
||||||
|
if sidebar is not None:
|
||||||
|
a = sidebar.find('a', href=lambda x: x and 'view-photo' in x)
|
||||||
|
if a is not None:
|
||||||
|
psoup = self.index_to_soup('http://www.nybooks.com'+a['href'])
|
||||||
|
cover = psoup.find('img', src=True)
|
||||||
|
self.cover_url = cover['src']
|
||||||
|
self.log('Found cover at:', self.cover_url)
|
||||||
|
|
||||||
|
# Find date
|
||||||
|
div = soup.find(id='page-title')
|
||||||
|
if div is not None:
|
||||||
|
h5 = div.find('h5')
|
||||||
|
if h5 is not None:
|
||||||
|
text = self.tag_to_string(h5)
|
||||||
|
date = text.partition(u'\u2022')[0].strip()
|
||||||
|
self.timefmt = u' [%s]'%date
|
||||||
|
self.log('Issue date:', date)
|
||||||
|
|
||||||
|
# Find TOC
|
||||||
|
toc = soup.find('ul', attrs={'class':'issue-article-list'})
|
||||||
articles = []
|
articles = []
|
||||||
for tag in date.itersiblings():
|
for li in toc.findAll('li'):
|
||||||
if tag.tag == 'h4': break
|
h3 = li.find('h3')
|
||||||
if tag.tag == 'p':
|
title = self.tag_to_string(h3)
|
||||||
if tag.get('class') == 'indented':
|
author = self.tag_to_string(li.find('h4'))
|
||||||
articles[-1]['description'] += html.tostring(tag)
|
title = title + u' (%s)'%author
|
||||||
else:
|
url = 'http://www.nybooks.com'+h3.find('a', href=True)['href']
|
||||||
href = tag.xpath('descendant::a[@href]')[0].get('href')
|
desc = ''
|
||||||
article = {
|
for p in li.findAll('p'):
|
||||||
'title': u''.join(tag.xpath('descendant::text()')),
|
desc += self.tag_to_string(p)
|
||||||
'date' : '',
|
self.log('Found article:', title)
|
||||||
'url' : 'http://www.nybooks.com'+href,
|
self.log('\t', url)
|
||||||
'description': '',
|
self.log('\t', desc)
|
||||||
}
|
articles.append({'title':title, 'url':url, 'date':'',
|
||||||
articles.append(article)
|
'description':desc})
|
||||||
|
|
||||||
return [('Current Issue', articles)]
|
return [('Current Issue', articles)]
|
||||||
|
|
||||||
|
|||||||
@ -6,10 +6,9 @@ __docformat__ = 'restructuredtext en'
|
|||||||
'''
|
'''
|
||||||
nybooks.com
|
nybooks.com
|
||||||
'''
|
'''
|
||||||
|
import re
|
||||||
|
|
||||||
from calibre.web.feeds.news import BasicNewsRecipe
|
from calibre.web.feeds.news import BasicNewsRecipe
|
||||||
from lxml import html
|
|
||||||
from calibre.constants import preferred_encoding
|
|
||||||
|
|
||||||
class NewYorkReviewOfBooks(BasicNewsRecipe):
|
class NewYorkReviewOfBooks(BasicNewsRecipe):
|
||||||
|
|
||||||
@ -17,57 +16,61 @@ class NewYorkReviewOfBooks(BasicNewsRecipe):
|
|||||||
description = u'Book reviews'
|
description = u'Book reviews'
|
||||||
language = 'en'
|
language = 'en'
|
||||||
|
|
||||||
__author__ = 'Kovid Goyal and Sujata Raman'
|
__author__ = 'Kovid Goyal'
|
||||||
|
|
||||||
no_stylesheets = True
|
no_stylesheets = True
|
||||||
no_javascript = True
|
no_javascript = True
|
||||||
remove_tags_before = {'id':'container'}
|
|
||||||
remove_tags = [{'class':['noprint', 'ad', 'footer']}, {'id':'right-content'},
|
|
||||||
dict(name='img', attrs={'src':"/images/1x1-clear.gif"}),
|
|
||||||
|
|
||||||
]
|
keep_only_tags = [dict(id='article-body')]
|
||||||
|
remove_tags = [dict(attrs={'class':['article-tools', 'article-links',
|
||||||
|
'center advertisement']})]
|
||||||
|
|
||||||
extra_css = '''
|
preprocess_regexps = [(re.compile(r'<head>.*?</head>', re.DOTALL), lambda
|
||||||
p{font-family:"Times New Roman",Georgia,serif; font-size: 60%;}
|
m:'<head></head>')]
|
||||||
.caption{ font-family:"Times New Roman",Georgia,serif; font-size:40%;}
|
|
||||||
h2{font-family:"Times New Roman",Georgia,serif; font-size:90%;}
|
|
||||||
a{ color:#003399; }
|
|
||||||
.reviewed-title{font-family:"Times New Roman",Georgia,serif;font-size : 50%; font-style:italic;}
|
|
||||||
.reviewed-author{font-family:"Times New Roman",Georgia,serif;font-size : 50%;}
|
|
||||||
.reviewed-info{font-family:"Times New Roman",Georgia,serif;font-size : 50%;}
|
|
||||||
h5{font-family:"Times New Roman",Georgia,serif;font-size : 50%;}
|
|
||||||
.date{font-family:"Times New Roman",Georgia,serif;font-variant:small-caps;font-size : 50%;}
|
|
||||||
h4{font-family:"Times New Roman",Georgia,serif;font-size : 50%;}
|
|
||||||
'''
|
|
||||||
|
|
||||||
def preprocess_html(self, soup):
|
def print_version(self, url):
|
||||||
|
return url+'?pagination=false'
|
||||||
for tag in soup.findAll(name=['span',]):
|
|
||||||
tag.name = 'div'
|
|
||||||
for tag in soup.findAll(name=['blockquote',]):
|
|
||||||
tag.name = 'p'
|
|
||||||
|
|
||||||
return soup
|
|
||||||
|
|
||||||
def parse_index(self):
|
def parse_index(self):
|
||||||
root = html.fromstring(self.browser.open('http://www.nybooks.com/current-issue').read())
|
soup = self.index_to_soup('http://www.nybooks.com/current-issue')
|
||||||
date = root.xpath('//h4[@class = "date"]')[0]
|
|
||||||
self.timefmt = ' ['+date.text.encode(preferred_encoding)+']'
|
# Find cover
|
||||||
|
sidebar = soup.find(id='sidebar')
|
||||||
|
if sidebar is not None:
|
||||||
|
a = sidebar.find('a', href=lambda x: x and 'view-photo' in x)
|
||||||
|
if a is not None:
|
||||||
|
psoup = self.index_to_soup('http://www.nybooks.com'+a['href'])
|
||||||
|
cover = psoup.find('img', src=True)
|
||||||
|
self.cover_url = cover['src']
|
||||||
|
self.log('Found cover at:', self.cover_url)
|
||||||
|
|
||||||
|
# Find date
|
||||||
|
div = soup.find(id='page-title')
|
||||||
|
if div is not None:
|
||||||
|
h5 = div.find('h5')
|
||||||
|
if h5 is not None:
|
||||||
|
text = self.tag_to_string(h5)
|
||||||
|
date = text.partition(u'\u2022')[0].strip()
|
||||||
|
self.timefmt = u' [%s]'%date
|
||||||
|
self.log('Issue date:', date)
|
||||||
|
|
||||||
|
# Find TOC
|
||||||
|
toc = soup.find('ul', attrs={'class':'issue-article-list'})
|
||||||
articles = []
|
articles = []
|
||||||
for tag in date.itersiblings():
|
for li in toc.findAll('li'):
|
||||||
if tag.tag == 'h4': break
|
h3 = li.find('h3')
|
||||||
if tag.tag == 'p':
|
title = self.tag_to_string(h3)
|
||||||
if tag.get('class') == 'indented':
|
author = self.tag_to_string(li.find('h4'))
|
||||||
articles[-1]['description'] += html.tostring(tag)
|
title = title + u' (%s)'%author
|
||||||
else:
|
url = 'http://www.nybooks.com'+h3.find('a', href=True)['href']
|
||||||
href = tag.xpath('descendant::a[@href]')[0].get('href')
|
desc = ''
|
||||||
article = {
|
for p in li.findAll('p'):
|
||||||
'title': u''.join(tag.xpath('descendant::text()')),
|
desc += self.tag_to_string(p)
|
||||||
'date' : '',
|
self.log('Found article:', title)
|
||||||
'url' : 'http://www.nybooks.com'+href,
|
self.log('\t', url)
|
||||||
'description': '',
|
self.log('\t', desc)
|
||||||
}
|
articles.append({'title':title, 'url':url, 'date':'',
|
||||||
articles.append(article)
|
'description':desc})
|
||||||
|
|
||||||
return [('Current Issue', articles)]
|
return [('Current Issue', articles)]
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user