mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-09-29 15:31:08 -04:00
136 lines
4.9 KiB
Python
136 lines
4.9 KiB
Python
#!/usr/bin/env python
|
|
|
|
__license__ = 'GPL v3'
|
|
__copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>'
|
|
|
|
'''
|
|
time.com
|
|
'''
|
|
|
|
import re
|
|
from calibre.web.feeds.news import BasicNewsRecipe
|
|
from lxml import html
|
|
|
|
class Time(BasicNewsRecipe):
|
|
title = u'Time'
|
|
__author__ = 'Kovid Goyal, Rick Shang'
|
|
description = ('Weekly US magazine.')
|
|
encoding = 'utf-8'
|
|
no_stylesheets = True
|
|
language = 'en'
|
|
remove_javascript = True
|
|
needs_subscription = True
|
|
|
|
keep_only_tags = [
|
|
{
|
|
'class':['primary-col', 'tout1']
|
|
},
|
|
]
|
|
remove_tags = [
|
|
{'class':['button', 'entry-sharing group', 'wp-paginate',
|
|
'moving-markup', 'entry-comments']},
|
|
|
|
]
|
|
extra_css = '.entry-date { padding-left: 2ex }'
|
|
|
|
preprocess_regexps = [(re.compile(
|
|
r'<meta .+/>'), lambda m:'')]
|
|
|
|
def get_browser(self):
|
|
br = BasicNewsRecipe.get_browser(self)
|
|
# This site uses javascript in its login process
|
|
if self.username is not None and self.password is not None:
|
|
br.open('http://www.time.com/time/magazine')
|
|
br.select_form(predicate=lambda f: 'action' in f.attrs and f.attrs['action'] == 'https://auth.time.com/login.php')
|
|
br['username'] = self.username
|
|
br['password'] = self.password
|
|
# br['magcode'] = ['TD']
|
|
br.find_control('turl').readonly = False
|
|
br['turl'] = 'http://www.time.com/time/magazine'
|
|
br.find_control('rurl').readonly = False
|
|
br['rurl'] = 'http://www.time.com/time/magazine'
|
|
br['remember'] = False
|
|
raw = br.submit().read()
|
|
if False and '>Log Out<' not in raw:
|
|
# This check is disabled as it does not work (there is probably
|
|
# some cookie missing) however, the login is "sufficient" for
|
|
# the actual article downloads to work.
|
|
raise ValueError('Failed to login to time.com, check'
|
|
' your username and password')
|
|
return br
|
|
|
|
def parse_index(self):
|
|
raw = self.index_to_soup('http://www.time.com/time/magazine', raw=True)
|
|
root = html.fromstring(raw)
|
|
img = root.xpath('//a[.="View Large Cover" and @href]')
|
|
if img:
|
|
cover_url = 'http://www.time.com' + img[0].get('href')
|
|
try:
|
|
nsoup = self.index_to_soup(cover_url)
|
|
img = nsoup.find('img', src=re.compile('archive/covers'))
|
|
if img is not None:
|
|
self.cover_url = img['src']
|
|
except:
|
|
self.log.exception('Failed to fetch cover')
|
|
|
|
dates = ''.join(root.xpath('//time[@class="updated"]/text()'))
|
|
if dates:
|
|
self.timefmt = ' [%s]'%dates
|
|
|
|
feeds = []
|
|
parent = root.xpath('//div[@class="content-main-aside"]')[0]
|
|
for sec in parent.xpath(
|
|
'descendant::section[contains(@class, "sec-mag-section")]'):
|
|
h3 = sec.xpath('./h3')
|
|
if h3:
|
|
section = html.tostring(h3[0], encoding=unicode,
|
|
method='text').strip().capitalize()
|
|
self.log('Found section', section)
|
|
articles = list(self.find_articles(sec))
|
|
if articles:
|
|
feeds.append((section, articles))
|
|
|
|
return feeds
|
|
|
|
def find_articles(self, sec):
|
|
|
|
for article in sec.xpath('./article'):
|
|
h2 = article.xpath('./*[@class="entry-title"]')
|
|
if not h2: continue
|
|
a = h2[0].xpath('./a[@href]')
|
|
if not a: continue
|
|
title = html.tostring(a[0], encoding=unicode,
|
|
method='text').strip()
|
|
if not title: continue
|
|
url = a[0].get('href')
|
|
if url.startswith('/'):
|
|
url = 'http://www.time.com'+url
|
|
if '/article/0,' in url:
|
|
soup = self.index_to_soup(url)
|
|
a = soup.find('a', href=lambda x:x and '/printout/' in x)
|
|
url = a['href'].replace('/printout', '/subscriber/printout')
|
|
else:
|
|
url += 'print/' if url.endswith('/') else '/print/'
|
|
if url.startswith('/'):
|
|
url = 'http://www.time.com'+url
|
|
desc = ''
|
|
p = article.xpath('./*[@class="entry-content"]')
|
|
if p:
|
|
desc = html.tostring(p[0], encoding=unicode,
|
|
method='text')
|
|
self.log('\t', title, ':\n\t\t', url)
|
|
yield {
|
|
'title' : title,
|
|
'url' : url,
|
|
'date' : '',
|
|
'description' : desc
|
|
}
|
|
|
|
def preprocess_html(self, soup):
|
|
for fig in soup.findAll('figure'):
|
|
img = fig.find('img')
|
|
if img is not None:
|
|
fig.replaceWith(img)
|
|
return soup
|
|
|