From 99fec62ddc38e19dc15eb840fa19cde6183ce867 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Fri, 5 Aug 2011 14:08:25 -0600 Subject: [PATCH] ... --- recipes/economist_free.recipe | 156 ++++++++++++++++++++++++++++++++++ 1 file changed, 156 insertions(+) diff --git a/recipes/economist_free.recipe b/recipes/economist_free.recipe index 71c323d6a3..73fb7b8891 100644 --- a/recipes/economist_free.recipe +++ b/recipes/economist_free.recipe @@ -1,3 +1,157 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal ' +''' +economist.com +''' +from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import Tag, NavigableString +from collections import OrderedDict + +import time, re + +class Economist(BasicNewsRecipe): + + title = 'The Economist' + language = 'en' + + __author__ = "Kovid Goyal" + INDEX = 'http://www.economist.com/printedition' + description = ('Global news and current affairs from a European' + ' perspective. Best downloaded on Friday mornings (GMT)') + extra_css = '.headline {font-size: x-large;} \n h2 { font-size: small; } \n h1 { font-size: medium; }' + oldest_article = 7.0 + cover_url = 'http://media.economist.com/sites/default/files/imagecache/print-cover-thumbnail/print-covers/currentcoverus_large.jpg' + #cover_url = 'http://www.economist.com/images/covers/currentcoverus_large.jpg' + remove_tags = [ + dict(name=['script', 'noscript', 'title', 'iframe', 'cf_floatingcontent']), + dict(attrs={'class':['dblClkTrk', 'ec-article-info', + 'share_inline_header', 'related-items']}), + {'class': lambda x: x and 'share-links-header' in x}, + ] + keep_only_tags = [dict(id='ec-article-body')] + needs_subscription = False + no_stylesheets = True + preprocess_regexps = [(re.compile('.*', re.DOTALL), + lambda x:'')] + + ''' + def get_browser(self): + br = BasicNewsRecipe.get_browser() + br.open('http://www.economist.com') + req = mechanize.Request( + 'http://www.economist.com/members/members.cfm?act=exec_login', + headers = { + 'Referer':'http://www.economist.com/', + }, + data=urllib.urlencode({ + 'logging_in' : 'Y', + 'returnURL' : '/', + 'email_address': self.username, + 'fakepword' : 'Password', + 'pword' : self.password, + 'x' : '0', + 'y' : '0', + })) + br.open(req).read() + return br + ''' + + def parse_index(self): + try: + return self.economist_parse_index() + except: + raise + self.log.warn( + 'Initial attempt to parse index failed, retrying in 30 seconds') + time.sleep(30) + return self.economist_parse_index() + + def economist_parse_index(self): + soup = self.index_to_soup(self.INDEX) + div = soup.find('div', attrs={'class':'issue-image'}) + if div is not None: + img = div.find('img', src=True) + if img is not None: + self.cover_url = img['src'] + feeds = OrderedDict() + for section in soup.findAll(attrs={'class':lambda x: x and 'section' in + x}): + h4 = section.find('h4') + if h4 is None: + continue + section_title = self.tag_to_string(h4).strip() + if not section_title: + continue + self.log('Found section: %s'%section_title) + articles = [] + for h5 in section.findAll('h5'): + article_title = self.tag_to_string(h5).strip() + if not article_title: + continue + data = h5.findNextSibling(attrs={'class':'article'}) + if data is None: continue + a = data.find('a', href=True) + if a is None: continue + url = a['href'] + if url.startswith('/'): url = 'http://www.economist.com'+url + url += '/print' + article_title += ': %s'%self.tag_to_string(a).strip() + articles.append({'title':article_title, 'url':url, + 'description':'', 'date':''}) + if not articles: + # We have last or first section + for art in section.findAll(attrs={'class':'article'}): + a = art.find('a', href=True) + if a is not None: + url = a['href'] + if url.startswith('/'): url = 'http://www.economist.com'+url + url += '/print' + title = self.tag_to_string(a) + if title: + articles.append({'title':title, 'url':url, + 'description':'', 'date':''}) + + if articles: + if section_title not in feeds: + feeds[section_title] = [] + feeds[section_title] += articles + + ans = [(key, val) for key, val in feeds.iteritems()] + if not ans: + raise Exception('Could not find any articles, either the ' + 'economist.com server is having trouble and you should ' + 'try later or the website format has changed and the ' + 'recipe needs to be updated.') + return ans + + def eco_find_image_tables(self, soup): + for x in soup.findAll('table', align=['right', 'center']): + if len(x.findAll('font')) in (1,2) and len(x.findAll('img')) == 1: + yield x + + def postprocess_html(self, soup, first): + body = soup.find('body') + for name, val in body.attrs: + del body[name] + + for table in list(self.eco_find_image_tables(soup)): + caption = table.find('font') + img = table.find('img') + div = Tag(soup, 'div') + div['style'] = 'text-align:left;font-size:70%' + ns = NavigableString(self.tag_to_string(caption)) + div.insert(0, ns) + div.insert(1, Tag(soup, 'br')) + del img['width'] + del img['height'] + img.extract() + div.insert(2, img) + table.replaceWith(div) + return soup + +''' from calibre.web.feeds.news import BasicNewsRecipe from calibre.utils.threadpool import ThreadPool, makeRequests from calibre.ebooks.BeautifulSoup import Tag, NavigableString @@ -145,3 +299,5 @@ class Economist(BasicNewsRecipe): div.insert(2, img) table.replaceWith(div) return soup +''' +