diff --git a/Changelog.yaml b/Changelog.yaml index 43eb775233..17f3ebcf97 100644 --- a/Changelog.yaml +++ b/Changelog.yaml @@ -19,6 +19,51 @@ # new recipes: # - title: +- version: 0.8.47 + date: 2012-04-13 + + new features: + - title: "Conversion pipeline: Add support for all the named entities in the HTML 5 spec." + tickets: [976056] + + - title: "Support for viewing and converting the Haodoo PDB ebook format" + tickets: [976478] + + - title: "Device driver for Laser EB720" + + bug fixes: + - title: "Fix regression in automatic adding in 0.8.46 that broke automatic adding if adding of duplicates is enabled and auto convert is also enabled" + tickets: [976336] + + - title: 'Fix "Tags" field in advanced search does not obey regex setting' + tickets: [980221] + + - title: "EPUB Input: Automatically extract cover image from simple HTML title page that consists of only a single tag, instead of rendering the page" + + - title: "Prevent errors when both author and author_sort are used in a template for reading metadata from filenames for files on a device" + + - title: "Amazon metadata download: Handle books whose titles start with a bracket." + tickets: [976365] + + - title: "Get Books: Fix downloading of purchased books from Baen" + tickets: [975929] + + + improved recipes: + - Forbes + - Caros Amigos + - Trouw + - Sun UK + - Metro + - Daily Mirror + + new recipes: + - title: "Melbourne Herald Sun" + author: Ray Hartley + + - title: "Editoriali and Zerocalcare" + author: faber1971 + - version: 0.8.46 date: 2012-04-06 diff --git a/recipes/caros_amigos.recipe b/recipes/caros_amigos.recipe index 48edceacba..7edcfb07c8 100644 --- a/recipes/caros_amigos.recipe +++ b/recipes/caros_amigos.recipe @@ -1,7 +1,5 @@ __copyright__ = '2011, Pablo Aldama ' - from calibre.web.feeds.news import BasicNewsRecipe - class AdvancedUserRecipe1311839910(BasicNewsRecipe): title = u'Caros Amigos' oldest_article = 20 @@ -9,9 +7,8 @@ class AdvancedUserRecipe1311839910(BasicNewsRecipe): language = 'pt_BR' __author__ = 'Pablo Aldama' - feeds = [(u'Caros Amigos', u'http://carosamigos.terra.com.br/index/index.php?format=feed&type=rss')] + feeds = [(u'Caros Amigos', u'http://carosamigos.terra.com.br/index2/index.php?format=feed&type=rss')] keep_only_tags = [dict(name='div', attrs={'class':['blog']}) ,dict(name='div', attrs={'class':['blogcontent']}) ] remove_tags = [dict(name='div', attrs={'class':'addtoany'})] - diff --git a/recipes/daily_mirror.recipe b/recipes/daily_mirror.recipe index d6794b1d97..8bac57951c 100644 --- a/recipes/daily_mirror.recipe +++ b/recipes/daily_mirror.recipe @@ -1,20 +1,21 @@ + from calibre.web.feeds.news import BasicNewsRecipe import re +import mechanize class AdvancedUserRecipe1306061239(BasicNewsRecipe): title = u'The Daily Mirror' description = 'News as provide by The Daily Mirror -UK' __author__ = 'Dave Asbury' - # last updated 11/2/12 + # last updated 7/4/12 language = 'en_GB' - - cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' + #cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' masthead_url = 'http://www.nmauk.co.uk/nma/images/daily_mirror.gif' oldest_article = 1 - max_articles_per_feed = 5 + max_articles_per_feed = 10 remove_empty_feeds = True remove_javascript = True no_stylesheets = True @@ -75,3 +76,28 @@ class AdvancedUserRecipe1306061239(BasicNewsRecipe): img { display:block} ''' + def get_cover_url(self): + soup = self.index_to_soup('http://www.politicshome.com/uk/latest_frontpage.html') + # look for the block containing the mirror button and url + cov = soup.find(attrs={'style' : 'background-image: url(http://www.politicshome.com/images/sources/source_frontpage_button_92.gif);'}) + cov2 = str(cov) + cov2='http://www.politicshome.com'+cov2[9:-142] + #cov2 now contains url of the page containing pic + soup = self.index_to_soup(cov2) + cov = soup.find(attrs={'id' : 'large'}) + cov2 = str(cov) + cov2=cov2[27:-18] + #cov2 now is pic url, now go back to original function + br = mechanize.Browser() + br.set_handle_redirect(False) + try: + br.open_novisit(cov2) + cover_url = cov2 + except: + cover_url = 'http://yookeo.com/screens/m/i/mirror.co.uk.jpg' + + #cover_url = cov2 + #cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png' + return cover_url + + diff --git a/recipes/editoriali.recipe b/recipes/editoriali.recipe new file mode 100644 index 0000000000..1b0c558df4 --- /dev/null +++ b/recipes/editoriali.recipe @@ -0,0 +1,16 @@ +__version__ = 'v1.0' +__date__ = '7, April 2012' + +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1332847053(BasicNewsRecipe): + title = u'Editoriali' + __author__ = 'faber1971' + description = 'Leading articles on Italy by the best Italian editorials' + + oldest_article = 1 + max_articles_per_feed = 100 + auto_cleanup = True + conversion_options = {'linearize_tables': True} + masthead_url = 'http://folkbulletin.folkest.com/wp-content/uploads/editoriale1.jpg' + feeds = [(u'Micromega', u'http://temi.repubblica.it/micromega-online/feed/'), (u'Corriere della Sera', u'http://xml.corriereobjects.it/rss/editoriali.xml'), (u'La Stampa', u'http://www.lastampa.it/cmstp/rubriche/oggetti/rss.asp?ID_blog=25'), (u"Italia dall'estero", u'http://italiadallestero.info/feed')] diff --git a/recipes/forbes.recipe b/recipes/forbes.recipe index a633d0f543..fe72fda536 100644 --- a/recipes/forbes.recipe +++ b/recipes/forbes.recipe @@ -1,39 +1,49 @@ -from calibre.ebooks.BeautifulSoup import BeautifulSoup +import re from calibre.web.feeds.news import BasicNewsRecipe class Forbes(BasicNewsRecipe): title = u'Forbes' description = 'Business and Financial News' - __author__ = 'Darko Miletic' + __author__ = 'Kovid Goyal' oldest_article = 30 - max_articles_per_feed = 100 + max_articles_per_feed = 20 language = 'en' + encoding = 'utf-8' + recursions = 1 no_stylesheets = True - html2lrf_options = ['--base-font-size', '10'] cover_url = u'http://www.forbes.com/media/current_covers/forbes_120_160.gif' - - feeds = [(u'Latest', u'http://www.forbes.com/news/index.xml'), - (u'Most Popular', u'http://www.forbes.com/feeds/popstories.xml'), - (u'Most Emailed', u'http://www.forbes.com/feeds/mostemailed.xml'), - (u'Faces', u'http://www.forbes.com/facesscan/index.xml'), - (u'Technology', u'http://www.forbes.com/technology/index.xml'), - (u'Personal Tech', u'http://www.forbes.com/personaltech/index.xml'), - (u'Wireless', u'http://www.forbes.com/wireless/index.xml'), - (u'Business', u'http://www.forbes.com/business/index.xml'), - (u'Sports Money', u'http://www.forbes.com/sportsmoney/index.xml'), - (u'Sports', u'http://www.forbes.com/forbeslife/sports/index.xml'), - (u'Vehicles', u'http://www.forbes.com/forbeslife/vehicles/index.xml'), - (u'Leadership', u'http://www.forbes.com/leadership/index.xml'), - (u'Careers', u'http://www.forbes.com/leadership/careers/index.xml'), - (u'Compensation', u'http://www.forbes.com/leadership/compensation/index.xml'), - (u'Managing', u'http://www.forbes.com/leadership/managing/index.xml')] - def print_version(self, url): - raw = self.browser.open(url).read() - soup = BeautifulSoup(raw.decode('latin1', 'replace')) - print_link = soup.find('a', {'onclick':"s_linkTrackVars='prop18';s_linkType='o';s_linkName='Print';if(typeof(globalPageName)!='undefined')s_prop18=globalPageName;s_lnk=s_co(this);s_gs(s_account);"}) - if print_link is None: - return '' - return 'http://www.forbes.com' + print_link['href'] \ No newline at end of file + feeds = [(u'Latest', u'http://www.forbes.com/news/index.xml'), + (u'Most Popular', u'http://www.forbes.com/feeds/popstories.xml'), + (u'Technology', u'http://www.forbes.com/technology/index.xml'), + (u'Business', u'http://www.forbes.com/business/index.xml'), + (u'Sports Money', u'http://www.forbes.com/sportsmoney/index.xml'), + (u'Leadership', u'http://www.forbes.com/leadership/index.xml'),] + + keep_only_tags = \ + {'class':lambda x: x and (set(x.split()) & {'body', 'pagination', + 'articleHead', 'article_head'})} + remove_tags_before = {'name':'h1'} + remove_tags = [ + {'class':['comment_bug', 'engagement_block', + 'video_promo_block', 'article_actions']}, + {'id':'comments'} + ] + + def is_link_wanted(self, url, tag): + ans = re.match(r'http://.*/[2-9]/', url) is not None + if ans: + self.log('Following multipage link: %s'%url) + return ans + + def postprocess_html(self, soup, first_fetch): + for pag in soup.findAll(True, 'pagination'): + pag.extract() + if not first_fetch: + h1 = soup.find('h1') + if h1 is not None: + h1.extract() + return soup + diff --git a/recipes/melbourne_herald_sun.recipe b/recipes/melbourne_herald_sun.recipe new file mode 100644 index 0000000000..c24a4563af --- /dev/null +++ b/recipes/melbourne_herald_sun.recipe @@ -0,0 +1,85 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__copyright__ = '2009, Matthew Briggs' +__docformat__ = 'restructuredtext en' + +''' +http://www.herald sun.com.au/ +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class DailyTelegraph(BasicNewsRecipe): + title = u'Melbourne Herald Sun' + __author__ = u'Ray Hartley' + description = (u'Victorian and National News' + '. You will need to have a subscription to ' + 'http://www.heraldsun.com.au to get full articles.') + language = 'en_AU' + + oldest_article = 2 + needs_subscription = 'optional' + max_articles_per_feed = 30 + remove_javascript = True + no_stylesheets = True + encoding = 'utf8' + use_embedded_content = False + language = 'en_AU' + remove_empty_feeds = True + publication_type = 'newspaper' + masthead_url = 'http://resources2.news.com.au/cs/heraldsun/images/header-and-footer/logo.gif' + extra_css = """ + body{font-family: Arial,Helvetica,sans-serif } + img{margin-bottom: 0.4em; display:block} + .caption{display: inline; font-size: x-small} + """ + + conversion_options = { + 'comment' : description + , 'language' : language + } + + keep_only_tags = [dict(attrs={'id':'story'})] + remove_tags_before=dict(attrs={'class':'story-header'}) + remove_tags_after=dict(attrs={'class':'story-footer'}) + remove_tags = [ + dict(name=['meta','link','base','iframe','embed','object','media-metadata','media-reference','media-producer']) + ,dict(attrs={'class':['story-header-tools','story-sidebar','story-footer','story-summary-list']}) + ] + remove_attributes=['lang'] + + + feeds = [(u'Breaking News' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_breakingnews_206.xml' ) + ,(u'Business' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_business_207.xml' ) + ,(u'Entertainment' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_entertainment_208.xml' ) + ,(u'Health Science' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_health_212.xml' ) + ,(u'Music' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_music_449.xml' ) + ,(u'National News' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_national_209.xml' ) + ,(u'Sport News' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_sport_213.xml' ) + ,(u'AFL News' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_afl_205.xml' ) + ,(u'State News' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_vic_214.xml' ) + ,(u'Technology' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_tech_215.xml' ) + ,(u'World News' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_world_216.xml' ) + ,(u'Opinion', u'http://feeds.news.com.au/public/rss/2.0/heraldsun_opinion_210.xml' ) + ,(u'Andrew Bolt' , u'http://blogs.news.com.au/heraldsun/andrewbolt/index.php/xml/rss_2.0/heraldsun/hs_andrewbolt/') + ,(u'Afl - St Kilda' , u'http://feeds.news.com.au/public/rss/2.0/heraldsun_afl_stkilda_565.xml') + ,(u'Terry McCrann' ,u'http://feeds.news.com.au/public/rss/2.0/heraldsun_tmccrann_224.xml' ) + ,(u'The Other side' ,u'http://feeds.news.com.au/public/rss/2.0/heraldsun_otherside_211.xml')] + + def get_browser(self): + br = BasicNewsRecipe.get_browser(self) + if self.username and self.password: + br.open('http://www.heraldsun.com.au') + br.select_form(nr=0) + br['username'] = self.username + br['password'] = self.password + raw = br.submit().read() + if '>log out' not in raw.lower(): + raise ValueError('Failed to log in to www.heralsun' + ' are your username and password correct?') + return br + + def get_article_url(self, article): + return article.id + + diff --git a/recipes/metro_uk.recipe b/recipes/metro_uk.recipe index 8dc7008a68..c30f81c019 100644 --- a/recipes/metro_uk.recipe +++ b/recipes/metro_uk.recipe @@ -1,52 +1,30 @@ -import re from calibre.web.feeds.news import BasicNewsRecipe class AdvancedUserRecipe1306097511(BasicNewsRecipe): title = u'Metro UK' description = 'News as provide by The Metro -UK' - + #timefmt = '' __author__ = 'Dave Asbury' - #last update 3/12/11 cover_url = 'http://profile.ak.fbcdn.net/hprofile-ak-snc4/276636_117118184990145_2132092232_n.jpg' - no_stylesheets = True + #no_stylesheets = True oldest_article = 1 - max_articles_per_feed = 20 + max_articles_per_feed = 10 remove_empty_feeds = True remove_javascript = True + auto_cleanup = True - #preprocess_regexps = [(re.compile(r'Tweet'), lambda a : '')] - preprocess_regexps = [ - (re.compile(r'', re.IGNORECASE | re.DOTALL), lambda match: '

')] - preprocess_regexps = [ - (re.compile(r'tweet', re.IGNORECASE | re.DOTALL), lambda match: '')] language = 'en_GB' - - masthead_url = 'http://e-edition.metro.co.uk/images/metro_logo.gif' - - keep_only_tags = [ - dict(name='h1'),dict(name='h2', attrs={'class':'h2'}), - dict(attrs={'class':['img-cnt figure']}), - dict(attrs={'class':['art-img']}), - dict(name='div', attrs={'class':'art-lft'}), - dict(name='p') + ] remove_tags = [ - dict(name = 'div',attrs={'id' : ['comments-news','formSubmission']}), - dict(name='div', attrs={'class':[ 'news m12 clrd clr-b p5t shareBtm', 'commentForm', 'metroCommentInnerWrap', - 'art-rgt','pluck-app pluck-comm','news m12 clrd clr-l p5t', 'flt-r','username','clrd' ]}), - dict(attrs={'class':['username', 'metroCommentFormWrap','commentText','commentsNav','avatar','submDateAndTime','addYourComment','displayName']}) - ,dict(name='div', attrs={'class' : 'clrd art-fd fd-gr1-b'}) + ] + + feeds = [ (u'News', u'http://www.metro.co.uk/rss/news/'), (u'Money', u'http://www.metro.co.uk/rss/money/'), (u'Sport', u'http://www.metro.co.uk/rss/sport/'), (u'Film', u'http://www.metro.co.uk/rss/metrolife/film/'), (u'Music', u'http://www.metro.co.uk/rss/metrolife/music/'), (u'TV', u'http://www.metro.co.uk/rss/tv/'), (u'Showbiz', u'http://www.metro.co.uk/rss/showbiz/'), (u'Weird News', u'http://www.metro.co.uk/rss/weird/'), (u'Travel', u'http://www.metro.co.uk/rss/travel/'), (u'Lifestyle', u'http://www.metro.co.uk/rss/lifestyle/'), (u'Books', u'http://www.metro.co.uk/rss/lifestyle/books/'), (u'Food', u'http://www.metro.co.uk/rss/lifestyle/restaurants/')] - extra_css = ''' - body {font: sans-serif medium;}' - h1 {text-align : center; font-family:Arial,Helvetica,sans-serif; font-size:20px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold;} - h2 {text-align : center;color:#4D4D4D;font-family:Arial,Helvetica,sans-serif; font-size:15px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:bold; } - span{ font-size:9.5px; font-weight:bold;font-style:italic} - p { text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} - - ''' + body{ text-align: justify; font-family:Arial,Helvetica,sans-serif; font-size:11px; font-size-adjust:none; font-stretch:normal; font-style:normal; font-variant:normal; font-weight:normal;} + ''' diff --git a/recipes/the_sun.recipe b/recipes/the_sun.recipe index 80b37f329a..db74e003a0 100644 --- a/recipes/the_sun.recipe +++ b/recipes/the_sun.recipe @@ -1,24 +1,23 @@ -import re +import re, mechanize from calibre.web.feeds.recipes import BasicNewsRecipe - class AdvancedUserRecipe1325006965(BasicNewsRecipe): title = u'The Sun UK' - cover_url = 'http://www.thesun.co.uk/img/global/new-masthead-logo.png' - description = 'A Recipe for The Sun tabloid UK - uses feed43' + description = 'A Recipe for The Sun tabloid UK' __author__ = 'Dave Asbury' - # last updated 20/2/12 + # last updated 7/4/12 language = 'en_GB' oldest_article = 1 max_articles_per_feed = 15 remove_empty_feeds = True no_stylesheets = True + #auto_cleanup = True + #articles_are_obfuscated = True masthead_url = 'http://www.thesun.co.uk/sol/img/global/Sun-logo.gif' - encoding = 'cp1251' + encoding = 'UTF-8' - encoding = 'cp1252' remove_empty_feeds = True remove_javascript = True no_stylesheets = True @@ -30,13 +29,14 @@ class AdvancedUserRecipe1325006965(BasicNewsRecipe): preprocess_regexps = [ (re.compile(r'