diff --git a/recipes/abc_au.recipe b/recipes/abc_au.recipe index 1330f8e4b5..f08beb4dae 100644 --- a/recipes/abc_au.recipe +++ b/recipes/abc_au.recipe @@ -1,5 +1,5 @@ __license__ = 'GPL v3' -__copyright__ = '2010, Dean Cording' +__copyright__ = '2011, Pat Stapleton ' ''' abc.net.au/news ''' @@ -8,7 +8,7 @@ from calibre.web.feeds.recipes import BasicNewsRecipe class ABCNews(BasicNewsRecipe): title = 'ABC News' - __author__ = 'Dean Cording' + __author__ = 'Pat Stapleton, Dean Cording' description = 'News from Australia' masthead_url = 'http://www.abc.net.au/news/assets/v5/images/common/logo-news.png' cover_url = 'http://www.abc.net.au/news/assets/v5/images/common/logo-news.png' @@ -23,7 +23,9 @@ class ABCNews(BasicNewsRecipe): category = 'News, Australia, World' language = 'en_AU' publication_type = 'newsportal' - preprocess_regexps = [(re.compile(r'', re.DOTALL), lambda m: '')] +# preprocess_regexps = [(re.compile(r'', re.DOTALL), lambda m: '')] +#Remove annoying map links (inline-caption class is also used for some image captions! hence regex to match maps.google) + preprocess_regexps = [(re.compile(r'", which sometimes + # cause a section of the ebook to start in an unsightly fashion or, more + # frequently, a "
" will muck up the formatting of a correspondant's byline. + # "
" and "
" are far more frequently used on the table formatted + # style of pages, and really spoil the look of the ebook pages. + preprocess_regexps = [(re.compile(r'', re.IGNORECASE), lambda m: ''), + (re.compile(r'', re.IGNORECASE), lambda m: '')] + + # Create regular expressions for tag keeping and removal to make the matches more + # robust against minor changes and errors in the HTML, Eg. double spaces, leading + # and trailing spaces, missing hyphens, and such like. + # Python regular expression ('re' class) page: http://docs.python.org/library/re.html + + # *************************************** + # Regular expressions for keep_only_tags: + # *************************************** + + # The BBC News HTML pages use variants of 'storybody' to denote the section of a HTML + # page which contains the main text of the article. Match storybody variants: 'storybody', + # 'story-body', 'story body','storybody ', etc. + storybody_reg_exp = '^.*story[_ -]*body.*$' + + # The BBC sport and 'newsbeat' (features) HTML pages use 'blq_content' to hold the title + # and published date. This is one level above the usual news pages which have the title + # and date within 'story-body'. This is annoying since 'blq_content' must also be kept, + # resulting in a lot of extra things to be removed by remove_tags. + blq_content_reg_exp = '^.*blq[_ -]*content.*$' + + # The BBC has an alternative page design structure, which I suspect is an out-of-date + # design but which is still used in some articles, Eg. 'Click' (technology), 'FastTrack' + # (travel), and in some sport pages. These alternative pages are table based (which is + # why I think they are an out-of-date design) and account for -I'm guesstimaking- less + # than 1% of all articles. They use a table class 'storycontent' to hold the article + # and like blq_content (above) have required lots of extra removal by remove_tags. + story_content_reg_exp = '^.*story[_ -]*content.*$' + + # Keep the sections of the HTML which match the list below. The HTML page created by + # Calibre will fill with those sections which are matched. Note that the + # blq_content_reg_exp must be listed before storybody_reg_exp in keep_only_tags due to + # it being the parent of storybody_reg_exp, that is to say the div class/id 'story-body' + # will be inside div class/id 'blq_content' in the HTML (if 'blq_content' is there at + # all). If they are the other way around in keep_only_tags then blq_content_reg_exp + # will end up being discarded. + keep_only_tags = [ dict(name='table', attrs={'class':re.compile(story_content_reg_exp, re.IGNORECASE)}), + dict(name='div', attrs={'class':re.compile(blq_content_reg_exp, re.IGNORECASE)}), + dict(name='div', attrs={'id':re.compile(blq_content_reg_exp, re.IGNORECASE)}), + dict(name='div', attrs={'class':re.compile(storybody_reg_exp, re.IGNORECASE)}), + dict(name='div', attrs={'id':re.compile(storybody_reg_exp, re.IGNORECASE)}) ] + + # ************************************ + # Regular expressions for remove_tags: + # ************************************ + + # Regular expression to remove share-help and variant tags. The share-help class + # is used by the site for a variety of 'sharing' type links, Eg. Facebook, delicious, + # twitter, email. Removed to avoid page clutter. + share_help_reg_exp = '^.*share[_ -]*help.*$' + + # Regular expression to remove embedded-hyper and variant tags. This class is used to + # display links to other BBC News articles on the same/similar subject. + embedded_hyper_reg_exp = '^.*embed*ed[_ -]*hyper.*$' + + # Regular expression to remove hypertabs and variant tags. This class is used to + # display a tab bar at the top of an article which allows the user to switch to + # an article (viewed on the same page) providing further info., 'in depth' analysis, + # an editorial, a correspondant's blog entry, and such like. The ability to handle + # a tab bar of this nature is currently beyond the scope of this recipe and + # possibly of Calibre itself (not sure about that - TO DO - check!). + hypertabs_reg_exp = '^.*hyper[_ -]*tabs.*$' + + # Regular expression to remove story-feature and variant tags. Eg. 'story-feature', + # 'story-feature related narrow', 'story-feature wide', 'story-feature narrow'. + # This class is used to add additional info. boxes, or small lists, outside of + # the main story. TO DO: Work out a way to incorporate these neatly. + story_feature_reg_exp = '^.*story[_ -]*feature.*$' + + # Regular expression to remove video and variant tags, Eg. 'videoInStoryB', + # 'videoInStoryC'. This class is used to embed video. + video_reg_exp = '^.*video.*$' + + # Regular expression to remove audio and variant tags, Eg. 'audioInStoryD'. + # This class is used to embed audio. + audio_reg_exp = '^.*audio.*$' + + # Regular expression to remove pictureGallery and variant tags, Eg. 'pictureGallery'. + # This class is used to embed a photo slideshow. See also 'slideshow' below. + picture_gallery_reg_exp = '^.*picture.*$' + + # Regular expression to remove slideshow and variant tags, Eg. 'dslideshow-enclosure'. + # This class is used to embed a slideshow (not necessarily photo) but both + # 'slideshow' and 'pictureGallery' are used for slideshows. + slideshow_reg_exp = '^.*slide[_ -]*show.*$' + + # Regular expression to remove social-links and variant tags. This class is used to + # display links to a BBC bloggers main page, used in various columnist's blogs + # (Eg. Nick Robinson, Robert Preston). + social_links_reg_exp = '^.*social[_ -]*links.*$' + + # Regular expression to remove quote and (multi) variant tags, Eg. 'quote', + # 'endquote', 'quote-credit', 'quote-credit-title', etc. These are usually + # removed by 'story-feature' removal (as they are usually within them), but + # not always. The quotation removed is always (AFAICT) in the article text + # as well but a 2nd copy is placed in a quote tag to draw attention to it. + # The quote class tags may or may not appear in div's. + quote_reg_exp = '^.*quote.*$' + + # Regular expression to remove hidden and variant tags, Eg. 'hidden'. + # The purpose of these is unclear, they seem to be an internal link to a + # section within the article, but the text of the link (Eg. 'Continue reading + # the main story') never seems to be displayed anyway. Removed to avoid clutter. + # The hidden class tags may or may not appear in div's. + hidden_reg_exp = '^.*hidden.*$' + + # Regular expression to remove comment and variant tags, Eg. 'comment-introduction'. + # Used on the site to display text about registered users entering comments. + comment_reg_exp = '^.*comment.*$' + + # Regular expression to remove form and variant tags, Eg. 'comment-form'. + # Used on the site to allow registered BBC users to fill in forms, typically + # for entering comments about an article. + form_reg_exp = '^.*form.*$' + + # Extra things to remove due to the addition of 'blq_content' in keep_only_tags. + + #
Used on sports pages for 'email' and 'print'. + story_actions_reg_exp = '^.*story[_ -]*actions.*$' + + #
Used on sports pages instead of 'share-help' (for + # social networking links). + bookmark_list_reg_exp = '^.*bookmark[_ -]*list.*$' + + #
+ # NOTE: Don't remove class="content-group" that is needed. + # Used on sports pages to link to 'similar stories'. + secondary_content_reg_exp = '^.*secondary[_ -]*content.*$' + + #
+ # NOTE: Don't remove class="content-group" that is needed. + # Used on sports pages to link to pages like 'tables', 'fixtures', etc. + featured_content_reg_exp = '^.*featured[_ -]*content.*$' + + #