calibre/recipes/satmagazine.recipe

#!/usr/bin/env python
##
# Title:        SatMagazine
##
# License:      GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html
##
# Written:      Feb 2012
# Last Edited:  Mar 2012
##

# Feb 2012: Initial release

__license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html'

'''
satmagazine.com
'''

import re
from calibre.web.feeds.news import BasicNewsRecipe


class SatMagazine(BasicNewsRecipe):

    title = u'SatMagazine'
    description = u'North American Satellite Markets...'
    publisher = 'Satnews Publishers'
    publication_type = 'magazine'
    INDEX = 'http://www.satmagazine.com/cgi-bin/display_edition.cgi'
    __author__ = 'kiavash'

    language = 'en'
    asciiize = True
    timeout = 120
    simultaneous_downloads = 2

    # Flattens all the tables to make it compatible with Nook
    conversion_options = {'linearize_tables': True}

    keep_only_tags = [dict(name='span', attrs={'class': 'story'})]

    no_stylesheets = True
    remove_javascript = True

    remove_attributes = ['border', 'cellspacing', 'align', 'cellpadding', 'colspan',
                         'valign', 'vspace', 'hspace', 'alt', 'width', 'height']

    # Specify extra CSS - overrides ALL other CSS (IE. Added last).
    extra_css = 'body { font-family: verdana, helvetica, sans-serif; } \
                 .introduction, .first { font-weight: bold; } \
                 .cross-head { font-weight: bold; font-size: 125%; } \
                 .cap, .caption { display: block; font-size: 80%; font-style: italic; } \
                 .cap, .caption, .caption img, .caption span { display: block; margin: 5px auto; } \
                 .byl, .byd, .byline img, .byline-name, .byline-title, .author-name, .author-position, \
                    .correspondent-portrait img, .byline-lead-in, .name, .bbc-role { display: block; \
                    font-size: 80%; font-style: italic; margin: 1px auto; } \
                 .story-date, .published { font-size: 80%; } \
                 table { width: 100%; } \
                 td img { display: block; margin: 5px auto; } \
                 ul { padding-top: 10px; } \
                 ol { padding-top: 10px; } \
                 li { padding-top: 5px; padding-bottom: 5px; } \
                 h1 { font-size: 175%; font-weight: bold; } \
                 h2 { font-size: 150%; font-weight: bold; } \
                 h3 { font-size: 125%; font-weight: bold; } \
                 h4, h5, h6 { font-size: 100%; font-weight: bold; }'

    # Remove the line breaks, href links and float left/right and picture
    # width/height.
    preprocess_regexps = [(re.compile(r'<br[ ]*/>', re.IGNORECASE), lambda m: ''),
                          (re.compile(r'<br[ ]*clear.*/>',
                                      re.IGNORECASE), lambda m: ''),
                          (re.compile(r'<a.*?>'), lambda h1: ''),
                          (re.compile(r'</a>'), lambda h2: ''),
                          (re.compile(r'float:.*?'), lambda h3: ''),
                          (re.compile(r'width:.*?px'), lambda h4: ''),
                          (re.compile(r'height:.*?px'), lambda h5: '')
                          ]

    def parse_index(self):

        article_info = []
        feeds = []

        soup = self.index_to_soup(self.INDEX)

        # Find Cover image
        cover = soup.find('img', src=True, alt='Cover Image')
        if cover is not None:
            self.cover_url = cover['src']
            self.log('Found Cover image:', self.cover_url)

        # main part of the site that has the articles
        soup = soup.find('div', attrs={'id': 'middlecontent'})

        # Find the Magazine date
        # contains the string with the date
        ts = soup.find('span', attrs={'class': 'master_heading'})
        ds = ' '.join(self.tag_to_string(ts).strip().split()[:2])
        self.log('Found Current Issue:', ds)
        self.timefmt = ' [%s]' % ds

        articles = soup.findAll('span', attrs={'class': 'heading'})

        descriptions = soup.findAll('span', attrs={'class': 'story'})

        title_number = 0

        # Goes thru all the articles one by one and sort them out
        for article in articles:

            title = self.tag_to_string(article)
            url = article.find('a').get('href')

            self.log('\tFound article:', title, 'at', url)
            desc = self.tag_to_string(descriptions[title_number])

            article_info.append({'title': title, 'url': url, 'description': desc,
                                 'date': self.timefmt})

            title_number = title_number + 1

        if article_info:
            feeds.append((self.title, article_info))

        return feeds

    def preprocess_html(self, soup):

        # Finds all the images
        for figure in soup.findAll('img', attrs={'src': True}):

            # if the image is an ad then remove it.
            if (figure['alt'].find('_ad_') >= 0) or (figure['alt'].find('_snipe_') >= 0):
                del figure['src']
                del figure['alt']
                del figure['border']
                del figure['hspace']
                del figure['vspace']
                del figure['align']
                del figure['size']
                figure.name = 'font'
                continue

            # adds /n before and after the image
            figure['style'] = 'display:block'

        # Makes the title standing out
        for title in soup.findAll('b'):
            title.name = 'h3'

        # Removes all unrelated links
        for link in soup.findAll('a', attrs={'href': True}):
            link.name = 'font'
            del link['href']
            del link['target']

        return soup