title

__license__ = 'GPL v3' __copyright__ = '2011, Darko Miletic ' ''' frontlineonnet.com ''' import re from calibre.web.feeds.news import BasicNewsRecipe class Frontlineonnet(BasicNewsRecipe): title = 'Frontline' __author__ = 'Darko Miletic' description = "India's national magazine" publisher = 'Frontline' category = 'news, politics, India' no_stylesheets = True delay = 1 INDEX = 'http://frontlineonnet.com/' use_embedded_content = False encoding = 'utf-8' language = 'en_IN' publication_type = 'magazine' masthead_url = 'http://frontlineonnet.com/images/newfline.jpg' extra_css = """ body{font-family: Verdana,Arial,Helvetica,sans-serif} img{margin-top:0.5em; margin-bottom: 0.7em; display: block} """ conversion_options = { 'comment': description, 'tags': category, 'publisher': publisher, 'language': language, 'linearize_tables': True } preprocess_regexps = [ (re.compile(r'.*?title', re.DOTALL | re.IGNORECASE), lambda match: ''), (re.compile(r'', re.DOTALL | re.IGNORECASE), lambda match: '

'), (re.compile(r'', re.DOTALL | re.IGNORECASE), lambda match: '

'), (re.compile(r'

', re.DOTALL | re.IGNORECASE), lambda match: '') # noqa ] keep_only_tags = [ dict(name='div', attrs={'id': 'content'}) ] remove_attributes = ['size', 'noshade', 'border'] use_javascript_to_login = True needs_subscription = True def javascript_login(self, browser, username, password): browser.visit('http://www.frontline.in/profile/login.do') browser.wait_for_element('form#loginForm', timeout=180) # Select the first form on the page form = browser.select_form('#loginForm') form['userName'] = username form['password'] = password browser.submit(timeout=120) def parse_index(self): articles = [] current_section = None feeds = [] soup = self.index_to_soup(self.INDEX) for h3 in soup.findAll('h3'): if h3.get('class', None) == 'artListSec': if articles: feeds.append((current_section, articles)) articles = [] current_section = self.tag_to_string(h3).strip() self.log(current_section) elif h3.get('id', None) in {'headseccol', 'headsec'}: a = h3.find('a', href=True) if a is not None: title = self.tag_to_string(a) url = a['href'] articles.append({ 'title': title, 'date': '', 'url': url, 'description': '' }) self.log('\t', title, url) if articles: feeds.append((current_section, articles)) return feeds