diff --git a/resources/recipes/lenta_ru.recipe b/resources/recipes/lenta_ru.recipe new file mode 100644 index 0000000000..d400bc5886 --- /dev/null +++ b/resources/recipes/lenta_ru.recipe @@ -0,0 +1,177 @@ +#!/usr/bin/env python + +''' +Lenta.ru +''' + +from calibre.web.feeds.feedparser import parse +from calibre.ebooks.BeautifulSoup import Tag +from calibre.web.feeds.news import BasicNewsRecipe +import re + +class LentaRURecipe(BasicNewsRecipe): + title = u'Lenta.ru: \u041d\u043e\u0432\u043e\u0441\u0442\u0438' + __author__ = 'Nikolai Kotchetkov' + publisher = 'lenta.ru' + category = 'news, Russia' + description = u'''\u0415\u0436\u0435\u0434\u043d\u0435\u0432\u043d\u0430\u044f + \u0438\u043d\u0442\u0435\u0440\u043d\u0435\u0442-\u0433\u0430\u0437\u0435\u0442\u0430. + \u041d\u043e\u0432\u043e\u0441\u0442\u0438 \u0441\u043e + \u0432\u0441\u0435\u0433\u043e \u043c\u0438\u0440\u0430 \u043d\u0430 + \u0440\u0443\u0441\u0441\u043a\u043e\u043c + \u044f\u0437\u044b\u043a\u0435''' + description = u'Ежедневная интернет-газета. Новости со всего мира на русском языке' + oldest_article = 3 + max_articles_per_feed = 100 + + masthead_url = u'http://img.lenta.ru/i/logowrambler.gif' + cover_url = u'http://img.lenta.ru/i/logowrambler.gif' + + #Add feed names if you want them to be sorted (feeds of this list appear first) + sortOrder = [u'_default', u'В России', u'б.СССР', u'В мире'] + + encoding = 'cp1251' + language = 'ru' + no_stylesheets = True + remove_javascript = True + recursions = 0 + + conversion_options = { + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + } + + + keep_only_tags = [dict(name='td', attrs={'class':['statya','content']})] + + remove_tags_after = [dict(name='p', attrs={'class':'links'}), dict(name='div', attrs={'id':'readers-block'})] + + remove_tags = [dict(name='table', attrs={'class':['vrezka','content']}), dict(name='div', attrs={'class':'b240'}), dict(name='div', attrs={'id':'readers-block'}), dict(name='p', attrs={'class':'links'})] + + feeds = [u'http://lenta.ru/rss/'] + + extra_css = 'h1 {font-size: 1.2em; margin: 0em 0em 0em 0em;} h2 {font-size: 1.0em; margin: 0em 0em 0em 0em;} h3 {font-size: 0.8em; margin: 0em 0em 0em 0em;}' + + def parse_index(self): + try: + feedData = parse(self.feeds[0]) + if not feedData: + raise NotImplementedError + self.log("parse_index: Feed loaded successfully.") + if feedData.feed.has_key('title'): + self.title = feedData.feed.title + self.log("parse_index: Title updated to: ", self.title) + if feedData.feed.has_key('image'): + self.log("HAS IMAGE!!!!") + + def get_virtual_feed_articles(feed): + if feeds.has_key(feed): + return feeds[feed][1] + self.log("Adding new feed: ", feed) + articles = [] + feeds[feed] = (feed, articles) + return articles + + feeds = {} + + #Iterate feed items and distribute articles using tags + for item in feedData.entries: + link = item.get('link', ''); + title = item.get('title', ''); + if '' == link or '' == title: + continue + article = {'title':title, 'url':link, 'description':item.get('description', ''), 'date':item.get('date', ''), 'content':''}; + if not item.has_key('tags'): + get_virtual_feed_articles('_default').append(article) + continue + for tag in item.tags: + addedToDefault = False + term = tag.get('term', '') + if '' == term: + if (not addedToDefault): + get_virtual_feed_articles('_default').append(article) + continue + get_virtual_feed_articles(term).append(article) + + #Get feed list + #Select sorted feeds first of all + result = [] + for feedName in self.sortOrder: + if (not feeds.has_key(feedName)): continue + result.append(feeds[feedName]) + del feeds[feedName] + result = result + feeds.values() + + return result + + except Exception, err: + self.log(err) + raise NotImplementedError + + def preprocess_html(self, soup): + return self.adeify_images(soup) + + def postprocess_html(self, soup, first_fetch): + #self.log('Original: ', soup.prettify()) + + contents = Tag(soup, 'div') + + #Extract tags with given attributes + extractElements = {'div' : [{'id' : 'readers-block'}]} + + #Remove all elements that were not extracted before + for tag, attrs in extractElements.iteritems(): + for attr in attrs: + garbage = soup.findAll(tag, attr) + if garbage: + for pieceOfGarbage in garbage: + pieceOfGarbage.extract() + + #Find article text using header + #and add all elements to contents + element = soup.find({'h1' : True, 'h2' : True}) + if (element): + element.name = 'h1' + while element: + nextElement = element.nextSibling + element.extract() + contents.insert(len(contents.contents), element) + element = nextElement + + #Place article date after header + dates = soup.findAll(text=re.compile('\d{2}\.\d{2}\.\d{4}, \d{2}:\d{2}:\d{2}')) + if dates: + for date in dates: + for string in date: + parent = date.parent + if (parent and isinstance(parent, Tag) and 'div' == parent.name and 'dt' == parent['class']): + #Date div found + parent.extract() + parent['style'] = 'font-size: 0.5em; color: gray; font-family: monospace;' + contents.insert(1, parent) + break + + #Place article picture after date + pic = soup.find('img') + if pic: + picDiv = Tag(soup, 'div') + picDiv['style'] = 'width: 100%; text-align: center;' + pic.extract() + picDiv.insert(0, pic) + title = pic.get('title', None) + if title: + titleDiv = Tag(soup, 'div') + titleDiv['style'] = 'font-size: 0.5em;' + titleDiv.insert(0, title) + picDiv.insert(1, titleDiv) + contents.insert(2, picDiv) + + body = soup.find('td', {'class':['statya','content']}) + if body: + body.replaceWith(contents) + + #self.log('Result: ', soup.prettify()) + return soup +