import re from calibre.web.feeds.news import BasicNewsRecipe from calibre.ebooks.BeautifulSoup import Comment class EchoDnia(BasicNewsRecipe): title = u'Echo Dnia' __author__ = 'fenuks' description = u'Echo Dnia - portal regionalny świętokrzyskiego radomskiego i podkarpackiego. Najnowsze wiadomości z Twojego regionu, galerie, video, mp3.' category = 'newspaper' language = 'pl' encoding = 'iso-8859-2' extra_css = 'ul {list-style: none; padding:0; margin:0;}' INDEX = 'http://www.echodnia.eu' masthead_url = INDEX + '/images/top_logo.png' oldest_article = 7 max_articles_per_feed = 100 remove_empty_feeds = True no_stylesheets = True use_embedded_content = False ignore_duplicate_articles = {'title', 'url'} preprocess_regexps = [(re.compile(u'Czytaj:.*?', re.DOTALL), lambda match: ''), (re.compile(u'Przeczytaj także:.*?', re.DOTALL | re.IGNORECASE), lambda match: ''), # noqa (re.compile(u'Przeczytaj również:.*?', re.DOTALL | re.IGNORECASE), lambda match: ''), (re.compile(u'Zobacz też:.*?', re.DOTALL | re.IGNORECASE), lambda match: '')] # noqa keep_only_tags = [dict(id=['article', 'cover', 'photostory'])] remove_tags = [dict(id=['articleTags', 'articleMeta', 'boxReadIt', 'articleGalleries', 'articleConnections', 'ForumArticleComments', 'articleRecommend', 'jedynkiLinks', 'articleGalleryConnections', 'photostoryConnections', 'articleEpaper', 'articlePoll', 'articleAlarm', 'articleByline', 'articleZoomText']), dict(attrs={'class': 'articleFunctions'})] feeds = [(u'Wszystkie', u'http://www.echodnia.eu/rss.xml'), (u'Świętokrzyskie', u'http://www.echodnia.eu/swietokrzyskie.xml'), (u'Radomskie', u'http://www.echodnia.eu/radomskie.xml'), (u'Podkarpackie', u'http://www.echodnia.eu/podkarpackie.xml'), (u'Sport \u015bwi\u0119tokrzyski', u'http://www.echodnia.eu/sport_swi.xml'), (u'Sport radomski', u'http://www.echodnia.eu/sport_rad.xml'), (u'Sport podkarpacki', u'http://www.echodnia.eu/sport_pod.xml'), (u'Pi\u0142ka no\u017cna', u'http://www.echodnia.eu/pilka.xml'), (u'Praca', u'http://www.echodnia.eu/praca.xml'), (u'Dom', u'http://www.echodnia.eu/dom.xml'), (u'Auto', u'http://www.echodnia.eu/auto.xml'), (u'Zdrowie', u'http://www.echodnia.eu/zdrowie.xml')] def get_cover_url(self): soup = self.index_to_soup( self.INDEX + '/apps/pbcs.dll/section?Category=JEDYNKI') nexturl = self.INDEX + soup.find(id='covers').find('a')['href'] soup = self.index_to_soup(nexturl) self.cover_url = self.INDEX + \ soup.find(id='cover').find(name='img')['src'] return getattr(self, 'cover_url', self.cover_url) def append_page(self, soup, appendtag): tag = soup.find('span', attrs={'class': 'photoNavigationPages'}) if tag: number = int(tag.string.rpartition('/')[-1].replace(' ', '')) baseurl = self.INDEX + \ soup.find(attrs={'class': 'photoNavigationNext'})['href'][:-1] for r in appendtag.findAll(attrs={'class': 'photoNavigation'}): r.extract() for nr in range(2, number + 1): soup2 = self.index_to_soup(baseurl + str(nr)) pagetext = soup2.find(id='photoContainer') if pagetext: pos = len(appendtag.contents) appendtag.insert(pos, pagetext) pagetext = soup2.find(attrs={'class': 'photoMeta'}) if pagetext: pos = len(appendtag.contents) appendtag.insert(pos, pagetext) pagetext = soup2.find(attrs={'class': 'photoStoryText'}) if pagetext: pos = len(appendtag.contents) appendtag.insert(pos, pagetext) comments = appendtag.findAll( text=lambda text: isinstance(text, Comment)) for comment in comments: comment.extract() def preprocess_html(self, soup): self.append_page(soup, soup.body) return soup