diff --git a/recipes/respekt_magazine.recipe b/recipes/respekt_magazine.recipe index d4deb1d6e4..eaa66e0edb 100644 --- a/recipes/respekt_magazine.recipe +++ b/recipes/respekt_magazine.recipe @@ -12,137 +12,178 @@ from calibre.ebooks.BeautifulSoup import BeautifulSoup,Tag #This imports the version bundled with Calibre import lxml from lxml.builder import E +respekt_url = 'http://www.respekt.cz' class respektRecipe(BasicNewsRecipe): - __author__ = u'Tomáš Hnyk' - title = u'Respekt - Magazine' + __author__ = 'Tomáš Hnyk' publisher = u'Respekt Publishing a. s.' - description = u'Articles from the printed edition, password needed for full access' - encoding = 'cp1250' + description = u'Articles from the print edition' + encoding = 'utf-8' language = 'cs' remove_javascript = True - extra_css = 'p {text-align:justify} \ - ul {color:black} \ - .image_caption {font-size:50%;font-style:italic;} \ - .author {text-align:left;} \ - p.indent_first_line {text-indent:30px;}' - remove_tags_before = dict(name='div',attrs={'class':['l']}) - remove_tags_after = dict(id='text') - remove_tags = [dict(name='ul', attrs={'class':['tabs-d'],'id':['comm']}), \ - dict(name='div',attrs={'class':['slot','reklama','date']}), \ - dict(name='span', attrs={'class':['detail-vykrik']}), \ - dict(name='p', attrs={'class':['detail-vykrik']}), \ - dict(name='div', attrs={'id':['col123d-video','col123d-infographic','col123d-gallery','col12d-discussion']}), # soup>lxml>soup in preprocess requires this - dict(name='strong', attrs={'class':['detail-vykrik']}), + remove_tags_before = dict(name='h1') + remove_tags_after = [dict(id='postcontent')] + remove_tags = [dict(name='div',attrs={'id':['postsharepopup','survey-respondents']}), \ + dict(name='div',attrs={'class':['ad','ad-content','adinarticle','ad-caption','post-actions','authorship-note','quote','postgallery']}), \ + dict(name='a',attrs={'class':['quote','authorship-face']}), \ + dict(name='span',attrs={'class':'embed'}), \ + dict(name='svg'), \ dict(name='script')] - # this makes authors left-aligned by not using the author class) - preprocess_regexps = [(re.compile(r'
', re.DOTALL|re.IGNORECASE), lambda match: '
')] - # remove empty tags - preprocess_regexps.append((re.compile(r' ', re.DOTALL|re.IGNORECASE), lambda match: ' ')) - preprocess_regexps.append((re.compile(r' ', re.DOTALL|re.IGNORECASE), lambda match: ' ')) - preprocess_regexps.append((re.compile(r'

', re.DOTALL|re.IGNORECASE), lambda match: '')) - preprocess_regexps.append((re.compile(r'font-size: 12px', re.DOTALL|re.IGNORECASE), lambda match: '')) - preprocess_regexps.append((re.compile(r'color: #[0-9]*', re.DOTALL|re.IGNORECASE), lambda match: '')) - def get_cover_url(self): - soup = self.index_to_soup('http://respekt.ihned.cz/') - cover = soup.findAll('div', attrs={'class':'cover'})[0].find('img')['src'] - return cover + extra_css = 'p {text-align:justify;margin-top:0;margin-bottom:0} \ + ul {color:black} \ + .frame-caption {font-weight:normal;font-size:50%;font-style:italic;} \ + h1 {font-size:150%;margin-bottom:0;} \ + h2 {font-size:100%;margin-bottom:0;} \ + .post-subtitle {margin-top:0;} \ + h3 {font-size:100%;margin-bottom:0;margin-top:0;} \ + .box-title {background-color: lightgray;font-size:150%;font-weight:bold;margin-left:12%;margin-right:12%;margin-top:12%;margin-bottom:0;} \ + .box-content {background-color:lightgray;margin-left:12%;margin-right:12%;margin-top:0;margin-bottom:12%;} \ + p.indent_first_line {text-indent:30px;} \ + a {text-decoration:none;color:black;}' needs_subscription = True def get_browser(self): br = BasicNewsRecipe.get_browser(self) if self.username is not None and self.password is not None: - br.open('http://muj-ucet.ihned.cz/') - br.select_form(name='login') - br['login[nick]'] = self.username - br['login[pass]'] = self.password + br.open('https://www.respekt.cz/') + for form in br.forms(): + if form.attrs.get('id') == 'frm-authBox-loginForm': + br.form = form + break + for c in form.controls: + if 'name' in c.attrs: + if c.attrs['name'] == 'username': + c.value = self.username + if c.attrs['name'] == 'password': + c.value = self.password br.submit() return br + + # So that remove_tags_before works for this section + def preprocess_raw_html(self, raw_html, url): + root = lxml.html.fromstring(raw_html) + if root.xpath("//title")[0].text == (u"Respekt • Despekt • RESPEKT"): + raw_html = re.sub("h2","h1",raw_html) + return raw_html def parse_index(self): - raw = self.index_to_soup('http://respekt.ihned.cz/aktualni-cislo/', raw=True) - root = lxml.html.fromstring(raw) + raw1 = self.index_to_soup('http://www.respekt.cz/tydenik/', raw=True) + root1 = lxml.html.fromstring(raw1) + current_edition_url = root1.xpath("//div[@class='heroissue']/a")[0].items()[0][1] + raw2 = self.index_to_soup('http://www.respekt.cz/' + current_edition_url, raw=True) + root2 = lxml.html.fromstring(raw2) + self.cover_url = root2.xpath("//i[contains(@class, 'heroissue-cover')]")[0].get("data-src") + # Fetch date + date_text = root2.xpath("//time[@class='heroissue-date']")[0].text.split(',')[1] + s = date_text.split(" ") + # Are the dates of the issue in the same month and year? + if len(s) == 4 or len(s) == 7: + date = "/".join([s[1].split(".")[0],s[2].split(".")[0],s[-1]]) + elif len(s) == 8: + date = "/".join([s[1].split(".")[0],s[2].split(".")[0],s[3]]) + self.conversion_options = {'pubdate':date} + self.title = "Respekt magazine #" + "/".join(current_edition_url.split("/")[-1:-3:-1]) ans = [] - for article in root.xpath("//div[@class='ow-enclose']/div[@class='ow']"): - section_title = article.xpath(".//span[text()='(rubrika: ']")[0].find("a").text - date = article.xpath("span[@class='date-author']")[0].text[:-3] - title = article.find("h2").find("a").text - url = article.find('h2').find('a').get('href') - link = {'title':title,'url':url,'date':date} - for section in ans: - if section[0] == section_title: - section[1].append(link) - break + for section in root2.xpath("//div[@class='col-md-6']/div[@class='issuedetail-categorized-sectionname']"): + section_name = section.text + articles = [] + article = section.getnext() + while hasattr(article, 'text') and not article.text.strip(): + title = article.xpath("span[@class='issuedetail-categorized-title']")[0].text + url = respekt_url + article.xpath("@href")[0] + articles.append({'title':title,'url':url}) + article = article.getnext() + ans.append((section_name,articles)) + highlights = zip(root2.xpath("//a[@class='issuedetail-highlighted-item']"),root2.xpath("//div[@class='issuedetail-highlighted-title']")) + highlights.reverse() + sections = [i[0] for i in ans] + for l,t in highlights: + title = t.text + link = l.xpath('@href')[0] + raw3 = self.index_to_soup(respekt_url + link, raw=True) + root3 = lxml.html.fromstring(raw3) + topics = [i.text.strip() for i in root3.xpath("//div[contains(@class, 'post-topics')]/a")] + # The name of the section changes its position + if u"Téma" in topics: + section_name = "Fokus" + elif u"Rozhovor" in topics: + section_name = "Rozhovor" else: - ans.append((section_title,[link])) + for t in topics: + if t in sections: + section_name = t + break + for i in ans: + if i[0] == section_name: + i[1].insert(-(len(i[1])),{'title':title,'url':respekt_url+link}) + if section_name == u"Rozhovor": + ans.insert(sections.index(u'Fokus')+1,(section_name,[{'title':title,'url':respekt_url+link}])) return ans def cleanup(self): - self.browser.open('http://muj-ucet.ihned.cz/?login[logout]=1') + self.browser.open('https://www.respekt.cz/?do=logout') - def preprocess_html(self,soup): raw = u''.join(unicode(a) for a in soup.contents) root = lxml.html.fromstring(raw) - - # Make image captions visible - body = root.xpath("//div[@id='text']")[0] - add = 0 - for index, element in enumerate(body): - try: - if element.tag == 'img': - body.insert(index+add+1,E.p(element.get('title'),{"class":"image_caption"})) - add += 1 - except: - pass - - # Add length of the articles in words after author - article_length = str(len(body.text_content().split(' '))) + ' slov' - root.xpath("//div[@class='author-image']/div[@class='']/ul")[0].append(E.li(article_length)) - - # Make perex (subheading) start on a new line - root.xpath("//h1")[0].append(E.br('')) - - # Indent paragraphs when typographically suitable - parse = True - # There are only single paragraphs in these sections - if root.xpath("//title")[0].text == u"Deset českých zpráv, které by vás neměly minout | Deset českých zpráv - RESPEKT.IHNED.CZ": - parse = False - if root.xpath("//title")[0].text == u"Deset zahraničních zpráv, které by vás neměly minout | Deset světových zpráv - RESPEKT.IHNED.CZ": - parse = False - if parse: - # First paragraph is never indented - paragraphs = root.xpath('//p') - # Clear the formatting a little bit by removing these attributes - for par in paragraphs: - if 'class' in par.keys(): - if par.attrib['class'] == 'detail-odstavec': - par.attrib.pop('class') - paragraphs.reverse() - for par in paragraphs[:-1]: - try: - # in the beginning of this paragraph means no indenting as well as ellipses as the only text in paragraph - if len(par) > 0: - if (par.text is None and par.getchildren()[0].tag == 'strong'): - continue - elif par.getprevious().text == u'\u2026': - continue - indent = False - # Either indent if the paragraphs are the same - if par.getprevious().attrib == par.attrib: - indent = True - # Or else if the first paragraph of the text was special - if 'class' in par.getprevious().keys(): - par_name = par.getprevious().attrib['class'] - if par_name == '01prvniodstavecrepublicblok' or par_name == 'Zkladnodstavec' or par_name == '01titulekhlavn': - indent = True - if indent: - for key in par.keys(): - par.attrib.pop(key) - par.attrib['class']="indent_first_line" - except: - pass - + #Fix Letem světem + if "Letem sv" in root.xpath("//title")[0].text: + p = root.xpath("//p") + for par in p[:]: + next = par.getnext() + if par.getchildren(): + child = par.getchildren()[0] + if hasattr(next,"tag") and next.tag == "h2" and hasattr(child,"tag") and child.tag == "strong": + text = child.text_content() + if next.text: + next.text = next.text + u" • " + text + else: + if next.getchildren(): + next_child = next.getchildren()[0] + next_child.text = next_child.text + u" • " + text + par.getparent().remove(par) + # Insert text length + text = root.xpath("//div[@id='postcontent']")[0] + article_length = u" • " + str(len(text.text_content().split(' '))) + ' slov' + try: + aut = root.xpath("//div[@class='authorship-names']")[0] + if aut.getchildren() and aut.getchildren()[0].tag == 'a': + t = aut.getchildren()[0] + t.text = 'Autor: ' + t.text + ' ' + # Remove link + e = E.span(t.text) + t.getparent().replace(t,e) + else: + t = root.xpath("//span[@class='post-author-name']")[0] + t.text = ('Autor: ' + t.text + ' ') + root.xpath("//div[@class='authorship-names']")[0].append(E.span(article_length)) + except: + pass + # Make images visible + pictures = root.xpath("//picture") + for picture in pictures: + image = picture.xpath("//source")[0] + image_link = [a for a in image.get('srcset').split(' ') if a[:4] == "http"][-1] + e=E.img({"src":image_link}) + picture.getparent().replace(picture,e) + # Properly indent + paragraphs = root.xpath('//p') + paragraphs.reverse() + # First paragraph is never indented + for par in paragraphs[:-1]: + prev = par.getprevious() + # Do not indent after headings + if hasattr(prev,'tag') and not (prev.tag in ['h2','h3']): + par.attrib['class']="indent_first_line" + # Fix subtitle for Téma + try: + o = root.xpath("//p[@class='post-perex']")[0] + e = E.h2({"class":"post-subtitle"}) + e.text = o.text + o.getparent().replace(o,e) + except: + pass return(BeautifulSoup(lxml.etree.tostring(root,encoding=unicode))) + diff --git a/recipes/respekt_web.recipe b/recipes/respekt_web.recipe deleted file mode 100644 index 0ddf21c327..0000000000 --- a/recipes/respekt_web.recipe +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/python -# -*- coding: utf-8 -*- -# License: GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html -# Copyright: tomashnyk@gmail.com - -__license__ = 'GNU General Public License v3 - http://www.gnu.org/copyleft/gpl.html' -__copyright__ = 'tomashnyk@gmail.com' - -import re,os,datetime -from calibre.web.feeds.recipes import BasicNewsRecipe -from calibre.ebooks.BeautifulSoup import BeautifulSoup,Tag -from calibre.constants import config_dir, CONFIG_DIR_MODE -#This imports the version bundled with Calibre -import lxml -from lxml.builder import E - -class respektWebRecipe(BasicNewsRecipe): - __author__ = u'Tomáš Hnyk' - title = u'Respekt - Web' - publisher = u'Respekt Publishing a. s.' - description = u'Free articles from respekt.cz website' - encoding = 'cp1250' - language = 'cs' - remove_javascript = True - cover_url = 'http://respekt.ihned.cz/img/R/respekt_logo.png' - extra_css = 'p {text-align:justify} \ - ul {color:black} \ - .image_caption {font-size:50%;font-style:italic;} \ - .author {text-align:left;} \ - p.indent_first_line {text-indent:30px;}' - remove_tags_before = dict(name='div',attrs={'class':['l']}) - remove_tags_after = dict(id='text') - remove_tags = [dict(name='ul', attrs={'class':['tabs-d'],'id':['comm']}), \ - dict(name='div',attrs={'class':['slot','reklama','date']}), \ - dict(name='span', attrs={'class':['detail-vykrik']}), \ - dict(name='p', attrs={'class':['detail-vykrik']}), \ - dict(name='div', attrs={'id':['col123d-video','col123d-infographic','col123d-gallery','col12d-discussion']}), # soup>lxml>soup in prprocess requires this - dict(name='strong', attrs={'class':['detail-vykrik']}), - dict(name='script')] - # this makes authors left-aligned by not using the author class) - preprocess_regexps = [(re.compile(r'
', re.DOTALL|re.IGNORECASE), lambda match: '
')] - # remove empty tags - preprocess_regexps.append((re.compile(r' ', re.DOTALL|re.IGNORECASE), lambda match: ' ')) - preprocess_regexps.append((re.compile(r' ', re.DOTALL|re.IGNORECASE), lambda match: ' ')) - preprocess_regexps.append((re.compile(r'

', re.DOTALL|re.IGNORECASE), lambda match: '')) - preprocess_regexps.append((re.compile(r'font-size: 12px', re.DOTALL|re.IGNORECASE), lambda match: '')) - preprocess_regexps.append((re.compile(r'color: #[0-9]*', re.DOTALL|re.IGNORECASE), lambda match: '')) - - def parse_index(self): - # Read already downloaded articles - recipe_dir = os.path.join(config_dir,'recipes') - old_articles = os.path.join(recipe_dir,self.title.encode('utf-8').replace('/',':')) - past_items = [] - if os.path.exists(old_articles): - with file(old_articles) as f: - for h in f: - l = h.strip().split(" ") - past_items.append((l[0]," ".join(l[1:]))) - old_urls = [x[0] for x in past_items] - count_items = {} - current_items = [] - # Keep a list of only 20 latest articles for each section - past_items.reverse() - for item in past_items: - if item[1] in count_items.keys(): - if count_items[item[1]] < 20: - count_items[item[1]] += 1 - current_items.append(item) - else: - count_items[item[1]] = 1 - current_items.append(item) - current_items.reverse() - - sections = [] - # Get the webpages to download lists of articles from - raw = self.index_to_soup('http://respekt.ihned.cz/sloupky-redaktoru/', raw=True) - root = lxml.html.fromstring(raw) - sections = [] - for section in root.xpath("//div[@class='ow-enclose sr']/table/tr/td"): - try: - url = section.find('a').get('href') - if not ('?m=authors&person[id]=' in url): - sections.append((url,section.find('a').find('b').text)) - except: - pass - sections.append(('http://respekt.ihned.cz/respekt-dj/','Respekt DJ')) - sections.append(('http://respekt.ihned.cz/fokus/','Fokus')) - sections.append(('http://respekt.ihned.cz/respekt-hub/','Respekt Hub')) - sections.append(('http://respekt.ihned.cz/rozhovory/','Rozhovory')) - sections.append(('http://respekt.ihned.cz/glosy/','Glosy')) - - # Get the list of articles - ans = [] - for section in sections: - raw = self.index_to_soup(section[0], raw=True) - root = lxml.html.fromstring(raw) - list_of_articles = [] - articles = root.xpath("//div[@class='ow-enclose']/div[@class='ow']") - # Sort the articles in a section from oldest to newest - articles.reverse() - for article in articles: - date = getattr(article.xpath("span[@class='date-author']")[0],'text','')[:-3] - author = getattr(article.xpath("span[@class='date-author']")[0].find("a"),'text','') - title = getattr(article.find("h2").find("a"),'text','') - url = article.find('h2').find('a').get('href') - # Only download new articles - if url not in old_urls: - old_urls.append(url) - current_items.append((url,section[1])) - list_of_articles.append({'title':title,'url':url,'date':date,'author':author}) - # Redownload this page next time if it is still being updated (between 7 and 17 GMT generally, so make the limits a little bit bigger):wq - if section[1] == 'Respekt DJ': - if list_of_articles: - if datetime.datetime.today().weekday() in range(0,5) and 6 < datetime.datetime.utcnow().hour < 17: - #list_of_articles = list_of_articles[:-1] - current_items = current_items[:-1] - if list_of_articles: - ans.append((section[1],list_of_articles)) - # Write already downloaded articles - with file(old_articles,'w') as f: - f.write('\n'.join('{} {}'.format(*x) for x in current_items)) - return ans - - # For some reason, the following does not work: - # preprocess_regexps.append((re.compile(r'

', re.DOTALL|re.IGNORECASE), lambda match: '

')) - def preprocess_raw_html(self, raw_html, url): - return re.sub("

","

",raw_html) - - def preprocess_html(self,soup): - raw = u''.join(unicode(a) for a in soup.contents) - root = lxml.html.fromstring(raw) - # Make image captions visible - body = root.xpath("//div[@id='text']")[0] - add = 0 - for index, element in enumerate(body): - try: - if element.tag == 'img': - body.insert(index+add+1,E.p(element.get('title'),{"class":"image_caption"})) - add += 1 - except: - pass - # Make captions visible on the website have the same style - try: - root.xpath("//div[@class='hlavni-obrazek-popis']")[0].attrib['class'] = 'image_caption' - except: - pass - # For DJ, the perex is always the same, so remove it - if root.xpath("//title")[0].text.split("|")[-1] == u' Respekt DJ - RESPEKT.CZ': - - perex = root.xpath("//div[@id='perex']")[0] - clean = root.xpath("//div[@class='clean']")[0] - perex.getparent().remove(perex) - clean.getparent().remove(clean) - - # DJ section gets mal-formatted on kindle otherwise - for i in root.xpath("//h2[@class='d-dj-t']"): - i.attrib['class'] = '' - E.style = "font-size:60%;font-weight:normal;" - time = E('span',i.getprevious().text_content(),style = E.style) - # Time should be ahead of the title - time.tail = ' ' + i.text - i.text = '' - i.insert(0,time) - for i in root.xpath("//div[@class='d-dj-d']"): - i.attrib['class'] = '' - i.xpath("div/span")[0].text = '' - for i in root.xpath("//div[@class='d-dj-b']"): - i.attrib['class'] = '' - - # Make captions visible on the website have the same style - root.xpath("//div[@class='hlavni-obrazekDJ-popis']")[0].attrib['class'] = 'image_caption' - - # Reverse the entries so that the earliest are at the top - entries = root.xpath("//div[@class='d-dj-i']") - entries.reverse() - dj_body = entries[0].getparent() - for entry in entries: - dj_body.remove(entry) - dj_body.append(entry) - - # We are not interested in this paragraph as it stays the same and is essentialy an ad - if root.xpath("//title")[0].text.split("|")[-1] == u' Audit Jana Macháčka - Respekt.iHNed.cz': - ad = root.xpath("//p[@id='ajmonf']")[0] - ad.getparent().remove(ad) - - # Add length of the articles in words after author - article_length = str(len(body.text_content().split(' '))) + ' slov' - root.xpath("//div[@class='author-image']/div[@class='']/ul")[0].append(E.li(article_length)) - - # Make perex (subheading) start on a new line - root.xpath("//h1")[0].append(E.br('')) - - # Indent paragraphs when typographically suitable - # First paragraph is never indented - paragraphs = root.xpath('//p') - # Clear the formatting a little bit by removing these attributes - for par in paragraphs: - if 'class' in par.keys(): - if par.attrib['class'] == 'detail-odstavec': - par.attrib.pop('class') - paragraphs.reverse() - for par in paragraphs[:-1]: - try: - # in the beginning of this paragraph means no indenting as well as ellipses as the only text in paragraph - if len(par) > 0: - if (par.text is None and par.getchildren()[0].tag == 'strong'): - continue - elif par.getprevious().text == u'\u2026': - continue - indent = False - # Either indent if the paragraphs are the same - if par.getprevious().attrib == par.attrib: - indent = True - # Or else if the first paragraph of the text was special - if 'class' in par.getprevious().keys(): - par_name = par.getprevious().attrib['class'] - if par_name == '01prvniodstavecrepublicblok' or par_name == 'Zkladnodstavec' or par_name == '01titulekhlavn': - indent = True - if indent: - for key in par.keys(): - par.attrib.pop(key) - par.attrib['class']="indent_first_line" - except: - pass - return(BeautifulSoup(lxml.etree.tostring(root,encoding=unicode)))