From 13ad540a05aa1c6bb6b69a0b3ebbe32206755a5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Tue, 18 Feb 2014 20:55:51 +0100 Subject: [PATCH 1/6] squash bugs originating from HEAD --- recipes/gosc_full.recipe | 1 + 1 file changed, 1 insertion(+) diff --git a/recipes/gosc_full.recipe b/recipes/gosc_full.recipe index 5903e809e1..ee7c37c49f 100644 --- a/recipes/gosc_full.recipe +++ b/recipes/gosc_full.recipe @@ -88,6 +88,7 @@ class GN(BasicNewsRecipe): dict(name='p', attrs={'class':['r tr', 'l l-2', 'wykop', 'tags']}), dict(name='div', attrs={'class':['doc_actions', 'cf', 'fr1_cl']}), dict(name='div', attrs={'id':'vote'}), + dict(name='link'), dict(name='a', attrs={'class':'img_enlarge'}) ] From c54d832d0613c0b9a9b4b75445a9325994a92406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Tue, 18 Feb 2014 21:42:02 +0100 Subject: [PATCH 2/6] add articles' descriptions and dates --- recipes/gosc_full.recipe | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/recipes/gosc_full.recipe b/recipes/gosc_full.recipe index ee7c37c49f..7d24505cbd 100644 --- a/recipes/gosc_full.recipe +++ b/recipes/gosc_full.recipe @@ -31,10 +31,12 @@ class GN(BasicNewsRecipe): self.cover_url = 'http://www.gosc.pl' + soup.find('div',attrs={'class':'fl-w100 release-wp'}).findAll('a')[-4].contents[0]['src'] feeds = [] # wstepniak - a = soup.find('div',attrs={'class':'release-wp-b'}).find('a') + a = soup.find('div',attrs={'class':'release-wp-b'}) + art = a.find('a') articles = [ - {'title' : self.tag_to_string(a), - 'url' : 'http://www.gosc.pl' + a['href'] + {'title' : self.tag_to_string(art), + 'url' : 'http://www.gosc.pl' + art['href'], + 'description' : self.tag_to_string(a.find('p',attrs={'class':'b lead'})) }] feeds.append((u'Wstępniak',articles)) # kategorie @@ -52,7 +54,9 @@ class GN(BasicNewsRecipe): art = a.find('a') yield { 'title' : self.tag_to_string(art), - 'url' : 'http://www.gosc.pl' + art['href'] + 'url' : 'http://www.gosc.pl' + art['href'], + 'date' : self.tag_to_string(a.find('p', attrs={'class':'sr-date'})), + 'description' : self.tag_to_string(a.find('p', attrs={'class':'sr-lead'})) } def append_page(self, soup, appendtag): From f468a0c4fd0e970a07444ebd75ac10daf6f8524b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Wed, 19 Feb 2014 23:29:18 +0100 Subject: [PATCH 3/6] improve image captions further --- recipes/gosc_full.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/gosc_full.recipe b/recipes/gosc_full.recipe index 7d24505cbd..614a31f232 100644 --- a/recipes/gosc_full.recipe +++ b/recipes/gosc_full.recipe @@ -81,7 +81,7 @@ class GN(BasicNewsRecipe): for r in soup.findAll(attrs={'class':['di_dr', 'doc_image']}): del r['style'] for r in soup.findAll(attrs={'class':'cm-i-a'}): - r.replaceWith( r.prettify() + '
') + r.replaceWith( '
' + r.prettify() + '
') return soup keep_only_tags = [ From 3dccff1f9462cd77ab58e3fcf3bbac713787d404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Thu, 20 Feb 2014 00:43:35 +0100 Subject: [PATCH 4/6] serve correct issue every day of the week --- recipes/gosc_full.recipe | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/recipes/gosc_full.recipe b/recipes/gosc_full.recipe index 614a31f232..c14129c816 100644 --- a/recipes/gosc_full.recipe +++ b/recipes/gosc_full.recipe @@ -24,7 +24,10 @@ class GN(BasicNewsRecipe): doc = html.fromstring(raw) page = doc.xpath('//div[@class="c"]//div[@class="search-result"]/div[1]/div[2]/h1//a/@href') - return page[4] + if time.strftime("%w") in ['3','4']: + return page[5] + else: + return page[4] def parse_index(self): soup = self.index_to_soup('http://gosc.pl' + self.find_last_issue()) From e5cc63b1c655b0480407712f1b8c688e31d25bbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Thu, 20 Feb 2014 23:27:52 +0100 Subject: [PATCH 5/6] translate comments --- recipes/gosc_full.recipe | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes/gosc_full.recipe b/recipes/gosc_full.recipe index c14129c816..96d8e362e5 100644 --- a/recipes/gosc_full.recipe +++ b/recipes/gosc_full.recipe @@ -33,7 +33,7 @@ class GN(BasicNewsRecipe): soup = self.index_to_soup('http://gosc.pl' + self.find_last_issue()) self.cover_url = 'http://www.gosc.pl' + soup.find('div',attrs={'class':'fl-w100 release-wp'}).findAll('a')[-4].contents[0]['src'] feeds = [] - # wstepniak + # editorial: a = soup.find('div',attrs={'class':'release-wp-b'}) art = a.find('a') articles = [ @@ -42,7 +42,7 @@ class GN(BasicNewsRecipe): 'description' : self.tag_to_string(a.find('p',attrs={'class':'b lead'})) }] feeds.append((u'Wstępniak',articles)) - # kategorie + # columns: for addr in soup.findAll('a',attrs={'href':re.compile('kategoria')}): if addr.string != u'wszystkie artyku\u0142y z tej kategorii \xbb': main_block = self.index_to_soup('http://www.gosc.pl' + addr['href']) From 23249ad434428bc8c0f5456035c7396bad56483d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20D=C5=82ugosz?= Date: Fri, 21 Feb 2014 03:07:32 +0100 Subject: [PATCH 6/6] add articles not assigned to any category --- recipes/gosc_full.recipe | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/recipes/gosc_full.recipe b/recipes/gosc_full.recipe index 96d8e362e5..07c36141eb 100644 --- a/recipes/gosc_full.recipe +++ b/recipes/gosc_full.recipe @@ -33,6 +33,7 @@ class GN(BasicNewsRecipe): soup = self.index_to_soup('http://gosc.pl' + self.find_last_issue()) self.cover_url = 'http://www.gosc.pl' + soup.find('div',attrs={'class':'fl-w100 release-wp'}).findAll('a')[-4].contents[0]['src'] feeds = [] + enlisted = [] # editorial: a = soup.find('div',attrs={'class':'release-wp-b'}) art = a.find('a') @@ -42,6 +43,7 @@ class GN(BasicNewsRecipe): 'description' : self.tag_to_string(a.find('p',attrs={'class':'b lead'})) }] feeds.append((u'Wstępniak',articles)) + enlisted.append(articles[0].get('url')) # columns: for addr in soup.findAll('a',attrs={'href':re.compile('kategoria')}): if addr.string != u'wszystkie artyku\u0142y z tej kategorii \xbb': @@ -50,6 +52,20 @@ class GN(BasicNewsRecipe): if len(articles) > 0: section = addr.string feeds.append((section, articles)) + enlisted.extend(list(article.get('url') for article in articles)) + # not assigned content: + page = 1 + not_assigned = [] + while True: + soup = self.index_to_soup('http://gosc.pl' + self.find_last_issue().replace('przeglad','wszystko') + '/' + str(page)) + articles = list(self.find_articles(soup)) + not_assigned.extend([ x for x in articles if x.get('url') not in enlisted ]) + page+=1 + pages = soup.find('span', attrs={'class':'pgr_nrs'}) + if str(page) not in [self.tag_to_string(x)[1] for x in pages.findAll('a')]: + break + + feeds.insert(1,(u'Nieprzypisane', not_assigned)) return feeds def find_articles(self, main_block):