From a1a97acb07f5e72e939a62d53960a275dcb895f6 Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Mon, 22 Jul 2024 12:16:47 +0530 Subject: [PATCH] Update DW group of recipes add recipe_specific_options --- recipes/deutsche_welle_bs.recipe | 18 ++++++++++++++++-- recipes/deutsche_welle_de.recipe | 14 ++++++++++++++ recipes/deutsche_welle_en.recipe | 16 +++++++++++++++- recipes/deutsche_welle_es.recipe | 18 ++++++++++++++++-- recipes/deutsche_welle_hr.recipe | 20 +++++++++++++++++--- recipes/deutsche_welle_pt.recipe | 21 +++++++++++++++++---- recipes/deutsche_welle_ru.recipe | 16 +++++++++++++++- recipes/deutsche_welle_sr.recipe | 20 +++++++++++++++++--- 8 files changed, 127 insertions(+), 16 deletions(-) diff --git a/recipes/deutsche_welle_bs.recipe b/recipes/deutsche_welle_bs.recipe index e638560cf8..a81ee32935 100644 --- a/recipes/deutsche_welle_bs.recipe +++ b/recipes/deutsche_welle_bs.recipe @@ -23,13 +23,27 @@ class DeutscheWelle_bs(BasicNewsRecipe): keep_only_tags = [ dict(name='article') ] - + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + remove_tags = [ dict(name=['footer', 'source']), dict(attrs={'data-tracking-name':'sharing-icons-inline'}), classes('kicker advertisement vjs-wrapper') ] - + feeds = [ (u'Politika', u'http://rss.dw-world.de/rdf/rss-bos-pol'), (u'Evropa', u'http://rss.dw-world.de/rdf/rss-bos-eu'), diff --git a/recipes/deutsche_welle_de.recipe b/recipes/deutsche_welle_de.recipe index e9a5bf1921..500e0bb9cc 100644 --- a/recipes/deutsche_welle_de.recipe +++ b/recipes/deutsche_welle_de.recipe @@ -21,6 +21,20 @@ class DeutscheWelle(BasicNewsRecipe): dict(name='article') ] + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + remove_tags = [ dict(name=['footer', 'source']), dict(attrs={'data-tracking-name':'sharing-icons-inline'}), diff --git a/recipes/deutsche_welle_en.recipe b/recipes/deutsche_welle_en.recipe index 00679daea4..634eab9b84 100644 --- a/recipes/deutsche_welle_en.recipe +++ b/recipes/deutsche_welle_en.recipe @@ -15,7 +15,21 @@ class DeutscheWelle_en(BasicNewsRecipe): remove_empty_feeds = True ignore_duplicate_articles = {'title', 'url'} remove_attributes = ['height', 'width', 'style'] - + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + keep_only_tags = [ dict(name='article') ] diff --git a/recipes/deutsche_welle_es.recipe b/recipes/deutsche_welle_es.recipe index b28c085e17..5076f1425d 100644 --- a/recipes/deutsche_welle_es.recipe +++ b/recipes/deutsche_welle_es.recipe @@ -17,10 +17,24 @@ class DeutscheWelle_es(BasicNewsRecipe): remove_attributes = ['height', 'width', 'style'] + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + keep_only_tags = [ dict(name='article') ] - + remove_tags = [ dict(name=['footer', 'source']), dict(attrs={'data-tracking-name':'sharing-icons-inline'}), @@ -40,7 +54,7 @@ class DeutscheWelle_es(BasicNewsRecipe): ('Conozca Alemania', 'http://rss.dw-world.de/rdf/rss-sp-con') ] - + def preprocess_html(self, soup): for img in soup.findAll('img', srcset=True): img['src'] = img['srcset'].split()[6] diff --git a/recipes/deutsche_welle_hr.recipe b/recipes/deutsche_welle_hr.recipe index 8edc24a618..bf56e72386 100644 --- a/recipes/deutsche_welle_hr.recipe +++ b/recipes/deutsche_welle_hr.recipe @@ -16,20 +16,34 @@ class DeutscheWelle_hr(BasicNewsRecipe): remove_empty_feeds = True masthead_url = 'http://www.dw-world.de/skins/std/channel1/pics/dw_logo1024.gif' remove_javascript = True - + ignore_duplicate_articles = {'title', 'url'} remove_attributes = ['height', 'width', 'style'] + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + keep_only_tags = [ dict(name='article') ] - + remove_tags = [ dict(name=['footer', 'source']), dict(attrs={'data-tracking-name':'sharing-icons-inline'}), classes('kicker advertisement vjs-wrapper') ] - + feeds = [ (u'Svijet', u'http://rss.dw-world.de/rdf/rss-cro-svijet'), (u'Europa', u'http://rss.dw-world.de/rdf/rss-cro-eu'), diff --git a/recipes/deutsche_welle_pt.recipe b/recipes/deutsche_welle_pt.recipe index afafe67a28..d2e5082854 100644 --- a/recipes/deutsche_welle_pt.recipe +++ b/recipes/deutsche_welle_pt.recipe @@ -15,12 +15,25 @@ class DeutscheWelle_pt(BasicNewsRecipe): publication_type = 'newsportal' remove_empty_feeds = True masthead_url = 'http://www.dw-world.de/skins/std/channel1/pics/dw_logo1024.gif' - - + remove_javascript = True ignore_duplicate_articles = {'title', 'url'} remove_attributes = ['height', 'width', 'style'] - + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + def preprocess_html(self, soup): for img in soup.findAll('img', srcset=True): img['src'] = img['srcset'].split()[6] @@ -29,7 +42,7 @@ class DeutscheWelle_pt(BasicNewsRecipe): keep_only_tags = [ dict(name='article') ] - + remove_tags = [ dict(name=['footer', 'source']), dict(attrs={'data-tracking-name':'sharing-icons-inline'}), diff --git a/recipes/deutsche_welle_ru.recipe b/recipes/deutsche_welle_ru.recipe index ec4e838af0..38df7884b1 100644 --- a/recipes/deutsche_welle_ru.recipe +++ b/recipes/deutsche_welle_ru.recipe @@ -16,7 +16,21 @@ class DeutscheWelle(BasicNewsRecipe): remove_empty_feeds = True ignore_duplicate_articles = {'title', 'url'} remove_attributes = ['height', 'width', 'style'] - + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + def preprocess_html(self, soup): for img in soup.findAll('img', srcset=True): img['src'] = img['srcset'].split()[6] diff --git a/recipes/deutsche_welle_sr.recipe b/recipes/deutsche_welle_sr.recipe index 9cb2aaa482..134ea368fc 100644 --- a/recipes/deutsche_welle_sr.recipe +++ b/recipes/deutsche_welle_sr.recipe @@ -18,7 +18,21 @@ class DeutscheWelle_sr(BasicNewsRecipe): remove_javascript = True ignore_duplicate_articles = {'title', 'url'} remove_attributes = ['height', 'width', 'style'] - + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article) + } + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + def preprocess_html(self, soup): for img in soup.findAll('img', srcset=True): img['src'] = img['srcset'].split()[6] @@ -27,13 +41,13 @@ class DeutscheWelle_sr(BasicNewsRecipe): keep_only_tags = [ dict(name='article') ] - + remove_tags = [ dict(name=['footer', 'source']), dict(attrs={'data-tracking-name':'sharing-icons-inline'}), classes('kicker advertisement vjs-wrapper') ] - + feeds = [ (u'Politika', u'http://rss.dw-world.de/rdf/rss-ser-pol'), (u'Srbija', u'http://rss.dw-world.de/rdf/rss-ser-pol-ser'),