diff --git a/recipes/icons/nyt_tmag.png b/recipes/icons/nyt_tmag.png new file mode 100644 index 0000000000..2d170d68f4 Binary files /dev/null and b/recipes/icons/nyt_tmag.png differ diff --git a/recipes/icons/nytimes_tech.png b/recipes/icons/nytimes_tech.png index 0f36015d2d..2d170d68f4 100644 Binary files a/recipes/icons/nytimes_tech.png and b/recipes/icons/nytimes_tech.png differ diff --git a/recipes/nyt_tmag.recipe b/recipes/nyt_tmag.recipe new file mode 100644 index 0000000000..bcda8e1d5f --- /dev/null +++ b/recipes/nyt_tmag.recipe @@ -0,0 +1,127 @@ +#!/usr/bin/env python + +from calibre.web.feeds.news import BasicNewsRecipe + + +use_wayback_machine = False + + +class NytMag(BasicNewsRecipe): + title = 'NYT T Magazine' + __author__ = 'unkn0wn' + description = 'The latest from The New York Times Style Magazine.' + oldest_article = 30 # days + encoding = 'utf-8' + use_embedded_content = False + language = 'en_US' + remove_empty_feeds = True + resolve_internal_links = True + ignore_duplicate_articles = {'title', 'url'} + masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png' + + feeds = [ + 'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml', + 'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml', + ] + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article), + }, + 'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'}, + 'rev': { + 'short': 'Reverse the order of articles in each feed?', + 'long': 'enter yes', + 'default': 'no', + }, + 'res': { + 'short': ( + 'For hi-res images, select a resolution from the following\noptions: ' + 'popup, jumbo, mobileMasterAt3x, superJumbo' + ), + 'long': ( + 'This is useful for non e-ink devices, and for a lower file size\nthan ' + 'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.' + ), + }, + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + r = self.recipe_specific_options.get('rev') + if r and isinstance(r, str): + if r.lower() == 'yes': + self.reverse_article_order = True + c = self.recipe_specific_options.get('comp') + if c and isinstance(c, str): + if c.lower() == 'yes': + self.compress_news_images = True + + extra_css = """ + .byl, .time { font-size:small; color:#202020; } + .cap { font-size:small; text-align:center; } + .cred { font-style:italic; font-size:small; } + em, blockquote { color: #202020; } + .sc { font-variant: small-caps; } + .lbl { font-size:small; color:#404040; } + img { display:block; margin:0 auto; } + """ + + @property + def nyt_parser(self): + ans = getattr(self, '_nyt_parser', None) + if ans is None: + from calibre.live import load_module + + self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes') + return ans + + def get_nyt_page(self, url, skip_wayback=False): + if use_wayback_machine and not skip_wayback: + from calibre import browser + + return self.nyt_parser.download_url(url, browser()) + return self.index_to_soup(url, raw=True) + + articles_are_obfuscated = use_wayback_machine + + if use_wayback_machine: + + def get_obfuscated_article(self, url): + from calibre.ptempfile import PersistentTemporaryFile + + with PersistentTemporaryFile() as tf: + tf.write(self.get_nyt_page(url)) + return tf.name + + def preprocess_raw_html(self, raw_html, url): + return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url) + + def get_browser(self, *args, **kwargs): + kwargs['user_agent'] = ( + 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' + ) + br = BasicNewsRecipe.get_browser(self, *args, **kwargs) + br.addheaders += [ + ('Referer', 'https://www.google.com/'), + ('X-Forwarded-For', '66.249.66.1'), + ] + return br + + def preprocess_html(self, soup): + w = self.recipe_specific_options.get('res') + if w and isinstance(w, str): + res = '-' + w + for img in soup.findAll('img', attrs={'src': True}): + if '-article' in img['src']: + ext = img['src'].split('?')[0].split('.')[-1] + img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext + for c in soup.findAll('div', attrs={'class': 'cap'}): + for p in c.findAll(['p', 'div']): + p.name = 'span' + return soup diff --git a/recipes/nytfeeds.recipe b/recipes/nytfeeds.recipe index 6214051f19..257d48647b 100644 --- a/recipes/nytfeeds.recipe +++ b/recipes/nytfeeds.recipe @@ -1,179 +1,9 @@ #!/usr/bin/env python -import json import re - -from calibre.utils.iso8601 import parse_iso8601 from calibre.web.feeds.news import BasicNewsRecipe -def extract_json(raw): - pre = re.search(r'