diff --git a/recipes/icons/nyt_magazine.png b/recipes/icons/nyt_magazine.png new file mode 100644 index 0000000000..2d170d68f4 Binary files /dev/null and b/recipes/icons/nyt_magazine.png differ diff --git a/recipes/nyt_magazine.recipe b/recipes/nyt_magazine.recipe new file mode 100644 index 0000000000..52c4a38b0d --- /dev/null +++ b/recipes/nyt_magazine.recipe @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +from calibre.web.feeds.news import BasicNewsRecipe + +use_wayback_machine = False + + +class NytMag(BasicNewsRecipe): + title = 'NYT Magazine' + __author__ = 'unkn0wn' + description = 'The latest from The New York Times Magazine.' + oldest_article = 30 # days + encoding = 'utf-8' + use_embedded_content = False + language = 'en_US' + remove_empty_feeds = True + resolve_internal_links = True + ignore_duplicate_articles = {'title', 'url'} + masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png' + + def get_cover_url(self): + soup = self.index_to_soup('https://www.nytimes.com/section/magazine') + issue = soup.find(attrs={'class': 'issue-promo'}) + return issue.find(attrs={'class': 'promo-image'}).img['src'] + + feeds = [ + 'https://rss.nytimes.com/services/xml/rss/nyt/Magazine.xml', + 'https://rss.nytimes.com/services/xml/rss/nyt/well.xml', + ] + + recipe_specific_options = { + 'days': { + 'short': 'Oldest article to download from this news source. In days ', + 'long': 'For example, 0.5, gives you articles from the past 12 hours', + 'default': str(oldest_article), + }, + 'comp': {'short': 'Compress News Images?', 'long': 'enter yes', 'default': 'no'}, + 'rev': { + 'short': 'Reverse the order of articles in each feed?', + 'long': 'enter yes', + 'default': 'no', + }, + 'res': { + 'short': ( + 'For hi-res images, select a resolution from the following\noptions: ' + 'popup, jumbo, mobileMasterAt3x, superJumbo' + ), + 'long': ( + 'This is useful for non e-ink devices, and for a lower file size\nthan ' + 'the default, use mediumThreeByTwo440, mediumThreeByTwo225, articleInline.' + ), + }, + } + + def __init__(self, *args, **kwargs): + BasicNewsRecipe.__init__(self, *args, **kwargs) + d = self.recipe_specific_options.get('days') + if d and isinstance(d, str): + self.oldest_article = float(d) + r = self.recipe_specific_options.get('rev') + if r and isinstance(r, str): + if r.lower() == 'yes': + self.reverse_article_order = True + c = self.recipe_specific_options.get('comp') + if c and isinstance(c, str): + if c.lower() == 'yes': + self.compress_news_images = True + + extra_css = """ + .byl, .time { font-size:small; color:#202020; } + .cap { font-size:small; text-align:center; } + .cred { font-style:italic; font-size:small; } + em, blockquote { color: #202020; } + .sc { font-variant: small-caps; } + .lbl { font-size:small; color:#404040; } + img { display:block; margin:0 auto; } + """ + + @property + def nyt_parser(self): + ans = getattr(self, '_nyt_parser', None) + if ans is None: + from calibre.live import load_module + + self._nyt_parser = ans = load_module('calibre.web.site_parsers.nytimes') + return ans + + def get_nyt_page(self, url, skip_wayback=False): + if use_wayback_machine and not skip_wayback: + from calibre import browser + + return self.nyt_parser.download_url(url, browser()) + return self.index_to_soup(url, raw=True) + + articles_are_obfuscated = use_wayback_machine + + if use_wayback_machine: + + def get_obfuscated_article(self, url): + from calibre.ptempfile import PersistentTemporaryFile + + with PersistentTemporaryFile() as tf: + tf.write(self.get_nyt_page(url)) + return tf.name + + def preprocess_raw_html(self, raw_html, url): + return self.nyt_parser.extract_html(self.index_to_soup(raw_html), url) + + def get_browser(self, *args, **kwargs): + kwargs['user_agent'] = ( + 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' + ) + br = BasicNewsRecipe.get_browser(self, *args, **kwargs) + br.addheaders += [ + ('Referer', 'https://www.google.com/'), + ('X-Forwarded-For', '66.249.66.1'), + ] + return br + + def preprocess_html(self, soup): + w = self.recipe_specific_options.get('res') + if w and isinstance(w, str): + res = '-' + w + for img in soup.findAll('img', attrs={'src': True}): + if '-article' in img['src']: + ext = img['src'].split('?')[0].split('.')[-1] + img['src'] = img['src'].rsplit('-article', 1)[0] + res + '.' + ext + for c in soup.findAll('div', attrs={'class': 'cap'}): + for p in c.findAll(['p', 'div']): + p.name = 'span' + return soup diff --git a/recipes/nyt_tmag.recipe b/recipes/nyt_tmag.recipe index 1d557061eb..b4b0a25a18 100644 --- a/recipes/nyt_tmag.recipe +++ b/recipes/nyt_tmag.recipe @@ -18,6 +18,11 @@ class NytMag(BasicNewsRecipe): ignore_duplicate_articles = {'title', 'url'} masthead_url = 'https://static01.nytimes.com/newsgraphics/2015-12-23-masthead-2016/b15c3d81d3d7b59065fff9a3f3afe85aa2e2dff5/_assets/nyt-logo.png' + def get_cover_url(self): + soup = self.index_to_soup('https://www.nytimes.com/section/t-magazine') + issue = soup.find(attrs={'class':'issue-promo'}) + return issue.a.img['src'] + feeds = [ 'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml', 'https://rss.nytimes.com/services/xml/rss/nyt/FashionandStyle.xml', diff --git a/recipes/nytfeeds.recipe b/recipes/nytfeeds.recipe index ce90125826..bf05758c1e 100644 --- a/recipes/nytfeeds.recipe +++ b/recipes/nytfeeds.recipe @@ -41,8 +41,9 @@ class NytFeeds(BasicNewsRecipe): # 'https://rss.nytimes.com/services/xml/rss/nyt/tmagazine.xml', # 'https://rss.nytimes.com/services/xml/rss/nyt/books.xml', 'https://www.nytimes.com/services/xml/rss/nyt/Travel.xml', - 'https://rss.nytimes.com/services/xml/rss/nyt/well.xml', + # 'https://rss.nytimes.com/services/xml/rss/nyt/well.xml', 'https://rss.nytimes.com/services/xml/rss/nyt/Sports.xml', + # 'https://rss.nytimes.com/services/xml/rss/nyt/Magazine.xml', # 'http://nytimes.com/timeswire/feeds/', ] diff --git a/recipes/nytimes.recipe b/recipes/nytimes.recipe index ba80a60857..8e6013895b 100644 --- a/recipes/nytimes.recipe +++ b/recipes/nytimes.recipe @@ -31,7 +31,7 @@ web_sections = [ ('Health', 'health'), ('Opinion', 'opinion'), ('Arts', 'arts'), - ('Books', 'books'), + # ('Books', 'books'), ('Movies', 'movies'), ('Music', 'arts/music'), ('Television', 'arts/television'), diff --git a/recipes/nytimes_sub.recipe b/recipes/nytimes_sub.recipe index 35f7505350..ba28033827 100644 --- a/recipes/nytimes_sub.recipe +++ b/recipes/nytimes_sub.recipe @@ -31,7 +31,7 @@ web_sections = [ ('Health', 'health'), ('Opinion', 'opinion'), ('Arts', 'arts'), - ('Books', 'books'), + # ('Books', 'books'), ('Movies', 'movies'), ('Music', 'arts/music'), ('Television', 'arts/television'), diff --git a/src/calibre/web/site_parsers/nytimes.py b/src/calibre/web/site_parsers/nytimes.py index 78715fa687..56b1db6f1d 100644 --- a/src/calibre/web/site_parsers/nytimes.py +++ b/src/calibre/web/site_parsers/nytimes.py @@ -9,7 +9,7 @@ from xml.sax.saxutils import escape, quoteattr from calibre.utils.iso8601 import parse_iso8601 -module_version = 9 # needed for live updates +module_version = 10 # needed for live updates pprint @@ -144,8 +144,10 @@ def parse_types(x): htag = 'h' + re.match(r'Heading([1-6])Block', typename).group(1) yield f'<{htag}{align}>{"".join(parse_cnt(x))}{htag}>' - elif typename in {'ParagraphBlock', 'DetailBlock', 'TextRunKV'}: + elif typename == 'ParagraphBlock': yield f'
{"".join(parse_cnt(x))}
' + elif typename in {'DetailBlock', 'TextRunKV'}: + yield f'{"".join(parse_cnt(x))}
' elif typename == 'BylineBlock': yield f'