mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-08 10:44:09 -04:00
Merge branch 'recipe-newrepublicmag' of https://github.com/ping/calibre
This commit is contained in:
Kovid Goyal
commit
472c1f0a83
BIN
recipes/icons/newrepublicmag.png
Normal file
BIN
recipes/icons/newrepublicmag.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 370 B |
314
recipes/newrepublicmag.recipe
Normal file
314
recipes/newrepublicmag.recipe
Normal file
@ -0,0 +1,314 @@
|
||||
"""
|
||||
newrepublic.com
|
||||
"""
|
||||
import json
|
||||
from functools import cmp_to_key
|
||||
from urllib.parse import urljoin, urlencode, urlsplit, urlparse
|
||||
|
||||
from calibre import iswindows
|
||||
from calibre.ebooks.BeautifulSoup import BeautifulSoup
|
||||
from calibre.utils.date import parse_date
|
||||
from calibre.web.feeds.news import BasicNewsRecipe
|
||||
|
||||
|
||||
_issue_url = "" # example: https://newrepublic.com/magazine/may-2023
|
||||
|
||||
|
||||
def sort_section(a, b, sections_sort):
|
||||
try:
|
||||
a_index = sections_sort.index(a["section"])
|
||||
except ValueError:
|
||||
a_index = 999
|
||||
try:
|
||||
b_index = sections_sort.index(b["section"])
|
||||
except ValueError:
|
||||
b_index = 999
|
||||
|
||||
if a_index < b_index:
|
||||
return -1
|
||||
if a_index > b_index:
|
||||
return 1
|
||||
if a["section"] == b["section"]:
|
||||
return -1 if a["date"] < b["date"] else 1
|
||||
return -1 if a["section"] < b["section"] else 1
|
||||
|
||||
|
||||
class NewRepublicMagazine(BasicNewsRecipe):
|
||||
title = "The New Republic Magazine"
|
||||
language = "en"
|
||||
__author__ = "ping"
|
||||
description = (
|
||||
"Founded in 1914, The New Republic is a media organization dedicated to addressing "
|
||||
"today’s most critical issues. https://newrepublic.com/magazine"
|
||||
)
|
||||
publication_type = "magazine"
|
||||
use_embedded_content = False
|
||||
masthead_url = "https://images.newrepublic.com/f5acdc0030e3212e601040dd24d5c2c0c684b15f.png?w=512&q=65&dpi=1&fit=crop&crop=faces&h=256"
|
||||
remove_attributes = ["height", "width"]
|
||||
ignore_duplicate_articles = {"title", "url"}
|
||||
remove_empty_feeds = True
|
||||
compress_news_images_auto_size = 6
|
||||
requires_version = (5, 0, 0)
|
||||
|
||||
BASE_URL = "https://newrepublic.com"
|
||||
|
||||
extra_css = """
|
||||
h1.headline { margin-bottom: 0.4rem; }
|
||||
h2.subheadline { font-style: italic; margin-bottom: 1rem; font-weight: normal; }
|
||||
.article-meta { margin-bottom: 1rem; }
|
||||
.article-meta span { display: inline-block; font-weight: bold; margin-right: 0.5rem; }
|
||||
.article-meta span:last-child { font-weight: normal; }
|
||||
div.pullquote { font-size: 1.25rem; margin-left: 0; text-align: center; }
|
||||
.lede-media img, .article-embed img, img {
|
||||
display: block; margin-bottom: 0.3rem; max-width: 100%; height: auto;
|
||||
box-sizing: border-box;
|
||||
}
|
||||
.lede-media .caption, .article-embed .caption { font-size: 0.8rem; }
|
||||
div.author-bios { margin-top: 2rem; font-style: italic; border-top: solid 1px dimgray; }
|
||||
"""
|
||||
|
||||
def _article_endpoint(self, nid):
|
||||
"""
|
||||
Graphql endpoint to fetch full article
|
||||
:param nid:
|
||||
:return:
|
||||
"""
|
||||
query = """
|
||||
query ($id: ID, $nid: ID) {
|
||||
Article(id: $id, nid: $nid) {
|
||||
...ArticlePageFields
|
||||
}
|
||||
}
|
||||
fragment ArticlePageFields on Article {
|
||||
id
|
||||
nid
|
||||
slug
|
||||
title
|
||||
cleanTitle
|
||||
badge
|
||||
frontPage {
|
||||
id
|
||||
slug
|
||||
title
|
||||
}
|
||||
LinkedSeriesId
|
||||
authors {
|
||||
id
|
||||
name
|
||||
slug
|
||||
blurb
|
||||
meta {
|
||||
twitter
|
||||
}
|
||||
}
|
||||
body
|
||||
publishedAt
|
||||
displayAt
|
||||
publicPublishedDate
|
||||
status
|
||||
ledeImage {
|
||||
id
|
||||
src
|
||||
format
|
||||
width
|
||||
height
|
||||
alt
|
||||
}
|
||||
ledeAltImage {
|
||||
id
|
||||
src
|
||||
format
|
||||
width
|
||||
height
|
||||
alt
|
||||
}
|
||||
url
|
||||
urlFull
|
||||
meta {
|
||||
wordCount
|
||||
template
|
||||
navigationTheme
|
||||
bigLede
|
||||
hideLede
|
||||
cropModeFronts
|
||||
ledeOverrideSource
|
||||
disableAds
|
||||
}
|
||||
ledeImageCredit
|
||||
ledeImageCreditBottom
|
||||
ledeImageRealCaption
|
||||
bylines
|
||||
deck
|
||||
type
|
||||
galleries {
|
||||
id
|
||||
galleryData {
|
||||
captionText
|
||||
creditText
|
||||
image {
|
||||
id
|
||||
src
|
||||
width
|
||||
height
|
||||
}
|
||||
}
|
||||
}
|
||||
tags {
|
||||
id
|
||||
slug
|
||||
label
|
||||
}
|
||||
}"""
|
||||
params = {"query": query, "variables": json.dumps({"nid": str(nid)})}
|
||||
return f"https://newrepublic.com/graphql?{urlencode(params)}"
|
||||
|
||||
def _resize_image(self, image_url, width, height):
|
||||
"""
|
||||
Rewrite the image url to fetch a device appropriate sized one instead
|
||||
of the full-res one
|
||||
|
||||
:param image_url:
|
||||
:param width:
|
||||
:param height:
|
||||
:return:
|
||||
"""
|
||||
crop_params = {
|
||||
"auto": "compress",
|
||||
"ar": f"{width}:{height}",
|
||||
"fm": "jpg",
|
||||
"fit": "crop",
|
||||
"crop": "faces",
|
||||
"ixlib": "react-9.0.2",
|
||||
"dpr": 1,
|
||||
"q": 65,
|
||||
"w": self.scale_news_images[0] if self.scale_news_images else 800,
|
||||
}
|
||||
url_tuple = urlsplit(image_url)
|
||||
return f"{url_tuple.scheme}://{url_tuple.netloc}{url_tuple.path}?{urlencode(crop_params)}"
|
||||
|
||||
def populate_article_metadata(self, article, soup, first):
|
||||
# pick up the og link from preprocess_raw_html() and set it as url instead of the api endpoint
|
||||
og_link = soup.select("[data-og-link]")
|
||||
if og_link:
|
||||
article.url = og_link[0]["data-og-link"]
|
||||
|
||||
def preprocess_raw_html(self, raw_html, url):
|
||||
# formulate the api response into html
|
||||
article = json.loads(raw_html)["data"]["Article"]
|
||||
# Example: 2022-08-12T10:00:00.000Z
|
||||
date_published_loc = parse_date(article["publishedAt"])
|
||||
# authors
|
||||
author_bios_html = ""
|
||||
post_authors = []
|
||||
try:
|
||||
post_authors = [a["name"] for a in article.get("authors", [])]
|
||||
if post_authors:
|
||||
author_bios_html = "".join(
|
||||
[a.get("blurb", "") for a in article.get("authors", [])]
|
||||
)
|
||||
author_bios_html = f'<div class="author-bios">{author_bios_html}</div>'
|
||||
except (KeyError, TypeError):
|
||||
pass
|
||||
|
||||
# lede image
|
||||
lede_image_html = ""
|
||||
if article.get("ledeImage"):
|
||||
img = article["ledeImage"]
|
||||
lede_img_url = self._resize_image(
|
||||
urljoin(self.BASE_URL, img["src"]), img["width"], img["height"]
|
||||
)
|
||||
lede_image_caption = ""
|
||||
if article.get("ledeImageRealCaption"):
|
||||
lede_image_caption = (
|
||||
f'<span class="caption">{article["ledeImageRealCaption"]}>/span>'
|
||||
)
|
||||
lede_image_html = f"""<p class="lede-media">
|
||||
<img src="{lede_img_url}">{lede_image_caption}
|
||||
</p>"""
|
||||
|
||||
body_soup = BeautifulSoup(article["body"], features="html.parser")
|
||||
for img in body_soup.find_all("img", attrs={"data-serialized": True}):
|
||||
try:
|
||||
img_info = json.loads(img["data-serialized"])
|
||||
img_src = self._resize_image(
|
||||
urljoin(self.BASE_URL, img_info["src"]),
|
||||
img_info["width"],
|
||||
img_info["height"],
|
||||
)
|
||||
img["src"] = img_src
|
||||
del img["data-serialized"]
|
||||
except: # noqa
|
||||
pass
|
||||
|
||||
return f"""<html>
|
||||
<head><title>{article["cleanTitle"]}</title></head>
|
||||
<body>
|
||||
<article data-og-link="{article["urlFull"]}">
|
||||
<h1 class="headline">{article["cleanTitle"]}</h1>
|
||||
{('<h2 class="subheadline">' + article["deck"] + "</h2>") if article.get("deck") else ""}
|
||||
<div class="article-meta">
|
||||
{f'<span class="author">{", ".join(post_authors)}</span>' if post_authors else ""}
|
||||
<span class="published-dt">
|
||||
{date_published_loc:{"%b %d, %Y" if iswindows else "%b %-d, %Y"}}
|
||||
</span>
|
||||
</div>
|
||||
{lede_image_html}
|
||||
{str(body_soup)}
|
||||
{author_bios_html}
|
||||
</article>
|
||||
</body></html>"""
|
||||
|
||||
def parse_index(self):
|
||||
br = self.get_browser()
|
||||
params = ""
|
||||
if _issue_url:
|
||||
month = urlparse(_issue_url).path.split("/")[-1]
|
||||
params = f'?{urlencode({"magazineTag": month})}'
|
||||
res = br.open_novisit(f"https://newrepublic.com/api/content/magazine{params}")
|
||||
magazine = json.loads(res.read().decode("utf-8"))["data"]
|
||||
self.log.debug(f'Found issue: {magazine["metaData"]["issueTag"]["text"]}')
|
||||
self.timefmt = f': {magazine["metaData"]["issueTag"]["text"]}'
|
||||
self.cover_url = urljoin(self.BASE_URL, magazine["metaData"]["image"]["src"])
|
||||
|
||||
feed_articles = []
|
||||
for k, articles in magazine.items():
|
||||
if not (k.startswith("magazine") and articles):
|
||||
continue
|
||||
try:
|
||||
for article in articles:
|
||||
self.log.debug(f'Found article: {article["title"]}')
|
||||
feed_articles.append(
|
||||
{
|
||||
"url": self._article_endpoint(article["nid"]),
|
||||
"title": article["title"].replace("\n", " "),
|
||||
"description": article.get("deck", ""),
|
||||
"date": article["publishedAt"],
|
||||
"section": k[len("magazine") :],
|
||||
}
|
||||
)
|
||||
except TypeError:
|
||||
# not iterable
|
||||
pass
|
||||
|
||||
sort_sections = [
|
||||
"Cover",
|
||||
"Editorsnote",
|
||||
"Features",
|
||||
"StateOfTheNation",
|
||||
"ResPublica",
|
||||
"Columns",
|
||||
"Upfront",
|
||||
"Backstory",
|
||||
"SignsAndWonders",
|
||||
"Usandtheworld",
|
||||
"Booksandthearts",
|
||||
"Poetry",
|
||||
"Exposure",
|
||||
]
|
||||
sort_category_key = cmp_to_key(lambda a, b: sort_section(a, b, sort_sections))
|
||||
return [
|
||||
(
|
||||
magazine["metaData"]["issueTag"]["text"],
|
||||
sorted(feed_articles, key=sort_category_key),
|
||||
)
|
||||
]
|
||||
Loading…
x
Reference in New Issue
Block a user