mirror of
https://github.com/searxng/searxng.git
synced 2025-10-24 23:39:01 -04:00
078c9fcb68
Revision of the engine / use of the result type Paper as well as other typifications. Signed-off-by: Markus Heiser <markus.heiser@darmarit.de>
98 lines
3.2 KiB
Python
98 lines
3.2 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
"""Crossref_ is the sustainable source of community-owned scholarly metadata and
|
|
is relied upon by thousands of systems across the research ecosystem and the
|
|
globe.
|
|
|
|
.. _Crossref: https://www.crossref.org/documentation/retrieve-metadata/
|
|
|
|
"""
|
|
|
|
import typing as t
|
|
|
|
from urllib.parse import urlencode
|
|
from datetime import datetime
|
|
from searx.result_types import EngineResults
|
|
|
|
if t.TYPE_CHECKING:
|
|
from searx.extended_types import SXNG_Response
|
|
from searx.search.processors import OnlineParams
|
|
|
|
about = {
|
|
"website": "https://www.crossref.org/",
|
|
"wikidata_id": "Q5188229",
|
|
"official_api_documentation": "https://api.crossref.org/swagger-ui/",
|
|
"use_official_api": True,
|
|
"require_api_key": False,
|
|
"results": "JSON",
|
|
}
|
|
|
|
categories = ["science", "scientific publications"]
|
|
paging = True
|
|
search_url = "https://api.crossref.org/works"
|
|
"""Returns a list of all works (journal articles, conference proceedings, books,
|
|
components, etc), 20 per page (`Works/get_works`_).
|
|
|
|
.. _Works/get_works: https://api.crossref.org/swagger-ui/index.html#/Works/get_works
|
|
"""
|
|
|
|
|
|
def request(query: str, params: "OnlineParams") -> None:
|
|
args = {
|
|
"query": query,
|
|
"offset": 20 * (params["pageno"] - 1),
|
|
}
|
|
params["url"] = f"{search_url}?{urlencode(args)}"
|
|
|
|
|
|
def response(resp: "SXNG_Response") -> EngineResults:
|
|
res = EngineResults()
|
|
json_data = resp.json()
|
|
|
|
def field(k: str) -> str:
|
|
return str(record.get(k, ""))
|
|
|
|
for record in json_data["message"]["items"]:
|
|
|
|
if record["type"] == "component":
|
|
# These seem to be files published along with papers. Not something
|
|
# you'd search for.
|
|
continue
|
|
title: str = ""
|
|
journal: str = ""
|
|
|
|
if record["type"] == "book-chapter":
|
|
title = record["container-title"][0]
|
|
if record["title"][0].lower().strip() != title.lower().strip():
|
|
title += f" ({record['title'][0]})"
|
|
else:
|
|
title = record["title"][0] if "title" in record else record.get("container-title", [None])[0]
|
|
journal = record.get("container-title", [None])[0] if "title" in record else ""
|
|
|
|
item = res.types.Paper(
|
|
title=title,
|
|
journal=journal,
|
|
content=field("abstract"),
|
|
doi=field("DOI"),
|
|
pages=field("page"),
|
|
publisher=field("publisher"),
|
|
tags=record.get("subject"),
|
|
type=field("type"),
|
|
url=field("URL"),
|
|
volume=field("volume"),
|
|
)
|
|
res.add(item)
|
|
|
|
if "resource" in record and "primary" in record["resource"] and "URL" in record["resource"]["primary"]:
|
|
item.url = record["resource"]["primary"]["URL"]
|
|
|
|
if "published" in record and "date-parts" in record["published"]:
|
|
item.publishedDate = datetime(*(record["published"]["date-parts"][0] + [1, 1][:3]))
|
|
|
|
item.authors = [a.get("given", "") + " " + a.get("family", "") for a in record.get("author", [])]
|
|
item.isbn = record.get("isbn") or [i["value"] for i in record.get("isbn-type", [])]
|
|
|
|
# All the links are not PDFs, even if the URL ends with ".pdf"
|
|
# item.pdf_url = record.get("link", [{"URL": None}])[0]["URL"]
|
|
|
|
return res
|