diff --git a/docs/troubleshooting/index.md b/docs/troubleshooting/index.md index 7f73f2dd5..815504390 100644 --- a/docs/troubleshooting/index.md +++ b/docs/troubleshooting/index.md @@ -70,7 +70,9 @@ If, after correctly adding and mapping your custom icons via the [Icons](../conf ## Disabling IPv6 -If you are having issues with certain widgets that are unable to reach public APIs (e.g. weather), you may need to disable IPv6 on your host machine. This can be done by adding the following to your `docker-compose.yml` file (or for docker run, the equivalent flag): +If you are having issues with certain widgets that are unable to reach public APIs (e.g. weather), in certain setups you may need to disable IPv6. You can set the environment variable `HOMEPAGE_PROXY_DISABLE_IPV6` to `true` to disable IPv6 for the homepage proxy. + +Alternatively, you can use the `sysctls` option in your docker-compose file to disable IPv6 for the homepage container completely: ```yaml services: @@ -79,12 +81,3 @@ services: sysctls: - net.ipv6.conf.all.disable_ipv6=1 ``` - -or disable IPv6 for the docker network: - -```yaml -networks: - some_network: - driver: bridge - enable_ipv6: false -``` diff --git a/src/pages/api/releases.js b/src/pages/api/releases.js index f15930c21..372ace9d5 100644 --- a/src/pages/api/releases.js +++ b/src/pages/api/releases.js @@ -1,4 +1,4 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import createLogger from "utils/logger"; const logger = createLogger("releases"); @@ -6,7 +6,7 @@ const logger = createLogger("releases"); export default async function handler(req, res) { const releasesURL = "https://api.github.com/repos/gethomepage/homepage/releases"; try { - return res.send(await cachedFetch(releasesURL, 5)); + return res.send(await cachedRequest(releasesURL, 5)); } catch (e) { logger.error(`Error checking GitHub releases: ${e}`); return res.send([]); diff --git a/src/pages/api/search/searchSuggestion.js b/src/pages/api/search/searchSuggestion.js index dbe072ea7..209d1f2cf 100644 --- a/src/pages/api/search/searchSuggestion.js +++ b/src/pages/api/search/searchSuggestion.js @@ -1,7 +1,7 @@ import { searchProviders } from "components/widgets/search/search"; import { getSettings } from "utils/config/config"; -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import { widgetsFromConfig } from "utils/config/widget-helpers"; export default async function handler(req, res) { @@ -29,5 +29,5 @@ export default async function handler(req, res) { return res.json([query, []]); // Responde with the same array format but with no suggestions. } - return res.send(await cachedFetch(`${provider.suggestionUrl}${encodeURIComponent(query)}`, 5, "Mozilla/5.0")); + return res.send(await cachedRequest(`${provider.suggestionUrl}${encodeURIComponent(query)}`, 5, "Mozilla/5.0")); } diff --git a/src/pages/api/widgets/openmeteo.js b/src/pages/api/widgets/openmeteo.js index e63847b43..28f2e4f00 100644 --- a/src/pages/api/widgets/openmeteo.js +++ b/src/pages/api/widgets/openmeteo.js @@ -1,9 +1,9 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; export default async function handler(req, res) { const { latitude, longitude, units, cache, timezone } = req.query; const degrees = units === "metric" ? "celsius" : "fahrenheit"; const timezeone = timezone ?? "auto"; const apiUrl = `https://api.open-meteo.com/v1/forecast?latitude=${latitude}&longitude=${longitude}&daily=sunrise,sunset¤t_weather=true&temperature_unit=${degrees}&timezone=${timezeone}`; - return res.send(await cachedFetch(apiUrl, cache)); + return res.send(await cachedRequest(apiUrl, cache)); } diff --git a/src/pages/api/widgets/openweathermap.js b/src/pages/api/widgets/openweathermap.js index 089ee804e..3bdc7a822 100644 --- a/src/pages/api/widgets/openweathermap.js +++ b/src/pages/api/widgets/openweathermap.js @@ -1,4 +1,4 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import { getSettings } from "utils/config/config"; import { getPrivateWidgetOptions } from "utils/config/widget-helpers"; @@ -26,5 +26,5 @@ export default async function handler(req, res) { const apiUrl = `https://api.openweathermap.org/data/2.5/weather?lat=${latitude}&lon=${longitude}&appid=${apiKey}&units=${units}&lang=${lang}`; - return res.send(await cachedFetch(apiUrl, cache)); + return res.send(await cachedRequest(apiUrl, cache)); } diff --git a/src/pages/api/widgets/stocks.js b/src/pages/api/widgets/stocks.js index 3941a773d..4e9f3f55e 100644 --- a/src/pages/api/widgets/stocks.js +++ b/src/pages/api/widgets/stocks.js @@ -1,4 +1,4 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import { getSettings } from "utils/config/config"; import createLogger from "utils/logger"; @@ -60,7 +60,7 @@ export default async function handler(req, res) { const apiUrl = `https://finnhub.io/api/v1/quote?symbol=${ticker}&token=${apiKey}`; // Finnhub free accounts allow up to 60 calls/minute // https://finnhub.io/pricing - const { c, dp } = await cachedFetch(apiUrl, cache || 1); + const { c, dp } = await cachedRequest(apiUrl, cache || 1); logger.debug("Finnhub API response for %s: %o", ticker, { c, dp }); // API sometimes returns 200, but values returned are `null` diff --git a/src/pages/api/widgets/weather.js b/src/pages/api/widgets/weather.js index 9d0451ce9..9e63e48d9 100644 --- a/src/pages/api/widgets/weather.js +++ b/src/pages/api/widgets/weather.js @@ -1,4 +1,4 @@ -import cachedFetch from "utils/proxy/cached-fetch"; +import { cachedRequest } from "utils/proxy/http"; import { getSettings } from "utils/config/config"; import { getPrivateWidgetOptions } from "utils/config/widget-helpers"; @@ -26,5 +26,5 @@ export default async function handler(req, res) { const apiUrl = `http://api.weatherapi.com/v1/current.json?q=${latitude},${longitude}&key=${apiKey}&lang=${lang}`; - return res.send(await cachedFetch(apiUrl, cache)); + return res.send(await cachedRequest(apiUrl, cache)); } diff --git a/src/utils/config/widget-helpers.js b/src/utils/config/widget-helpers.js index 3b1355d6b..93f71194c 100644 --- a/src/utils/config/widget-helpers.js +++ b/src/utils/config/widget-helpers.js @@ -56,21 +56,22 @@ export async function cleanWidgetGroups(widgets) { export async function getPrivateWidgetOptions(type, widgetIndex) { const widgets = await widgetsFromConfig(); - const privateOptions = widgets.map((widget) => { - const { index, url, username, password, key, apiKey } = widget.options; + const privateOptions = + widgets.map((widget) => { + const { index, url, username, password, key, apiKey } = widget.options; - return { - type: widget.type, - options: { - index, - url, - username, - password, - key, - apiKey, - }, - }; - }); + return { + type: widget.type, + options: { + index, + url, + username, + password, + key, + apiKey, + }, + }; + }) || {}; return type !== undefined && widgetIndex !== undefined ? privateOptions.find((o) => o.type === type && o.options.index === parseInt(widgetIndex, 10))?.options diff --git a/src/utils/proxy/cached-fetch.js b/src/utils/proxy/cached-fetch.js deleted file mode 100644 index ae3c46108..000000000 --- a/src/utils/proxy/cached-fetch.js +++ /dev/null @@ -1,25 +0,0 @@ -import cache from "memory-cache"; - -const defaultDuration = 5; - -export default async function cachedFetch(url, duration, ua) { - const cached = cache.get(url); - - // eslint-disable-next-line no-param-reassign - duration = duration || defaultDuration; - - if (cached) { - return cached; - } - - // wrapping text in JSON.parse to handle utf-8 issues - const options = {}; - if (ua) { - options.headers = { - "User-Agent": ua, - }; - } - const data = await fetch(url, options).then((res) => res.json()); - cache.put(url, data, duration * 1000 * 60); - return data; -} diff --git a/src/utils/proxy/http.js b/src/utils/proxy/http.js index 3743515b4..f8d2dcce9 100644 --- a/src/utils/proxy/http.js +++ b/src/utils/proxy/http.js @@ -3,6 +3,7 @@ import { createUnzip, constants as zlibConstants } from "node:zlib"; import { http, https } from "follow-redirects"; +import cache from "memory-cache"; import { addCookieToJar, setCookieHeader } from "./cookie-jar"; import { sanitizeErrorURL } from "./api-helpers"; @@ -81,20 +82,46 @@ export function httpRequest(url, params) { return handleRequest(http, url, params); } +export async function cachedRequest(url, duration = 5, ua = "homepage") { + const cached = cache.get(url); + + if (cached) { + return cached; + } + + const options = { + headers: { + "User-Agent": ua, + Accept: "application/json", + }, + }; + let [, , data] = await httpProxy(url, options); + if (Buffer.isBuffer(data)) { + try { + data = JSON.parse(Buffer.from(data).toString()); + } catch (e) { + logger.debug("Error parsing cachedRequest data for %s: %s %s", url, Buffer.from(data).toString(), e); + data = Buffer.from(data).toString(); + } + } + cache.put(url, data, duration * 1000 * 60); + return data; +} + export async function httpProxy(url, params = {}) { const constructedUrl = new URL(url); + const disableIpv6 = process.env.HOMEPAGE_PROXY_DISABLE_IPV6 === "true"; + const agentOptions = disableIpv6 ? { family: 4, autoSelectFamily: false } : {}; let request = null; if (constructedUrl.protocol === "https:") { request = httpsRequest(constructedUrl, { - agent: new https.Agent({ - rejectUnauthorized: false, - }), + agent: new https.Agent({ ...agentOptions, rejectUnauthorized: false }), ...params, }); } else { request = httpRequest(constructedUrl, { - agent: new http.Agent(), + agent: new http.Agent(agentOptions), ...params, }); }