Cutlery/mealie
1
0
Fork 0
mirror of https://github.com/mealie-recipes/mealie.git synced 2025-07-09 03:04:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix: Follow redirects during scraping (#3875)

Browse Source
This commit is contained in:
zeskeertwee 2024-07-10 18:49:13 +02:00 committed by GitHub
parent 47124488bb
commit fd2dc15a15
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
mealie/services/scraper/scraper_strategies.py
View File

@ -44,7 +44,9 @@ async def safe_scrape_html(url: str) -> str:
""" """
async with AsyncClient(transport=safehttp.AsyncSafeTransport()) as client: async with AsyncClient(transport=safehttp.AsyncSafeTransport()) as client:
html_bytes = b"" html_bytes = b""
async with client.stream("GET", url, timeout=SCRAPER_TIMEOUT, headers={"User-Agent": _FIREFOX_UA}) as resp: async with client.stream(
"GET", url, timeout=SCRAPER_TIMEOUT, headers={"User-Agent": _FIREFOX_UA}, follow_redirects=True
) as resp:
start_time = time.time() start_time = time.time()
async for chunk in resp.aiter_bytes(chunk_size=1024): async for chunk in resp.aiter_bytes(chunk_size=1024):