From 46f0877f7e33602aeb941e55abfe25a83e5992c5 Mon Sep 17 00:00:00 2001 From: unkn0w7n <51942695+unkn0w7n@users.noreply.github.com> Date: Sun, 15 Oct 2023 15:10:35 +0530 Subject: [PATCH] Update spectator_magazine.recipe --- recipes/spectator_magazine.recipe | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/recipes/spectator_magazine.recipe b/recipes/spectator_magazine.recipe index 2debf2eab3..d8a9963bcb 100644 --- a/recipes/spectator_magazine.recipe +++ b/recipes/spectator_magazine.recipe @@ -46,10 +46,14 @@ class spectator(BasicNewsRecipe): fc['id'] = 'fig-c' return soup - # the print_version loads all articles but sometimes it might fail due to too many requests - def print_version(self, url): - from urllib.parse import quote - return 'https://webcache.googleusercontent.com/search?q=cache:' + quote(url, safe='') + def get_browser(self, *args, **kwargs): + kwargs['user_agent'] = 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' + br = BasicNewsRecipe.get_browser(self, *args, **kwargs) + br.addheaders += [ + ('Referer', 'https://www.google.com/'), + ('X-Forwarded-For', '66.249.66.1') + ] + return br def parse_index(self): soup = self.index_to_soup('https://www.spectator.co.uk/magazine')