Merge branch 'master' of https://github.com/unkn0w7n/calibre

recipes/caravan_magazine.recipe

@@ -4,33 +4,36 @@
 
 import json
 
-from calibre.ebooks.BeautifulSoup import Tag
+
-from calibre.web.feeds.recipes import BasicNewsRecipe
+from calibre.web.feeds.news import BasicNewsRecipe, classes
 from mechanize import Request
 
+def absurl(x):
+    if x.startswith('//'):
+        x = 'https:' + x
+    elif not x.startswith('http'):
+        x = 'https://caravanmagazine.in' + x
+    return x
 
-def classes(classes):
+def safe_dict(data, *names):
-    q = frozenset(classes.split(' '))
+    ans = data
-    return dict(attrs={
+    for x in names:
-        'class': lambda x: x and frozenset(x.split()).intersection(q)})
+        ans = ans.get(x) or ''
-
+    return ans
-
-def new_tag(soup, name, attrs=()):
-    impl = getattr(soup, 'new_tag', None)
-    if impl is not None:
-        return impl(name, attrs=dict(attrs))
-    return Tag(soup, name, attrs=attrs or None)
-
 
 class CaravanMagazine(BasicNewsRecipe):
 
     title = 'Caravan Magazine'
     __author__ = 'Kovid Goyal, Gobelinus, unkn0wn'
-    description = 'An Indian Journal of politics and culture'
+    description = (
+        'The Caravan has established itself as one of the country’s most respected and intellectually agile publications, '
+        'setting new benchmarks for the Indian and South Asian media. We publish immersive reportage, daring commentary, '
+        'path-breaking investigations, insightful literary criticism and more, spanning the worlds of politics, culture, '
+        'business, society, media, the environment and the arts.'
+    )
     language = 'en_IN'
     timefmt = ' [%b, %Y]'
     encoding = 'utf-8'
-    needs_subscription = 'optional'
 
     no_stylesheets = True
 
@@ -39,20 +42,12 @@ class CaravanMagazine(BasicNewsRecipe):
     resolve_internal_links = True
 
     extra_css = '''
-        blockquote {color:#202020;}
+        blockquote, em {color:#202020;}
-        #fig-c {text-align:center; font-size:small;}
+        .article_subtitle {font-style:italic; color:#202020;}
-        em {color:#202020;}
+        #fig-c, .photo_wrapper, .cover_figure_element {text-align:center; font-size:small;}
-        .article-footer {font-size:small;}
+        .pre-title, .text_wrapper {font-size:small; color:#404040;}
-        .date, .pre-title {font-size:small; color:#404040;}
-        .authors {font-size:small; font-weight:bold;}
     '''
 
-    remove_tags = [
-        classes('related-articles'),
-        dict(name='meta'),
-        dict(attrs={'class': ['share-with', 'img-wrap abs']}),
-    ]
-
     def get_browser(self, *args, **kw):
         br = BasicNewsRecipe.get_browser(self, *args, **kw)
         if not self.username or not self.password:
@@ -79,75 +74,51 @@ class CaravanMagazine(BasicNewsRecipe):
             raise ValueError('Login failed, check your username and password')
         return br
 
-    # To parse article toc
+    keep_only_tags = [
-    def parse_index(self):
+        classes('text_wrapper cover_figure_element article_content')
-        base_url = 'https://www.caravanmagazine.in/'
+    ]
-        soup = self.index_to_soup('{0}magazine'.format(base_url))
-        if magdate := soup.find('h6', attrs={'class':'magazine-date'}):
-            self.timefmt = ' [' + self.tag_to_string(magdate).strip() + ']'
-
-        # find current issue cover
-        feeds = []
-        sections = soup.find(attrs={'class': lambda x: x and 'current-magazine-issue' in x.split()}).find(
-                attrs={'class': lambda x: x and 'sections' in x.split()})
-        for section in sections.findAll(attrs={'class': lambda x: x and 'section' in x.split()}):
-            a = section.find('a')
-            section_title = self.tag_to_string(a)
-            self.log('\nSection:', section_title)
-            articles = []
-            for article in section.findAll('article'):
-                details = article.find(attrs={'class': lambda x: x and 'details' in x.split()})
-                pre = details.find(attrs={'class': lambda x: x and 'pre-heading' in x.split()})
-                if pre is not None:
-                    pre.extract()
-                a = details.find('a')
-                url = base_url + a['href'].lstrip('/')
-                title = self.tag_to_string(a)
-                desc = self.tag_to_string(details.find('div'))
-                self.log('\t', title, url)
-                articles.append({'title': title, 'description': desc, 'url': url})
-            if articles:
-                feeds.append((section_title, articles))
-
-        return feeds
-
-    def get_cover_url(self):
-        soup = self.index_to_soup(
-            'https://www.readwhere.com/magazine/delhi-press/The-Caravan/5326'
-        )
-        for citem in soup.findAll(
-            'meta', content=lambda s: s and s.endswith('/magazine/300/new')
-        ):
-            return citem['content'].replace('300', '600')
-
-    def print_version(self, url):
-        if not self.username or not self.password:
-            return url.replace('.in/','.in/amp/')
-        return url
 
     def preprocess_html(self, soup):
-        if not self.username or not self.password:
+        h2 = soup.find('h2')
-            keep_only_tags = [classes('main-content')]
+        if h2:
-            for fc in soup.findAll('figcaption'):
+            h2.name = 'p'
-                fc['id'] = 'fig-c'
+        for fc in soup.findAll('figcaption'):
-            for img in soup.findAll('amp-img'):
+            fc['id'] = 'fig-c'
-                img.name = 'img'
-            if h6 := soup.find('h6'):
-                h6.name = 'h4'
-        else:
-            keep_only_tags = [
-                classes('post-title short-desc author-details cover'),
-                dict(itemprop='articleBody'),
-            ]
-            for div in soup.findAll(itemprop='image'):
-                for img in div.findAll('img'):
-                    img['src'] = div['content']
-            for img in soup.findAll(attrs={'data-src': True}):
-                img['src'] = img['data-src']
-
-        body = new_tag(soup, 'body')
-        for spec in keep_only_tags:
-            for tag in soup.find('body').findAll(**spec):
-                body.insert(len(body.contents), tag)
-        soup.find('body').replaceWith(body)
         return soup
+
+    def parse_index(self):
+        self.log(
+            '\n***\nif this recipe fails, report it on: '
+            'https://www.mobileread.com/forums/forumdisplay.php?f=228\n***\n'
+        )
+        api = 'https://api.caravanmagazine.in/api/trpc/magazines.getLatestIssue'
+        # https://api.caravanmagazine.in/api/trpc/magazines.getForMonthAndYear?batch=1&input=
+        # %7B%220%22%3A%7B%22json%22%3A%7B%22month%22%3A 2 %2C%22year%22%3A 2024 %7D%7D%7D
+        # input={"0":{"json":{"month":2,"year":2024}}}
+        raw = self.index_to_soup(api, raw=True)
+        data = json.loads(raw)['result']['data']['json']
+        cover = safe_dict(data, 'issue', 'cover', 'data', 'url').replace('=s0', '=s768-rw')
+        self.cover_url = absurl(cover)
+
+        feeds = []
+
+        for sec in data['categories']:
+            section = sec['name']
+            self.log(section)
+            articles = []
+            for arts in sec['amc']:
+                title = safe_dict(arts, 'article', 'title')
+                desc = safe_dict(arts, 'article', 'theme', 'name') + ' | ' + safe_dict(arts, 'article', 'printTitle')
+                names = []
+                for auth in arts['article']['authors']:
+                    name = safe_dict(auth, 'profile', 'name')
+                    if name != '':
+                        names.append(name)
+                if names:
+                    desc = desc + ' | ' + ', '.join(names)
+                url = absurl(arts['article']['slug'])
+                self.log('\t', title, url, '\n\t', desc)
+                articles.append({'title': title, 'description': desc, 'url': url})
+            if articles:
+                feeds.append((section, articles))
+        return feeds

recipes/globaltimes.recipe

@@ -51,6 +51,10 @@ class GlobalTimes(BasicNewsRecipe):
         return soup
 
     def parse_index(self):
+        self.log(
+            '\n***\nif this recipe fails, report it on: '
+            'https://www.mobileread.com/forums/forumdisplay.php?f=228\n***\n'
+        )
         sec_url = index + '{}/index.html'
 
         section_list = [