Merge branch 'master' of https://github.com/h-holm/calibre

recipes/fokus.recipe

@@ -2,30 +2,10 @@
 # vim:fileencoding=utf-8
 from datetime import datetime, timezone
 
-from calibre.web.feeds.news import BasicNewsRecipe
+from bs4 import BeautifulSoup
+from bs4.element import Tag
 
-WEB_SECTIONS = [
+from calibre.web.feeds.news import BasicNewsRecipe
-    ('Inrikes', 'inrikes'),
-    ('Utrikes', 'utrikes'),
-    ('Aktuellt', 'aktuellt'),
-    ('Politik', 'politik'),
-    ('Ekonomi', 'ekonomi'),
-    ('Kultur', 'kultur'),
-    ('Analys', 'analys'),
-    ('Vetenskap', 'vetenskap'),
-    ('Krönikor', 'kronika'),
-    ('Opinion', 'opinion'),
-    ('Veckans Fokus', 'veckans-fokus'),
-    ('Synvinkel', 'synvinkel'),
-    ('Minnesord', 'minnesord'),
-    ('Debatt', 'debatt'),
-    ('Andra kammaren', 'andra-kammaren'),
-    ('Skuggkabinettet', 'skuggkabinettet'),
-    ('Intervju', 'intervju'),
-    ('Mötet', 'motet'),
-    ('Veckans bråk', 'veckans-brak'),
-    ('Johans Blogg', 'johans-blogg'),
-]
 
 
 class NoArticles(Exception):
@@ -38,7 +18,7 @@ class Fokus(BasicNewsRecipe):
     description = "The last 7 days of news and articles from the Swedish current-affairs magazine 'Fokus'"
     encoding = 'utf-8'
     __author__ = 'Henrik Holm (https://github.com/h-holm)'
-    language = 'se'
+    language = 'sv'
     ignore_duplicate_articles = {'title', 'url'}
     masthead_url = 'https://cdn.fokus.se/app/uploads/fokus/2022/05/12214931/fokus-logo.svg'
     no_stylesheets = True
@@ -46,7 +26,7 @@ class Fokus(BasicNewsRecipe):
     needs_subscription = 'optional'
     max_age = 7  # days
     remove_empty_feeds = True
-    extra_css = 'img { display: block; width: 100%; height: auto }'
+    extra_css = 'img { display: block; width: 75%; height: auto }'
 
     remove_tags = [
         dict(name='div', attrs={'class': 'External-ad'}),
@@ -82,7 +62,8 @@ class Fokus(BasicNewsRecipe):
         # dict(name='p', class_='Meta__author'),  # Author.
         # dict(name='time', class_='Meta__updated'),  # Last updated.
         # Main article.
-        dict(name='div', class_='mediaconnect-protected-content'),
+        dict(name='div', class_='sesamy-protected-content'),
+        dict(name='div', class_='wp-block-core-paragraph'),
     ]
 
     def get_browser(self):
@@ -95,65 +76,211 @@ class Fokus(BasicNewsRecipe):
             br.submit()
         return br
 
-    def parse_article_blurb(self, article_blurb):
+    def get_web_sections(self, main_url: str) -> dict[str, str]:
-        desc = ''
+        """Return a dict of (1) section URL and (2) section name key-value pairs found at `main_url`.
+
+        For example, if the Fokus website currently includes an 'Aktuellt' section, the dict should include an entry on
+        the form: `{'https://www.fokus.se/aktuellt': 'Aktuellt'}`.
+
+        Args:
+            main_url (str): The entrypoint URL of the Fokus website.
+
+        Yields:
+            dict[str, str]: (1) URLs and (2) human-readable names of Fokus sections.
+        """
+        self.log(f"Identifying all sections under '{main_url}'...")
+        soup = self.index_to_soup(main_url)
+
+        # Identify all unique <li> tags of class 'menu-item-type-taxonomy'. The class subsetting excludes sections that
+        # are not suited for reading, e.g., the "Podcast" and "Läs E-Tidningen" sections.
+        section_urls_and_names = {}
+        for li_tag in soup.find_all('li', class_='menu-item-type-taxonomy'):
+            # The <li> tag contains (should contain) an <a> anchor that in turn contains the URL and link name.
+            a_tag = li_tag.find('a')
+            url = a_tag.get('href').rstrip('/')
+            section_name = a_tag.text.strip()
+
+            if url in section_urls_and_names:
+                # If this section URL has already been extracted from another <li> tag, it can be the case that the
+                # section name differs within this duplicate pair. In this case, use whichever section name is longer.
+                if len(section_name) >= len(section_urls_and_names[url]):
+                    section_urls_and_names[url] = section_name
+
+            self.log(f"Identified section '{section_name}' at URL '{url}'.")
+            section_urls_and_names[url] = section_name
+
+        self.log(f'Identified a total of {len(section_urls_and_names)} unique sections.')
+        return section_urls_and_names
+
+    def parse_article_blurb(self, article_blurb: Tag) -> dict[str, str, str, str] | None:
+        """Given a <article> tag of class 'Blurb', parse it into a dict.
+
+        Args:
+            article_blurb (Tag): An <article> tag hosting metadata and the URL of an article.
+
+        Returns:
+            dict[str, str, str, str]: A dict on a `{'url': str, 'title': str, 'description': str, 'date': str}` format.
+        """
         if a_tag := article_blurb.find('a', href=True):
-            url = a_tag['href']
+            url = a_tag['href'].strip().rstrip('/')
             if url.startswith('/'):
                 url = f'{self.main_url}{url}'
+
             if title_tag := a_tag.find('h2', {'class': 'Blurb__title'}):
-                title = self.tag_to_string(title_tag)
+                title = self.tag_to_string(title_tag).strip()
                 if time_tag := a_tag.find('time', {'class': 'Blurb__date'}):
-                    swedish_date_str = self.tag_to_string(time_tag)
+                    swedish_date_str = self.tag_to_string(time_tag).rstrip()
+
+                    # Skip articles older than `self.max_age`.
                     datetime_str = time_tag['datetime']
-                    datetime_time = datetime.strptime(
+                    datetime_time = datetime.strptime(datetime_str, '%Y-%m-%dT%H:%M:%S%z')
-                        datetime_str, '%Y-%m-%dT%H:%M:%S%z')
                     now = datetime.now(timezone.utc)
                     delta = now - datetime_time
                     if delta.days > self.max_age:
-                        self.log.debug(
+                        self.log.debug(f"\tSkipping article as it is too old: '{title}'")
-                            f"\tSkipping article '{title}' as it is too old")
+                        return
-                    else:
-                        if desc_tag := a_tag.find('div', {'class': 'Blurb__summary'}):
-                            desc = self.tag_to_string(desc_tag)
-                            if in_cooperation_with_tag := a_tag.find('p', {'class': 'Blurb__meta'}):
-                                desc += f' ({self.tag_to_string(in_cooperation_with_tag)})'
-                        return {'title': title, 'url': url, 'description': desc, 'date': swedish_date_str}
 
-    def parse_web_section(self, soup, slug):
+                    desc = ''
-        def log(article):
+                    if desc_tag := a_tag.find('div', {'class': 'Blurb__summary'}):
+                        desc = self.tag_to_string(desc_tag).strip()
+                        if in_cooperation_with_tag := a_tag.find('p', {'class': 'Blurb__meta'}):
+                            desc += f' ({self.tag_to_string(in_cooperation_with_tag).strip()})'
+
+                    return {'url': url, 'title': title, 'description': desc, 'date': swedish_date_str}
+        return
+
+    def _get_article_blurbs(self, soup: BeautifulSoup) -> dict[str, dict[str, str, str, str]]:
+        """Given a Fokus webpage `soup`, return a dict of unique article entries found on the page.
+
+        The key of a given entry in the output dictionary is the article URL. The corresponding value is a dictionary
+        on a `{'url': str, 'title': str, 'description': str, 'date': str}` format.
+
+        Args:
+            soup (BeautifulSoup): The `bs4.BeautifulSoup` soup of a Fokus webpage.
+
+        Returns:
+            dict[str, dict[str, str, str, str]]: A dict with article URLs as keys and 'article dicts' as values.
+        """
+
+        def _log(article) -> None:
+            """Log a digestible summary of the input `article` blurb."""
             log_message = f"\t{article['title']} : {article['date']} : {article['url']}"
             if article.get('description'):
                 log_message += f" : {article['description']}"
-            self.log(log_message)
+            self.log.debug(log_message)
+
         try:
             article_blurbs = soup.find_all('article', {'class': 'Blurb'})
         except AttributeError:
             article_blurbs = []
+
         if not article_blurbs:
-            raise ValueError(f'Failed to find article blurbs for slug: {slug}')
+            raise ValueError('Failed to identify any article blurbs.')
+
+        parsed_blurbs = {}
         for article_blurb in article_blurbs:
-            if (article := self.parse_article_blurb(article_blurb)):
+            if article := self.parse_article_blurb(article_blurb):
-                log(article)
+                _log(article)
-                yield article
+                # If an entry with the same URL already exists, keep whichever entry has the longer description.
+                if article['url'] in article_blurbs:
+                    if len(article['description']) <= len(parsed_blurbs[article['url']]['description']):
+                        continue
+                parsed_blurbs[article['url']] = article
+
+        return parsed_blurbs
+
+    def get_article_blurbs(self, sections: dict[str, str]) -> dict[str, dict[str, str, str, str]]:
+        """Create and return a dict of all unique article blurbs found in all `sections`.
+
+        The key of a given entry in the output dictionary is the article URL. The corresponding value is a dictionary
+        on a `{'url': str, 'title': str, 'description': str, 'date': str}` format.
+
+        Args:
+            sections (dict[str, str]): A dict on a `{section_url: section_name}` format.
+
+        Returns:
+            dict[str, dict[str, str, str, str]]: A dict with article URLs as keys and 'article dicts' as values.
+        """
+        self.log(f'Identifying all articles under all {len(sections)} sections...')
+
+        article_blurbs = {}
+        for section_url, section_title in sections.items():
+            try:
+                section_soup = self.index_to_soup(section_url)
+            except Exception:
+                self.log.error(f"Failed to download section '{section_title}' via URL '{section_url}'")
+                continue
+            self.log(f"Identifying all articles under '{section_url}'...")
+            for article_url, article_blurb in self._get_article_blurbs(section_soup).items():
+                # If the article URL has already been encountered, keep only the article blurb with the longer
+                # description string.
+                if article_url not in article_blurbs:
+                    article_blurbs[article_url] = article_blurb
+                elif len(article_blurb['description']) > len(article_blurbs[article_url]['description']):
+                    article_blurbs[article_url] = article_blurb
+
+        self.log(f'A total of {len(article_blurbs)} articles were identified in the {len(sections)} sections.')
+        return article_blurbs
+
+    def assign_articles_to_sections(
+        self,
+        sections: dict[str, str],
+        articles: dict[str, dict[str, str, str, str]],
+    ) -> dict[str, list[dict[str, str, str, str]]]:
+        """Assign each article in `articles` to a section in `sections`.
+
+        Args:
+            sections (dict[str, str]): A dict of section URLs as keys and section titles as values.
+            articles (dict[str, dict[str, str, str, str]]): A dict of article URLs as keys and article dicts as values.
+
+        Returns:
+            dict[str, list[dict[str, str, str, str]]]: A dict on a `{section_title: list[article_dict]}` format.
+        """
+        self.log(f'Assigning each of the {len(articles)} articles to either of the {len(sections)} sections...')
+        section_to_articles = {}
+        for article_url, article_dict in articles.items():
+            last_url = article_url
+            while article_url not in sections and len(article_url) > len(self.main_url):
+                article_url = article_url.rsplit('/', 1)[0]
+
+                # Prevent an infinite loop.
+                if article_url == last_url:
+                    break
+                last_url = article_url
+
+            # If no section corresponding to the URL exists, default to the 'Home Page' section.
+            section_title = sections[article_url] if article_url in sections else sections[self.main_url]
+            if section_title not in section_to_articles:
+                section_to_articles[section_title] = []
+            section_to_articles[section_title].append(article_dict)
+
+        # Log how many sections contained no articles younger than `self.max_age`.
+        if diff := len(sections) - len(section_to_articles):
+            self.log(f'{diff} sections contained no articles younger than {self.max_age} days.')
+
+        return section_to_articles
 
     def parse_index(self):
-        feeds = []
+        # Identify all sections in the web version of Fokus.
-        for section_title, slug in WEB_SECTIONS:
+        sections = self.get_web_sections(self.main_url)
-            url = f'{self.main_url}/{slug}'
+
-            try:
+        # Add an entry for the start page.
-                soup = self.index_to_soup(url)
+        sections[self.main_url] = 'Home Page'
-            except Exception:
+
-                self.log.error(f'Failed to download section: {url}')
+        # From the section URLs and the main URL, identify all unique articles.
-                continue
+        articles = self.get_article_blurbs(sections)
-            self.log(f'Found section: {section_title}')
+        if not articles:
-            articles = list(self.parse_web_section(soup, slug))
-            if articles:
-                feeds.append((section_title, articles))
-        if not feeds:
             raise NoArticles(
-                'Could not find any articles. Either the fokus.se server is having issues and '
+                f"Could not find any articles. Either the '{self.main_url}' server is experiencing issues, in which "
-                'you should try later or the website format has changed and the recipe needs '
+                'case you should try again later, or the website format has changed and the recipe needs updating.'
-                'to be updated.'
             )
+
+        # Assign each identified article to a section based on its URL.
+        section_to_articles = self.assign_articles_to_sections(sections, articles)
+
+        # Convert to the expected `list[tuple[str, dict[str, str, str, str]]]` format.
+        feeds = [(section_url, article_dicts) for section_url, article_dicts in section_to_articles.items()]
+        num_articles = sum(len(article_dicts) for article_dicts in section_to_articles.values())
+        self.log(f'A total of {num_articles} articles belonging to {len(section_to_articles)} sections were kept.')
+
         return feeds