From 2c37b1c36bcca29da702244ce592a7f472669c0a Mon Sep 17 00:00:00 2001
From: Kovid Goyal <kovid@kovidgoyal.net>
Date: Thu, 8 Apr 2010 17:19:47 +0530
Subject: [PATCH] PC Quest and Living Digital. News download: Implement
 is_link_wanted a method for context sensitive link filtering

---
 resources/recipes/living_digital.recipe | 14 ++++++++++++++
 resources/recipes/pc_quest_india.recipe | 14 ++++++++++++++
 src/calibre/ebooks/mobi/reader.py       |  5 ++++-
 src/calibre/web/feeds/news.py           | 20 +++++++++++++++++---
 src/calibre/web/fetch/simple.py         | 15 +++++++++++++--
 5 files changed, 62 insertions(+), 6 deletions(-)
 create mode 100644 resources/recipes/living_digital.recipe
 create mode 100644 resources/recipes/pc_quest_india.recipe

diff --git a/resources/recipes/living_digital.recipe b/resources/recipes/living_digital.recipe
new file mode 100644
index 0000000000..d8bba7b9f8
--- /dev/null
+++ b/resources/recipes/living_digital.recipe
@@ -0,0 +1,14 @@
+from calibre.web.feeds.news import CalibrePeriodical
+
+class LivingDigital(CalibrePeriodical):
+
+    title = 'Living Digital'
+    calibre_periodicals_slug = 'living-digital'
+
+    description = '''
+    Catch the latest buzz in the digital world with Living Digital. Enjoy
+    reviews, news, features and recommendations on a wide range of consumer
+    technology products - from smartphones to flat panel TVs, netbooks to
+    cameras, and many more consumer lifestyle gadgets.
+    '''
+    language = 'en_IN'
diff --git a/resources/recipes/pc_quest_india.recipe b/resources/recipes/pc_quest_india.recipe
new file mode 100644
index 0000000000..3528731d47
--- /dev/null
+++ b/resources/recipes/pc_quest_india.recipe
@@ -0,0 +1,14 @@
+from calibre.web.feeds.news import CalibrePeriodical
+
+class PCQ(CalibrePeriodical):
+
+    title = 'PCQuest'
+    calibre_periodicals_slug = 'pc-quest-india'
+
+    description = '''
+    Buying a tech product? Seeking a tech solution? Consult PCQuest, India's
+    market-leading selection and implementation guide for the latest
+    technologies: servers, business apps, security, open source, gadgets and
+    more.
+    '''
+    language = 'en_IN'
diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py
index e84ae2547c..84e6208086 100644
--- a/src/calibre/ebooks/mobi/reader.py
+++ b/src/calibre/ebooks/mobi/reader.py
@@ -296,6 +296,10 @@ class MobiReader(object):
         self.add_anchors()
         self.processed_html = self.processed_html.decode(self.book_header.codec,
             'ignore')
+        self.processed_html = self.processed_html.replace('</</', '</')
+        self.processed_html = re.sub(r'</([a-zA-Z]+)<', r'</\1><',
+                self.processed_html)
+
         for pat in ENCODING_PATS:
             self.processed_html = pat.sub('', self.processed_html)
         e2u = functools.partial(entity_to_unicode,
@@ -320,7 +324,6 @@ class MobiReader(object):
             from lxml.html import soupparser
             self.log.warning('Malformed markup, parsing using BeautifulSoup')
             try:
-                self.processed_html = self.processed_html.replace('</</', '</')
                 root = soupparser.fromstring(self.processed_html)
             except Exception:
                 self.log.warning('MOBI markup appears to contain random bytes. Stripping.')
diff --git a/src/calibre/web/feeds/news.py b/src/calibre/web/feeds/news.py
index c4f1138ea4..4bf7950d7f 100644
--- a/src/calibre/web/feeds/news.py
+++ b/src/calibre/web/feeds/news.py
@@ -150,7 +150,8 @@ class BasicNewsRecipe(Recipe):
     remove_empty_feeds = False
 
     #: List of regular expressions that determines which links to follow
-    #: If empty, it is ignored. For example::
+    #: If empty, it is ignored. Used only if is_link_wanted is
+    #: not implemented. For example::
     #:
     #:     match_regexps = [r'page=[0-9]+']
     #:
@@ -161,7 +162,8 @@ class BasicNewsRecipe(Recipe):
     match_regexps         = []
 
     #: List of regular expressions that determines which links to ignore
-    #: If empty it is ignored. For example::
+    #: If empty it is ignored. Used only if is_link_wanted is not
+    #: implemented. For example::
     #:
     #:     filter_regexps = [r'ads\.doubleclick\.net']
     #:
@@ -291,6 +293,17 @@ class BasicNewsRecipe(Recipe):
     def short_title(self):
         return self.title
 
+    def is_link_wanted(self, url, tag):
+        '''
+        Return True if the link should be followed or False otherwise. By
+        default, raises NotImplementedError which causes the downloader to
+        ignore it.
+
+        :param url: The URL to be followed
+        :param tag: The Tag from which the URL was derived
+        '''
+        raise NotImplementedError
+
     def get_cover_url(self):
         '''
         Return a :term:`URL` to the cover image for this issue or `None`.
@@ -575,7 +588,8 @@ class BasicNewsRecipe(Recipe):
 
         self.web2disk_options = web2disk_option_parser().parse_args(web2disk_cmdline)[0]
         for extra in ('keep_only_tags', 'remove_tags', 'preprocess_regexps',
-                      'preprocess_html', 'remove_tags_after', 'remove_tags_before'):
+                      'preprocess_html', 'remove_tags_after',
+                      'remove_tags_before', 'is_link_wanted'):
             setattr(self.web2disk_options, extra, getattr(self, extra))
         self.web2disk_options.postprocess_html = self._postprocess_html
         self.web2disk_options.encoding = self.encoding
diff --git a/src/calibre/web/fetch/simple.py b/src/calibre/web/fetch/simple.py
index fc87e7fdcd..5e785bc29a 100644
--- a/src/calibre/web/fetch/simple.py
+++ b/src/calibre/web/fetch/simple.py
@@ -91,6 +91,9 @@ class DummyLock(object):
     def __enter__(self, *args): return self
     def __exit__(self, *args): pass
 
+def default_is_link_wanted(url, tag):
+    raise NotImplementedError()
+
 class RecursiveFetcher(object):
     LINK_FILTER = tuple(re.compile(i, re.IGNORECASE) for i in
                 ('.exe\s*$', '.mp3\s*$', '.ogg\s*$', '^\s*mailto:', '^\s*$'))
@@ -134,6 +137,8 @@ class RecursiveFetcher(object):
         self.keep_only_tags      = getattr(options, 'keep_only_tags', [])
         self.preprocess_html_ext = getattr(options, 'preprocess_html', lambda soup: soup)
         self.postprocess_html_ext= getattr(options, 'postprocess_html', None)
+        self._is_link_wanted     = getattr(options, 'is_link_wanted',
+                default_is_link_wanted)
         self.download_stylesheets = not options.no_stylesheets
         self.show_progress = True
         self.failed_links = []
@@ -233,7 +238,13 @@ class RecursiveFetcher(object):
                 return False
         return True
 
-    def is_link_wanted(self, url):
+    def is_link_wanted(self, url, tag):
+        try:
+            return self._is_link_wanted(url, tag)
+        except NotImplementedError:
+            pass
+        except:
+            return False
         if self.filter_regexps:
             for f in self.filter_regexps:
                 if f.search(url):
@@ -342,7 +353,7 @@ class RecursiveFetcher(object):
         if not self.is_link_ok(iurl):
             self.log.debug('Skipping invalid link:', iurl)
             return None
-        if filter and not self.is_link_wanted(iurl):
+        if filter and not self.is_link_wanted(iurl, tag):
             self.log.debug('Filtered link: '+iurl)
             return None
         return iurl