Remove inline navars from downloaded recipes when converting to indexed MOBI. Also fix various typos.

src/calibre/customize/profiles.py

@@ -143,6 +143,9 @@ class OutputProfile(Plugin):
     # The image size for comics
     comic_screen_size = (584, 754)
 
+    # If True the MOBI renderer on the device supports MOBI indexing
+    supports_mobi_indexing = False
+
     @classmethod
     def tags_to_string(cls, tags):
         return ', '.join(tags)
@@ -230,6 +233,7 @@ class KindleOutput(OutputProfile):
     dpi                       = 168.451
     fbase                     = 16
     fsizes                    = [12, 12, 14, 16, 18, 20, 22, 24]
+    supports_mobi_indexing = True
 
     @classmethod
     def tags_to_string(cls, tags):
@@ -245,6 +249,7 @@ class KindleDXOutput(OutputProfile):
     screen_size               = (744, 1022)
     dpi                       = 150.0
     comic_screen_size         = (741, 1022)
+    supports_mobi_indexing = True
 
     @classmethod
     def tags_to_string(cls, tags):

src/calibre/ebooks/conversion/plumber.py

@@ -550,6 +550,8 @@ OptionRecommendation(name='list_recipes',
                 break
 
         self.read_user_metadata()
+        self.opts.no_inline_navbars = self.opts.output_profile.supports_mobi_indexing \
+                and self.output_fmt == 'mobi'
 
     def flush(self):
         try:

src/calibre/ebooks/mobi/output.py

@@ -79,10 +79,11 @@ class MOBIOutput(OutputFormatPlugin):
             start_href = self.oeb.spine[0].href
             self.log('Converting TOC for MOBI periodical indexing...')
             articles = {}
-            if toc.depth < 3:
+            if toc.depth() < 3:
-                sections = [TOC(klass='section')]
+                sections = [TOC(klass='section', title=_('All articles'),
+                    href=start_href)]
                 for x in toc:
-                    sections[0].append(x)
+                    sections[0].nodes.append(x)
             else:
                 sections = list(toc)
                 for x in sections:
@@ -99,7 +100,7 @@ class MOBIOutput(OutputFormatPlugin):
                 if articles[id(s)]:
                     for a in articles[id(s)]:
                         s.nodes.append(a)
-            root.nodes.append(s)
+                    root.nodes.append(s)
 
             for x in list(toc.nodes):
                 toc.nodes.remove(x)

src/calibre/ebooks/mobi/writer.py

@@ -379,7 +379,7 @@ class MobiWriter(object):
             try:
                 self._generate_index()
             except:
-                self.oeb.log.exception('Failed to generate index')
+                self._oeb.log.exception('Failed to generate index')
 
         self._generate_images()
 
@@ -1178,40 +1178,32 @@ class MobiWriter(object):
         '''
         toc = self._oeb.toc
         nodes = list(toc.iter())[1:]
+        toc_conforms = True
         for (i, child) in enumerate(nodes) :
-            if self.opts.verbose > 3 :
+            if child.klass == "periodical" and child.depth() != 3 or    \
-                self._oeb.logger.info("  <title>: %-25.25s \tklass=%-15.15s \tdepth:%d  playOrder=%03d" % \
+               child.klass == "section" and child.depth() != 2 or       \
-                    (child.title, child.klass, child.depth(), child.play_order) )
+               child.klass == "article" and child.depth() != 1 :
 
-            if child.klass == "periodical" and child.depth() != 3 :
+                self._oeb.logger.warn('Nonconforming TOC entry: "%s" found at depth %d' % \
-                    self._oeb.logger.info('<navPoint class="periodical"> found at depth %d, nonconforming TOC' % \
+                        (child.klass, child.depth()) )
-                                            child.depth() )
+                self._oeb.logger.warn("  <title>: '%-25.25s...' \t\tklass=%-15.15s \tdepth:%d  \tplayOrder=%03d" % \
-                    return False
+                        (child.title, child.klass, child.depth(), child.play_order) )
-
+                toc_conforms = False
-            if child.klass == "section" and child.depth() != 2 :
-                    self._oeb.logger.info('<navPoint class="section"> found at depth %d, nonconforming TOC' % \
-                                            child.depth() )
-                    return False
-
-            if child.klass == "article" and child.depth() != 1 :
-                    self._oeb.logger.info('<navPoint class="article"> found at depth %d, nonconforming TOC' % \
-                                            child.depth() )
-                    return False
 
         # We also need to know that we have a pubdate or timestamp in the metadata, which the Kindle needs
         if self._oeb.metadata['date'] == [] and self._oeb.metadata['timestamp'] == [] :
-            self._oeb.logger.info('metadata missing timestamp needed for periodical')
+            self._oeb.logger.info('metadata missing date/timestamp')
-            return False
+            toc_conforms = False
 
         # Periodicals also need a mastheadImage in the manifest
         has_mastheadImage = 'masthead' in self._oeb.guide
 
         if not has_mastheadImage :
-            self._oeb.logger.info('mastheadImage missing from manifest, aborting periodical indexing')
+            self._oeb.logger.info('mastheadImage missing from manifest')
-            return False
+            toc_conforms = False
 
-        self._oeb.logger.info('TOC structure and pubdate verified')
+        self._oeb.logger.info("%s" % "TOC structure conforms" if toc_conforms else "TOC structure non-conforming")
-        return True
+        return toc_conforms
 
 
     def _generate_text(self):
@@ -1236,7 +1228,7 @@ class MobiWriter(object):
 
         # Evaluate toc for conformance
         if self.opts.mobi_periodical :
-            self._oeb.logger.info('--mobi-periodical specified, evaluating TOC for periodical conformance ...')
+            self._oeb.logger.info('MOBI periodical specified, evaluating TOC for periodical conformance ...')
             self._conforming_periodical_toc = self._evaluate_periodical_toc()
 
         # This routine decides whether to build flat or structured based on self._conforming_periodical_toc
@@ -1545,7 +1537,7 @@ class MobiWriter(object):
                 exth.write(data)
                 nrecs += 1
             if term == 'rights' :
-                rights = unicode(oeb.metadata.rights[0])
+                rights = unicode(oeb.metadata.rights[0]).encode('utf-8')
                 exth.write(pack('>II', EXTH_CODES['rights'], len(rights) + 8))
                 exth.write(rights)
 

src/calibre/ebooks/oeb/base.py

@@ -1468,7 +1468,9 @@ class TOC(object):
             node.to_opf1(tour)
         return tour
 
-    def to_ncx(self, parent):
+    def to_ncx(self, parent=None):
+        if parent is None:
+            parent = etree.Element(NCX('navMap'))
         for node in self.nodes:
             id = node.id or unicode(uuid.uuid4())
             attrib = {'id': id, 'playOrder': str(node.play_order)}

src/calibre/ebooks/oeb/stylizer.py

@@ -169,7 +169,8 @@ class Stylizer(object):
             if not matches and class_sel_pat.match(text):
                 found = False
                 for x in tree.xpath('//*[@class]'):
-                    if text.lower().endswith('.'+x.get('class').lower()):
+                    if text.lower().endswith('.'+x.get('class').lower()) and \
+                            text.lower() != text:
                         matches.append(x)
                         found = True
                 if found:

src/calibre/web/feeds/news.py

@@ -469,6 +469,7 @@ class BasicNewsRecipe(Recipe):
         self.username = options.username
         self.password = options.password
         self.lrf = options.lrf
+        self.include_navbars = not options.no_inline_navbars
 
         self.output_dir = os.path.abspath(self.output_dir)
         if options.test:
@@ -539,7 +540,7 @@ class BasicNewsRecipe(Recipe):
         if first_fetch and job_info:
             url, f, a, feed_len = job_info
             body = soup.find('body')
-            if body is not None:
+            if body is not None and self.include_navbars:
                 templ = self.navbar.generate(False, f, a, feed_len,
                                              not self.has_single_feed,
                                              url, __appname__,
@@ -907,12 +908,13 @@ class BasicNewsRecipe(Recipe):
                         body = soup.find('body')
                         if body is not None:
                             prefix = '/'.join('..'for i in range(2*len(re.findall(r'link\d+', last))))
-                            templ = self.navbar.generate(True, num, j, len(f),
+                            if self.include_navbars:
-                                             not self.has_single_feed,
+                                templ = self.navbar.generate(True, num, j, len(f),
-                                             a.orig_url, __appname__, prefix=prefix,
+                                                not self.has_single_feed,
-                                             center=self.center_navbar)
+                                                a.orig_url, __appname__, prefix=prefix,
-                            elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div')
+                                                center=self.center_navbar)
-                            body.insert(len(body.contents), elem)
+                                elem = BeautifulSoup(templ.render(doctype='xhtml').decode('utf-8')).find('div')
+                                body.insert(len(body.contents), elem)
                             with open(last, 'wb') as fi:
                                 fi.write(unicode(soup).encode('utf-8'))
 
@@ -923,7 +925,7 @@ class BasicNewsRecipe(Recipe):
                 if po is None:
                     self.play_order_counter += 1
                     po = self.play_order_counter
-                desc = f.description
+                desc = getattr(f, 'description', None)
                 if not desc:
                     desc = None
                 feed_index(i, toc.add_item('feed_%d/index.html'%i, None,