From 8676ddd30fba0df90eb62e7c1c84c3fd3dc13f39 Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Fri, 14 Jan 2011 18:12:17 +0800
Subject: [PATCH 1/6] updated heuristics help messages

---
 src/calibre/ebooks/conversion/plumber.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py
index 3ec4e104f9..50d0646c7d 100644
--- a/src/calibre/ebooks/conversion/plumber.py
+++ b/src/calibre/ebooks/conversion/plumber.py
@@ -491,8 +491,8 @@ OptionRecommendation(name='enable_heuristics',
 
 OptionRecommendation(name='markup_chapter_headings',
     recommended_value=False, level=OptionRecommendation.LOW,
-    help=_('Detect chapter headings and sub headings. Change ' 
-           'them to h1 and h2 tags.')),
+    help=_('Detect unformatted chapter headings and sub headings. Change ' 
+           'them to h2 and h3 tags.')),
            
 OptionRecommendation(name='italicize_common_cases',
     recommended_value=False, level=OptionRecommendation.LOW,
@@ -508,26 +508,30 @@ OptionRecommendation(name='html_unwrap_factor',
     recommended_value=0.40, level=OptionRecommendation.LOW,
     help=_('Scale used to determine the length at which a line should '
             'be unwrapped. Valid values are a decimal between 0 and 1. The '
-            'default is 0.4, just below the median line length.')),
+            'default is 0.4, just below the median line length.  If only a '
+            'few lines in the document require unwrapping this value should '
+            'be reduced')),
             
 OptionRecommendation(name='unwrap_lines',
     recommended_value=False, level=OptionRecommendation.LOW,
-    help=_('Unwrap lines.')),
+    help=_('Unwrap lines using punctuation and other formatting clues.')),
     
 OptionRecommendation(name='delete_blank_paragraphs',
     recommended_value=True, level=OptionRecommendation.LOW,
-    help=_('Remove empyt paragraphs from the document')),
+    help=_('Remove empty paragraphs from the document when they exist between '
+           'every other paragraph')),
     
 OptionRecommendation(name='format_scene_breaks',
     recommended_value=False, level=OptionRecommendation.LOW,
-    help=_('Replace soft scene breaks that use multiple blank lines '
-           'with horizontal rules.')),
+    help=_('Detects left aligned scene break markers and center aligns them. '
+           'Replace soft scene breaks that use multiple blank lines with'
+           'horizontal rules.')),
 
 OptionRecommendation(name='dehyphenate',
     recommended_value=True, level=OptionRecommendation.LOW,
-    help=_('Combine words that are separated by a hyphen. '
-           'This is for cases where a word is hyphenated across '
-           'two lines to denote the characters from a single word.')),
+    help=_('Analyses hyphenated words throughout the document.  The '
+           'document itself is used as a dictionary to determine whether hyphens '
+           'should be retained or removed.')),
     
 OptionRecommendation(name='sr1_search',
     recommended_value='', level=OptionRecommendation.LOW,

From 90c978bb1076e7afb7843df96959cc365d17332d Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Fri, 14 Jan 2011 21:33:47 +0800
Subject: [PATCH 2/6] tied enable heuristics to preprocess, moved various
 pieces to functions

---
 src/calibre/customize/conversion.py      |   2 +-
 src/calibre/ebooks/conversion/plumber.py |   4 +-
 src/calibre/ebooks/conversion/utils.py   | 117 ++++++++++++++---------
 src/calibre/ebooks/lit/input.py          |   2 +-
 4 files changed, 78 insertions(+), 47 deletions(-)

diff --git a/src/calibre/customize/conversion.py b/src/calibre/customize/conversion.py
index ec83600a49..a9e573ffa0 100644
--- a/src/calibre/customize/conversion.py
+++ b/src/calibre/customize/conversion.py
@@ -160,7 +160,7 @@ class InputFormatPlugin(Plugin):
         '''
         raise NotImplementedError()
 
-    def preprocess_html(self, opts, html):
+    def heuristics(self, opts, html):
         '''
         This method is called by the conversion pipeline on all HTML before it
         is parsed. It is meant to be used to do any required preprocessing on
diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py
index 50d0646c7d..a40c17a743 100644
--- a/src/calibre/ebooks/conversion/plumber.py
+++ b/src/calibre/ebooks/conversion/plumber.py
@@ -1012,8 +1012,8 @@ def create_oebbook(log, path_or_stream, opts, input_plugin, reader=None,
     Create an OEBBook.
     '''
     from calibre.ebooks.oeb.base import OEBBook
-    html_preprocessor = HTMLPreProcessor(input_plugin.preprocess_html,
-            opts.preprocess_html, opts)
+    html_preprocessor = HTMLPreProcessor(input_plugin.heuristics,
+            opts.enable_heuristics, opts)
     if not encoding:
         encoding = None
     oeb = OEBBook(log, html_preprocessor,
diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index dac93fa2e2..44d4235b6c 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -113,6 +113,11 @@ class PreProcessor(object):
         return wordcount.words
 
     def markup_chapters(self, html, wordcount, blanks_between_paragraphs):
+        '''
+        Searches for common chapter headings throughout the document
+        attempts multiple patterns based on likelihood of a match
+        with minimum false positives.  Exits after finding a successful pattern
+        '''
         # Typical chapters are between 2000 and 7000 words, use the larger number to decide the
         # minimum of chapters to search for
         self.min_chapters = 1
@@ -185,6 +190,10 @@ class PreProcessor(object):
         return html
 
     def punctuation_unwrap(self, length, content, format):
+        '''
+        Unwraps lines based on line length and punctuation
+        supports range of potential html markup and text files
+        '''
         # define the pieces of the regex
         lookahead = "(?<=.{"+str(length)+"}([a-zäëïöüàèìòùáćéíóńśúâêîôûçąężıãõñæøþðß,:)\IA\u00DF]|(?<!\&\w{4});))" # (?<!\&\w{4});) is a semicolon not part of an entity
         line_ending = "\s*</(span|p|div)>\s*(</(p|span|div)>)?"
@@ -201,53 +210,38 @@ class PreProcessor(object):
         return content
 
 
-    def __call__(self, html):
-        self.log("*********  Preprocessing HTML  *********")
+    def text_process_pre(self, html):
+        pre = re.compile(r'<pre>', re.IGNORECASE)
+        if len(pre.findall(html)) == 1:
+            self.log("Running Text Processing")
+            from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
+            separate_paragraphs_single_line
+            outerhtml = re.compile(r'.*?(?<=<pre>)(?P<text>.*)(?=</pre>).*', re.IGNORECASE|re.DOTALL)
+            html = outerhtml.sub('\g<text>', html)
+            html = separate_paragraphs_single_line(html)
+            html = preserve_spaces(html)
+            html = convert_basic(html, epub_split_size_kb=0)
+        else:
+            # Add markup naively
+            # TODO - find out if there are cases where there are more than one <pre> tag or
+            # other types of unmarked html and handle them in some better fashion
+            add_markup = re.compile('(?<!>)(\n)')
+            html = add_markup.sub('</p>\n<p>', html)
+        return html
 
-        # Count the words in the document to estimate how many chapters to look for and whether
-        # other types of processing are attempted
-        totalwords = 0
-        totalwords = self.get_word_count(html)
-
-        if totalwords < 50:
-            self.log("not enough text, not preprocessing")
-            return html
-
-        # Arrange line feeds and </p> tags so the line_length and no_markup functions work correctly
+    def arrange_htm_line_endings(self, html):
         html = re.sub(r"\s*</(?P<tag>p|div)>", "</"+"\g<tag>"+">\n", html)
         html = re.sub(r"\s*<(?P<tag>p|div)(?P<style>[^>]*)>\s*", "\n<"+"\g<tag>"+"\g<style>"+">", html)
+        return html
 
-        ###### Check Markup ######
-        #
-        # some lit files don't have any <p> tags or equivalent (generally just plain text between
-        # <pre> tags), check and  mark up line endings if required before proceeding
-        if self.no_markup(html, 0.1):
-            self.log("not enough paragraph markers, adding now")
-            # check if content is in pre tags, use txt processor to mark up if so
-            pre = re.compile(r'<pre>', re.IGNORECASE)
-            if len(pre.findall(html)) == 1:
-                self.log("Running Text Processing")
-                from calibre.ebooks.txt.processor import convert_basic, preserve_spaces, \
-                separate_paragraphs_single_line
-                outerhtml = re.compile(r'.*?(?<=<pre>)(?P<text>.*)(?=</pre>).*', re.IGNORECASE|re.DOTALL)
-                html = outerhtml.sub('\g<text>', html)
-                html = separate_paragraphs_single_line(html)
-                html = preserve_spaces(html)
-                html = convert_basic(html, epub_split_size_kb=0)
-            else:
-                # Add markup naively
-                # TODO - find out if there are cases where there are more than one <pre> tag or
-                # other types of unmarked html and handle them in some better fashion
-                add_markup = re.compile('(?<!>)(\n)')
-                html = add_markup.sub('</p>\n<p>', html)
-
-        ###### Mark Indents/Cleanup ######
-        #
-        # Replace series of non-breaking spaces with text-indent
+    def fix_nbsp_indents(self, html):
         txtindent = re.compile(ur'<p(?P<formatting>[^>]*)>\s*(?P<span>(<span[^>]*>\s*)+)?\s*(\u00a0){2,}', re.IGNORECASE)
         html = txtindent.sub(self.insert_indent, html)
         if self.found_indents > 1:
             self.log("replaced "+unicode(self.found_indents)+ " nbsp indents with inline styles")
+        return html
+
+    def cleanup_markup(self, html):
         # remove remaining non-breaking spaces
         html = re.sub(ur'\u00a0', ' ', html)
         # Get rid of various common microsoft specific tags which can cause issues later
@@ -259,27 +253,64 @@ class PreProcessor(object):
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]*>\s*</span>){0,2}\s*</span>\s*", " ", html)
         html = re.sub(r"\s*<[ibu][^>]*>\s*(<[ibu][^>]*>\s*</[ibu]>\s*){0,2}\s*</[ibu]>", " ", html)
         html = re.sub(r"\s*<span[^>]*>\s*(<span[^>]>\s*</span>){0,2}\s*</span>\s*", " ", html)
+        return html
+
+
+    def __call__(self, html):
+        self.log("*********  Preprocessing HTML  *********")
+
+        # Count the words in the document to estimate how many chapters to look for and whether
+        # other types of processing are attempted
+        totalwords = 0
+        totalwords = self.get_word_count(html)
+
+        if totalwords < 50:
+            self.log("flow is too short, not running heuristics")
+            return html
+
+        # Arrange line feeds and </p> tags so the line_length and no_markup functions work correctly
+        html = self.arrange_htm_line_endings(html)
+
+
+        ###### Check Markup ######
+        #
+        # some lit files don't have any <p> tags or equivalent (generally just plain text between
+        # <pre> tags), check and  mark up line endings if required before proceeding
+        if self.no_markup(html, 0.1):
+            self.log("not enough paragraph markers, adding now")
+            # check if content is in pre tags, use txt processor to mark up if so
+            html = self.text_process_pre(html)
+
+        ###### Mark Indents/Cleanup ######
+        #
+        # Replace series of non-breaking spaces with text-indent
+        html = self.fix_nbsp_indents(html)
+        
+        html = self.cleanup_markup(html)
+
         # ADE doesn't render <br />, change to empty paragraphs
         #html = re.sub('<br[^>]*>', u'<p>\u00a0</p>', html)
 
-        # If more than 40% of the lines are empty paragraphs and the user has enabled remove
-        # paragraph spacing then delete blank lines to clean up spacing
+        # If more than 40% of the lines are empty paragraphs and the user has enabled delete
+        # blank paragraphs then delete blank lines to clean up spacing
         linereg = re.compile('(?<=<p).*?(?=</p>)', re.IGNORECASE|re.DOTALL)
         blankreg = re.compile(r'\s*(?P<openline><p[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
         #multi_blank = re.compile(r'(\s*<p[^>]*>\s*(<(b|i|u)>)?\s*(</(b|i|u)>)?\s*</p>){2,}', re.IGNORECASE)
         blanklines = blankreg.findall(html)
         lines = linereg.findall(html)
         blanks_between_paragraphs = False
+        if getattr(self.extra_opts, 'delete_blank_paragraphs', False):
+            print "configured to delete blank paragraphs"
         if len(lines) > 1:
             self.log("There are " + unicode(len(blanklines)) + " blank lines. " +
                     unicode(float(len(blanklines)) / float(len(lines))) + " percent blank")
             if float(len(blanklines)) / float(len(lines)) > 0.40 and getattr(self.extra_opts,
-            'remove_paragraph_spacing', False):
+            'delete_blank_paragraphs', False):
                 self.log("deleting blank lines")
                 html = blankreg.sub('', html)
             elif float(len(blanklines)) / float(len(lines)) > 0.40:
                 blanks_between_paragraphs = True
-                #print "blanks between paragraphs is marked True"
+                print "blanks between paragraphs is marked True"
             else:
                 blanks_between_paragraphs = False
 
diff --git a/src/calibre/ebooks/lit/input.py b/src/calibre/ebooks/lit/input.py
index 46a5e75977..d0ecf008b7 100644
--- a/src/calibre/ebooks/lit/input.py
+++ b/src/calibre/ebooks/lit/input.py
@@ -53,7 +53,7 @@ class LITInput(InputFormatPlugin):
                         pre.append(ne)
 
 
-    def preprocess_html(self, options, html):
+    def heuristics(self, options, html):
         self.options = options
         preprocessor = PreProcessor(self.options, log=getattr(self, 'log', None))
         return preprocessor(html)

From 4893fa5d3a5ff8f4a3e3ebf8915ff6611c9c3921 Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Fri, 14 Jan 2011 22:18:44 +0800
Subject: [PATCH 3/6] fixed the other plugins using preprocess

---
 src/calibre/ebooks/html/input.py | 2 +-
 src/calibre/ebooks/lrf/input.py  | 2 +-
 src/calibre/ebooks/mobi/input.py | 2 +-
 src/calibre/ebooks/pdb/input.py  | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py
index 1f07f4ca41..479f852c77 100644
--- a/src/calibre/ebooks/html/input.py
+++ b/src/calibre/ebooks/html/input.py
@@ -486,7 +486,7 @@ class HTMLInput(InputFormatPlugin):
             return (None, None)
         return (None, raw)
 
-    def preprocess_html(self, options, html):
+    def heuristics(self, options, html):
         self.options = options
         preprocessor = PreProcessor(self.options, log=getattr(self, 'log', None))
         return preprocessor(html)
diff --git a/src/calibre/ebooks/lrf/input.py b/src/calibre/ebooks/lrf/input.py
index 70529c0a04..05c8731da5 100644
--- a/src/calibre/ebooks/lrf/input.py
+++ b/src/calibre/ebooks/lrf/input.py
@@ -420,7 +420,7 @@ class LRFInput(InputFormatPlugin):
         styles.write()
         return os.path.abspath('content.opf')
 
-    def preprocess_html(self, options, html):
+    def heuristics(self, options, html):
         self.options = options
         preprocessor = PreProcessor(self.options, log=getattr(self, 'log', None))
         return preprocessor(html)
diff --git a/src/calibre/ebooks/mobi/input.py b/src/calibre/ebooks/mobi/input.py
index 9ab7996a74..584be71fe4 100644
--- a/src/calibre/ebooks/mobi/input.py
+++ b/src/calibre/ebooks/mobi/input.py
@@ -39,7 +39,7 @@ class MOBIInput(InputFormatPlugin):
                 accelerators['pagebreaks'] = '//h:div[@class="mbp_pagebreak"]'
         return mr.created_opf_path
 
-    def preprocess_html(self, options, html):
+    def heuristics(self, options, html):
         # search for places where a first or second level heading is immediately followed by another
         # top level heading.  demote the second heading to h3 to prevent splitting between chapter
         # headings and titles, images, etc
diff --git a/src/calibre/ebooks/pdb/input.py b/src/calibre/ebooks/pdb/input.py
index 1b665bf94e..b0e7746c7e 100644
--- a/src/calibre/ebooks/pdb/input.py
+++ b/src/calibre/ebooks/pdb/input.py
@@ -33,7 +33,7 @@ class PDBInput(InputFormatPlugin):
 
         return opf
 
-    def preprocess_html(self, options, html):
+    def heuristics(self, options, html):
         self.options = options
         preprocessor = PreProcessor(self.options, log=getattr(self, 'log', None))
         return preprocessor(html)

From 80ed2e7d4ee1f94b6f6ffe1297b7058345a8d22a Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Fri, 14 Jan 2011 22:32:03 +0800
Subject: [PATCH 4/6] ...

---
 src/calibre/ebooks/conversion/utils.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index 44d4235b6c..bfb23c45aa 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -299,8 +299,7 @@ class PreProcessor(object):
         blanklines = blankreg.findall(html)
         lines = linereg.findall(html)
         blanks_between_paragraphs = False
-        if getattr(self.extra_opts, 'delete_blank_paragraphs', False):
-            print "configured to delete blank paragraphs"
+        print "delete blank paragraphs is "+str(getattr(self.extra_opts, 'delete_blank_paragraphs', True))
         if len(lines) > 1:
             self.log("There are " + unicode(len(blanklines)) + " blank lines. " +
                     unicode(float(len(blanklines)) / float(len(lines))) + " percent blank")

From 65f9eff665042099269114f2905bfc30eef0a456 Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Fri, 14 Jan 2011 23:43:38 +0800
Subject: [PATCH 5/6] remove heuristics from pdb input

---
 src/calibre/ebooks/conversion/utils.py | 2 +-
 src/calibre/ebooks/pdb/input.py        | 5 -----
 2 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index bfb23c45aa..286fad1aaa 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -299,7 +299,7 @@ class PreProcessor(object):
         blanklines = blankreg.findall(html)
         lines = linereg.findall(html)
         blanks_between_paragraphs = False
-        print "delete blank paragraphs is "+str(getattr(self.extra_opts, 'delete_blank_paragraphs', True))
+        print "delete blank paragraphs is "+str(getattr(self.extra_opts, 'delete_blank_paragraphs', False))
         if len(lines) > 1:
             self.log("There are " + unicode(len(blanklines)) + " blank lines. " +
                     unicode(float(len(blanklines)) / float(len(lines))) + " percent blank")
diff --git a/src/calibre/ebooks/pdb/input.py b/src/calibre/ebooks/pdb/input.py
index b0e7746c7e..de210e0a6d 100644
--- a/src/calibre/ebooks/pdb/input.py
+++ b/src/calibre/ebooks/pdb/input.py
@@ -32,8 +32,3 @@ class PDBInput(InputFormatPlugin):
         opf = reader.extract_content(os.getcwd())
 
         return opf
-
-    def heuristics(self, options, html):
-        self.options = options
-        preprocessor = PreProcessor(self.options, log=getattr(self, 'log', None))
-        return preprocessor(html)

From f46749863850242e58e92a2f337a6abb1be03486 Mon Sep 17 00:00:00 2001
From: ldolse <ldolse@yahoo.com>
Date: Sat, 15 Jan 2011 00:11:21 +0800
Subject: [PATCH 6/6] preserve soft breaks when deleting blank paragraphs

---
 src/calibre/ebooks/conversion/utils.py | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index 286fad1aaa..96bd303933 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -294,8 +294,8 @@ class PreProcessor(object):
         # If more than 40% of the lines are empty paragraphs and the user has enabled delete
         # blank paragraphs then delete blank lines to clean up spacing
         linereg = re.compile('(?<=<p).*?(?=</p>)', re.IGNORECASE|re.DOTALL)
-        blankreg = re.compile(r'\s*(?P<openline><p[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
-        #multi_blank = re.compile(r'(\s*<p[^>]*>\s*(<(b|i|u)>)?\s*(</(b|i|u)>)?\s*</p>){2,}', re.IGNORECASE)
+        blankreg = re.compile(r'\s*(?P<openline><p(?!\sid=\"softbreak\")[^>]*>)\s*(?P<closeline></p>)', re.IGNORECASE)
+        multi_blank = re.compile(r'(\s*<p[^>]*>\s*</p>){2,}', re.IGNORECASE)
         blanklines = blankreg.findall(html)
         lines = linereg.findall(html)
         blanks_between_paragraphs = False
@@ -303,11 +303,8 @@ class PreProcessor(object):
         if len(lines) > 1:
             self.log("There are " + unicode(len(blanklines)) + " blank lines. " +
                     unicode(float(len(blanklines)) / float(len(lines))) + " percent blank")
-            if float(len(blanklines)) / float(len(lines)) > 0.40 and getattr(self.extra_opts,
-            'delete_blank_paragraphs', False):
-                self.log("deleting blank lines")
-                html = blankreg.sub('', html)
-            elif float(len(blanklines)) / float(len(lines)) > 0.40:
+                    
+            if float(len(blanklines)) / float(len(lines)) > 0.40:
                 blanks_between_paragraphs = True
                 print "blanks between paragraphs is marked True"
             else:
@@ -319,7 +316,12 @@ class PreProcessor(object):
 
         html = self.markup_chapters(html, totalwords, blanks_between_paragraphs)
 
-
+        if blanks_between_paragraphs and getattr(self.extra_opts,
+        'delete_blank_paragraphs', False):
+            self.log("deleting blank lines")
+            html = multi_blank.sub('\n<p id="softbreak" style="margin-top:1.5em; margin-bottom:1.5em"> </p>', html)
+            html = blankreg.sub('', html)
+            
         ###### Unwrap lines ######
         #
         # Some OCR sourced files have line breaks in the html using a combination of span & p tags