diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py
index ad2214fcb5..d4af7797ae 100644
--- a/src/calibre/ebooks/conversion/utils.py
+++ b/src/calibre/ebooks/conversion/utils.py
@@ -24,10 +24,11 @@ class HeuristicProcessor(object):
self.chapters_no_title = 0
self.chapters_with_title = 0
self.blanks_deleted = False
+ self.blanks_between_paragraphs = False
self.linereg = re.compile('(?<=
)', re.IGNORECASE|re.DOTALL)
- self.blankreg = re.compile(r'\s*(?P]*>)\s*(?P
)', re.IGNORECASE)
- self.softbreak = re.compile(r'\s*(?P]*>)\s*(?P
)', re.IGNORECASE)
- self.multi_blank = re.compile(r'(\s*]*>\s*
){2,}', re.IGNORECASE)
+ self.blankreg = re.compile(r'\s*(?P]*>)\s*(?P
)', re.IGNORECASE)
+ self.anyblank = re.compile(r'\s*(?P]*>)\s*(?P
)', re.IGNORECASE)
+ self.multi_blank = re.compile(r'(\s*]*>\s*
){2,}(?!\s*' in src[:1000]
@@ -42,8 +43,10 @@ class HeuristicProcessor(object):
" chapters. - " + unicode(chap))
return ''+chap+'
\n'
else:
- txt_chap = html2text(chap)
- txt_title = html2text(title)
+ delete_whitespace = re.compile('^\s*(?P.*?)\s*$')
+ delete_quotes = re.compile('\'\"')
+ txt_chap = delete_quotes.sub('', delete_whitespace.sub('\g', html2text(chap)))
+ txt_title = delete_quotes.sub('', delete_whitespace.sub('\g', html2text(title)))
self.html_preprocess_sections = self.html_preprocess_sections + 1
self.log.debug("marked " + unicode(self.html_preprocess_sections) +
" chapters & titles. - " + unicode(chap) + ", " + unicode(title))
@@ -375,9 +378,9 @@ class HeuristicProcessor(object):
html = re.sub('', '', html)
# Get rid of empty span, bold, font, em, & italics tags
html = re.sub(r"\s*]*>\s*(]*>\s*){0,2}\s*\s*", " ", html)
- html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*(font|[ibu]|em)>\s*){0,2}\s*(font|[ibu]|em)>", " ", html)
+ html = re.sub(r"\s*<(font|[ibu]|em|strong)[^>]*>\s*(<(font|[ibu]|em|strong)[^>]*>\s*(font|[ibu]|em|strong)>\s*){0,2}\s*(font|[ibu]|em|strong)>", " ", html)
html = re.sub(r"\s*]*>\s*(]>\s*){0,2}\s*\s*", " ", html)
- html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*(font|[ibu]|em)>\s*){0,2}\s*(font|[ibu]|em)>", " ", html)
+ html = re.sub(r"\s*<(font|[ibu]|em|strong)[^>]*>\s*(<(font|[ibu]|em|strong)[^>]*>\s*(font|[ibu]|em|strong)>\s*){0,2}\s*(font|[ibu]|em|strong)>", " ", html)
self.deleted_nbsps = True
return html
@@ -416,6 +419,28 @@ class HeuristicProcessor(object):
return True
return False
+ def detect_blank_formatting(self, html):
+ blanks_before_headings = re.compile(r'(\s*]*>\s*
){1,}(?=\s*)(\s*]*>\s*
){1,}', re.IGNORECASE)
+
+ def markup_spacers(match):
+ blanks = match.group(0)
+ blanks = self.blankreg.sub('\n
', blanks)
+ return blanks
+ html = blanks_before_headings.sub(markup_spacers, html)
+ html = blanks_after_headings.sub(markup_spacers, html)
+ if self.html_preprocess_sections > self.min_chapters:
+ html = re.sub('(?si)^.*?(?=
', html)
+ else:
+ html = self.blankreg.sub('\n
', html)
+ return html
+
+
def __call__(self, html):
self.log.debug("********* Heuristic processing HTML *********")
@@ -457,23 +482,23 @@ class HeuristicProcessor(object):
#html = re.sub('
]*>', u'\u00a0
', html)
# Determine whether the document uses interleaved blank lines
- blanks_between_paragraphs = self.analyze_blanks(html)
+ self.blanks_between_paragraphs = self.analyze_blanks(html)
#self.dump(html, 'before_chapter_markup')
# detect chapters/sections to match xpath or splitting logic
if getattr(self.extra_opts, 'markup_chapter_headings', False):
- html = self.markup_chapters(html, self.totalwords, blanks_between_paragraphs)
+ html = self.markup_chapters(html, self.totalwords, self.blanks_between_paragraphs)
if getattr(self.extra_opts, 'italicize_common_cases', False):
html = self.markup_italicis(html)
# If more than 40% of the lines are empty paragraphs and the user has enabled delete
# blank paragraphs then delete blank lines to clean up spacing
- if blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False):
+ if self.blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False):
self.log.debug("deleting blank lines")
self.blanks_deleted = True
- html = self.multi_blank.sub('\n
', html)
+ html = self.multi_blank.sub('\n
', html)
html = self.blankreg.sub('', html)
# Determine line ending type
@@ -525,14 +550,13 @@ class HeuristicProcessor(object):
html = doubleheading.sub('\g'+'\n'+'
', html)
if getattr(self.extra_opts, 'format_scene_breaks', False):
+ html = self.detect_blank_formatting(html)
+ html = self.detect_soft_breaks(html)
# Center separator lines
- html = re.sub(u'<(?Pp|div)[^>]*>\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(?P([*#•=✦]+\s*)+)\s*((?P=inner3)>)?\s*((?P=inner2)>)?\s*((?P=inner1)>)?\s*(?P=outer)>', '' + '\g' + '
', html)
- if not self.blanks_deleted:
- html = self.multi_blank.sub('\n
', html)
- html = re.sub(']*>\s*
', '
', html)
+ html = re.sub(u'<(?Pp|div)[^>]*>\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(?P([*#•=✦]+\s*)+)\s*((?P=inner3)>)?\s*((?P=inner2)>)?\s*((?P=inner1)>)?\s*(?P=outer)>', '' + '\g' + '
', html)
+ #html = re.sub(']*>\s*
', '
', html)
if self.deleted_nbsps:
# put back non-breaking spaces in empty paragraphs to preserve original formatting
- html = self.blankreg.sub('\n'+r'\g'+u'\u00a0'+r'\g', html)
- html = self.softbreak.sub('\n'+r'\g'+u'\u00a0'+r'\g', html)
+ html = self.anyblank.sub('\n'+r'\g'+u'\u00a0'+r'\g', html)
return html