diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index ad2214fcb5..d4af7797ae 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -24,10 +24,11 @@ class HeuristicProcessor(object): self.chapters_no_title = 0 self.chapters_with_title = 0 self.blanks_deleted = False + self.blanks_between_paragraphs = False self.linereg = re.compile('(?<=)', re.IGNORECASE|re.DOTALL) - self.blankreg = re.compile(r'\s*(?P]*>)\s*(?P

)', re.IGNORECASE) - self.softbreak = re.compile(r'\s*(?P]*>)\s*(?P

)', re.IGNORECASE) - self.multi_blank = re.compile(r'(\s*]*>\s*

){2,}', re.IGNORECASE) + self.blankreg = re.compile(r'\s*(?P]*>)\s*(?P

)', re.IGNORECASE) + self.anyblank = re.compile(r'\s*(?P]*>)\s*(?P

)', re.IGNORECASE) + self.multi_blank = re.compile(r'(\s*]*>\s*

){2,}(?!\s*' in src[:1000] @@ -42,8 +43,10 @@ class HeuristicProcessor(object): " chapters. - " + unicode(chap)) return '

'+chap+'

\n' else: - txt_chap = html2text(chap) - txt_title = html2text(title) + delete_whitespace = re.compile('^\s*(?P.*?)\s*$') + delete_quotes = re.compile('\'\"') + txt_chap = delete_quotes.sub('', delete_whitespace.sub('\g', html2text(chap))) + txt_title = delete_quotes.sub('', delete_whitespace.sub('\g', html2text(title))) self.html_preprocess_sections = self.html_preprocess_sections + 1 self.log.debug("marked " + unicode(self.html_preprocess_sections) + " chapters & titles. - " + unicode(chap) + ", " + unicode(title)) @@ -375,9 +378,9 @@ class HeuristicProcessor(object): html = re.sub('', '', html) # Get rid of empty span, bold, font, em, & italics tags html = re.sub(r"\s*]*>\s*(]*>\s*){0,2}\s*\s*", " ", html) - html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*\s*){0,2}\s*", " ", html) + html = re.sub(r"\s*<(font|[ibu]|em|strong)[^>]*>\s*(<(font|[ibu]|em|strong)[^>]*>\s*\s*){0,2}\s*", " ", html) html = re.sub(r"\s*]*>\s*(]>\s*){0,2}\s*\s*", " ", html) - html = re.sub(r"\s*<(font|[ibu]|em)[^>]*>\s*(<(font|[ibu]|em)[^>]*>\s*\s*){0,2}\s*", " ", html) + html = re.sub(r"\s*<(font|[ibu]|em|strong)[^>]*>\s*(<(font|[ibu]|em|strong)[^>]*>\s*\s*){0,2}\s*", " ", html) self.deleted_nbsps = True return html @@ -416,6 +419,28 @@ class HeuristicProcessor(object): return True return False + def detect_blank_formatting(self, html): + blanks_before_headings = re.compile(r'(\s*]*>\s*

){1,}(?=\s*)(\s*]*>\s*

){1,}', re.IGNORECASE) + + def markup_spacers(match): + blanks = match.group(0) + blanks = self.blankreg.sub('\n

', blanks) + return blanks + html = blanks_before_headings.sub(markup_spacers, html) + html = blanks_after_headings.sub(markup_spacers, html) + if self.html_preprocess_sections > self.min_chapters: + html = re.sub('(?si)^.*?(?=

', html) + else: + html = self.blankreg.sub('\n

', html) + return html + + def __call__(self, html): self.log.debug("********* Heuristic processing HTML *********") @@ -457,23 +482,23 @@ class HeuristicProcessor(object): #html = re.sub(']*>', u'

\u00a0

', html) # Determine whether the document uses interleaved blank lines - blanks_between_paragraphs = self.analyze_blanks(html) + self.blanks_between_paragraphs = self.analyze_blanks(html) #self.dump(html, 'before_chapter_markup') # detect chapters/sections to match xpath or splitting logic if getattr(self.extra_opts, 'markup_chapter_headings', False): - html = self.markup_chapters(html, self.totalwords, blanks_between_paragraphs) + html = self.markup_chapters(html, self.totalwords, self.blanks_between_paragraphs) if getattr(self.extra_opts, 'italicize_common_cases', False): html = self.markup_italicis(html) # If more than 40% of the lines are empty paragraphs and the user has enabled delete # blank paragraphs then delete blank lines to clean up spacing - if blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False): + if self.blanks_between_paragraphs and getattr(self.extra_opts, 'delete_blank_paragraphs', False): self.log.debug("deleting blank lines") self.blanks_deleted = True - html = self.multi_blank.sub('\n

', html) + html = self.multi_blank.sub('\n

', html) html = self.blankreg.sub('', html) # Determine line ending type @@ -525,14 +550,13 @@ class HeuristicProcessor(object): html = doubleheading.sub('\g'+'\n'+'', html) if getattr(self.extra_opts, 'format_scene_breaks', False): + html = self.detect_blank_formatting(html) + html = self.detect_soft_breaks(html) # Center separator lines - html = re.sub(u'<(?Pp|div)[^>]*>\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(?P([*#•=✦]+\s*)+)\s*()?\s*()?\s*()?\s*', '

' + '\g' + '

', html) - if not self.blanks_deleted: - html = self.multi_blank.sub('\n

', html) - html = re.sub(']*>\s*

', '

', html) + html = re.sub(u'<(?Pp|div)[^>]*>\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(<(?Pfont|span|[ibu])[^>]*>)?\s*(?P([*#•=✦]+\s*)+)\s*()?\s*()?\s*()?\s*', '

' + '\g' + '

', html) + #html = re.sub(']*>\s*

', '

', html) if self.deleted_nbsps: # put back non-breaking spaces in empty paragraphs to preserve original formatting - html = self.blankreg.sub('\n'+r'\g'+u'\u00a0'+r'\g', html) - html = self.softbreak.sub('\n'+r'\g'+u'\u00a0'+r'\g', html) + html = self.anyblank.sub('\n'+r'\g'+u'\u00a0'+r'\g', html) return html