From 8cac314ffe204be4ea946d69dd4c0ac7368db0f1 Mon Sep 17 00:00:00 2001 From: ldolse Date: Thu, 16 Sep 2010 16:48:59 +0800 Subject: [PATCH] adding html_unwrap_factor --- src/calibre/ebooks/conversion/cli.py | 2 +- src/calibre/ebooks/conversion/plumber.py | 9 ++++ src/calibre/ebooks/conversion/utils.py | 16 ++++-- .../gui2/convert/structure_detection.py | 7 ++- .../gui2/convert/structure_detection.ui | 53 ++++++++++++++----- 5 files changed, 67 insertions(+), 20 deletions(-) diff --git a/src/calibre/ebooks/conversion/cli.py b/src/calibre/ebooks/conversion/cli.py index 2ef633d0bb..62a941142b 100644 --- a/src/calibre/ebooks/conversion/cli.py +++ b/src/calibre/ebooks/conversion/cli.py @@ -137,7 +137,7 @@ def add_pipeline_options(parser, plumber): 'chapter', 'chapter_mark', 'prefer_metadata_cover', 'remove_first_image', 'insert_metadata', 'page_breaks_before', - 'preprocess_html', + 'preprocess_html', 'html_unwrap_factor', ] ), diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 16282dd28d..c8803fb922 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -362,6 +362,15 @@ OptionRecommendation(name='preprocess_html', ) ), +OptionRecommendation(name='html_unwrap_factor', + recommended_value=0.40, level=OptionRecommendation.LOW, + help=_('Scale used to determine the length at which a line should ' + 'be unwrapped if preprocess is enabled. Valid values are a decimal between 0 and 1. The ' + 'default is 0.40, just below the median line length. This will unwrap typical books ' + ' with hard line breaks, but should be reduced if the line length is variable.' + ) + ), + OptionRecommendation(name='smarten_punctuation', recommended_value=False, level=OptionRecommendation.LOW, help=_('Convert plain quotes, dashes and ellipsis to their ' diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index 5301f70a16..f9d16b428c 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -11,10 +11,11 @@ from calibre.utils.logging import default_log class PreProcessor(object): - def __init__(self, log=None): + def __init__(self, log=None, extra_opts=None): self.log = default_log if log is None else log self.html_preprocess_sections = 0 self.found_indents = 0 + self.extra_opts = extra_opts def chapter_head(self, match): chap = match.group('chap') @@ -91,6 +92,7 @@ class PreProcessor(object): # If more than 40% of the lines are empty paragraphs then delete them to clean up spacing linereg = re.compile('(?<=)', re.IGNORECASE|re.DOTALL) blankreg = re.compile(r'\s*]*>\s*(<(b|i|u)>)?\s*()?\s*

', re.IGNORECASE) + multi_blank = re.compile(r'(\s*]*>\s*(<(b|i|u)>)?\s*()?\s*

){2,}', re.IGNORECASE) blanklines = blankreg.findall(html) lines = linereg.findall(html) if len(lines) > 1: @@ -146,16 +148,20 @@ class PreProcessor(object): else: format = 'html' - # Calculate Length - length = line_length(format, html, 0.4) + # Calculate Length + #if getattr(self.extra_opts, 'html_unwrap_factor', 0.0) > 0.01: + length = line_length('pdf', html, getattr(self.extra_opts, 'html_unwrap_factor')) + #else: + # length = line_length(format, html, 0.4) + # self.log("#@#%!$@#$ - didn't find unwrap_factor") self.log("*** Median line length is " + str(length) + ",calculated with " + format + " format ***") # # Unwrap and/or delete soft-hyphens, hyphens html = re.sub(u'­\s*(\s*(\s*<[iubp][^>]*>\s*)?]*>|\s*<[iubp][^>]*>)?\s*', '', html) html = re.sub(u'(?<=[-–—])\s*(?=<)(\s*(\s*<[iubp][^>]*>\s*)?]*>|\s*<[iubp][^>]*>)?\s*(?=[[a-z\d])', '', html) - # Unwrap lines using punctation if the median length of all lines is less than 200 - unwrap = re.compile(r"(?<=.{%i}[a-z,;:\IA])\s*\s*()?\s*(?P<(p|span|div)[^>]*>\s*(<(p|span|div)[^>]*>\s*\s*)\s*){0,3}\s*<(span|div|p)[^>]*>\s*(<(span|div|p)[^>]*>)?\s*" % length, re.UNICODE) + # Unwrap lines using punctation and line length + unwrap = re.compile(r"(?<=.{%i}([a-z,;):\IA]|(?\s*()?\s*(?P<(p|span|div)[^>]*>\s*(<(p|span|div)[^>]*>\s*\s*)\s*){0,3}\s*<(span|div|p)[^>]*>\s*(<(span|div|p)[^>]*>)?\s*" % length, re.UNICODE) html = unwrap.sub(' ', html) # If still no sections after unwrapping mark split points on lines with no punctuation diff --git a/src/calibre/gui2/convert/structure_detection.py b/src/calibre/gui2/convert/structure_detection.py index f2ca49d1bd..68f820bda4 100644 --- a/src/calibre/gui2/convert/structure_detection.py +++ b/src/calibre/gui2/convert/structure_detection.py @@ -26,7 +26,7 @@ class StructureDetectionWidget(Widget, Ui_Form): 'remove_first_image', 'insert_metadata', 'page_breaks_before', 'preprocess_html', 'remove_header', 'header_regex', - 'remove_footer', 'footer_regex'] + 'remove_footer', 'footer_regex','html_unwrap_factor'] ) self.db, self.book_id = db, book_id for x in ('pagebreak', 'rule', 'both', 'none'): @@ -64,3 +64,8 @@ class StructureDetectionWidget(Widget, Ui_Form): _('The XPath expression %s is invalid.')%x.text).exec_() return False return True + + def set_value_handler(self, g, val): + if val is None and isinstance(g, QDoubleSpinBox): + g.setValue(0.0) + return True \ No newline at end of file diff --git a/src/calibre/gui2/convert/structure_detection.ui b/src/calibre/gui2/convert/structure_detection.ui index eb2892a07a..54534af950 100644 --- a/src/calibre/gui2/convert/structure_detection.ui +++ b/src/calibre/gui2/convert/structure_detection.ui @@ -48,17 +48,7 @@ - - - - &Preprocess input file to possibly improve structure detection - - - - - - - + Qt::Vertical @@ -88,8 +78,45 @@ - - + + + opt_page_breaks_before + + + + + opt_footer_regex + + + + + + &Preprocess input file to possibly improve structure detection + + + + + + + Qt::RightToLeft + + + Line Un-Wrapping Factor + + + + + + + 1.000000000000000 + + + 0.050000000000000 + + + 0.400000000000000 + +