diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py
index 03a0047927..a1e28b2554 100644
--- a/src/calibre/ebooks/conversion/preprocess.py
+++ b/src/calibre/ebooks/conversion/preprocess.py
@@ -106,6 +106,50 @@ def line_length(format, raw, percent):
return lengths[index]
+class Dehyphenator(object):
+ '''
+ Analyzes words to determine whether hyphens should be retained/removed. Uses the document
+ itself is as a dictionary. This method handles all languages along with uncommon, made-up, and
+ scientific words. The primary disadvantage is that words appearing only once in the document
+ retain hyphens.
+ '''
+
+ def dehyphenate(self, match):
+ firsthalf = match.group('firstpart')
+ secondhalf = match.group('secondpart')
+ hyphenated = str(firsthalf) + "-" + str(secondhalf)
+ dehyphenated = str(firsthalf) + str(secondhalf)
+ # Add common suffixes to the regex below to increase the likelihood of a match -
+ # don't add suffixes which are also complete words, such as 'able' or 'sex'
+ removesuffixes = re.compile(r"((ed)?ly|(')?s|a?(t|s)ion(s|al(ly)?)?|ings?|(i)?ous|(i|a)ty|(it)?ies|ive|gence|istic|(e|a)nce|ment(s)?|ism|ated|(e|u)ct(ed)?|ed|(i|ed)?ness|(e|a)ncy|ble|ier|al|ex)$", re.IGNORECASE)
+ lookupword = removesuffixes.sub('', dehyphenated)
+ # remove prefixes if the prefix was not already the point of hyphenation
+ prefixes = re.compile(r'^(un|in|ex)$', re.IGNORECASE)
+ removeprefix = re.compile(r'^(un|in|ex)', re.IGNORECASE)
+ if prefixes.match(firsthalf) is None:
+ lookupword = removeprefix.sub('', lookupword)
+ booklookup = re.compile(u'%s' % lookupword, re.IGNORECASE)
+ #print "lookup word is: "+str(lookupword)+", orig is: " + str(hyphenated)
+ match = booklookup.search(self.html)
+ if match:
+ #print "returned dehyphenated word: " + str(dehyphenated)
+ return dehyphenated
+ else:
+ #print "returned hyphenated word: " + str(hyphenated)
+ return hyphenated
+
+ def __call__(self, html, format, length=1):
+ self.html = html
+ if format == 'html':
+ intextmatch = re.compile(u'(?<=.{%i})(?P |[iub]>\s* \s*<[iub]>)\s*(?P \s*(?=[[a-z\d])'), lambda match: ''))
# unwrap/delete soft hyphens
end_rules.append((re.compile(u'[](\s* )+\s*(?=[[a-z\d])'), lambda match: ''))
# unwrap/delete soft hyphens with formatting
@@ -350,7 +393,7 @@ class HTMLPreProcessor(object):
# print "The pdf line length returned is " + str(length)
end_rules.append(
# Un wrap using punctuation
- (re.compile(r'(?<=.{%i}([a-z,:)\-IA]|(?(i|b|u)>)?\s*(
\s*", "\n
", html)
- #self.log("\n\n\n\n\n\n\n\n\n\n\n"+html+"\n\n\n\n\n\n\n\n\n\n\n\n\n")
# detect chapters/sections to match xpath or splitting logic
heading = re.compile('