mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
moved chapter markup to a function, tied preprocessing into word count
This commit is contained in:
ldolse
parent
b61cabc2ed
commit
a1dcbb33c1
@ -65,7 +65,7 @@ class PreProcessor(object):
|
|||||||
inspect. Percent is the minimum percent of line endings which should
|
inspect. Percent is the minimum percent of line endings which should
|
||||||
be marked up to return true.
|
be marked up to return true.
|
||||||
'''
|
'''
|
||||||
htm_end_ere = re.compile('</p>', re.DOTALL)
|
htm_end_ere = re.compile('</(p|div)>', re.DOTALL)
|
||||||
line_end_ere = re.compile('(\n|\r|\r\n)', re.DOTALL)
|
line_end_ere = re.compile('(\n|\r|\r\n)', re.DOTALL)
|
||||||
htm_end = htm_end_ere.findall(raw)
|
htm_end = htm_end_ere.findall(raw)
|
||||||
line_end = line_end_ere.findall(raw)
|
line_end = line_end_ere.findall(raw)
|
||||||
@ -102,12 +102,93 @@ class PreProcessor(object):
|
|||||||
with open(os.path.join(odir, name), 'wb') as f:
|
with open(os.path.join(odir, name), 'wb') as f:
|
||||||
f.write(raw.encode('utf-8'))
|
f.write(raw.encode('utf-8'))
|
||||||
|
|
||||||
|
def get_word_count(self, html):
|
||||||
|
totalwords = 0
|
||||||
|
word_count_text = re.sub(r'(?s)<head[^>]*>.*?</head>', '', html)
|
||||||
|
word_count_text = re.sub(r'<[^>]*>', '', word_count_text)
|
||||||
|
wordcount = get_wordcount_obj(word_count_text)
|
||||||
|
return wordcount.words
|
||||||
|
|
||||||
|
def markup_chapters(self, html, wordcount, blanks_between_paragraphs):
|
||||||
|
# Typical chapters are between 2000 and 7000 words, use the larger number to decide the
|
||||||
|
# minimum of chapters to search for
|
||||||
|
self.min_chapters = 1
|
||||||
|
if wordcount > 7000:
|
||||||
|
self.min_chapters = wordcount / 7000
|
||||||
|
print "minimum chapters required are: "+str(self.min_chapters)
|
||||||
|
heading = re.compile('<h[1-3][^>]*>', re.IGNORECASE)
|
||||||
|
self.html_preprocess_sections = len(heading.findall(html))
|
||||||
|
self.log("found " + unicode(self.html_preprocess_sections) + " pre-existing headings")
|
||||||
|
|
||||||
|
# Build the Regular Expressions in pieces
|
||||||
|
init_lookahead = "(?=<(p|div))"
|
||||||
|
chapter_line_open = "<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*"
|
||||||
|
title_line_open = "<(?P<outer2>p|div)[^>]*>\s*(<(?P<inner4>font|span|[ibu])[^>]*>)?\s*(<(?P<inner5>font|span|[ibu])[^>]*>)?\s*(<(?P<inner6>font|span|[ibu])[^>]*>)?\s*"
|
||||||
|
chapter_header_open = r"(?P<chap>"
|
||||||
|
title_header_open = r"(?P<title>"
|
||||||
|
chapter_header_close = ")\s*"
|
||||||
|
title_header_close = ")"
|
||||||
|
chapter_line_close = "(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>"
|
||||||
|
title_line_close = "(</(?P=inner6)>)?\s*(</(?P=inner5)>)?\s*(</(?P=inner4)>)?\s*</(?P=outer2)>"
|
||||||
|
|
||||||
|
if blanks_between_paragraphs:
|
||||||
|
blank_lines = "(\s*<p[^>]*>\s*</p>){0,2}\s*"
|
||||||
|
else:
|
||||||
|
blank_lines = ""
|
||||||
|
opt_title_open = "("
|
||||||
|
opt_title_close = ")?"
|
||||||
|
n_lookahead_open = "\s+(?!"
|
||||||
|
n_lookahead_close = ")"
|
||||||
|
|
||||||
|
default_title = r"\s{0,3}([\w\'\"-]+\s{0,3}){1,5}?(?=<)"
|
||||||
|
|
||||||
|
chapter_types = [
|
||||||
|
[r"[^'\"]?(Introduction|Synopsis|Acknowledgements|Chapter|Kapitel|Epilogue|Volume\s|Prologue|Book\s|Part\s|Dedication)\s*([\d\w-]+\:?\s*){0,4}", True, "Searching for common Chapter Headings"],
|
||||||
|
[r"[^'\"]?(\d+\.?|CHAPTER)\s*([\dA-Z\-\'\"\?\.!#,]+\s*){0,7}\s*", True, "Searching for numeric chapter headings"], # Numeric Chapters
|
||||||
|
[r"<b[^>]*>\s*(<span[^>]*>)?\s*(?!([*#•]+\s*)+)(\s*(?=[\w#\-*\s]+<)([\w#-*]+\s*){1,5}\s*)(</span>)?\s*</b>", True, "Searching for emphasized lines"], # Emphasized lines
|
||||||
|
[r"[^'\"]?(\d+\.?\s+([\d\w-]+\:?\'?-?\s?){0,5})\s*", True, "Searching for numeric chapters with titles"], # Numeric Titles
|
||||||
|
[r"\s*[^'\"]?([A-Z#]+(\s|-){0,3}){1,5}\s*", False, "Searching for chapters with Uppercase Characters" ] # Uppercase Chapters
|
||||||
|
]
|
||||||
|
|
||||||
|
# Start with most typical chapter headings, get more aggressive until one works
|
||||||
|
for [chapter_type, lookahead_ignorecase, log_message] in chapter_types:
|
||||||
|
if self.html_preprocess_sections >= self.min_chapters:
|
||||||
|
break
|
||||||
|
full_chapter_line = chapter_line_open+chapter_header_open+chapter_type+chapter_header_close+chapter_line_close
|
||||||
|
n_lookahead = re.sub("(ou|in|cha)", "lookahead_", full_chapter_line)
|
||||||
|
self.log("Marked " + unicode(self.html_preprocess_sections) + " headings, " + log_message)
|
||||||
|
if lookahead_ignorecase:
|
||||||
|
chapter_marker = init_lookahead+full_chapter_line+blank_lines+n_lookahead_open+n_lookahead+n_lookahead_close+opt_title_open+title_line_open+title_header_open+default_title+title_header_close+title_line_close+opt_title_close
|
||||||
|
chapdetect = re.compile(r'%s' % chapter_marker, re.IGNORECASE)
|
||||||
|
else:
|
||||||
|
chapter_marker = init_lookahead+full_chapter_line+blank_lines+opt_title_open+title_line_open+title_header_open+default_title+title_header_close+title_line_close+opt_title_close+n_lookahead_open+n_lookahead+n_lookahead_close
|
||||||
|
chapdetect = re.compile(r'%s' % chapter_marker, re.UNICODE)
|
||||||
|
|
||||||
|
html = chapdetect.sub(self.chapter_head, html)
|
||||||
|
|
||||||
|
words_per_chptr = wordcount
|
||||||
|
if words_per_chptr > 0 and self.html_preprocess_sections > 0:
|
||||||
|
words_per_chptr = wordcount / self.html_preprocess_sections
|
||||||
|
print "Total wordcount is: "+ str(wordcount)+", Average words per section is: "+str(words_per_chptr)+", Marked up "+str(self.html_preprocess_sections)+" chapters"
|
||||||
|
|
||||||
|
return html
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def __call__(self, html):
|
def __call__(self, html):
|
||||||
self.log("********* Preprocessing HTML *********")
|
self.log("********* Preprocessing HTML *********")
|
||||||
|
|
||||||
|
# Count the words in the document to estimate how many chapters to look for and whether
|
||||||
|
# other types of processing are attempted
|
||||||
|
totalwords = self.get_word_count(html)
|
||||||
|
|
||||||
|
if totalwords < 10:
|
||||||
|
print "not enough text, not preprocessing"
|
||||||
|
return html
|
||||||
|
|
||||||
# Arrange line feeds and </p> tags so the line_length and no_markup functions work correctly
|
# Arrange line feeds and </p> tags so the line_length and no_markup functions work correctly
|
||||||
html = re.sub(r"\s*</p>", "</p>\n", html)
|
html = re.sub(r"\s*</(?P<tag>p|div)>", "</"+"\g<tag>"+">\n", html)
|
||||||
html = re.sub(r"\s*<p(?P<style>[^>]*)>\s*", "\n<p"+"\g<style>"+">", html)
|
html = re.sub(r"\s*<(?P<tag>p|div)(?P<style>[^>]*)>\s*", "\n<"+"\g<tag>"+"\g<style>"+">", html)
|
||||||
|
|
||||||
###### Check Markup ######
|
###### Check Markup ######
|
||||||
#
|
#
|
||||||
@ -170,68 +251,12 @@ class PreProcessor(object):
|
|||||||
else:
|
else:
|
||||||
blanks_between_paragraphs = False
|
blanks_between_paragraphs = False
|
||||||
|
|
||||||
# Count the words in the document to estimate how many chapters to look for
|
|
||||||
word_count_text = re.sub(r'(?s)<head[^>]*>.*?</head>', '', html)
|
|
||||||
word_count_text = re.sub(r'<[^>]*>', '', word_count_text)
|
|
||||||
wordcount = get_wordcount_obj(word_count_text)
|
|
||||||
|
|
||||||
|
|
||||||
#self.dump(html, 'before_chapter_markup')
|
#self.dump(html, 'before_chapter_markup')
|
||||||
# detect chapters/sections to match xpath or splitting logic
|
# detect chapters/sections to match xpath or splitting logic
|
||||||
#
|
#
|
||||||
min_chapters = 10
|
|
||||||
heading = re.compile('<h[1-3][^>]*>', re.IGNORECASE)
|
|
||||||
self.html_preprocess_sections = len(heading.findall(html))
|
|
||||||
self.log("found " + unicode(self.html_preprocess_sections) + " pre-existing headings")
|
|
||||||
|
|
||||||
# Build the Regular Expressions in pieces
|
self.markup_chapters(html, totalwords, blanks_between_paragraphs)
|
||||||
init_lookahead = "(?=<(p|div))"
|
|
||||||
chapter_line_open = "<(?P<outer>p|div)[^>]*>\s*(<(?P<inner1>font|span|[ibu])[^>]*>)?\s*(<(?P<inner2>font|span|[ibu])[^>]*>)?\s*(<(?P<inner3>font|span|[ibu])[^>]*>)?\s*"
|
|
||||||
title_line_open = "<(?P<outer2>p|div)[^>]*>\s*(<(?P<inner4>font|span|[ibu])[^>]*>)?\s*(<(?P<inner5>font|span|[ibu])[^>]*>)?\s*(<(?P<inner6>font|span|[ibu])[^>]*>)?\s*"
|
|
||||||
chapter_header_open = r"(?P<chap>"
|
|
||||||
title_header_open = r"(?P<title>"
|
|
||||||
chapter_header_close = ")\s*"
|
|
||||||
title_header_close = ")"
|
|
||||||
chapter_line_close = "(</(?P=inner3)>)?\s*(</(?P=inner2)>)?\s*(</(?P=inner1)>)?\s*</(?P=outer)>"
|
|
||||||
title_line_close = "(</(?P=inner6)>)?\s*(</(?P=inner5)>)?\s*(</(?P=inner4)>)?\s*</(?P=outer2)>"
|
|
||||||
|
|
||||||
if blanks_between_paragraphs:
|
|
||||||
blank_lines = "(\s*<p[^>]*>\s*</p>){0,2}\s*"
|
|
||||||
else:
|
|
||||||
blank_lines = ""
|
|
||||||
opt_title_open = "("
|
|
||||||
opt_title_close = ")?"
|
|
||||||
n_lookahead_open = "\s+(?!"
|
|
||||||
n_lookahead_close = ")"
|
|
||||||
|
|
||||||
default_title = r"\s{0,3}([\w\'\"-]+\s{0,3}){1,5}?(?=<)"
|
|
||||||
|
|
||||||
chapter_types = [
|
|
||||||
[r"[^'\"]?(Introduction|Synopsis|Acknowledgements|Chapter|Kapitel|Epilogue|Volume\s|Prologue|Book\s|Part\s|Dedication)\s*([\d\w-]+\:?\s*){0,4}", True, "Searching for common Chapter Headings"],
|
|
||||||
[r"[^'\"]?(\d+\.?|CHAPTER)\s*([\dA-Z\-\'\"\?\.!#,]+\s*){0,7}\s*", True, "Searching for numeric chapter headings"], # Numeric Chapters
|
|
||||||
[r"<b[^>]*>\s*(<span[^>]*>)?\s*(?!([*#•]+\s*)+)(\s*(?=[\w#\-*\s]+<)([\w#-*]+\s*){1,5}\s*)(</span>)?\s*</b>", True, "Searching for emphasized lines"], # Emphasized lines
|
|
||||||
[r"[^'\"]?(\d+\.?\s+([\d\w-]+\:?\'?-?\s?){0,5})\s*", True, "Searching for numeric chapters with titles"], # Numeric Titles
|
|
||||||
[r"\s*[^'\"]?([A-Z#]+(\s|-){0,3}){1,5}\s*", False, "Searching for chapters with Uppercase Characters" ] # Uppercase Chapters
|
|
||||||
]
|
|
||||||
|
|
||||||
# Start with most typical chapter headings, get more aggressive until one works
|
|
||||||
for [chapter_type, lookahead_ignorecase, log_message] in chapter_types:
|
|
||||||
if self.html_preprocess_sections >= min_chapters:
|
|
||||||
break
|
|
||||||
full_chapter_line = chapter_line_open+chapter_header_open+chapter_type+chapter_header_close+chapter_line_close
|
|
||||||
n_lookahead = re.sub("(ou|in|cha)", "lookahead_", full_chapter_line)
|
|
||||||
self.log("Marked " + unicode(self.html_preprocess_sections) + " headings, " + log_message)
|
|
||||||
if lookahead_ignorecase:
|
|
||||||
chapter_marker = init_lookahead+full_chapter_line+blank_lines+n_lookahead_open+n_lookahead+n_lookahead_close+opt_title_open+title_line_open+title_header_open+default_title+title_header_close+title_line_close+opt_title_close
|
|
||||||
chapdetect = re.compile(r'%s' % chapter_marker, re.IGNORECASE)
|
|
||||||
else:
|
|
||||||
chapter_marker = init_lookahead+full_chapter_line+blank_lines+opt_title_open+title_line_open+title_header_open+default_title+title_header_close+title_line_close+opt_title_close+n_lookahead_open+n_lookahead+n_lookahead_close
|
|
||||||
chapdetect = re.compile(r'%s' % chapter_marker, re.UNICODE)
|
|
||||||
|
|
||||||
html = chapdetect.sub(self.chapter_head, html)
|
|
||||||
|
|
||||||
words_per_chptr = wordcount.words / self.html_preprocess_sections
|
|
||||||
print "wordcount is: "+ str(wordcount.words)+", Average words per chapter is: "+str(words_per_chptr)+", Marked "+str(self.html_preprocess_sections)+" chapters"
|
|
||||||
|
|
||||||
###### Unwrap lines ######
|
###### Unwrap lines ######
|
||||||
#
|
#
|
||||||
@ -257,7 +282,7 @@ class PreProcessor(object):
|
|||||||
# Calculate Length
|
# Calculate Length
|
||||||
unwrap_factor = getattr(self.extra_opts, 'html_unwrap_factor', 0.4)
|
unwrap_factor = getattr(self.extra_opts, 'html_unwrap_factor', 0.4)
|
||||||
length = docanalysis.line_length(unwrap_factor)
|
length = docanalysis.line_length(unwrap_factor)
|
||||||
self.log("*** Median line length is " + unicode(length) + ", calculated with " + format + " format ***")
|
self.log("Median line length is " + unicode(length) + ", calculated with " + format + " format")
|
||||||
# only go through unwrapping code if the histogram shows unwrapping is required or if the user decreased the default unwrap_factor
|
# only go through unwrapping code if the histogram shows unwrapping is required or if the user decreased the default unwrap_factor
|
||||||
if hardbreaks or unwrap_factor < 0.4:
|
if hardbreaks or unwrap_factor < 0.4:
|
||||||
self.log("Unwrapping required, unwrapping Lines")
|
self.log("Unwrapping required, unwrapping Lines")
|
||||||
@ -286,7 +311,7 @@ class PreProcessor(object):
|
|||||||
html = re.sub(u'\xad\s*(</span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*', '', html)
|
html = re.sub(u'\xad\s*(</span>\s*(</[iubp]>\s*<[iubp][^>]*>\s*)?<span[^>]*>|</[iubp]>\s*<[iubp][^>]*>)?\s*', '', html)
|
||||||
|
|
||||||
# If still no sections after unwrapping mark split points on lines with no punctuation
|
# If still no sections after unwrapping mark split points on lines with no punctuation
|
||||||
if self.html_preprocess_sections < 5:
|
if self.html_preprocess_sections < self.min_chapters:
|
||||||
self.log("Looking for more split points based on punctuation,"
|
self.log("Looking for more split points based on punctuation,"
|
||||||
" currently have " + unicode(self.html_preprocess_sections))
|
" currently have " + unicode(self.html_preprocess_sections))
|
||||||
chapdetect3 = re.compile(r'<(?P<styles>(p|div)[^>]*)>\s*(?P<section>(<span[^>]*>)?\s*(?!([*#•]+\s*)+)(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*.?(?=[a-z#\-*\s]+<)([a-z#-*]+\s*){1,5}\s*\s*(</span>)?(</[ibu]>){0,2}\s*(</span>)?\s*(</[ibu]>){0,2}\s*(</span>)?\s*</(p|div)>)', re.IGNORECASE)
|
chapdetect3 = re.compile(r'<(?P<styles>(p|div)[^>]*)>\s*(?P<section>(<span[^>]*>)?\s*(?!([*#•]+\s*)+)(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*(<[ibu][^>]*>){0,2}\s*(<span[^>]*>)?\s*.?(?=[a-z#\-*\s]+<)([a-z#-*]+\s*){1,5}\s*\s*(</span>)?(</[ibu]>){0,2}\s*(</span>)?\s*(</[ibu]>){0,2}\s*(</span>)?\s*</(p|div)>)', re.IGNORECASE)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user