From 333cb0652ef3f8ff4b2045db730650b78df92709 Mon Sep 17 00:00:00 2001 From: ldolse Date: Mon, 22 Nov 2010 18:35:44 +0800 Subject: [PATCH 1/5] fix for mobi files which put style tags outside of paragraph tags, which upsets lxml and BeautifulSoup --- src/calibre/ebooks/mobi/reader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index c55b94cac5..d2bc25f8eb 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -476,6 +476,9 @@ class MobiReader(object): self.processed_html = self.processed_html.replace('> <', '>\n<') self.processed_html = self.processed_html.replace(']*>', '', self.processed_html) + # Rearrange various style tags & paragraph tags so that lxml and BeautifulSoup don't get upset + self.processed_html = re.sub(r'(?i)(?P(<(i|b|u|em|small|tt)>\s*){1,})(?P]*>)', '\g'+'\g', self.processed_html) + self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P(\s*){1,})', '\g'+'\g', self.processed_html) def remove_random_bytes(self, html): return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08', From 0013d4e059ac50ae4e597db2511ba836e2994403 Mon Sep 17 00:00:00 2001 From: ldolse Date: Mon, 22 Nov 2010 18:49:52 +0800 Subject: [PATCH 2/5] ... --- src/calibre/ebooks/mobi/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index d2bc25f8eb..972b049643 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -477,8 +477,8 @@ class MobiReader(object): self.processed_html = self.processed_html.replace(']*>', '', self.processed_html) # Rearrange various style tags & paragraph tags so that lxml and BeautifulSoup don't get upset - self.processed_html = re.sub(r'(?i)(?P(<(i|b|u|em|small|tt)>\s*){1,})(?P]*>)', '\g'+'\g', self.processed_html) - self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P(\s*){1,})', '\g'+'\g', self.processed_html) + self.processed_html = re.sub(r'(?i)(?P(<(i|b|u|em|small|big|strong|tt)>\s*){1,})(?P]*>)', '\g'+'\g', self.processed_html) + self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P(\s*){1,})', '\g'+'\g', self.processed_html) def remove_random_bytes(self, html): return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08', From 38e08a7bd39b868ac496f0e0dcbf1bacb7f231b8 Mon Sep 17 00:00:00 2001 From: ldolse Date: Mon, 22 Nov 2010 19:19:47 +0800 Subject: [PATCH 3/5] had to include heading tags with the previous change, because lxml/beautifulsoup were equally unhappy when heading tags contained paragraph tags --- src/calibre/ebooks/mobi/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 972b049643..355fc87c0c 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -477,8 +477,8 @@ class MobiReader(object): self.processed_html = self.processed_html.replace(']*>', '', self.processed_html) # Rearrange various style tags & paragraph tags so that lxml and BeautifulSoup don't get upset - self.processed_html = re.sub(r'(?i)(?P(<(i|b|u|em|small|big|strong|tt)>\s*){1,})(?P]*>)', '\g'+'\g', self.processed_html) - self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P(\s*){1,})', '\g'+'\g', self.processed_html) + self.processed_html = re.sub(r'(?i)(?P(<(h\d+|i|b|u|em|small|big|strong|tt)>\s*){1,})(?P]*>)', '\g'+'\g', self.processed_html) + self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P(\s*){1,})', '\g'+'\g', self.processed_html) def remove_random_bytes(self, html): return re.sub('\x14|\x15|\x19|\x1c|\x1d|\xef|\x12|\x13|\xec|\x08', From b71a2874ffb0f41730839b29a74cae0ca7deb909 Mon Sep 17 00:00:00 2001 From: ldolse Date: Mon, 22 Nov 2010 19:54:51 +0800 Subject: [PATCH 4/5] ... --- src/calibre/ebooks/mobi/reader.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 355fc87c0c..733f79ee65 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -476,7 +476,8 @@ class MobiReader(object): self.processed_html = self.processed_html.replace('> <', '>\n<') self.processed_html = self.processed_html.replace(']*>', '', self.processed_html) - # Rearrange various style tags & paragraph tags so that lxml and BeautifulSoup don't get upset + # Swap inline and block level elements, and order block level elements according to priority + # - lxml and beautifulsoup live in the modern world and expect/assume a specific order self.processed_html = re.sub(r'(?i)(?P(<(h\d+|i|b|u|em|small|big|strong|tt)>\s*){1,})(?P]*>)', '\g'+'\g', self.processed_html) self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P(\s*){1,})', '\g'+'\g', self.processed_html) From 8c15219933df8eb59e59c8196695f3847d25ef10 Mon Sep 17 00:00:00 2001 From: ldolse Date: Mon, 22 Nov 2010 21:09:16 +0800 Subject: [PATCH 5/5] ... --- src/calibre/ebooks/mobi/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index 733f79ee65..c4845f9443 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -477,7 +477,7 @@ class MobiReader(object): self.processed_html = self.processed_html.replace(']*>', '', self.processed_html) # Swap inline and block level elements, and order block level elements according to priority - # - lxml and beautifulsoup live in the modern world and expect/assume a specific order + # - lxml and beautifulsoup expect/assume a specific order based on xhtml spec self.processed_html = re.sub(r'(?i)(?P(<(h\d+|i|b|u|em|small|big|strong|tt)>\s*){1,})(?P]*>)', '\g'+'\g', self.processed_html) self.processed_html = re.sub(r'(?i)(?P]*>)\s*(?P(\s*){1,})', '\g'+'\g', self.processed_html)