Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Cleanup HTML metadata parsing

Browse Source
This commit is contained in:
Kovid Goyal 2019-08-12 10:10:50 +05:30
parent 44e54bffc4
commit c5aeaa8c8a
No known key found for this signature in database
GPG Key ID: 06BC317B515ACE7C
2 changed files with 65 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
setup/test.py
View File

@ -110,6 +110,8 @@ def find_tests(which_tests=None):
if ok('ebooks'): if ok('ebooks'):
from calibre.ebooks.metadata.rtf import find_tests from calibre.ebooks.metadata.rtf import find_tests
a(find_tests()) a(find_tests())
from calibre.ebooks.metadata.html import find_tests
a(find_tests())
if ok('misc'): if ok('misc'):
from calibre.ebooks.metadata.tag_mapper import find_tests from calibre.ebooks.metadata.tag_mapper import find_tests
a(find_tests()) a(find_tests())

149
src/calibre/ebooks/metadata/html.py
View File

@ -12,7 +12,8 @@ import re
import unittest import unittest
from collections import defaultdict from collections import defaultdict
from HTMLParser import HTMLParser from html5_parser import parse
from lxml.etree import Comment
from calibre.ebooks.metadata import string_to_authors, authors_to_string from calibre.ebooks.metadata import string_to_authors, authors_to_string
from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.metadata.book.base import Metadata
@ -56,91 +57,75 @@ META_NAMES = {
'comments': ('comments', 'dc.description'), 'comments': ('comments', 'dc.description'),
'tags': ('tags',), 'tags': ('tags',),
} }
rmap_comment = {v:k for k, v in iteritems(COMMENT_NAMES)}
rmap_meta = {v:k for k, l in iteritems(META_NAMES) for v in l}
# Extract an HTML attribute value, supports both single and double quotes and # Extract an HTML attribute value, supports both single and double quotes and
# single quotes inside double quotes and vice versa. # single quotes inside double quotes and vice versa.
attr_pat = r'''(?:(?P<sq>')|(?P<dq>"))(?P<content>(?(sq)[^']+|[^"]+))(?(sq)'|")''' attr_pat = r'''(?:(?P<sq>')|(?P<dq>"))(?P<content>(?(sq)[^']+|[^"]+))(?(sq)'|")'''
def handle_comment(data, comment_tags):
if not hasattr(handle_comment, 'pat'):
handle_comment.pat = re.compile(r'''(?P<name>\S+)\s*=\s*%s''' % attr_pat)
for match in handle_comment.pat.finditer(data):
x = match.group('name')
field = None
try:
field = rmap_comment[x]
except KeyError:
pass
if field:
comment_tags[field].append(replace_entities(match.group('content')))
def parse_metadata(src): def parse_metadata(src):
class MetadataParser(HTMLParser): root = parse(src)
def __init__(self): comment_tags = defaultdict(list)
self.comment_tags = defaultdict(list) meta_tags = defaultdict(list)
self.meta_tag_ids = defaultdict(list) meta_tag_ids = defaultdict(list)
self.meta_tags = defaultdict(list) title = ''
self.title_tag = '' identifier_pat = re.compile(r'(?:dc|dcterms)[.:]identifier(?:\.|$)', flags=re.IGNORECASE)
id_pat2 = re.compile(r'(?:dc|dcterms)[.:]identifier$', flags=re.IGNORECASE)
self.recording = False for comment in root.iterdescendants(tag=Comment):
self.recorded = [] if comment.text:
handle_comment(comment.text, comment_tags)
self.rmap_comment = {v:k for k, v in iteritems(COMMENT_NAMES)} for q in root.iterdescendants(tag='title'):
self.rmap_meta = {v:k for k, l in iteritems(META_NAMES) for v in l} if q.text:
title = q.text
break
HTMLParser.__init__(self) for meta in root.iterdescendants(tag='meta'):
name, content = meta.get('name'), meta.get('content')
def handle_starttag(self, tag, attrs): if not name or not content:
attr_dict = dict(attrs) continue
if identifier_pat.match(name) is not None:
if tag == 'title': scheme = None
self.recording = True if id_pat2.match(name) is not None:
self.recorded = [] scheme = meta.get('scheme')
else:
elif tag == 'meta' and re.match(r'(?:dc|dcterms)[.:]identifier(?:\.|$)', attr_dict.get('name', ''), flags=re.IGNORECASE): elements = re.split(r'[.:]', name)
scheme = None if len(elements) == 3 and not meta.get('scheme'):
if re.match(r'(?:dc|dcterms)[.:]identifier$', attr_dict.get('name', ''), flags=re.IGNORECASE): scheme = elements[2].strip()
scheme = attr_dict.get('scheme', '').strip() if scheme:
elif 'scheme' not in attr_dict: meta_tag_ids[scheme.lower()].append(content)
elements = re.split(r'[.:]', attr_dict['name']) else:
if len(elements) == 3: x = name.lower()
scheme = elements[2].strip() field = None
if scheme: try:
self.meta_tag_ids[scheme.lower()].append(attr_dict.get('content', '')) field = rmap_meta[x]
except KeyError:
elif tag == 'meta':
x = attr_dict.get('name', '').lower()
field = None
try: try:
field = self.rmap_meta[x] field = rmap_meta[x.replace(':', '.')]
except KeyError:
try:
field = self.rmap_meta[x.replace(':', '.')]
except KeyError:
pass
if field:
self.meta_tags[field].append(attr_dict.get('content', ''))
def handle_data(self, data):
if self.recording:
self.recorded.append(data)
def handle_charref(self, ref):
if self.recording:
self.recorded.append(replace_entities("&#%s;" % ref))
def handle_entityref(self, ref):
if self.recording:
self.recorded.append(replace_entities("&%s;" % ref))
def handle_endtag(self, tag):
if tag == 'title':
self.recording = False
self.title_tag = ''.join(self.recorded)
def handle_comment(self, data):
for match in re.finditer(r'''(?P<name>\S+)\s*=\s*%s''' % (attr_pat), data):
x = match.group('name')
field = None
try:
field = self.rmap_comment[x]
except KeyError: except KeyError:
pass pass
if field: if field:
self.comment_tags[field].append(replace_entities(match.group('content'))) meta_tags[field].append(content)
parser = MetadataParser() return comment_tags, meta_tags, meta_tag_ids, title
parser.feed(src)
return (parser.comment_tags, parser.meta_tags, parser.meta_tag_ids, parser.title_tag)
def get_metadata_(src, encoding=None): def get_metadata_(src, encoding=None):
@ -153,7 +138,7 @@ def get_metadata_(src, encoding=None):
else: else:
src = src.decode(encoding, 'replace') src = src.decode(encoding, 'replace')
src = src[:150000] # Searching shouldn't take too long src = src[:150000] # Searching shouldn't take too long
(comment_tags, meta_tags, meta_tag_ids, title_tag) = parse_metadata(src) comment_tags, meta_tags, meta_tag_ids, title_tag = parse_metadata(src)
def get_all(field): def get_all(field):
ans = comment_tags.get(field, meta_tags.get(field, None)) ans = comment_tags.get(field, meta_tags.get(field, None))
@ -329,7 +314,7 @@ class MetadataHtmlTest(unittest.TestCase):
<!-- SERIES="Comment Series" --> <!-- SERIES="Comment Series" -->
<!-- SERIESNUMBER="3" --> <!-- SERIESNUMBER="3" -->
<!-- RATING="20" --> <!-- RATING="20" -->
<!-- COMMENTS="comment &quot;comments&quot; &#x2665; HTML too &amp;amp;" --> <!-- COMMENTS="comment &quot;comments&quot; &#x2665; HTML -- too &amp;amp;" -->
<!-- TAGS="tag d" --> <!-- TAGS="tag d" -->
''' '''
@ -345,7 +330,7 @@ class MetadataHtmlTest(unittest.TestCase):
<!-- SERIES="Comment Series 2" --> <!-- SERIES="Comment Series 2" -->
<!-- SERIESNUMBER="4" --> <!-- SERIESNUMBER="4" -->
<!-- RATING="1" --> <!-- RATING="1" -->
<!-- COMMENTS="comment &quot;comments&quot; &#x2665; HTML too &amp;amp; for sure" --> <!-- COMMENTS="comment &quot;comments&quot; &#x2665; HTML -- too &amp;amp; for sure" -->
<!-- TAGS="tag e, tag f" --> <!-- TAGS="tag e, tag f" -->
''' '''
@ -402,7 +387,7 @@ class MetadataHtmlTest(unittest.TestCase):
canon_meta.series = 'Comment Series' canon_meta.series = 'Comment Series'
canon_meta.series_index = float(3) canon_meta.series_index = float(3)
canon_meta.rating = float(0) canon_meta.rating = float(0)
canon_meta.comments = 'comment &quot;comments&quot; ♥ HTML too &amp;amp;' canon_meta.comments = 'comment &quot;comments&quot; ♥ HTML -- too &amp;amp;'
canon_meta.tags = ['tag d'] canon_meta.tags = ['tag d']
canon_meta.set_identifiers({'isbn': '3456789012', 'url': 'http://google.com/search?q=calibre'}) canon_meta.set_identifiers({'isbn': '3456789012', 'url': 'http://google.com/search?q=calibre'})
self.compare_metadata(stream_meta, canon_meta) self.compare_metadata(stream_meta, canon_meta)
@ -417,19 +402,11 @@ class MetadataHtmlTest(unittest.TestCase):
canon_meta.series = 'Comment Series' canon_meta.series = 'Comment Series'
canon_meta.series_index = float(3) canon_meta.series_index = float(3)
canon_meta.rating = float(0) canon_meta.rating = float(0)
canon_meta.comments = 'comment &quot;comments&quot; ♥ HTML too &amp;amp;' canon_meta.comments = 'comment &quot;comments&quot; ♥ HTML -- too &amp;amp;'
canon_meta.tags = ['tag d', 'tag e', 'tag f'] canon_meta.tags = ['tag d', 'tag e', 'tag f']
canon_meta.set_identifiers({'isbn': '3456789012', 'url': 'http://google.com/search?q=calibre'}) canon_meta.set_identifiers({'isbn': '3456789012', 'url': 'http://google.com/search?q=calibre'})
self.compare_metadata(stream_meta, canon_meta) self.compare_metadata(stream_meta, canon_meta)
def suite(): def find_tests():
return unittest.TestLoader().loadTestsFromTestCase(MetadataHtmlTest) return unittest.TestLoader().loadTestsFromTestCase(MetadataHtmlTest)
def test():
unittest.TextTestRunner(verbosity=2).run(suite())
if __name__ == '__main__':
test()