mirror of
https://github.com/kovidgoyal/calibre.git
synced 2025-07-09 03:04:10 -04:00
pep8
This commit is contained in:
Kovid Goyal
parent
4f5155d190
commit
4ae11fa295
@ -1,23 +1,20 @@
|
||||
#!/usr/bin/env python2
|
||||
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
|
||||
from __future__ import (unicode_literals, division, absolute_import,
|
||||
print_function)
|
||||
# License: GPLv3 Copyright: 2011, Kovid Goyal <kovid at kovidgoyal.net>
|
||||
from __future__ import absolute_import, division, print_function, unicode_literals
|
||||
|
||||
__license__ = 'GPL v3'
|
||||
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
|
||||
__docformat__ = 'restructuredtext en'
|
||||
|
||||
import time, hashlib
|
||||
import hashlib
|
||||
import time
|
||||
from functools import partial
|
||||
from Queue import Queue, Empty
|
||||
from Queue import Empty, Queue
|
||||
|
||||
from calibre.ebooks.metadata import check_isbn
|
||||
from calibre.ebooks.metadata.sources.base import Source
|
||||
from calibre.ebooks.metadata.book.base import Metadata
|
||||
from calibre import as_unicode
|
||||
from calibre.ebooks.chardet import xml_to_unicode
|
||||
from calibre.ebooks.metadata import check_isbn
|
||||
from calibre.ebooks.metadata.book.base import Metadata
|
||||
from calibre.ebooks.metadata.sources.base import Source
|
||||
from calibre.utils.cleantext import clean_ascii_chars
|
||||
from calibre.utils.localization import canonicalize_lang
|
||||
from calibre import as_unicode
|
||||
|
||||
NAMESPACES = {
|
||||
'openSearch': 'http://a9.com/-/spec/opensearchrss/1.0/',
|
||||
@ -39,6 +36,8 @@ def get_details(browser, url, timeout): # {{{
|
||||
raw = browser.open_novisit(url, timeout=timeout).read()
|
||||
|
||||
return raw
|
||||
|
||||
|
||||
# }}}
|
||||
|
||||
|
||||
@ -86,8 +85,9 @@ def to_metadata(browser, log, entry_, timeout): # {{{
|
||||
mi.identifiers = {'google': google_id}
|
||||
try:
|
||||
raw = get_details(browser, id_url, timeout)
|
||||
feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
|
||||
strip_encoding_pats=True)[0])
|
||||
feed = etree.fromstring(
|
||||
xml_to_unicode(clean_ascii_chars(raw), strip_encoding_pats=True)[0]
|
||||
)
|
||||
extra = entry(feed)[0]
|
||||
except:
|
||||
log.exception('Failed to get additional details for', mi.title)
|
||||
@ -149,11 +149,14 @@ def to_metadata(browser, log, entry_, timeout): # {{{
|
||||
# Cover
|
||||
mi.has_google_cover = None
|
||||
for x in extra.xpath(
|
||||
'//*[@href and @rel="http://schemas.google.com/books/2008/thumbnail"]'):
|
||||
'//*[@href and @rel="http://schemas.google.com/books/2008/thumbnail"]'
|
||||
):
|
||||
mi.has_google_cover = x.get('href')
|
||||
break
|
||||
|
||||
return mi
|
||||
|
||||
|
||||
# }}}
|
||||
|
||||
|
||||
@ -162,21 +165,23 @@ class GoogleBooks(Source):
|
||||
name = 'Google'
|
||||
description = _('Downloads metadata and covers from Google Books')
|
||||
|
||||
capabilities = frozenset(['identify', 'cover'])
|
||||
touched_fields = frozenset(['title', 'authors', 'tags', 'pubdate',
|
||||
'comments', 'publisher', 'identifier:isbn', 'rating',
|
||||
'identifier:google', 'languages'])
|
||||
capabilities = frozenset({'identify', 'cover'})
|
||||
touched_fields = frozenset({
|
||||
'title', 'authors', 'tags', 'pubdate', 'comments', 'publisher',
|
||||
'identifier:isbn', 'rating', 'identifier:google', 'languages'
|
||||
})
|
||||
supports_gzip_transfer_encoding = True
|
||||
cached_cover_url_is_reliable = False
|
||||
|
||||
GOOGLE_COVER = 'https://books.google.com/books?id=%s&printsec=frontcover&img=1'
|
||||
|
||||
DUMMY_IMAGE_MD5 = frozenset(['0de4383ebad0adad5eeb8975cd796657'])
|
||||
DUMMY_IMAGE_MD5 = frozenset({'0de4383ebad0adad5eeb8975cd796657'})
|
||||
|
||||
def get_book_url(self, identifiers): # {{{
|
||||
goog = identifiers.get('google', None)
|
||||
if goog is not None:
|
||||
return ('google', goog, 'https://books.google.com/books?id=%s' % goog)
|
||||
|
||||
# }}}
|
||||
|
||||
def create_query(self, log, title=None, authors=None, identifiers={}): # {{{
|
||||
@ -187,16 +192,16 @@ class GoogleBooks(Source):
|
||||
if isbn is not None:
|
||||
q += 'isbn:' + isbn
|
||||
elif title or authors:
|
||||
|
||||
def build_term(prefix, parts):
|
||||
return ' '.join('in' + prefix + ':' + x for x in parts)
|
||||
|
||||
title_tokens = list(self.get_title_tokens(title))
|
||||
if title_tokens:
|
||||
q += build_term('title', title_tokens)
|
||||
author_tokens = self.get_author_tokens(authors,
|
||||
only_first_author=True)
|
||||
author_tokens = self.get_author_tokens(authors, only_first_author=True)
|
||||
if author_tokens:
|
||||
q += ('+' if q else '') + build_term('author',
|
||||
author_tokens)
|
||||
q += ('+' if q else '') + build_term('author', author_tokens)
|
||||
|
||||
if isinstance(q, unicode):
|
||||
q = q.encode('utf-8')
|
||||
@ -208,16 +213,32 @@ class GoogleBooks(Source):
|
||||
'start-index': 1,
|
||||
'min-viewability': 'none',
|
||||
})
|
||||
|
||||
# }}}
|
||||
|
||||
def download_cover(self, log, result_queue, abort, # {{{
|
||||
title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False):
|
||||
def download_cover( # {{{
|
||||
self,
|
||||
log,
|
||||
result_queue,
|
||||
abort,
|
||||
title=None,
|
||||
authors=None,
|
||||
identifiers={},
|
||||
timeout=30,
|
||||
get_best_cover=False
|
||||
):
|
||||
cached_url = self.get_cached_cover_url(identifiers)
|
||||
if cached_url is None:
|
||||
log.info('No cached cover found, running identify')
|
||||
rq = Queue()
|
||||
self.identify(log, rq, abort, title=title, authors=authors,
|
||||
identifiers=identifiers)
|
||||
self.identify(
|
||||
log,
|
||||
rq,
|
||||
abort,
|
||||
title=title,
|
||||
authors=authors,
|
||||
identifiers=identifiers
|
||||
)
|
||||
if abort.is_set():
|
||||
return
|
||||
results = []
|
||||
@ -226,8 +247,11 @@ class GoogleBooks(Source):
|
||||
results.append(rq.get_nowait())
|
||||
except Empty:
|
||||
break
|
||||
results.sort(key=self.identify_results_keygen(
|
||||
title=title, authors=authors, identifiers=identifiers))
|
||||
results.sort(
|
||||
key=self.identify_results_keygen(
|
||||
title=title, authors=authors, identifiers=identifiers
|
||||
)
|
||||
)
|
||||
for mi in results:
|
||||
cached_url = self.get_cached_cover_url(mi.identifiers)
|
||||
if cached_url is not None:
|
||||
@ -263,10 +287,18 @@ class GoogleBooks(Source):
|
||||
url = self.cached_identifier_to_cover_url(goog)
|
||||
|
||||
return url
|
||||
|
||||
# }}}
|
||||
|
||||
def get_all_details(self, br, log, entries, abort, # {{{
|
||||
result_queue, timeout):
|
||||
def get_all_details( # {{{
|
||||
self,
|
||||
br,
|
||||
log,
|
||||
entries,
|
||||
abort,
|
||||
result_queue,
|
||||
timeout
|
||||
):
|
||||
from lxml import etree
|
||||
for relevance, i in enumerate(entries):
|
||||
try:
|
||||
@ -277,26 +309,37 @@ class GoogleBooks(Source):
|
||||
for isbn in getattr(ans, 'all_isbns', []):
|
||||
self.cache_isbn_to_identifier(isbn, goog)
|
||||
if getattr(ans, 'has_google_cover', False):
|
||||
self.cache_identifier_to_cover_url(goog,
|
||||
self.GOOGLE_COVER%goog)
|
||||
self.cache_identifier_to_cover_url(
|
||||
goog, self.GOOGLE_COVER % goog
|
||||
)
|
||||
self.clean_downloaded_metadata(ans)
|
||||
result_queue.put(ans)
|
||||
except:
|
||||
log.exception(
|
||||
'Failed to get metadata for identify entry:',
|
||||
etree.tostring(i))
|
||||
'Failed to get metadata for identify entry:', etree.tostring(i)
|
||||
)
|
||||
if abort.is_set():
|
||||
break
|
||||
|
||||
# }}}
|
||||
|
||||
def identify(self, log, result_queue, abort, title=None, authors=None, # {{{
|
||||
identifiers={}, timeout=30):
|
||||
def identify( # {{{
|
||||
self,
|
||||
log,
|
||||
result_queue,
|
||||
abort,
|
||||
title=None,
|
||||
authors=None,
|
||||
identifiers={},
|
||||
timeout=30
|
||||
):
|
||||
from lxml import etree
|
||||
XPath = partial(etree.XPath, namespaces=NAMESPACES)
|
||||
entry = XPath('//atom:entry')
|
||||
|
||||
query = self.create_query(log, title=title, authors=authors,
|
||||
identifiers=identifiers)
|
||||
query = self.create_query(
|
||||
log, title=title, authors=authors, identifiers=identifiers
|
||||
)
|
||||
if not query:
|
||||
log.error('Insufficient metadata to construct query')
|
||||
return
|
||||
@ -309,8 +352,10 @@ class GoogleBooks(Source):
|
||||
|
||||
try:
|
||||
parser = etree.XMLParser(recover=True, no_network=True)
|
||||
feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw),
|
||||
strip_encoding_pats=True)[0], parser=parser)
|
||||
feed = etree.fromstring(
|
||||
xml_to_unicode(clean_ascii_chars(raw), strip_encoding_pats=True)[0],
|
||||
parser=parser
|
||||
)
|
||||
entries = entry(feed)
|
||||
except Exception as e:
|
||||
log.exception('Failed to parse identify results')
|
||||
@ -318,34 +363,45 @@ class GoogleBooks(Source):
|
||||
|
||||
if not entries and identifiers and title and authors and \
|
||||
not abort.is_set():
|
||||
return self.identify(log, result_queue, abort, title=title,
|
||||
authors=authors, timeout=timeout)
|
||||
return self.identify(
|
||||
log,
|
||||
result_queue,
|
||||
abort,
|
||||
title=title,
|
||||
authors=authors,
|
||||
timeout=timeout
|
||||
)
|
||||
|
||||
# There is no point running these queries in threads as google
|
||||
# throttles requests returning 403 Forbidden errors
|
||||
self.get_all_details(br, log, entries, abort, result_queue, timeout)
|
||||
|
||||
return None
|
||||
|
||||
# }}}
|
||||
|
||||
|
||||
if __name__ == '__main__': # tests {{{
|
||||
# To run these test use: calibre-debug -e src/calibre/ebooks/metadata/sources/google.py
|
||||
from calibre.ebooks.metadata.sources.test import (test_identify_plugin,
|
||||
title_test, authors_test)
|
||||
test_identify_plugin(GoogleBooks.name,
|
||||
[
|
||||
|
||||
|
||||
(
|
||||
{'identifiers':{'isbn': '0743273567'}, 'title':'Great Gatsby',
|
||||
'authors':['Fitzgerald']},
|
||||
[title_test('The great gatsby', exact=True),
|
||||
authors_test(['F. Scott Fitzgerald'])]
|
||||
),
|
||||
|
||||
(
|
||||
{'title': 'Flatland', 'authors':['Abbott']},
|
||||
[title_test('Flatland', exact=False)]
|
||||
),
|
||||
])
|
||||
from calibre.ebooks.metadata.sources.test import (
|
||||
test_identify_plugin, title_test, authors_test
|
||||
)
|
||||
test_identify_plugin(
|
||||
GoogleBooks.name, [
|
||||
({
|
||||
'identifiers': {
|
||||
'isbn': '0743273567'
|
||||
},
|
||||
'title': 'Great Gatsby',
|
||||
'authors': ['Fitzgerald']
|
||||
}, [
|
||||
title_test('The great gatsby', exact=True),
|
||||
authors_test(['F. Scott Fitzgerald'])
|
||||
]),
|
||||
({
|
||||
'title': 'Flatland',
|
||||
'authors': ['Abbott']
|
||||
}, [title_test('Flatland', exact=False)]),
|
||||
]
|
||||
)
|
||||
# }}}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user