From cbaa443e065841705c8288281368c8e1fd52568b Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Feb 2011 13:09:59 -0700 Subject: [PATCH 1/5] EPUB Output: Do not set the file-as attribute on title elements in the OPF as the current OPF spec does not support file-as. Instead use a calibre extension to OPF. Fixes #9109 (Calibre Inserts "file-as" for author sort name and title sort name) --- src/calibre/ebooks/oeb/base.py | 2 +- src/calibre/ebooks/oeb/transforms/metadata.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 9389964962..ccc452f1f8 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -515,7 +515,7 @@ class Metadata(object): 'publisher', 'relation', 'rights', 'source', 'subject', 'title', 'type']) CALIBRE_TERMS = set(['series', 'series_index', 'rating', 'timestamp', - 'publication_type']) + 'publication_type', 'title_sort']) OPF_ATTRS = {'role': OPF('role'), 'file-as': OPF('file-as'), 'scheme': OPF('scheme'), 'event': OPF('event'), 'type': XSI('type'), 'lang': XML('lang'), 'id': 'id'} diff --git a/src/calibre/ebooks/oeb/transforms/metadata.py b/src/calibre/ebooks/oeb/transforms/metadata.py index f1ce31f25b..19c209b74d 100644 --- a/src/calibre/ebooks/oeb/transforms/metadata.py +++ b/src/calibre/ebooks/oeb/transforms/metadata.py @@ -18,7 +18,8 @@ def meta_info_to_oeb_metadata(mi, m, log, override_input_metadata=False): if mi.title_sort: if not m.title: m.add('title', mi.title_sort) - m.title[0].file_as = mi.title_sort + m.clear('title_sort') + m.add('title_sort', mi.title_sort) if not mi.is_null('authors'): m.filter('creator', lambda x : x.role.lower() in ['aut', '']) for a in mi.authors: From dee216e6bb3eed740613efea283c6e1b61d77cb1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Feb 2011 15:13:14 -0700 Subject: [PATCH 2/5] Support for the Archos 7 tablet --- src/calibre/devices/android/driver.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 6f77a4d96c..99679283a7 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -60,7 +60,12 @@ class ANDROID(USBMS): 0x1004 : { 0x61cc : [0x100] }, # Archos - 0x0e79 : { 0x1419: [0x0216], 0x1420 : [0x0216], 0x1422 : [0x0216]}, + 0x0e79 : { + 0x1400 : [0x0222, 0x0216], + 0x1419 : [0x0216], + 0x1420 : [0x0216], + 0x1422 : [0x0216] + }, # Huawei # Disabled as this USB id is used by various USB flash drives @@ -84,10 +89,10 @@ class ANDROID(USBMS): 'GT-I9000', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'SCH-I500_CARD', 'SPH-D700_CARD', 'MB810', 'GT-P1000', 'DESIRE', 'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H', - 'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD'] + 'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD', '7'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', - 'A70S', 'A101IT'] + 'A70S', 'A101IT', '7'] OSX_MAIN_MEM = 'Android Device Main Memory' From 6460f08b7fbb2365ecd1542bedb6ec55687297c0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Feb 2011 20:17:57 -0700 Subject: [PATCH 3/5] Only start emailer thread on demand --- src/calibre/gui2/email.py | 5 ++++- src/calibre/gui2/ui.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index 426747e044..c84b3180f7 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -209,7 +209,6 @@ class EmailMixin(object): # {{{ def __init__(self): self.emailer = Emailer(self.job_manager) - self.emailer.start() def send_by_mail(self, to, fmts, delete_from_library, send_ids=None, do_auto_convert=True, specific_format=None): @@ -255,6 +254,8 @@ class EmailMixin(object): # {{{ to_s = list(repeat(to, len(attachments))) if attachments: + if not self.emailer.is_alive(): + self.emailer.start() self.emailer.send_mails(jobnames, Dispatcher(partial(self.email_sent, remove=remove)), attachments, to_s, subjects, texts, attachment_names) @@ -325,6 +326,8 @@ class EmailMixin(object): # {{{ files, auto = self.library_view.model().\ get_preferred_formats_from_ids([id_], fmts) return files + if not self.emailer.is_alive(): + self.emailer.start() sent_mails = self.emailer.email_news(mi, remove, get_fmts, self.email_sent) if sent_mails: diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 9b9308d253..8844446de6 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -633,7 +633,8 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ mb.stop() self.hide_windows() - self.emailer.stop() + if self.emailer.is_alive(): + self.emailer.stop() try: try: if self.content_server is not None: From 52c19d7b9bdd15bf74e84744f61a5b22c151fdb0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Feb 2011 20:34:06 -0700 Subject: [PATCH 4/5] caching of google identifiers and logic to get cover url from google identifier --- src/calibre/ebooks/metadata/sources/base.py | 16 ++++- src/calibre/ebooks/metadata/sources/google.py | 66 +++++++++++++------ 2 files changed, 62 insertions(+), 20 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 74e184cc66..54d7d49d6d 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -7,7 +7,7 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import re +import re, threading from calibre.customize import Plugin from calibre.utils.logging import ThreadSafeLog, FileStream @@ -30,7 +30,21 @@ class Source(Plugin): touched_fields = frozenset() + def __init__(self, *args, **kwargs): + Plugin.__init__(self, *args, **kwargs) + self._isbn_to_identifier_cache = {} + self.cache_lock = threading.RLock() + # Utility functions {{{ + + def cache_isbn_to_identifier(self, isbn, identifier): + with self.cache_lock: + self._isbn_to_identifier_cache[isbn] = identifier + + def cached_isbn_to_identifier(self, isbn): + with self.cache_lock: + return self._isbn_to_identifier_cache.get(isbn, None) + def get_author_tokens(self, authors, only_first_author=True): ''' Take a list of authors and return a list of tokens useful for an diff --git a/src/calibre/ebooks/metadata/sources/google.py b/src/calibre/ebooks/metadata/sources/google.py index 498c7574ea..0720b21ded 100644 --- a/src/calibre/ebooks/metadata/sources/google.py +++ b/src/calibre/ebooks/metadata/sources/google.py @@ -13,6 +13,7 @@ from functools import partial from lxml import etree +from calibre.ebooks.metadata import check_isbn from calibre.ebooks.metadata.sources.base import Source from calibre.ebooks.metadata.book.base import Metadata from calibre.ebooks.chardet import xml_to_unicode @@ -69,6 +70,7 @@ def to_metadata(browser, log, entry_, timeout): id_url = entry_id(entry_)[0].text + google_id = id_url.split('/')[-1] title_ = ': '.join([x.text for x in title(entry_)]).strip() authors = [x.text.strip() for x in creator(entry_) if x.text] if not authors: @@ -78,6 +80,7 @@ def to_metadata(browser, log, entry_, timeout): return None mi = Metadata(title_, authors) + mi.identifiers = {'google':google_id} try: raw = get_details(browser, id_url, timeout) feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw), @@ -103,9 +106,12 @@ def to_metadata(browser, log, entry_, timeout): t = str(x.text).strip() if t[:5].upper() in ('ISBN:', 'LCCN:', 'OCLC:'): if t[:5].upper() == 'ISBN:': - isbns.append(t[5:]) + t = check_isbn(t[5:]) + if t: + isbns.append(t) if isbns: mi.isbn = sorted(isbns, key=len)[-1] + mi.all_isbns = isbns # Tags try: @@ -133,20 +139,6 @@ def to_metadata(browser, log, entry_, timeout): return mi -def get_all_details(br, log, entries, abort, result_queue, timeout): - for i in entries: - try: - ans = to_metadata(br, log, i, timeout) - if isinstance(ans, Metadata): - result_queue.put(ans) - except: - log.exception( - 'Failed to get metadata for identify entry:', - etree.tostring(i)) - if abort.is_set(): - break - - class GoogleBooks(Source): name = 'Google Books' @@ -185,6 +177,36 @@ class GoogleBooks(Source): 'min-viewability':'none', }) + def cover_url_from_identifiers(self, identifiers): + goog = identifiers.get('google', None) + if goog is None: + isbn = identifiers.get('isbn', None) + goog = self.cached_isbn_to_identifier(isbn) + if goog is not None: + return ('http://books.google.com/books?id=%s&printsec=frontcover&img=1' % + goog) + + def is_cover_image_valid(self, raw): + # When no cover is present, returns a PNG saying image not available + # Try for example google identifier llNqPwAACAAJ + # I have yet to see an actual cover in PNG format + return raw and len(raw) > 17000 and raw[1:4] != 'PNG' + + def get_all_details(self, br, log, entries, abort, result_queue, timeout): + for i in entries: + try: + ans = to_metadata(br, log, i, timeout) + if isinstance(ans, Metadata): + result_queue.put(ans) + for isbn in ans.all_isbns: + self.cache_isbn_to_identifier(isbn, + ans.identifiers['google']) + except: + log.exception( + 'Failed to get metadata for identify entry:', + etree.tostring(i)) + if abort.is_set(): + break def identify(self, log, result_queue, abort, title=None, authors=None, identifiers={}, timeout=5): @@ -207,8 +229,8 @@ class GoogleBooks(Source): return as_unicode(e) # There is no point running these queries in threads as google - # throttles requests returning Forbidden errors - get_all_details(br, log, entries, abort, result_queue, timeout) + # throttles requests returning 403 Forbidden errors + self.get_all_details(br, log, entries, abort, result_queue, timeout) return None @@ -218,8 +240,14 @@ if __name__ == '__main__': title_test) test_identify_plugin(GoogleBooks.name, [ + ( - {'title': 'Great Expectations', 'authors':['Charles Dickens']}, - [title_test('Great Expectations', exact=True)] + {'identifiers':{'isbn': '0743273567'}}, + [title_test('The great gatsby', exact=True)] ), + + #( + # {'title': 'Great Expectations', 'authors':['Charles Dickens']}, + # [title_test('Great Expectations', exact=True)] + #), ]) From 09d0c75755f07d063b20783c1ae54ed8b616db05 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Tue, 22 Feb 2011 22:38:00 -0700 Subject: [PATCH 5/5] ... --- src/calibre/gui2/dialogs/drm_error.ui | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/dialogs/drm_error.ui b/src/calibre/gui2/dialogs/drm_error.ui index ff28ef5a48..c4b9a1cfdb 100644 --- a/src/calibre/gui2/dialogs/drm_error.ui +++ b/src/calibre/gui2/dialogs/drm_error.ui @@ -44,7 +44,8 @@ <p>This book is locked by <b>DRM</b>. To learn more about DRM and why you cannot read or convert this book in calibre, -<a href="http://bugs.calibre-ebook.com/wiki/DRM">click here</a>. + <a href="http://drmfree.calibre-ebook.com/about#drm">click here</a>.<p>A large number of recent, DRM free releases are + available at <a href="http://drmfree.calibre-ebook.com">Open Books</a>. true