From 571f50be431f69504a0f72366be298ec84e5dc90 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 30 Apr 2011 13:14:23 -0600 Subject: [PATCH] Get rid of the test_eight_code tweak. That means the next release will be 0.8.0! --- src/calibre/customize/builtins.py | 36 +- src/calibre/customize/ui.py | 44 +- src/calibre/ebooks/metadata/amazon.py | 224 ---- src/calibre/ebooks/metadata/amazonfr.py | 516 ---------- src/calibre/ebooks/metadata/covers.py | 317 ------ src/calibre/ebooks/metadata/douban.py | 263 ----- src/calibre/ebooks/metadata/fetch.py | 523 ---------- src/calibre/ebooks/metadata/fictionwise.py | 390 ------- src/calibre/ebooks/metadata/google_books.py | 247 ----- src/calibre/ebooks/metadata/isbndb.py | 159 --- src/calibre/ebooks/metadata/nicebooks.py | 411 -------- src/calibre/ebooks/metadata/sources/base.py | 2 +- src/calibre/ebooks/metadata/sources/cli.py | 8 - src/calibre/gui2/actions/add.py | 34 +- src/calibre/gui2/actions/edit_metadata.py | 104 +- src/calibre/gui2/dialogs/fetch_metadata.py | 271 ----- src/calibre/gui2/dialogs/fetch_metadata.ui | 179 ---- src/calibre/gui2/dialogs/metadata_single.py | 1031 ------------------- src/calibre/gui2/dialogs/metadata_single.ui | 937 ----------------- src/calibre/gui2/metadata/bulk_download.py | 465 ++++----- src/calibre/gui2/metadata/bulk_download2.py | 195 ---- src/calibre/gui2/preferences/behavior.py | 17 +- src/calibre/gui2/preferences/behavior.ui | 51 +- src/calibre/gui2/preferences/social.py | 79 -- src/calibre/utils/config.py | 2 - 25 files changed, 222 insertions(+), 6283 deletions(-) delete mode 100644 src/calibre/ebooks/metadata/amazon.py delete mode 100644 src/calibre/ebooks/metadata/amazonfr.py delete mode 100644 src/calibre/ebooks/metadata/covers.py delete mode 100644 src/calibre/ebooks/metadata/douban.py delete mode 100644 src/calibre/ebooks/metadata/fetch.py delete mode 100644 src/calibre/ebooks/metadata/fictionwise.py delete mode 100644 src/calibre/ebooks/metadata/google_books.py delete mode 100644 src/calibre/ebooks/metadata/isbndb.py delete mode 100644 src/calibre/ebooks/metadata/nicebooks.py delete mode 100644 src/calibre/gui2/dialogs/fetch_metadata.py delete mode 100644 src/calibre/gui2/dialogs/fetch_metadata.ui delete mode 100644 src/calibre/gui2/dialogs/metadata_single.py delete mode 100644 src/calibre/gui2/dialogs/metadata_single.ui delete mode 100644 src/calibre/gui2/metadata/bulk_download2.py delete mode 100644 src/calibre/gui2/preferences/social.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index b3bca01c5b..36bcbdbfe2 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -9,7 +9,6 @@ from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \ from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf -from calibre.utils.config import test_eight_code # To archive plugins {{{ class HTML2ZIP(FileTypePlugin): @@ -621,28 +620,16 @@ from calibre.ebooks.epub.fix.epubcheck import Epubcheck plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested, Epubcheck, ] -if test_eight_code: # New metadata download plugins {{{ - from calibre.ebooks.metadata.sources.google import GoogleBooks - from calibre.ebooks.metadata.sources.amazon import Amazon - from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary - from calibre.ebooks.metadata.sources.isbndb import ISBNDB - from calibre.ebooks.metadata.sources.overdrive import OverDrive +from calibre.ebooks.metadata.sources.google import GoogleBooks +from calibre.ebooks.metadata.sources.amazon import Amazon +from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary +from calibre.ebooks.metadata.sources.isbndb import ISBNDB +from calibre.ebooks.metadata.sources.overdrive import OverDrive - plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive] +plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive] # }}} -else: - from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \ - KentDistrictLibrary - from calibre.ebooks.metadata.douban import DoubanBooks - from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers - from calibre.ebooks.metadata.covers import OpenLibraryCovers, \ - AmazonCovers, DoubanCovers - - plugins += [GoogleBooks, ISBNDB, Amazon, - OpenLibraryCovers, AmazonCovers, DoubanCovers, - NiceBooksCovers, KentDistrictLibrary, DoubanBooks, NiceBooks] plugins += [ ComicInput, @@ -867,10 +854,7 @@ plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, ActionRestart, ActionOpenFolder, ActionConnectShare, ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks, ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary, - ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch] - -if test_eight_code: - plugins += [ActionStore] + ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore] # }}} @@ -1096,10 +1080,8 @@ class Misc(PreferencesPlugin): plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions, CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard, - Email, Server, Plugins, Tweaks, Misc, TemplateFunctions] - -if test_eight_code: - plugins.append(MetadataSources) + Email, Server, Plugins, Tweaks, Misc, TemplateFunctions, + MetadataSources] #}}} diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index d3ecab7f16..151235cef9 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -15,10 +15,8 @@ from calibre.customize.profiles import InputProfile, OutputProfile from calibre.customize.builtins import plugins as builtin_plugins from calibre.devices.interface import DevicePlugin from calibre.ebooks.metadata import MetaInformation -from calibre.ebooks.metadata.covers import CoverDownload -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.utils.config import make_config_dir, Config, ConfigProxy, \ - plugin_dir, OptionParser, prefs +from calibre.utils.config import (make_config_dir, Config, ConfigProxy, + plugin_dir, OptionParser) from calibre.ebooks.epub.fix import ePubFixer from calibre.ebooks.metadata.sources.base import Source @@ -190,44 +188,6 @@ def output_profiles(): yield plugin # }}} -# Metadata sources {{{ -def metadata_sources(metadata_type='basic', customize=True, isbndb_key=None): - for plugin in _initialized_plugins: - if isinstance(plugin, MetadataSource) and \ - plugin.metadata_type == metadata_type: - if is_disabled(plugin): - continue - if customize: - customization = config['plugin_customization'] - plugin.site_customization = customization.get(plugin.name, None) - if plugin.name == 'IsbnDB' and isbndb_key is not None: - plugin.site_customization = isbndb_key - yield plugin - -def get_isbndb_key(): - return config['plugin_customization'].get('IsbnDB', None) - -def set_isbndb_key(key): - for plugin in _initialized_plugins: - if plugin.name == 'IsbnDB': - return customize_plugin(plugin, key) - -def migrate_isbndb_key(): - key = prefs['isbndb_com_key'] - if key: - prefs.set('isbndb_com_key', '') - set_isbndb_key(key) - -def cover_sources(): - customization = config['plugin_customization'] - for plugin in _initialized_plugins: - if isinstance(plugin, CoverDownload): - if not is_disabled(plugin): - plugin.site_customization = customization.get(plugin.name, '') - yield plugin - -# }}} - # Interface Actions # {{{ def interface_actions(): diff --git a/src/calibre/ebooks/metadata/amazon.py b/src/calibre/ebooks/metadata/amazon.py deleted file mode 100644 index 4100439feb..0000000000 --- a/src/calibre/ebooks/metadata/amazon.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Fetch metadata using Amazon AWS -''' -import sys, re -from threading import RLock - -from lxml import html -from lxml.html import soupparser - -from calibre import browser -from calibre.ebooks.metadata import check_isbn -from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.chardet import xml_to_unicode -from calibre.library.comments import sanitize_comments_html - -asin_cache = {} -cover_url_cache = {} -cache_lock = RLock() - -def find_asin(br, isbn): - q = 'http://www.amazon.com/s/?search-alias=aps&field-keywords='+isbn - res = br.open_novisit(q) - raw = res.read() - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - root = html.fromstring(raw) - revs = root.xpath('//*[@class="asinReviewsSummary" and @name]') - revs = [x.get('name') for x in revs] - if revs: - return revs[0] - -def to_asin(br, isbn): - with cache_lock: - ans = asin_cache.get(isbn, None) - if ans: - return ans - if ans is False: - return None - if len(isbn) == 13: - try: - asin = find_asin(br, isbn) - except: - import traceback - traceback.print_exc() - asin = None - else: - asin = isbn - with cache_lock: - asin_cache[isbn] = asin if asin else False - return asin - - -def get_social_metadata(title, authors, publisher, isbn): - mi = Metadata(title, authors) - if not isbn: - return mi - isbn = check_isbn(isbn) - if not isbn: - return mi - br = browser() - asin = to_asin(br, isbn) - if asin and get_metadata(br, asin, mi): - return mi - from calibre.ebooks.metadata.xisbn import xisbn - for i in xisbn.get_associated_isbns(isbn): - asin = to_asin(br, i) - if asin and get_metadata(br, asin, mi): - return mi - return mi - -def get_cover_url(isbn, br): - isbn = check_isbn(isbn) - if not isbn: - return None - with cache_lock: - ans = cover_url_cache.get(isbn, None) - if ans: - return ans - if ans is False: - return None - asin = to_asin(br, isbn) - if asin: - ans = _get_cover_url(br, asin) - if ans: - with cache_lock: - cover_url_cache[isbn] = ans - return ans - from calibre.ebooks.metadata.xisbn import xisbn - for i in xisbn.get_associated_isbns(isbn): - asin = to_asin(br, i) - if asin: - ans = _get_cover_url(br, asin) - if ans: - with cache_lock: - cover_url_cache[isbn] = ans - cover_url_cache[i] = ans - return ans - with cache_lock: - cover_url_cache[isbn] = False - return None - -def _get_cover_url(br, asin): - q = 'http://amzn.com/'+asin - try: - raw = br.open_novisit(q).read() - except Exception as e: - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return None - raise - if '404 - ' in raw: - return None - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - root = soupparser.fromstring(raw) - except: - return False - - imgs = root.xpath('//img[@id="prodImage" and @src]') - if imgs: - src = imgs[0].get('src') - parts = src.split('/') - if len(parts) > 3: - bn = parts[-1] - sparts = bn.split('_') - if len(sparts) > 2: - bn = sparts[0] + sparts[-1] - return ('/'.join(parts[:-1]))+'/'+bn - return None - - -def get_metadata(br, asin, mi): - q = 'http://amzn.com/'+asin - try: - raw = br.open_novisit(q).read() - except Exception as e: - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return False - raise - if '<title>404 - ' in raw: - return False - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - root = soupparser.fromstring(raw) - except: - return False - if root.xpath('//*[@id="errorMessage"]'): - return False - - ratings = root.xpath('//div[@class="jumpBar"]/descendant::span[@class="asinReviewsSummary"]') - pat = re.compile(r'([0-9.]+) out of (\d+) stars') - if ratings: - for elem in ratings[0].xpath('descendant::*[@title]'): - t = elem.get('title').strip() - m = pat.match(t) - if m is not None: - try: - mi.rating = float(m.group(1))/float(m.group(2)) * 5 - except: - pass - - desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]') - if desc: - desc = desc[0] - for c in desc.xpath('descendant::*[@class="seeAll" or' - ' @class="emptyClear" or @href]'): - c.getparent().remove(c) - desc = html.tostring(desc, method='html', encoding=unicode).strip() - # remove all attributes from tags - desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc) - # Collapse whitespace - #desc = re.sub('\n+', '\n', desc) - #desc = re.sub(' +', ' ', desc) - # Remove the notice about text referring to out of print editions - desc = re.sub(r'(?s)<em>--This text ref.*?</em>', '', desc) - # Remove comments - desc = re.sub(r'(?s)<!--.*?-->', '', desc) - mi.comments = sanitize_comments_html(desc) - - return True - - -def main(args=sys.argv): - import tempfile, os - tdir = tempfile.gettempdir() - br = browser() - for title, isbn in [ - ('The Heroes', '9780316044981'), # Test find_asin - ('Learning Python', '8324616489'), # Test xisbn - ('Angels & Demons', '9781416580829'), # Test sophisticated comment formatting - # Random tests - ('Star Trek: Destiny: Mere Mortals', '9781416551720'), - ('The Great Gatsby', '0743273567'), - ]: - cpath = os.path.join(tdir, title+'.jpg') - curl = get_cover_url(isbn, br) - if curl is None: - print 'No cover found for', title - else: - open(cpath, 'wb').write(br.open_novisit(curl).read()) - print 'Cover for', title, 'saved to', cpath - - #import time - #st = time.time() - mi = get_social_metadata(title, None, None, isbn) - if not mi.comments: - print 'Failed to downlaod social metadata for', title - return 1 - #print '\n\n', time.time() - st, '\n\n' - print mi - print '\n' - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/amazonfr.py b/src/calibre/ebooks/metadata/amazonfr.py deleted file mode 100644 index 248c8d9ed0..0000000000 --- a/src/calibre/ebooks/metadata/amazonfr.py +++ /dev/null @@ -1,516 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2010, sengian <sengian1@gmail.com>' - -import sys, textwrap, re, traceback -from urllib import urlencode -from math import ceil - -from lxml import html -from lxml.html import soupparser - -from calibre.utils.date import parse_date, utcnow, replace_months -from calibre.utils.cleantext import clean_ascii_chars -from calibre import browser, preferred_encoding -from calibre.ebooks.chardet import xml_to_unicode -from calibre.ebooks.metadata import MetaInformation, check_isbn, \ - authors_to_sort_string -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.utils.config import OptionParser -from calibre.library.comments import sanitize_comments_html - - -class AmazonFr(MetadataSource): - - name = 'Amazon French' - description = _('Downloads metadata from amazon.fr') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='fr') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class AmazonEs(MetadataSource): - - name = 'Amazon Spanish' - description = _('Downloads metadata from amazon.com in spanish') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='es') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class AmazonEn(MetadataSource): - - name = 'Amazon English' - description = _('Downloads metadata from amazon.com in english') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='en') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class AmazonDe(MetadataSource): - - name = 'Amazon German' - description = _('Downloads metadata from amazon.de') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='de') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class Amazon(MetadataSource): - - name = 'Amazon' - description = _('Downloads metadata from amazon.com') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Kovid Goyal & Sengian' - version = (1, 1, 0) - has_html_comments = True - - def fetch(self): - # if not self.site_customization: - # return - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='all') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - # @property - # def string_customization_help(self): - # return _('You can select here the language for metadata search with amazon.com') - - -def report(verbose): - if verbose: - traceback.print_exc() - - -class Query(object): - - BASE_URL_ALL = 'http://www.amazon.com' - BASE_URL_FR = 'http://www.amazon.fr' - BASE_URL_DE = 'http://www.amazon.de' - - def __init__(self, title=None, author=None, publisher=None, isbn=None, keywords=None, - max_results=20, rlang='all'): - assert not(title is None and author is None and publisher is None \ - and isbn is None and keywords is None) - assert (max_results < 21) - - self.max_results = int(max_results) - self.renbres = re.compile(u'\s*(\d+)\s*') - - q = { 'search-alias' : 'stripbooks' , - 'unfiltered' : '1', - 'field-keywords' : '', - 'field-author' : '', - 'field-title' : '', - 'field-isbn' : '', - 'field-publisher' : '' - #get to amazon detailed search page to get all options - # 'node' : '', - # 'field-binding' : '', - #before, during, after - # 'field-dateop' : '', - #month as number - # 'field-datemod' : '', - # 'field-dateyear' : '', - #french only - # 'field-collection' : '', - #many options available - } - - if rlang =='all': - q['sort'] = 'relevanceexprank' - self.urldata = self.BASE_URL_ALL - elif rlang =='es': - q['sort'] = 'relevanceexprank' - q['field-language'] = 'Spanish' - self.urldata = self.BASE_URL_ALL - elif rlang =='en': - q['sort'] = 'relevanceexprank' - q['field-language'] = 'English' - self.urldata = self.BASE_URL_ALL - elif rlang =='fr': - q['sort'] = 'relevancerank' - self.urldata = self.BASE_URL_FR - elif rlang =='de': - q['sort'] = 'relevancerank' - self.urldata = self.BASE_URL_DE - self.baseurl = self.urldata - - if isbn is not None: - q['field-isbn'] = isbn.replace('-', '') - else: - if title is not None: - q['field-title'] = title - if author is not None: - q['field-author'] = author - if publisher is not None: - q['field-publisher'] = publisher - if keywords is not None: - q['field-keywords'] = keywords - - if isinstance(q, unicode): - q = q.encode('utf-8') - self.urldata += '/gp/search/ref=sr_adv_b/?' + urlencode(q) - - def __call__(self, browser, verbose, timeout = 5.): - if verbose: - print 'Query:', self.urldata - - try: - raw = browser.open_novisit(self.urldata, timeout=timeout).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - raise - if '<title>404 - ' in raw: - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None, self.urldata - - #nb of page - try: - nbresults = self.renbres.findall(feed.xpath("//*[@class='resultCount']")[0].text) - except: - return None, self.urldata - - pages =[feed] - if len(nbresults) > 1: - nbpagetoquery = int(ceil(float(min(int(nbresults[2]), self.max_results))/ int(nbresults[1]))) - for i in xrange(2, nbpagetoquery + 1): - try: - urldata = self.urldata + '&page=' + str(i) - raw = browser.open_novisit(urldata, timeout=timeout).read() - except Exception as e: - continue - if '<title>404 - ' in raw: - continue - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_chars(raw)) - except: - continue - pages.append(feed) - - results = [] - for x in pages: - results.extend([i.getparent().get('href') \ - for i in x.xpath("//a/span[@class='srTitle']")]) - return results[:self.max_results], self.baseurl - -class ResultList(list): - - def __init__(self, baseurl, lang = 'all'): - self.baseurl = baseurl - self.lang = lang - self.repub = re.compile(u'\((.*)\)') - self.rerat = re.compile(u'([0-9.]+)') - self.reattr = re.compile(r'<([a-zA-Z0-9]+)\s[^>]+>') - self.reoutp = re.compile(r'(?s)<em>--This text ref.*?</em>') - self.recom = re.compile(r'(?s)<!--.*?-->') - self.republi = re.compile(u'(Editeur|Publisher|Verlag)', re.I) - self.reisbn = re.compile(u'(ISBN-10|ISBN-10|ASIN)', re.I) - self.relang = re.compile(u'(Language|Langue|Sprache)', re.I) - self.reratelt = re.compile(u'(Average\s*Customer\s*Review|Moyenne\s*des\s*commentaires\s*client|Durchschnittliche\s*Kundenbewertung)', re.I) - self.reprod = re.compile(u'(Product\s*Details|D.tails\s*sur\s*le\s*produit|Produktinformation)', re.I) - - def strip_tags_etree(self, etreeobj, invalid_tags): - for (itag, rmv) in invalid_tags.iteritems(): - if rmv: - for elts in etreeobj.getiterator(itag): - elts.drop_tree() - else: - for elts in etreeobj.getiterator(itag): - elts.drop_tag() - - def clean_entry(self, entry, invalid_tags = {'script': True}, - invalid_id = (), invalid_class=()): - #invalid_tags: remove tag and keep content if False else remove - #remove tags - if invalid_tags: - self.strip_tags_etree(entry, invalid_tags) - #remove id - if invalid_id: - for eltid in invalid_id: - elt = entry.get_element_by_id(eltid) - if elt is not None: - elt.drop_tree() - #remove class - if invalid_class: - for eltclass in invalid_class: - elts = entry.find_class(eltclass) - if elts is not None: - for elt in elts: - elt.drop_tree() - - def get_title(self, entry): - title = entry.get_element_by_id('btAsinTitle') - if title is not None: - title = title.text - return unicode(title.replace('\n', '').strip()) - - def get_authors(self, entry): - author = entry.get_element_by_id('btAsinTitle') - while author.getparent().tag != 'div': - author = author.getparent() - author = author.getparent() - authortext = [] - for x in author.getiterator('a'): - authortext.append(unicode(x.text_content().strip())) - return authortext - - def get_description(self, entry, verbose): - try: - description = entry.get_element_by_id("productDescription").find("div[@class='content']") - inv_class = ('seeAll', 'emptyClear') - inv_tags ={'img': True, 'a': False} - self.clean_entry(description, invalid_tags=inv_tags, invalid_class=inv_class) - description = html.tostring(description, method='html', encoding=unicode).strip() - # remove all attributes from tags - description = self.reattr.sub(r'<\1>', description) - # Remove the notice about text referring to out of print editions - description = self.reoutp.sub('', description) - # Remove comments - description = self.recom.sub('', description) - return unicode(sanitize_comments_html(description)) - except: - report(verbose) - return None - - def get_tags(self, entry, browser, verbose): - try: - tags = entry.get_element_by_id('tagContentHolder') - testptag = tags.find_class('see-all') - if testptag: - for x in testptag: - alink = x.xpath('descendant-or-self::a') - if alink: - if alink[0].get('class') == 'tgJsActive': - continue - link = self.baseurl + alink[0].get('href') - entry = self.get_individual_metadata(browser, link, verbose) - tags = entry.get_element_by_id('tagContentHolder') - break - tags = [a.text for a in tags.getiterator('a') if a.get('rel') == 'tag'] - except: - report(verbose) - tags = [] - return tags - - def get_book_info(self, entry, mi, verbose): - try: - entry = entry.get_element_by_id('SalesRank').getparent() - except: - try: - for z in entry.getiterator('h2'): - if self.reprod.search(z.text_content()): - entry = z.getparent().find("div[@class='content']/ul") - break - except: - report(verbose) - return mi - elts = entry.findall('li') - #pub & date - elt = filter(lambda x: self.republi.search(x.find('b').text), elts) - if elt: - pub = elt[0].find('b').tail - mi.publisher = unicode(self.repub.sub('', pub).strip()) - d = self.repub.search(pub) - if d is not None: - d = d.group(1) - try: - default = utcnow().replace(day=15) - if self.lang != 'all': - d = replace_months(d, self.lang) - d = parse_date(d, assume_utc=True, default=default) - mi.pubdate = d - except: - report(verbose) - #ISBN - elt = filter(lambda x: self.reisbn.search(x.find('b').text), elts) - if elt: - isbn = elt[0].find('b').tail.replace('-', '').strip() - if check_isbn(isbn): - mi.isbn = unicode(isbn) - elif len(elt) > 1: - isbn = elt[1].find('b').tail.replace('-', '').strip() - if check_isbn(isbn): - mi.isbn = unicode(isbn) - #Langue - elt = filter(lambda x: self.relang.search(x.find('b').text), elts) - if elt: - langue = elt[0].find('b').tail.strip() - if langue: - mi.language = unicode(langue) - #ratings - elt = filter(lambda x: self.reratelt.search(x.find('b').text), elts) - if elt: - ratings = elt[0].find_class('swSprite') - if ratings: - ratings = self.rerat.findall(ratings[0].get('title')) - if len(ratings) == 2: - mi.rating = float(ratings[0])/float(ratings[1]) * 5 - return mi - - def fill_MI(self, entry, title, authors, browser, verbose): - mi = MetaInformation(title, authors) - mi.author_sort = authors_to_sort_string(authors) - mi.comments = self.get_description(entry, verbose) - mi = self.get_book_info(entry, mi, verbose) - mi.tags = self.get_tags(entry, browser, verbose) - return mi - - def get_individual_metadata(self, browser, linkdata, verbose): - try: - raw = browser.open_novisit(linkdata).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - raise - if '<title>404 - ' in raw: - report(verbose) - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - return soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_chars(raw)) - except: - report(verbose) - return - - def populate(self, entries, browser, verbose=False): - for x in entries: - try: - entry = self.get_individual_metadata(browser, x, verbose) - # clean results - # inv_ids = ('divsinglecolumnminwidth', 'sims.purchase', 'AutoBuyXGetY', 'A9AdsMiddleBoxTop') - # inv_class = ('buyingDetailsGrid', 'productImageGrid') - # inv_tags ={'script': True, 'style': True, 'form': False} - # self.clean_entry(entry, invalid_id=inv_ids) - title = self.get_title(entry) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - print 'URL who failed:', x - report(verbose) - continue - self.append(self.fill_MI(entry, title, authors, browser, verbose)) - - -def search(title=None, author=None, publisher=None, isbn=None, - max_results=5, verbose=False, keywords=None, lang='all'): - br = browser() - entries, baseurl = Query(title=title, author=author, isbn=isbn, publisher=publisher, - keywords=keywords, max_results=max_results,rlang=lang)(br, verbose) - - if entries is None or len(entries) == 0: - return - - #List of entry - ans = ResultList(baseurl, lang) - ans.populate(entries, br, verbose) - return ans - -def option_parser(): - parser = OptionParser(textwrap.dedent(\ - _('''\ - %prog [options] - - Fetch book metadata from Amazon. You must specify one of title, author, - ISBN, publisher or keywords. Will fetch a maximum of 10 matches, - so you should make your query as specific as possible. - You can chose the language for metadata retrieval: - All & english & french & german & spanish - ''' - ))) - parser.add_option('-t', '--title', help='Book title') - parser.add_option('-a', '--author', help='Book author(s)') - parser.add_option('-p', '--publisher', help='Book publisher') - parser.add_option('-i', '--isbn', help='Book ISBN') - parser.add_option('-k', '--keywords', help='Keywords') - parser.add_option('-m', '--max-results', default=10, - help='Maximum number of results to fetch') - parser.add_option('-l', '--lang', default='all', - help='Chosen language for metadata search (all, en, fr, es, de)') - parser.add_option('-v', '--verbose', default=0, action='count', - help='Be more verbose about errors') - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, isbn=opts.isbn, publisher=opts.publisher, - keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results, - lang=opts.lang) - except AssertionError: - report(True) - parser.print_help() - return 1 - if results is None or len(results) == 0: - print 'No result found for this search!' - return 0 - for result in results: - print unicode(result).encode(preferred_encoding, 'replace') - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py deleted file mode 100644 index 10acff4e61..0000000000 --- a/src/calibre/ebooks/metadata/covers.py +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai - -__license__ = 'GPL v3' -__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -import traceback, socket, sys -from functools import partial -from threading import Thread, Event -from Queue import Queue, Empty -from lxml import etree - -import mechanize - -from calibre.customize import Plugin -from calibre import browser, prints -from calibre.constants import preferred_encoding, DEBUG - -class CoverDownload(Plugin): - ''' - These plugins are used to download covers for books. - ''' - - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Kovid Goyal' - type = _('Cover download') - - def has_cover(self, mi, ans, timeout=5.): - ''' - Check if the book described by mi has a cover. Call ans.set() if it - does. Do nothing if it doesn't. - - :param mi: MetaInformation object - :param timeout: timeout in seconds - :param ans: A threading.Event object - ''' - raise NotImplementedError() - - def get_covers(self, mi, result_queue, abort, timeout=5.): - ''' - Download covers for books described by the mi object. Downloaded covers - must be put into the result_queue. If more than one cover is available, - the plugin should continue downloading them and putting them into - result_queue until abort.is_set() returns True. - - :param mi: MetaInformation object - :param result_queue: A multithreaded Queue - :param abort: A threading.Event object - :param timeout: timeout in seconds - ''' - raise NotImplementedError() - - def exception_to_string(self, ex): - try: - return unicode(ex) - except: - try: - return str(ex).decode(preferred_encoding, 'replace') - except: - return repr(ex) - - def debug(self, *args, **kwargs): - if DEBUG: - prints('\t'+self.name+':', *args, **kwargs) - - - -class HeadRequest(mechanize.Request): - - def get_method(self): - return 'HEAD' - -class OpenLibraryCovers(CoverDownload): # {{{ - 'Download covers from openlibrary.org' - - # See http://openlibrary.org/dev/docs/api/covers - - OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false' - name = 'openlibrary.org covers' - description = _('Download covers from openlibrary.org') - author = 'Kovid Goyal' - - def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: - return False - from calibre.ebooks.metadata.library_thing import get_browser - br = get_browser() - br.set_handle_redirect(False) - try: - br.open_novisit(HeadRequest(self.OPENLIBRARY%mi.isbn), timeout=timeout) - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception as e: - if callable(getattr(e, 'getcode', None)) and e.getcode() == 302: - self.debug('cover for', mi.isbn, 'found') - ans.set() - else: - self.debug(e) - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - from calibre.ebooks.metadata.library_thing import get_browser - br = get_browser() - try: - ans = br.open(self.OPENLIBRARY%mi.isbn, timeout=timeout).read() - result_queue.put((True, ans, 'jpg', self.name)) - except Exception as e: - if callable(getattr(e, 'getcode', None)) and e.getcode() == 404: - result_queue.put((False, _('ISBN: %s not found')%mi.isbn, '', self.name)) - else: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) - -# }}} - -class AmazonCovers(CoverDownload): # {{{ - - name = 'amazon.com covers' - description = _('Download covers from amazon.com') - author = 'Kovid Goyal' - - - def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: - return False - from calibre.ebooks.metadata.amazon import get_cover_url - br = browser() - try: - get_cover_url(mi.isbn, br) - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception as e: - self.debug(e) - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - from calibre.ebooks.metadata.amazon import get_cover_url - br = browser() - try: - url = get_cover_url(mi.isbn, br) - if url is None: - raise ValueError('No cover found for ISBN: %s'%mi.isbn) - cover_data = br.open_novisit(url).read() - result_queue.put((True, cover_data, 'jpg', self.name)) - except Exception as e: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) - -# }}} - -def check_for_cover(mi, timeout=5.): # {{{ - from calibre.customize.ui import cover_sources - ans = Event() - checkers = [partial(p.has_cover, mi, ans, timeout=timeout) for p in - cover_sources()] - workers = [Thread(target=c) for c in checkers] - for w in workers: - w.daemon = True - w.start() - while not ans.is_set(): - ans.wait(0.1) - if sum([int(w.is_alive()) for w in workers]) == 0: - break - return ans.is_set() - -# }}} - -def download_covers(mi, result_queue, max_covers=50, timeout=5.): # {{{ - from calibre.customize.ui import cover_sources - abort = Event() - temp = Queue() - getters = [partial(p.get_covers, mi, temp, abort, timeout=timeout) for p in - cover_sources()] - workers = [Thread(target=c) for c in getters] - for w in workers: - w.daemon = True - w.start() - count = 0 - while count < max_covers: - try: - result = temp.get_nowait() - if result[0]: - count += 1 - result_queue.put(result) - except Empty: - pass - if sum([int(w.is_alive()) for w in workers]) == 0: - break - - abort.set() - - while True: - try: - result = temp.get_nowait() - count += 1 - result_queue.put(result) - except Empty: - break - -# }}} - -class DoubanCovers(CoverDownload): # {{{ - 'Download covers from Douban.com' - - DOUBAN_ISBN_URL = 'http://api.douban.com/book/subject/isbn/' - CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d' - name = 'Douban.com covers' - description = _('Download covers from Douban.com') - author = 'Li Fanxi' - - def get_cover_url(self, isbn, br, timeout=5.): - try: - url = self.DOUBAN_ISBN_URL + isbn + "?apikey=" + self.CALIBRE_DOUBAN_API_KEY - src = br.open(url, timeout=timeout).read() - except Exception as err: - if isinstance(getattr(err, 'args', [None])[0], socket.timeout): - err = Exception(_('Douban.com API timed out. Try again later.')) - raise err - else: - feed = etree.fromstring(src) - NAMESPACES = { - 'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/', - 'atom' : 'http://www.w3.org/2005/Atom', - 'db': 'http://www.douban.com/xmlns/' - } - XPath = partial(etree.XPath, namespaces=NAMESPACES) - entries = XPath('//atom:entry')(feed) - if len(entries) < 1: - return None - try: - cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href") - u = cover_url(entries[0])[0].replace('/spic/', '/lpic/'); - # If URL contains "book-default", the book doesn't have a cover - if u.find('book-default') != -1: - return None - except: - return None - return u - - def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: - return False - br = browser() - try: - if self.get_cover_url(mi.isbn, br, timeout=timeout) != None: - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception as e: - self.debug(e) - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - br = browser() - try: - url = self.get_cover_url(mi.isbn, br, timeout=timeout) - cover_data = br.open_novisit(url).read() - result_queue.put((True, cover_data, 'jpg', self.name)) - except Exception as e: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) -# }}} - -def download_cover(mi, timeout=5.): # {{{ - results = Queue() - download_covers(mi, results, max_covers=1, timeout=timeout) - errors, ans = [], None - while True: - try: - x = results.get_nowait() - if x[0]: - ans = x[1] - else: - errors.append(x) - except Empty: - break - return ans, errors - -# }}} - -def test(isbns): # {{{ - from calibre.ebooks.metadata import MetaInformation - mi = MetaInformation('test', ['test']) - for isbn in isbns: - prints('Testing ISBN:', isbn) - mi.isbn = isbn - found = check_for_cover(mi) - prints('Has cover:', found) - ans, errors = download_cover(mi) - if ans is not None: - prints('Cover downloaded') - else: - prints('Download failed:') - for err in errors: - prints('\t', err[-1]+':', err[1]) - print '\n' - -# }}} - -if __name__ == '__main__': - isbns = sys.argv[1:] + ['9781591025412', '9780307272119'] - #test(isbns) - - from calibre.ebooks.metadata import MetaInformation - oc = OpenLibraryCovers(None) - for isbn in isbns: - mi = MetaInformation('xx', ['yy']) - mi.isbn = isbn - rq = Queue() - oc.get_covers(mi, rq, Event()) - result = rq.get_nowait() - if not result[0]: - print 'Failed for ISBN:', isbn - print result diff --git a/src/calibre/ebooks/metadata/douban.py b/src/calibre/ebooks/metadata/douban.py deleted file mode 100644 index 98a51f69d1..0000000000 --- a/src/calibre/ebooks/metadata/douban.py +++ /dev/null @@ -1,263 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>; 2010, Li Fanxi <lifanxi@freemindworld.com>' -__docformat__ = 'restructuredtext en' - -import sys, textwrap -import traceback -from urllib import urlencode -from functools import partial -from lxml import etree - -from calibre import browser, preferred_encoding -from calibre.ebooks.metadata import MetaInformation -from calibre.utils.config import OptionParser -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.utils.date import parse_date, utcnow - -NAMESPACES = { - 'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/', - 'atom' : 'http://www.w3.org/2005/Atom', - 'db': 'http://www.douban.com/xmlns/' - } -XPath = partial(etree.XPath, namespaces=NAMESPACES) -total_results = XPath('//openSearch:totalResults') -start_index = XPath('//openSearch:startIndex') -items_per_page = XPath('//openSearch:itemsPerPage') -entry = XPath('//atom:entry') -entry_id = XPath('descendant::atom:id') -title = XPath('descendant::atom:title') -description = XPath('descendant::atom:summary') -publisher = XPath("descendant::db:attribute[@name='publisher']") -isbn = XPath("descendant::db:attribute[@name='isbn13']") -date = XPath("descendant::db:attribute[@name='pubdate']") -creator = XPath("descendant::db:attribute[@name='author']") -tag = XPath("descendant::db:tag") - -CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d' - -class DoubanBooks(MetadataSource): - - name = 'Douban Books' - description = _('Downloads metadata from Douban.com') - supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on - author = 'Li Fanxi <lifanxi@freemindworld.com>' # The author of this plugin - version = (1, 0, 1) # The version number of this plugin - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, - verbose=self.verbose) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -def report(verbose): - if verbose: - import traceback - traceback.print_exc() - -class Query(object): - - SEARCH_URL = 'http://api.douban.com/book/subjects?' - ISBN_URL = 'http://api.douban.com/book/subject/isbn/' - - type = "search" - - def __init__(self, title=None, author=None, publisher=None, isbn=None, - max_results=20, start_index=1, api_key=''): - assert not(title is None and author is None and publisher is None and \ - isbn is None) - assert (int(max_results) < 21) - q = '' - if isbn is not None: - q = isbn - self.type = 'isbn' - else: - def build_term(parts): - return ' '.join(x for x in parts) - if title is not None: - q += build_term(title.split()) - if author is not None: - q += (' ' if q else '') + build_term(author.split()) - if publisher is not None: - q += (' ' if q else '') + build_term(publisher.split()) - self.type = 'search' - - if isinstance(q, unicode): - q = q.encode('utf-8') - - if self.type == "isbn": - self.url = self.ISBN_URL + q - if api_key != '': - self.url = self.url + "?apikey=" + api_key - else: - self.url = self.SEARCH_URL+urlencode({ - 'q':q, - 'max-results':max_results, - 'start-index':start_index, - }) - if api_key != '': - self.url = self.url + "&apikey=" + api_key - - def __call__(self, browser, verbose): - if verbose: - print 'Query:', self.url - if self.type == "search": - feed = etree.fromstring(browser.open(self.url).read()) - total = int(total_results(feed)[0].text) - start = int(start_index(feed)[0].text) - entries = entry(feed) - new_start = start + len(entries) - if new_start > total: - new_start = 0 - return entries, new_start - elif self.type == "isbn": - feed = etree.fromstring(browser.open(self.url).read()) - entries = entry(feed) - return entries, 0 - -class ResultList(list): - - def get_description(self, entry, verbose): - try: - desc = description(entry) - if desc: - return 'SUMMARY:\n'+desc[0].text - except: - report(verbose) - - def get_title(self, entry): - candidates = [x.text for x in title(entry)] - return ': '.join(candidates) - - def get_authors(self, entry): - m = creator(entry) - if not m: - m = [] - m = [x.text for x in m] - return m - - def get_tags(self, entry, verbose): - try: - btags = [x.attrib["name"] for x in tag(entry)] - tags = [] - for t in btags: - tags.extend([y.strip() for y in t.split('/')]) - tags = list(sorted(list(set(tags)))) - except: - report(verbose) - tags = [] - return [x.replace(',', ';') for x in tags] - - def get_publisher(self, entry, verbose): - try: - pub = publisher(entry)[0].text - except: - pub = None - return pub - - def get_isbn(self, entry, verbose): - try: - isbn13 = isbn(entry)[0].text - except Exception: - isbn13 = None - return isbn13 - - def get_date(self, entry, verbose): - try: - d = date(entry) - if d: - default = utcnow().replace(day=15) - d = parse_date(d[0].text, assume_utc=True, default=default) - else: - d = None - except: - report(verbose) - d = None - return d - - def populate(self, entries, browser, verbose=False, api_key=''): - for x in entries: - try: - id_url = entry_id(x)[0].text - title = self.get_title(x) - except: - report(verbose) - mi = MetaInformation(title, self.get_authors(x)) - try: - if api_key != '': - id_url = id_url + "?apikey=" + api_key - raw = browser.open(id_url).read() - feed = etree.fromstring(raw) - x = entry(feed)[0] - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - mi.comments = self.get_description(x, verbose) - mi.tags = self.get_tags(x, verbose) - mi.isbn = self.get_isbn(x, verbose) - mi.publisher = self.get_publisher(x, verbose) - mi.pubdate = self.get_date(x, verbose) - self.append(mi) - -def search(title=None, author=None, publisher=None, isbn=None, - verbose=False, max_results=40, api_key=None): - br = browser() - start, entries = 1, [] - - if api_key is None: - api_key = CALIBRE_DOUBAN_API_KEY - - while start > 0 and len(entries) <= max_results: - new, start = Query(title=title, author=author, publisher=publisher, - isbn=isbn, max_results=max_results, start_index=start, api_key=api_key)(br, verbose) - if not new: - break - entries.extend(new) - - entries = entries[:max_results] - - ans = ResultList() - ans.populate(entries, br, verbose, api_key) - return ans - -def option_parser(): - parser = OptionParser(textwrap.dedent( - '''\ - %prog [options] - - Fetch book metadata from Douban. You must specify one of title, author, - publisher or ISBN. If you specify ISBN the others are ignored. Will - fetch a maximum of 100 matches, so you should make your query as - specific as possible. - ''' - )) - parser.add_option('-t', '--title', help='Book title') - parser.add_option('-a', '--author', help='Book author(s)') - parser.add_option('-p', '--publisher', help='Book publisher') - parser.add_option('-i', '--isbn', help='Book ISBN') - parser.add_option('-m', '--max-results', default=10, - help='Maximum number of results to fetch') - parser.add_option('-v', '--verbose', default=0, action='count', - help='Be more verbose about errors') - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, opts.publisher, opts.isbn, - verbose=opts.verbose, max_results=int(opts.max_results)) - except AssertionError: - report(True) - parser.print_help() - return 1 - for result in results: - print unicode(result).encode(preferred_encoding) - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py deleted file mode 100644 index e1fac50d16..0000000000 --- a/src/calibre/ebooks/metadata/fetch.py +++ /dev/null @@ -1,523 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -import traceback, sys, textwrap, re -from threading import Thread - -from calibre import prints -from calibre.utils.config import OptionParser -from calibre.utils.logging import default_log -from calibre.utils.titlecase import titlecase -from calibre.customize import Plugin -from calibre.ebooks.metadata.covers import check_for_cover -from calibre.utils.html2text import html2text - -metadata_config = None - -class MetadataSource(Plugin): # {{{ - ''' - Represents a source to query for metadata. Subclasses must implement - at least the fetch method. - - When :meth:`fetch` is called, the `self` object will have the following - useful attributes (each of which may be None):: - - title, book_author, publisher, isbn, log, verbose and extra - - Use these attributes to construct the search query. extra is reserved for - future use. - - The fetch method must store the results in `self.results` as a list of - :class:`Metadata` objects. If there is an error, it should be stored - in `self.exception` and `self.tb` (for the traceback). - ''' - - author = 'Kovid Goyal' - - supported_platforms = ['windows', 'osx', 'linux'] - - #: The type of metadata fetched. 'basic' means basic metadata like - #: title/author/isbn/etc. 'social' means social metadata like - #: tags/rating/reviews/etc. - metadata_type = 'basic' - - #: If not None, the customization dialog will allow for string - #: based customization as well the default customization. The - #: string customization will be saved in the site_customization - #: member. - string_customization_help = None - - #: Set this to true if your plugin returns HTML markup in comments. - #: Then if the user disables HTML, calibre will automagically convert - #: the HTML to Markdown. - has_html_comments = False - - type = _('Metadata download') - - def __call__(self, title, author, publisher, isbn, verbose, log=None, - extra=None): - self.worker = Thread(target=self._fetch) - self.worker.daemon = True - self.title = title - self.verbose = verbose - self.book_author = author - self.publisher = publisher - self.isbn = isbn - self.log = log if log is not None else default_log - self.extra = extra - self.exception, self.tb, self.results = None, None, [] - self.worker.start() - - def _fetch(self): - try: - self.fetch() - if self.results: - c = self.config_store().get(self.name, {}) - res = self.results - if hasattr(res, 'authors'): - res = [res] - for mi in res: - if not c.get('rating', True): - mi.rating = None - if not c.get('comments', True): - mi.comments = None - if not c.get('tags', True): - mi.tags = [] - if self.has_html_comments and mi.comments and \ - c.get('textcomments', False): - try: - mi.comments = html2text(mi.comments) - except: - traceback.print_exc() - mi.comments = None - - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - def fetch(self): - ''' - All the actual work is done here. - ''' - raise NotImplementedError - - def join(self): - return self.worker.join() - - def is_alive(self): - return self.worker.is_alive() - - def is_customizable(self): - return True - - def config_store(self): - global metadata_config - if metadata_config is None: - from calibre.utils.config import XMLConfig - metadata_config = XMLConfig('plugins/metadata_download') - return metadata_config - - def config_widget(self): - from PyQt4.Qt import QWidget, QVBoxLayout, QLabel, Qt, QLineEdit, \ - QCheckBox - from calibre.customize.ui import config - w = QWidget() - w._layout = QVBoxLayout(w) - w.setLayout(w._layout) - if self.string_customization_help is not None: - w._sc_label = QLabel(self.string_customization_help, w) - w._layout.addWidget(w._sc_label) - customization = config['plugin_customization'] - def_sc = customization.get(self.name, '') - if not def_sc: - def_sc = '' - w._sc = QLineEdit(def_sc, w) - w._layout.addWidget(w._sc) - w._sc_label.setWordWrap(True) - w._sc_label.setTextInteractionFlags(Qt.LinksAccessibleByMouse - | Qt.LinksAccessibleByKeyboard) - w._sc_label.setOpenExternalLinks(True) - c = self.config_store() - c = c.get(self.name, {}) - for x, l in {'rating':_('ratings'), 'tags':_('tags'), - 'comments':_('description/reviews')}.items(): - cb = QCheckBox(_('Download %s from %s')%(l, - self.name)) - setattr(w, '_'+x, cb) - cb.setChecked(c.get(x, True)) - w._layout.addWidget(cb) - - if self.has_html_comments: - cb = QCheckBox(_('Convert comments downloaded from %s to plain text')%(self.name)) - setattr(w, '_textcomments', cb) - cb.setChecked(c.get('textcomments', False)) - w._layout.addWidget(cb) - - return w - - def save_settings(self, w): - dl_settings = {} - for x in ('rating', 'tags', 'comments'): - dl_settings[x] = getattr(w, '_'+x).isChecked() - if self.has_html_comments: - dl_settings['textcomments'] = getattr(w, '_textcomments').isChecked() - c = self.config_store() - c.set(self.name, dl_settings) - if hasattr(w, '_sc'): - sc = unicode(w._sc.text()).strip() - from calibre.customize.ui import customize_plugin - customize_plugin(self, sc) - - def customization_help(self): - return 'This plugin can only be customized using the GUI' - - # }}} - -class GoogleBooks(MetadataSource): # {{{ - - name = 'Google Books' - description = _('Downloads metadata from Google Books') - - def fetch(self): - from calibre.ebooks.metadata.google_books import search - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, - verbose=self.verbose) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - # }}} - -class ISBNDB(MetadataSource): # {{{ - - name = 'IsbnDB' - description = _('Downloads metadata from isbndb.com') - - def fetch(self): - if not self.site_customization: - return - from calibre.ebooks.metadata.isbndb import option_parser, create_books - args = ['isbndb'] - if self.isbn: - args.extend(['--isbn', self.isbn]) - else: - if self.title: - args.extend(['--title', self.title]) - if self.book_author: - args.extend(['--author', self.book_author]) - if self.publisher: - args.extend(['--publisher', self.publisher]) - if self.verbose: - args.extend(['--verbose']) - args.append(self.site_customization) # IsbnDb key - try: - opts, args = option_parser().parse_args(args) - self.results = create_books(opts, args) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - @property - def string_customization_help(self): - ans = _('To use isbndb.com you must sign up for a %sfree account%s ' - 'and enter your access key below.') - return '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>') - - # }}} - -class Amazon(MetadataSource): # {{{ - - name = 'Amazon' - metadata_type = 'social' - description = _('Downloads social metadata from amazon.com') - - has_html_comments = True - - def fetch(self): - if not self.isbn: - return - from calibre.ebooks.metadata.amazon import get_social_metadata - try: - self.results = get_social_metadata(self.title, self.book_author, - self.publisher, self.isbn) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - # }}} - -class KentDistrictLibrary(MetadataSource): # {{{ - - name = 'Kent District Library' - metadata_type = 'social' - description = _('Downloads series information from ww2.kdl.org. ' - 'This website cannot handle large numbers of queries, ' - 'so the plugin is disabled by default.') - - def fetch(self): - if not self.title or not self.book_author: - return - from calibre.ebooks.metadata.kdl import get_series - try: - self.results = get_series(self.title, self.book_author) - except Exception as e: - import traceback - traceback.print_exc() - self.exception = e - self.tb = traceback.format_exc() - - # }}} - - -def result_index(source, result): - if not result.isbn: - return -1 - for i, x in enumerate(source): - if x.isbn == result.isbn: - return i - return -1 - -def merge_results(one, two): - if two is not None and one is not None: - for x in two: - idx = result_index(one, x) - if idx < 0: - one.append(x) - else: - one[idx].smart_update(x) - -class MetadataSources(object): - - def __init__(self, sources): - self.sources = sources - - def __enter__(self): - for s in self.sources: - s.__enter__() - return self - - def __exit__(self, *args): - for s in self.sources: - s.__exit__() - - def __call__(self, *args, **kwargs): - for s in self.sources: - s(*args, **kwargs) - - def join(self): - for s in self.sources: - s.join() - -def filter_metadata_results(item): - keywords = ["audio", "tape", "cassette", "abridged", "playaway"] - for keyword in keywords: - if item.publisher and keyword in item.publisher.lower(): - return False - return True - -def do_cover_check(item): - item.has_cover = False - try: - item.has_cover = check_for_cover(item) - except: - pass # Cover not found - -def check_for_covers(items): - threads = [Thread(target=do_cover_check, args=(item,)) for item in items] - for t in threads: t.start() - for t in threads: t.join() - -def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None, - verbose=0): - assert not(title is None and author is None and publisher is None and \ - isbn is None) - from calibre.customize.ui import metadata_sources, migrate_isbndb_key - migrate_isbndb_key() - if isbn is not None: - isbn = re.sub(r'[^a-zA-Z0-9]', '', isbn).upper() - fetchers = list(metadata_sources(isbndb_key=isbndb_key)) - with MetadataSources(fetchers) as manager: - manager(title, author, publisher, isbn, verbose) - manager.join() - - results = list(fetchers[0].results) if fetchers else [] - for fetcher in fetchers[1:]: - merge_results(results, fetcher.results) - - results = list(filter(filter_metadata_results, results)) - - check_for_covers(results) - - words = ("the", "a", "an", "of", "and") - prefix_pat = re.compile(r'^(%s)\s+'%("|".join(words))) - trailing_paren_pat = re.compile(r'\(.*\)$') - whitespace_pat = re.compile(r'\s+') - - def sort_func(x, y): - - def cleanup_title(s): - if s is None: - s = _('Unknown') - s = s.strip().lower() - s = prefix_pat.sub(' ', s) - s = trailing_paren_pat.sub('', s) - s = whitespace_pat.sub(' ', s) - return s.strip() - - t = cleanup_title(title) - x_title = cleanup_title(x.title) - y_title = cleanup_title(y.title) - - # prefer titles that start with the search title - tx = cmp(t, x_title) - ty = cmp(t, y_title) - result = 0 if abs(tx) == abs(ty) else abs(tx) - abs(ty) - - # then prefer titles that have a cover image - if result == 0: - result = -cmp(x.has_cover, y.has_cover) - - # then prefer titles with the longest comment, with in 10% - if result == 0: - cx = len(x.comments.strip() if x.comments else '') - cy = len(y.comments.strip() if y.comments else '') - t = (cx + cy) / 20 - result = cy - cx - if abs(result) < t: - result = 0 - - return result - - results = sorted(results, cmp=sort_func) - - # if for some reason there is no comment in the top selection, go looking for one - if len(results) > 1: - if not results[0].comments or len(results[0].comments) == 0: - for r in results[1:]: - try: - if title and title.lower() == r.title[:len(title)].lower() \ - and r.comments and len(r.comments): - results[0].comments = r.comments - break - except: - pass - # Find a pubdate - pubdate = None - for r in results: - if r.pubdate is not None: - pubdate = r.pubdate - break - if pubdate is not None: - for r in results: - if r.pubdate is None: - r.pubdate = pubdate - - def fix_case(x): - if x: - x = titlecase(x) - return x - - for r in results: - r.title = fix_case(r.title) - if r.authors: - r.authors = list(map(fix_case, r.authors)) - - return results, [(x.name, x.exception, x.tb) for x in fetchers] - -def get_social_metadata(mi, verbose=0): - from calibre.customize.ui import metadata_sources - fetchers = list(metadata_sources(metadata_type='social')) - with MetadataSources(fetchers) as manager: - manager(mi.title, mi.authors, mi.publisher, mi.isbn, verbose) - manager.join() - ratings, tags, comments, series, series_index = [], set([]), set([]), None, None - for fetcher in fetchers: - if fetcher.results: - dmi = fetcher.results - if dmi.rating is not None: - ratings.append(dmi.rating) - if dmi.tags: - for t in dmi.tags: - tags.add(t) - if mi.pubdate is None and dmi.pubdate is not None: - mi.pubdate = dmi.pubdate - if dmi.comments: - comments.add(dmi.comments) - if dmi.series is not None: - series = dmi.series - if dmi.series_index is not None: - series_index = dmi.series_index - if ratings: - rating = sum(ratings)/float(len(ratings)) - if mi.rating is None or mi.rating < 0.1: - mi.rating = rating - else: - mi.rating = (mi.rating + rating)/2.0 - if tags: - if not mi.tags: - mi.tags = [] - mi.tags += list(tags) - mi.tags = list(sorted(list(set(mi.tags)))) - if comments: - if not mi.comments or len(mi.comments)+20 < len(' '.join(comments)): - mi.comments = '' - for x in comments: - mi.comments += x+'\n\n' - if series and series_index is not None: - mi.series = series - mi.series_index = series_index - - return [(x.name, x.exception, x.tb) for x in fetchers if x.exception is not - None] - - - -def option_parser(): - parser = OptionParser(textwrap.dedent( - '''\ - %prog [options] - - Fetch book metadata from online sources. You must specify at least one - of title, author, publisher or ISBN. If you specify ISBN, the others - are ignored. - ''' - )) - parser.add_option('-t', '--title', help='Book title') - parser.add_option('-a', '--author', help='Book author(s)') - parser.add_option('-p', '--publisher', help='Book publisher') - parser.add_option('-i', '--isbn', help='Book ISBN') - parser.add_option('-m', '--max-results', default=10, - help='Maximum number of results to fetch') - parser.add_option('-k', '--isbndb-key', - help=('The access key for your ISBNDB.com account. ' - 'Only needed if you want to search isbndb.com ' - 'and you haven\'t customized the IsbnDB plugin.')) - parser.add_option('-v', '--verbose', default=0, action='count', - help='Be more verbose about errors') - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - results, exceptions = search(opts.title, opts.author, opts.publisher, - opts.isbn, opts.isbndb_key, opts.verbose) - social_exceptions = [] - for result in results: - social_exceptions.extend(get_social_metadata(result, opts.verbose)) - prints(unicode(result)) - print - - for name, exception, tb in exceptions+social_exceptions: - if exception is not None: - print 'WARNING: Fetching from', name, 'failed with error:' - print exception - print tb - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/fictionwise.py b/src/calibre/ebooks/metadata/fictionwise.py deleted file mode 100644 index 145e39768d..0000000000 --- a/src/calibre/ebooks/metadata/fictionwise.py +++ /dev/null @@ -1,390 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2010, sengian <sengian1@gmail.com>' -__docformat__ = 'restructuredtext en' - -import sys, textwrap, re, traceback, socket -from urllib import urlencode - -from lxml.html import soupparser, tostring - -from calibre import browser, preferred_encoding -from calibre.ebooks.chardet import xml_to_unicode -from calibre.ebooks.metadata import MetaInformation, check_isbn, \ - authors_to_sort_string -from calibre.library.comments import sanitize_comments_html -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.utils.config import OptionParser -from calibre.utils.date import parse_date, utcnow -from calibre.utils.cleantext import clean_ascii_chars - -class Fictionwise(MetadataSource): # {{{ - - author = 'Sengian' - name = 'Fictionwise' - description = _('Downloads metadata from Fictionwise') - - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - # }}} - -class FictionwiseError(Exception): - pass - -def report(verbose): - if verbose: - traceback.print_exc() - -class Query(object): - - BASE_URL = 'http://www.fictionwise.com/servlet/mw' - - def __init__(self, title=None, author=None, publisher=None, keywords=None, max_results=20): - assert not(title is None and author is None and publisher is None and keywords is None) - assert (max_results < 21) - - self.max_results = int(max_results) - q = { 'template' : 'searchresults_adv.htm' , - 'searchtitle' : '', - 'searchauthor' : '', - 'searchpublisher' : '', - 'searchkeyword' : '', - #possibilities startoflast, fullname, lastfirst - 'searchauthortype' : 'startoflast', - 'searchcategory' : '', - 'searchcategory2' : '', - 'searchprice_s' : '0', - 'searchprice_e' : 'ANY', - 'searchformat' : '', - 'searchgeo' : 'US', - 'searchfwdatetype' : '', - #maybe use dates fields if needed? - #'sortorder' : 'DESC', - #many options available: b.SortTitle, a.SortName, - #b.DateFirstPublished, b.FWPublishDate - 'sortby' : 'b.SortTitle' - } - if title is not None: - q['searchtitle'] = title - if author is not None: - q['searchauthor'] = author - if publisher is not None: - q['searchpublisher'] = publisher - if keywords is not None: - q['searchkeyword'] = keywords - - if isinstance(q, unicode): - q = q.encode('utf-8') - self.urldata = urlencode(q) - - def __call__(self, browser, verbose, timeout = 5.): - if verbose: - print _('Query: %s') % self.BASE_URL+self.urldata - - try: - raw = browser.open_novisit(self.BASE_URL, self.urldata, timeout=timeout).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - if isinstance(getattr(e, 'args', [None])[0], socket.timeout): - raise FictionwiseError(_('Fictionwise timed out. Try again later.')) - raise FictionwiseError(_('Fictionwise encountered an error.')) - if '<title>404 - ' in raw: - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None - - # get list of results as links - results = feed.xpath("//table[3]/tr/td[2]/table/tr/td/p/table[2]/tr[@valign]") - results = results[:self.max_results] - results = [i.xpath('descendant-or-self::a')[0].get('href') for i in results] - #return feed if no links ie normally a single book or nothing - if not results: - results = [feed] - return results - -class ResultList(list): - - BASE_URL = 'http://www.fictionwise.com' - COLOR_VALUES = {'BLUE': 4, 'GREEN': 3, 'YELLOW': 2, 'RED': 1, 'NA': 0} - - def __init__(self): - self.retitle = re.compile(r'\[[^\[\]]+\]') - self.rechkauth = re.compile(r'.*book\s*by', re.I) - self.redesc = re.compile(r'book\s*description\s*:\s*(<br[^>]+>)*(?P<desc>.*)<br[^>]*>.{,15}publisher\s*:', re.I) - self.repub = re.compile(r'.*publisher\s*:\s*', re.I) - self.redate = re.compile(r'.*release\s*date\s*:\s*', re.I) - self.retag = re.compile(r'.*book\s*category\s*:\s*', re.I) - self.resplitbr = re.compile(r'<br[^>]*>', re.I) - self.recomment = re.compile(r'(?s)<!--.*?-->') - self.reimg = re.compile(r'<img[^>]*>', re.I) - self.resanitize = re.compile(r'\[HTML_REMOVED\]\s*', re.I) - self.renbcom = re.compile('(?P<nbcom>\d+)\s*Reader Ratings:') - self.recolor = re.compile('(?P<ncolor>[^/]+).gif') - self.resplitbrdiv = re.compile(r'(<br[^>]+>|</?div[^>]*>)', re.I) - self.reisbn = re.compile(r'.*ISBN\s*:\s*', re.I) - - def strip_tags_etree(self, etreeobj, invalid_tags): - for (itag, rmv) in invalid_tags.iteritems(): - if rmv: - for elts in etreeobj.getiterator(itag): - elts.drop_tree() - else: - for elts in etreeobj.getiterator(itag): - elts.drop_tag() - - def clean_entry(self, entry, invalid_tags = {'script': True}, - invalid_id = (), invalid_class=(), invalid_xpath = ()): - #invalid_tags: remove tag and keep content if False else remove - #remove tags - if invalid_tags: - self.strip_tags_etree(entry, invalid_tags) - #remove xpath - if invalid_xpath: - for eltid in invalid_xpath: - elt = entry.xpath(eltid) - for el in elt: - el.drop_tree() - #remove id - if invalid_id: - for eltid in invalid_id: - elt = entry.get_element_by_id(eltid) - if elt is not None: - elt.drop_tree() - #remove class - if invalid_class: - for eltclass in invalid_class: - elts = entry.find_class(eltclass) - if elts is not None: - for elt in elts: - elt.drop_tree() - - def output_entry(self, entry, prettyout = True, htmlrm="\d+"): - out = tostring(entry, pretty_print=prettyout) - #try to work around tostring to remove this encoding for exemle - reclean = re.compile('(\n+|\t+|\r+|&#'+htmlrm+';)') - return reclean.sub('', out) - - def get_title(self, entry): - title = entry.findtext('./') - return self.retitle.sub('', title).strip() - - def get_authors(self, entry): - authortext = entry.find('./br').tail - if not self.rechkauth.search(authortext): - return [] - authortext = self.rechkauth.sub('', authortext) - return [a.strip() for a in authortext.split('&')] - - def get_rating(self, entrytable, verbose): - nbcomment = tostring(entrytable.getprevious()) - try: - nbcomment = self.renbcom.search(nbcomment).group("nbcom") - except: - report(verbose) - return None - hval = dict((self.COLOR_VALUES[self.recolor.search(image.get('src', default='NA.gif')).group("ncolor")], - float(image.get('height', default=0))) \ - for image in entrytable.getiterator('img')) - #ratings as x/5 - return float(1.25*sum(k*v for (k, v) in hval.iteritems())/sum(hval.itervalues())) - - def get_description(self, entry): - description = self.output_entry(entry.xpath('./p')[1],htmlrm="") - description = self.redesc.search(description) - if not description or not description.group("desc"): - return None - #remove invalid tags - description = self.reimg.sub('', description.group("desc")) - description = self.recomment.sub('', description) - description = self.resanitize.sub('', sanitize_comments_html(description)) - return _('SUMMARY:\n %s') % re.sub(r'\n\s+</p>','\n</p>', description) - - def get_publisher(self, entry): - publisher = self.output_entry(entry.xpath('./p')[1]) - publisher = filter(lambda x: self.repub.search(x) is not None, - self.resplitbr.split(publisher)) - if not len(publisher): - return None - publisher = self.repub.sub('', publisher[0]) - return publisher.split(',')[0].strip() - - def get_tags(self, entry): - tag = self.output_entry(entry.xpath('./p')[1]) - tag = filter(lambda x: self.retag.search(x) is not None, - self.resplitbr.split(tag)) - if not len(tag): - return [] - return map(lambda x: x.strip(), self.retag.sub('', tag[0]).split('/')) - - def get_date(self, entry, verbose): - date = self.output_entry(entry.xpath('./p')[1]) - date = filter(lambda x: self.redate.search(x) is not None, - self.resplitbr.split(date)) - if not len(date): - return None - try: - d = self.redate.sub('', date[0]) - if d: - default = utcnow().replace(day=15) - d = parse_date(d, assume_utc=True, default=default) - else: - d = None - except: - report(verbose) - d = None - return d - - def get_ISBN(self, entry): - isbns = self.output_entry(entry.xpath('./p')[2]) - isbns = filter(lambda x: self.reisbn.search(x) is not None, - self.resplitbrdiv.split(isbns)) - if not len(isbns): - return None - isbns = [self.reisbn.sub('', x) for x in isbns if check_isbn(self.reisbn.sub('', x))] - return sorted(isbns, cmp=lambda x,y:cmp(len(x), len(y)))[-1] - - def fill_MI(self, entry, title, authors, ratings, verbose): - mi = MetaInformation(title, authors) - mi.rating = ratings - mi.comments = self.get_description(entry) - mi.publisher = self.get_publisher(entry) - mi.tags = self.get_tags(entry) - mi.pubdate = self.get_date(entry, verbose) - mi.isbn = self.get_ISBN(entry) - mi.author_sort = authors_to_sort_string(authors) - return mi - - def get_individual_metadata(self, browser, linkdata, verbose): - try: - raw = browser.open_novisit(self.BASE_URL + linkdata).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - if isinstance(getattr(e, 'args', [None])[0], socket.timeout): - raise FictionwiseError(_('Fictionwise timed out. Try again later.')) - raise FictionwiseError(_('Fictionwise encountered an error.')) - if '<title>404 - ' in raw: - report(verbose) - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - return soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None - - def populate(self, entries, browser, verbose=False): - inv_tags ={'script': True, 'a': False, 'font': False, 'strong': False, 'b': False, - 'ul': False, 'span': False} - inv_xpath =('./table',) - #single entry - if len(entries) == 1 and not isinstance(entries[0], str): - try: - entry = entries.xpath("//table[3]/tr/td[2]/table[1]/tr/td/font/table/tr/td") - self.clean_entry(entry, invalid_tags=inv_tags, invalid_xpath=inv_xpath) - title = self.get_title(entry) - #maybe strenghten the search - ratings = self.get_rating(entry.xpath("./p/table")[1], verbose) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print _('Failed to get all details for an entry') - print e - return - self.append(self.fill_MI(entry, title, authors, ratings, verbose)) - else: - #multiple entries - for x in entries: - try: - entry = self.get_individual_metadata(browser, x, verbose) - entry = entry.xpath("//table[3]/tr/td[2]/table[1]/tr/td/font/table/tr/td")[0] - self.clean_entry(entry, invalid_tags=inv_tags, invalid_xpath=inv_xpath) - title = self.get_title(entry) - #maybe strenghten the search - ratings = self.get_rating(entry.xpath("./p/table")[1], verbose) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print _('Failed to get all details for an entry') - print e - continue - self.append(self.fill_MI(entry, title, authors, ratings, verbose)) - - -def search(title=None, author=None, publisher=None, isbn=None, - min_viewability='none', verbose=False, max_results=5, - keywords=None): - br = browser() - entries = Query(title=title, author=author, publisher=publisher, - keywords=keywords, max_results=max_results)(br, verbose, timeout = 15.) - - #List of entry - ans = ResultList() - ans.populate(entries, br, verbose) - return ans - - -def option_parser(): - parser = OptionParser(textwrap.dedent(\ - _('''\ - %prog [options] - - Fetch book metadata from Fictionwise. You must specify one of title, author, - or keywords. No ISBN specification possible. Will fetch a maximum of 20 matches, - so you should make your query as specific as possible. - ''') - )) - parser.add_option('-t', '--title', help=_('Book title')) - parser.add_option('-a', '--author', help=_('Book author(s)')) - parser.add_option('-p', '--publisher', help=_('Book publisher')) - parser.add_option('-k', '--keywords', help=_('Keywords')) - parser.add_option('-m', '--max-results', default=20, - help=_('Maximum number of results to fetch')) - parser.add_option('-v', '--verbose', default=0, action='count', - help=_('Be more verbose about errors')) - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, publisher=opts.publisher, - keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results) - except AssertionError: - report(True) - parser.print_help() - return 1 - if results is None or len(results) == 0: - print _('No result found for this search!') - return 0 - for result in results: - print unicode(result).encode(preferred_encoding, 'replace') - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/google_books.py b/src/calibre/ebooks/metadata/google_books.py deleted file mode 100644 index 2e52bf020d..0000000000 --- a/src/calibre/ebooks/metadata/google_books.py +++ /dev/null @@ -1,247 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -import sys, textwrap -from urllib import urlencode -from functools import partial - -from lxml import etree - -from calibre import browser, preferred_encoding -from calibre.ebooks.metadata import MetaInformation -from calibre.utils.config import OptionParser -from calibre.utils.date import parse_date, utcnow - -NAMESPACES = { - 'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/', - 'atom' : 'http://www.w3.org/2005/Atom', - 'dc': 'http://purl.org/dc/terms' - } -XPath = partial(etree.XPath, namespaces=NAMESPACES) - -total_results = XPath('//openSearch:totalResults') -start_index = XPath('//openSearch:startIndex') -items_per_page = XPath('//openSearch:itemsPerPage') -entry = XPath('//atom:entry') -entry_id = XPath('descendant::atom:id') -creator = XPath('descendant::dc:creator') -identifier = XPath('descendant::dc:identifier') -title = XPath('descendant::dc:title') -date = XPath('descendant::dc:date') -publisher = XPath('descendant::dc:publisher') -subject = XPath('descendant::dc:subject') -description = XPath('descendant::dc:description') -language = XPath('descendant::dc:language') - -def report(verbose): - if verbose: - import traceback - traceback.print_exc() - - -class Query(object): - - BASE_URL = 'http://books.google.com/books/feeds/volumes?' - - def __init__(self, title=None, author=None, publisher=None, isbn=None, - max_results=20, min_viewability='none', start_index=1): - assert not(title is None and author is None and publisher is None and \ - isbn is None) - assert (max_results < 21) - assert (min_viewability in ('none', 'partial', 'full')) - q = '' - if isbn is not None: - q += 'isbn:'+isbn - else: - def build_term(prefix, parts): - return ' '.join('in'+prefix + ':' + x for x in parts) - if title is not None: - q += build_term('title', title.split()) - if author is not None: - q += ('+' if q else '')+build_term('author', author.split()) - if publisher is not None: - q += ('+' if q else '')+build_term('publisher', publisher.split()) - - if isinstance(q, unicode): - q = q.encode('utf-8') - self.url = self.BASE_URL+urlencode({ - 'q':q, - 'max-results':max_results, - 'start-index':start_index, - 'min-viewability':min_viewability, - }) - - def __call__(self, browser, verbose): - if verbose: - print 'Query:', self.url - feed = etree.fromstring(browser.open(self.url).read()) - #print etree.tostring(feed, pretty_print=True) - total = int(total_results(feed)[0].text) - start = int(start_index(feed)[0].text) - entries = entry(feed) - new_start = start + len(entries) - if new_start > total: - new_start = 0 - return entries, new_start - - -class ResultList(list): - - def get_description(self, entry, verbose): - try: - desc = description(entry) - if desc: - return 'SUMMARY:\n'+desc[0].text - except: - report(verbose) - - def get_language(self, entry, verbose): - try: - l = language(entry) - if l: - return l[0].text - except: - report(verbose) - - def get_title(self, entry): - candidates = [x.text for x in title(entry)] - return ': '.join(candidates) - - def get_authors(self, entry): - m = creator(entry) - if not m: - m = [] - m = [x.text for x in m] - return m - - def get_author_sort(self, entry, verbose): - for x in creator(entry): - for key, val in x.attrib.items(): - if key.endswith('file-as'): - return val - - def get_identifiers(self, entry, mi): - isbns = [] - for x in identifier(entry): - t = str(x.text).strip() - if t[:5].upper() in ('ISBN:', 'LCCN:', 'OCLC:'): - if t[:5].upper() == 'ISBN:': - isbns.append(t[5:]) - if isbns: - mi.isbn = sorted(isbns, cmp=lambda x,y:cmp(len(x), len(y)))[-1] - - def get_tags(self, entry, verbose): - try: - btags = [x.text for x in subject(entry)] - tags = [] - for t in btags: - tags.extend([y.strip() for y in t.split('/')]) - tags = list(sorted(list(set(tags)))) - except: - report(verbose) - tags = [] - return [x.replace(',', ';') for x in tags] - - def get_publisher(self, entry, verbose): - try: - pub = publisher(entry)[0].text - except: - pub = None - return pub - - def get_date(self, entry, verbose): - try: - d = date(entry) - if d: - default = utcnow().replace(day=15) - d = parse_date(d[0].text, assume_utc=True, default=default) - else: - d = None - except: - report(verbose) - d = None - return d - - def populate(self, entries, browser, verbose=False): - for x in entries: - try: - id_url = entry_id(x)[0].text - title = self.get_title(x) - except: - report(verbose) - mi = MetaInformation(title, self.get_authors(x)) - try: - raw = browser.open(id_url).read() - feed = etree.fromstring(raw) - x = entry(feed)[0] - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - mi.author_sort = self.get_author_sort(x, verbose) - mi.comments = self.get_description(x, verbose) - self.get_identifiers(x, mi) - mi.tags = self.get_tags(x, verbose) - mi.publisher = self.get_publisher(x, verbose) - mi.pubdate = self.get_date(x, verbose) - mi.language = self.get_language(x, verbose) - self.append(mi) - - -def search(title=None, author=None, publisher=None, isbn=None, - min_viewability='none', verbose=False, max_results=40): - br = browser() - br.set_handle_gzip(True) - start, entries = 1, [] - while start > 0 and len(entries) <= max_results: - new, start = Query(title=title, author=author, publisher=publisher, - isbn=isbn, min_viewability=min_viewability)(br, verbose) - if not new: - break - entries.extend(new) - - entries = entries[:max_results] - - ans = ResultList() - ans.populate(entries, br, verbose) - return ans - -def option_parser(): - parser = OptionParser(textwrap.dedent( - '''\ - %prog [options] - - Fetch book metadata from Google. You must specify one of title, author, - publisher or ISBN. If you specify ISBN the others are ignored. Will - fetch a maximum of 100 matches, so you should make your query as - specific as possible. - ''' - )) - parser.add_option('-t', '--title', help='Book title') - parser.add_option('-a', '--author', help='Book author(s)') - parser.add_option('-p', '--publisher', help='Book publisher') - parser.add_option('-i', '--isbn', help='Book ISBN') - parser.add_option('-m', '--max-results', default=10, - help='Maximum number of results to fetch') - parser.add_option('-v', '--verbose', default=0, action='count', - help='Be more verbose about errors') - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, opts.publisher, opts.isbn, - verbose=opts.verbose, max_results=opts.max_results) - except AssertionError: - report(True) - parser.print_help() - return 1 - for result in results: - print unicode(result).encode(preferred_encoding) - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/isbndb.py b/src/calibre/ebooks/metadata/isbndb.py deleted file mode 100644 index 54cd403c62..0000000000 --- a/src/calibre/ebooks/metadata/isbndb.py +++ /dev/null @@ -1,159 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' -''' -Interface to isbndb.com. My key HLLXQX2A. -''' - -import sys, re -from urllib import quote - -from calibre.utils.config import OptionParser -from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup -from calibre import browser - -BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%(key)s&page_number=1&results=subjects,authors,texts&' - -class ISBNDBError(Exception): - pass - -def fetch_metadata(url, max=3, timeout=5.): - books = [] - page_number = 1 - total_results = 31 - br = browser() - while len(books) < total_results and max > 0: - try: - raw = br.open(url, timeout=timeout).read() - except Exception as err: - raise ISBNDBError('Could not fetch ISBNDB metadata. Error: '+str(err)) - soup = BeautifulStoneSoup(raw, - convertEntities=BeautifulStoneSoup.XML_ENTITIES) - book_list = soup.find('booklist') - if book_list is None: - errmsg = soup.find('errormessage').string - raise ISBNDBError('Error fetching metadata: '+errmsg) - total_results = int(book_list['total_results']) - page_number += 1 - np = '&page_number=%s&'%page_number - url = re.sub(r'\&page_number=\d+\&', np, url) - books.extend(book_list.findAll('bookdata')) - max -= 1 - return books - - -class ISBNDBMetadata(Metadata): - - def __init__(self, book): - Metadata.__init__(self, None) - - def tostring(e): - if not hasattr(e, 'string'): - return None - ans = e.string - if ans is not None: - ans = unicode(ans).strip() - if not ans: - ans = None - return ans - - self.isbn = unicode(book.get('isbn13', book.get('isbn'))) - title = tostring(book.find('titlelong')) - if not title: - title = tostring(book.find('title')) - self.title = title - self.title = unicode(self.title).strip() - authors = [] - au = tostring(book.find('authorstext')) - if au: - au = au.strip() - temp = au.split(',') - for au in temp: - if not au: continue - authors.extend([a.strip() for a in au.split('&')]) - if authors: - self.authors = authors - try: - self.author_sort = tostring(book.find('authors').find('person')) - if self.authors and self.author_sort == self.authors[0]: - self.author_sort = None - except: - pass - self.publisher = tostring(book.find('publishertext')) - - summ = tostring(book.find('summary')) - if summ: - self.comments = 'SUMMARY:\n'+summ - - -def build_isbn(base_url, opts): - return base_url + 'index1=isbn&value1='+opts.isbn - -def build_combined(base_url, opts): - query = ' '.join([e for e in (opts.title, opts.author, opts.publisher) \ - if e is not None ]) - query = query.strip() - if len(query) == 0: - raise ISBNDBError('You must specify at least one of --author, --title or --publisher') - - query = re.sub(r'\s+', '+', query) - if isinstance(query, unicode): - query = query.encode('utf-8') - return base_url+'index1=combined&value1='+quote(query, '+') - - -def option_parser(): - parser = OptionParser(usage=\ -_(''' -%prog [options] key - -Fetch metadata for books from isndb.com. You can specify either the -books ISBN ID or its title and author. If you specify the title and author, -then more than one book may be returned. - -key is the account key you generate after signing up for a free account from isbndb.com. - -''')) - parser.add_option('-i', '--isbn', default=None, dest='isbn', - help=_('The ISBN ID of the book you want metadata for.')) - parser.add_option('-a', '--author', dest='author', - default=None, help=_('The author whose book to search for.')) - parser.add_option('-t', '--title', dest='title', - default=None, help=_('The title of the book to search for.')) - parser.add_option('-p', '--publisher', default=None, dest='publisher', - help=_('The publisher of the book to search for.')) - parser.add_option('-v', '--verbose', default=False, - action='store_true', help=_('Verbose processing')) - - return parser - - -def create_books(opts, args, timeout=5.): - base_url = BASE_URL%dict(key=args[1]) - if opts.isbn is not None: - url = build_isbn(base_url, opts) - else: - url = build_combined(base_url, opts) - - if opts.verbose: - print ('ISBNDB query: '+url) - - tans = [ISBNDBMetadata(book) for book in fetch_metadata(url, timeout=timeout)] - #remove duplicates ISBN - return list(dict((book.isbn, book) for book in tans).values()) - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - if len(args) != 2: - parser.print_help() - print ('You must supply the isbndb.com key') - return 1 - - for book in create_books(opts, args): - print unicode(book).encode('utf-8') - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/nicebooks.py b/src/calibre/ebooks/metadata/nicebooks.py deleted file mode 100644 index 2afa6c018a..0000000000 --- a/src/calibre/ebooks/metadata/nicebooks.py +++ /dev/null @@ -1,411 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2010, sengian <sengian1@gmail.com>' -__docformat__ = 'restructuredtext en' - -import sys, textwrap, re, traceback, socket -from urllib import urlencode -from math import ceil -from copy import deepcopy - -from lxml.html import soupparser - -from calibre.utils.date import parse_date, utcnow, replace_months -from calibre.utils.cleantext import clean_ascii_chars -from calibre import browser, preferred_encoding -from calibre.ebooks.chardet import xml_to_unicode -from calibre.ebooks.metadata import MetaInformation, check_isbn, \ - authors_to_sort_string -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.ebooks.metadata.covers import CoverDownload -from calibre.utils.config import OptionParser - -class NiceBooks(MetadataSource): - - name = 'Nicebooks' - description = _('Downloads metadata from french Nicebooks') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class NiceBooksCovers(CoverDownload): - - name = 'Nicebooks covers' - description = _('Downloads covers from french Nicebooks') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - type = _('Cover download') - version = (1, 0, 0) - - def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: - return False - br = browser() - try: - entry = Query(isbn=mi.isbn, max_results=1)(br, False, timeout)[0] - if Covers(mi.isbn)(entry).check_cover(): - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception as e: - self.debug(e) - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - br = browser() - try: - entry = Query(isbn=mi.isbn, max_results=1)(br, False, timeout)[0] - cover_data, ext = Covers(mi.isbn)(entry).get_cover(br, timeout) - if not ext: - ext = 'jpg' - result_queue.put((True, cover_data, ext, self.name)) - except Exception as e: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) - - -class NiceBooksError(Exception): - pass - -class ISBNNotFound(NiceBooksError): - pass - -def report(verbose): - if verbose: - traceback.print_exc() - -class Query(object): - - BASE_URL = 'http://fr.nicebooks.com/' - - def __init__(self, title=None, author=None, publisher=None, isbn=None, keywords=None, max_results=20): - assert not(title is None and author is None and publisher is None \ - and isbn is None and keywords is None) - assert (max_results < 21) - - self.max_results = int(max_results) - - if isbn is not None: - q = isbn - else: - q = ' '.join([i for i in (title, author, publisher, keywords) \ - if i is not None]) - - if isinstance(q, unicode): - q = q.encode('utf-8') - self.urldata = 'search?' + urlencode({'q':q,'s':'Rechercher'}) - - def __call__(self, browser, verbose, timeout = 5.): - if verbose: - print _('Query: %s') % self.BASE_URL+self.urldata - - try: - raw = browser.open_novisit(self.BASE_URL+self.urldata, timeout=timeout).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - if isinstance(getattr(e, 'args', [None])[0], socket.timeout): - raise NiceBooksError(_('Nicebooks timed out. Try again later.')) - raise NiceBooksError(_('Nicebooks encountered an error.')) - if '<title>404 - ' in raw: - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None - - #nb of page to call - try: - nbresults = int(feed.xpath("//div[@id='topbar']/b")[0].text) - except: - #direct hit - return [feed] - - nbpagetoquery = int(ceil(float(min(nbresults, self.max_results))/10)) - pages =[feed] - if nbpagetoquery > 1: - for i in xrange(2, nbpagetoquery + 1): - try: - urldata = self.urldata + '&p=' + str(i) - raw = browser.open_novisit(self.BASE_URL+urldata, timeout=timeout).read() - except Exception as e: - continue - if '<title>404 - ' in raw: - continue - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_chars(raw)) - except: - continue - pages.append(feed) - - results = [] - for x in pages: - results.extend([i.find_class('title')[0].get('href') \ - for i in x.xpath("//ul[@id='results']/li")]) - return results[:self.max_results] - -class ResultList(list): - - BASE_URL = 'http://fr.nicebooks.com' - - def __init__(self): - self.repub = re.compile(u'\s*.diteur\s*', re.I) - self.reauteur = re.compile(u'\s*auteur.*', re.I) - self.reautclean = re.compile(u'\s*\(.*\)\s*') - - def get_title(self, entry): - title = deepcopy(entry) - title.remove(title.find("dl[@title='Informations sur le livre']")) - title = ' '.join([i.text_content() for i in title.iterchildren()]) - return unicode(title.replace('\n', '')) - - def get_authors(self, entry): - author = entry.find("dl[@title='Informations sur le livre']") - authortext = [] - for x in author.getiterator('dt'): - if self.reauteur.match(x.text): - elt = x.getnext() - while elt.tag == 'dd': - authortext.append(unicode(elt.text_content())) - elt = elt.getnext() - break - if len(authortext) == 1: - authortext = [self.reautclean.sub('', authortext[0])] - return authortext - - def get_description(self, entry, verbose): - try: - return u'RESUME:\n' + unicode(entry.getparent().xpath("//p[@id='book-description']")[0].text) - except: - report(verbose) - return None - - def get_book_info(self, entry, mi, verbose): - entry = entry.find("dl[@title='Informations sur le livre']") - for x in entry.getiterator('dt'): - if x.text == 'ISBN': - isbntext = x.getnext().text_content().replace('-', '') - if check_isbn(isbntext): - mi.isbn = unicode(isbntext) - elif self.repub.match(x.text): - mi.publisher = unicode(x.getnext().text_content()) - elif x.text == 'Langue': - mi.language = unicode(x.getnext().text_content()) - elif x.text == 'Date de parution': - d = x.getnext().text_content() - try: - default = utcnow().replace(day=15) - d = replace_months(d, 'fr') - d = parse_date(d, assume_utc=True, default=default) - mi.pubdate = d - except: - report(verbose) - return mi - - def fill_MI(self, entry, title, authors, verbose): - mi = MetaInformation(title, authors) - mi.author_sort = authors_to_sort_string(authors) - mi.comments = self.get_description(entry, verbose) - return self.get_book_info(entry, mi, verbose) - - def get_individual_metadata(self, browser, linkdata, verbose): - try: - raw = browser.open_novisit(self.BASE_URL + linkdata).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - if isinstance(getattr(e, 'args', [None])[0], socket.timeout): - raise NiceBooksError(_('Nicebooks timed out. Try again later.')) - raise NiceBooksError(_('Nicebooks encountered an error.')) - if '<title>404 - ' in raw: - report(verbose) - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None - - # get results - return feed.xpath("//div[@id='container']")[0] - - def populate(self, entries, browser, verbose=False): - #single entry - if len(entries) == 1 and not isinstance(entries[0], str): - try: - entry = entries[0].xpath("//div[@id='container']")[0] - entry = entry.find("div[@id='book-info']") - title = self.get_title(entry) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - return - self.append(self.fill_MI(entry, title, authors, verbose)) - else: - #multiple entries - for x in entries: - try: - entry = self.get_individual_metadata(browser, x, verbose) - entry = entry.find("div[@id='book-info']") - title = self.get_title(entry) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - continue - self.append(self.fill_MI(entry, title, authors, verbose)) - -class Covers(object): - - def __init__(self, isbn = None): - assert isbn is not None - self.urlimg = '' - self.isbn = isbn - self.isbnf = False - - def __call__(self, entry = None): - try: - self.urlimg = entry.xpath("//div[@id='book-picture']/a")[0].get('href') - except: - return self - isbno = entry.get_element_by_id('book-info').find("dl[@title='Informations sur le livre']") - for x in isbno.getiterator('dt'): - if x.text == 'ISBN' and check_isbn(x.getnext().text_content()): - self.isbnf = True - break - return self - - def check_cover(self): - return True if self.urlimg else False - - def get_cover(self, browser, timeout = 5.): - try: - cover, ext = browser.open_novisit(self.urlimg, timeout=timeout).read(), \ - self.urlimg.rpartition('.')[-1] - return cover, ext if ext else 'jpg' - except Exception as err: - if isinstance(getattr(err, 'args', [None])[0], socket.timeout): - raise NiceBooksError(_('Nicebooks timed out. Try again later.')) - if not len(self.urlimg): - if not self.isbnf: - raise ISBNNotFound(_('ISBN: %s not found.') % self.isbn) - raise NiceBooksError(_('An errror occured with Nicebooks cover fetcher')) - - -def search(title=None, author=None, publisher=None, isbn=None, - max_results=5, verbose=False, keywords=None): - br = browser() - entries = Query(title=title, author=author, isbn=isbn, publisher=publisher, - keywords=keywords, max_results=max_results)(br, verbose,timeout = 10.) - - if entries is None or len(entries) == 0: - return None - - #List of entry - ans = ResultList() - ans.populate(entries, br, verbose) - return ans - -def check_for_cover(isbn): - br = browser() - entry = Query(isbn=isbn, max_results=1)(br, False)[0] - return Covers(isbn)(entry).check_cover() - -def cover_from_isbn(isbn, timeout = 5.): - br = browser() - entry = Query(isbn=isbn, max_results=1)(br, False, timeout)[0] - return Covers(isbn)(entry).get_cover(br, timeout) - - -def option_parser(): - parser = OptionParser(textwrap.dedent(\ - _('''\ - %prog [options] - - Fetch book metadata from Nicebooks. You must specify one of title, author, - ISBN, publisher or keywords. Will fetch a maximum of 20 matches, - so you should make your query as specific as possible. - It can also get covers if the option is activated. - ''') - )) - parser.add_option('-t', '--title', help=_('Book title')) - parser.add_option('-a', '--author', help=_('Book author(s)')) - parser.add_option('-p', '--publisher', help=_('Book publisher')) - parser.add_option('-i', '--isbn', help=_('Book ISBN')) - parser.add_option('-k', '--keywords', help=_('Keywords')) - parser.add_option('-c', '--covers', default=0, - help=_('Covers: 1-Check/ 2-Download')) - parser.add_option('-p', '--coverspath', default='', - help=_('Covers files path')) - parser.add_option('-m', '--max-results', default=20, - help=_('Maximum number of results to fetch')) - parser.add_option('-v', '--verbose', default=0, action='count', - help=_('Be more verbose about errors')) - return parser - -def main(args=sys.argv): - import os - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, isbn=opts.isbn, publisher=opts.publisher, - keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results) - except AssertionError: - report(True) - parser.print_help() - return 1 - if results is None or len(results) == 0: - print _('No result found for this search!') - return 0 - for result in results: - print unicode(result).encode(preferred_encoding, 'replace') - covact = int(opts.covers) - if covact == 1: - textcover = _('No cover found!') - if check_for_cover(result.isbn): - textcover = _('A cover was found for this book') - print textcover - elif covact == 2: - cover_data, ext = cover_from_isbn(result.isbn) - cpath = result.isbn - if len(opts.coverspath): - cpath = os.path.normpath(opts.coverspath + '/' + result.isbn) - oname = os.path.abspath(cpath+'.'+ext) - open(oname, 'wb').write(cover_data) - print _('Cover saved to file '), oname - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index e67b87efbd..3eff9b11b3 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -307,7 +307,7 @@ class Source(Plugin): title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in [ # Remove things like: (2010) (Omnibus) etc. - (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|mass\s*market|edition|ed\.)[\])}]', ''), + (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|turtleback|mass\s*market|edition|ed\.)[\])}]', ''), # Remove any strings that contain the substring edition inside # parentheses (r'(?i)[({\[].*?(edition|ed.).*?[\]})]', ''), diff --git a/src/calibre/ebooks/metadata/sources/cli.py b/src/calibre/ebooks/metadata/sources/cli.py index cb422f939d..f8b9c6b7a9 100644 --- a/src/calibre/ebooks/metadata/sources/cli.py +++ b/src/calibre/ebooks/metadata/sources/cli.py @@ -19,13 +19,8 @@ from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.metadata.sources.base import create_log from calibre.ebooks.metadata.sources.identify import identify from calibre.ebooks.metadata.sources.covers import download_cover -from calibre.utils.config import test_eight_code def option_parser(): - if not test_eight_code: - from calibre.ebooks.metadata.fetch import option_parser - return option_parser() - parser = OptionParser(textwrap.dedent( '''\ %prog [options] @@ -48,9 +43,6 @@ def option_parser(): return parser def main(args=sys.argv): - if not test_eight_code: - from calibre.ebooks.metadata.fetch import main - return main(args) parser = option_parser() opts, args = parser.parse_args(args) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index f8dd0693ea..737bf38a56 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -20,9 +20,8 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.utils.filenames import ascii_filename from calibre.constants import preferred_encoding, filesystem_encoding from calibre.gui2.actions import InterfaceAction -from calibre.gui2 import config, question_dialog +from calibre.gui2 import question_dialog from calibre.ebooks.metadata import MetaInformation -from calibre.utils.config import test_eight_code from calibre.ebooks.metadata.sources.base import msprefs def get_filters(): @@ -180,26 +179,17 @@ class AddAction(InterfaceAction): except IndexError: self.gui.library_view.model().books_added(self.isbn_add_dialog.value) self.isbn_add_dialog.accept() - if test_eight_code: - orig = msprefs['ignore_fields'] - new = list(orig) - for x in ('title', 'authors'): - if x in new: - new.remove(x) - msprefs['ignore_fields'] = new - try: - self.gui.iactions['Edit Metadata'].download_metadata( - ids=self.add_by_isbn_ids) - finally: - msprefs['ignore_fields'] = orig - else: - orig = config['overwrite_author_title_metadata'] - config['overwrite_author_title_metadata'] = True - try: - self.gui.iactions['Edit Metadata'].do_download_metadata( - self.add_by_isbn_ids) - finally: - config['overwrite_author_title_metadata'] = orig + orig = msprefs['ignore_fields'] + new = list(orig) + for x in ('title', 'authors'): + if x in new: + new.remove(x) + msprefs['ignore_fields'] = new + try: + self.gui.iactions['Edit Metadata'].download_metadata( + ids=self.add_by_isbn_ids) + finally: + msprefs['ignore_fields'] = orig return diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 3349be8d80..4ab4950179 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -10,15 +10,13 @@ from functools import partial from PyQt4.Qt import Qt, QMenu, QModelIndex, QTimer -from calibre.gui2 import error_dialog, config, Dispatcher, question_dialog -from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog +from calibre.gui2 import error_dialog, Dispatcher, question_dialog from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog from calibre.gui2.dialogs.confirm_delete import confirm from calibre.gui2.dialogs.tag_list_editor import TagListEditor from calibre.gui2.actions import InterfaceAction from calibre.ebooks.metadata import authors_to_string from calibre.utils.icu import sort_key -from calibre.utils.config import test_eight_code class EditMetadataAction(InterfaceAction): @@ -36,22 +34,8 @@ class EditMetadataAction(InterfaceAction): md.addAction(_('Edit metadata in bulk'), partial(self.edit_metadata, False, bulk=True)) md.addSeparator() - if test_eight_code: - dall = self.download_metadata - else: - dall = partial(self.download_metadata_old, False, covers=True) - dident = partial(self.download_metadata_old, False, covers=False) - dcovers = partial(self.download_metadata_old, False, covers=True, - set_metadata=False, set_social_metadata=False) - - md.addAction(_('Download metadata and covers'), dall, + md.addAction(_('Download metadata and covers'), self.download_metadata, Qt.ControlModifier+Qt.Key_D) - if not test_eight_code: - md.addAction(_('Download only metadata'), dident) - md.addAction(_('Download only covers'), dcovers) - md.addAction(_('Download only social metadata'), - partial(self.download_metadata_old, False, covers=False, - set_metadata=False, set_social_metadata=True)) self.metadata_menu = md mb = QMenu() @@ -88,7 +72,7 @@ class EditMetadataAction(InterfaceAction): _('No books selected'), show=True) db = self.gui.library_view.model().db ids = [db.id(row.row()) for row in rows] - from calibre.gui2.metadata.bulk_download2 import start_download + from calibre.gui2.metadata.bulk_download import start_download start_download(self.gui, ids, Dispatcher(self.metadata_downloaded)) @@ -96,7 +80,7 @@ class EditMetadataAction(InterfaceAction): if job.failed: self.gui.job_exception(job, dialog_title=_('Failed to download metadata')) return - from calibre.gui2.metadata.bulk_download2 import get_job_details + from calibre.gui2.metadata.bulk_download import get_job_details id_map, failed_ids, failed_covers, all_failed, det_msg = \ get_job_details(job) if all_failed: @@ -112,8 +96,9 @@ class EditMetadataAction(InterfaceAction): show_copy_button = False if failed_ids or failed_covers: show_copy_button = True + num = len(failed_ids.union(failed_covers)) msg += '<p>'+_('Could not download metadata and/or covers for %d of the books. Click' - ' "Show details" to see which books.')%len(failed_ids) + ' "Show details" to see which books.')%num payload = (id_map, failed_ids, failed_covers) from calibre.gui2.dialogs.message_box import ProceedNotification @@ -158,49 +143,6 @@ class EditMetadataAction(InterfaceAction): self.apply_metadata_changes(id_map) - def download_metadata_old(self, checked, covers=True, set_metadata=True, - set_social_metadata=None): - rows = self.gui.library_view.selectionModel().selectedRows() - if not rows or len(rows) == 0: - d = error_dialog(self.gui, _('Cannot download metadata'), - _('No books selected')) - d.exec_() - return - db = self.gui.library_view.model().db - ids = [db.id(row.row()) for row in rows] - self.do_download_metadata(ids, covers=covers, - set_metadata=set_metadata, - set_social_metadata=set_social_metadata) - - def do_download_metadata(self, ids, covers=True, set_metadata=True, - set_social_metadata=None): - m = self.gui.library_view.model() - db = m.db - if set_social_metadata is None: - get_social_metadata = config['get_social_metadata'] - else: - get_social_metadata = set_social_metadata - from calibre.gui2.metadata.bulk_download import DoDownload - if set_social_metadata is not None and set_social_metadata: - x = _('social metadata') - else: - x = _('covers') if covers and not set_metadata else _('metadata') - title = _('Downloading {0} for {1} book(s)').format(x, len(ids)) - self._download_book_metadata = DoDownload(self.gui, title, db, ids, - get_covers=covers, set_metadata=set_metadata, - get_social_metadata=get_social_metadata) - m.stop_metadata_backup() - try: - self._download_book_metadata.exec_() - finally: - m.start_metadata_backup() - cr = self.gui.library_view.currentIndex().row() - x = self._download_book_metadata - if x.updated: - self.gui.library_view.model().refresh_ids( - x.updated, cr) - if self.gui.cover_flow: - self.gui.cover_flow.dataChanged() # }}} def edit_metadata(self, checked, bulk=None): @@ -227,9 +169,7 @@ class EditMetadataAction(InterfaceAction): list(range(self.gui.library_view.model().rowCount(QModelIndex()))) current_row = row_list.index(cr) - func = (self.do_edit_metadata if test_eight_code else - self.do_edit_metadata_old) - changed, rows_to_refresh = func(row_list, current_row) + changed, rows_to_refresh = self.do_edit_metadata(row_list, current_row) m = self.gui.library_view.model() @@ -244,36 +184,6 @@ class EditMetadataAction(InterfaceAction): m.current_changed(current, previous) self.gui.tags_view.recount() - def do_edit_metadata_old(self, row_list, current_row): - changed = set([]) - db = self.gui.library_view.model().db - - while True: - prev = next_ = None - if current_row > 0: - prev = db.title(row_list[current_row-1]) - if current_row < len(row_list) - 1: - next_ = db.title(row_list[current_row+1]) - - d = MetadataSingleDialog(self.gui, row_list[current_row], db, - prev=prev, next_=next_) - d.view_format.connect(lambda - fmt:self.gui.iactions['View'].view_format(row_list[current_row], - fmt)) - ret = d.exec_() - d.break_cycles() - if ret != d.Accepted: - break - - changed.add(d.id) - self.gui.library_view.model().refresh_ids(list(d.books_to_refresh)) - if d.row_delta == 0: - break - current_row += d.row_delta - self.gui.library_view.set_current_row(current_row) - self.gui.library_view.scroll_to_row(current_row) - return changed, set() - def do_edit_metadata(self, row_list, current_row): from calibre.gui2.metadata.single import edit_metadata db = self.gui.library_view.model().db diff --git a/src/calibre/gui2/dialogs/fetch_metadata.py b/src/calibre/gui2/dialogs/fetch_metadata.py deleted file mode 100644 index 426c7b1d60..0000000000 --- a/src/calibre/gui2/dialogs/fetch_metadata.py +++ /dev/null @@ -1,271 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' -''' -GUI for fetching metadata from servers. -''' - -import time -from threading import Thread - -from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, pyqtSignal, \ - QAbstractTableModel, QCoreApplication, QTimer -from PyQt4.QtGui import QDialog, QItemSelectionModel, QIcon - -from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata -from calibre.gui2 import error_dialog, NONE, info_dialog, config -from calibre.gui2.widgets import ProgressIndicator -from calibre import strftime, force_unicode -from calibre.customize.ui import get_isbndb_key, set_isbndb_key -from calibre.utils.icu import sort_key - -_hung_fetchers = set([]) - -class Fetcher(Thread): - - def __init__(self, title, author, publisher, isbn, key): - Thread.__init__(self) - self.daemon = True - self.title = title - self.author = author - self.publisher = publisher - self.isbn = isbn - self.key = key - self.results, self.exceptions = [], [] - - def run(self): - from calibre.ebooks.metadata.fetch import search - self.results, self.exceptions = search(self.title, self.author, - self.publisher, self.isbn, - self.key if self.key else None) - - -class Matches(QAbstractTableModel): - - def __init__(self, matches): - self.matches = matches - self.yes_icon = QVariant(QIcon(I('ok.png'))) - QAbstractTableModel.__init__(self) - - def rowCount(self, *args): - return len(self.matches) - - def columnCount(self, *args): - return 8 - - def headerData(self, section, orientation, role): - if role != Qt.DisplayRole: - return NONE - text = "" - if orientation == Qt.Horizontal: - if section == 0: text = _("Title") - elif section == 1: text = _("Author(s)") - elif section == 2: text = _("Author Sort") - elif section == 3: text = _("Publisher") - elif section == 4: text = _("ISBN") - elif section == 5: text = _("Published") - elif section == 6: text = _("Has Cover") - elif section == 7: text = _("Has Summary") - - return QVariant(text) - else: - return QVariant(section+1) - - def summary(self, row): - return self.matches[row].comments - - def data_as_text(self, book, col): - if col == 0 and book.title is not None: - return book.title - elif col == 1: - return ', '.join(book.authors) - elif col == 2 and book.author_sort is not None: - return book.author_sort - elif col == 3 and book.publisher is not None: - return book.publisher - elif col == 4 and book.isbn is not None: - return book.isbn - elif col == 5 and hasattr(book.pubdate, 'timetuple'): - return strftime('%b %Y', book.pubdate.timetuple()) - elif col == 6 and book.has_cover: - return 'y' - elif col == 7 and book.comments: - return 'y' - return '' - - def data(self, index, role): - row, col = index.row(), index.column() - book = self.matches[row] - if role == Qt.DisplayRole: - res = self.data_as_text(book, col) - if col <= 5 and res: - return QVariant(res) - return NONE - elif role == Qt.DecorationRole: - if col == 6 and book.has_cover: - return self.yes_icon - if col == 7 and book.comments: - return self.yes_icon - return NONE - - def sort(self, col, order, reset=True): - if not self.matches: - return - descending = order == Qt.DescendingOrder - self.matches.sort(None, - lambda x: sort_key(unicode(force_unicode(self.data_as_text(x, col)))), - descending) - if reset: - self.reset() - -class FetchMetadata(QDialog, Ui_FetchMetadata): - - HANG_TIME = 75 #seconds - - queue_reject = pyqtSignal() - - def __init__(self, parent, isbn, title, author, publisher, timeout): - QDialog.__init__(self, parent) - Ui_FetchMetadata.__init__(self) - self.setupUi(self) - - for fetcher in list(_hung_fetchers): - if not fetcher.is_alive(): - _hung_fetchers.remove(fetcher) - - self.pi = ProgressIndicator(self) - self.timeout = timeout - QObject.connect(self.fetch, SIGNAL('clicked()'), self.fetch_metadata) - self.queue_reject.connect(self.reject, Qt.QueuedConnection) - - isbndb_key = get_isbndb_key() - if not isbndb_key: - isbndb_key = '' - self.key.setText(isbndb_key) - - self.setWindowTitle(title if title else _('Unknown')) - self.isbn = isbn - self.title = title - self.author = author.strip() - self.publisher = publisher - self.previous_row = None - self.warning.setVisible(False) - self.connect(self.matches, SIGNAL('activated(QModelIndex)'), self.chosen) - self.connect(self.matches, SIGNAL('entered(QModelIndex)'), - self.show_summary) - self.matches.setMouseTracking(True) - # Enabling sorting and setting a sort column will not change the initial - # order of the results, as they are filled in later - self.matches.setSortingEnabled(True) - self.matches.horizontalHeader().sectionClicked.connect(self.show_sort_indicator) - self.matches.horizontalHeader().setSortIndicatorShown(False) - self.fetch_metadata() - self.opt_get_social_metadata.setChecked(config['get_social_metadata']) - self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata']) - self.opt_auto_download_cover.setChecked(config['auto_download_cover']) - - def show_summary(self, current, *args): - row = current.row() - if row != self.previous_row: - summ = self.model.summary(row) - self.summary.setText(summ if summ else '') - self.previous_row = row - - def fetch_metadata(self): - self.warning.setVisible(False) - key = str(self.key.text()) - if key: - set_isbndb_key(key) - else: - key = None - title = author = publisher = isbn = None - if self.isbn: - isbn = self.isbn - if self.title: - title = self.title - if self.author and not self.author == _('Unknown'): - author = self.author - self.fetch.setEnabled(False) - self.setCursor(Qt.WaitCursor) - QCoreApplication.instance().processEvents() - self.fetcher = Fetcher(title, author, publisher, isbn, key) - self.fetcher.start() - self.pi.start(_('Finding metadata...')) - self._hangcheck = QTimer(self) - self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck, - Qt.QueuedConnection) - self.start_time = time.time() - self._hangcheck.start(100) - - def hangcheck(self): - if self.fetcher.is_alive() and \ - time.time() - self.start_time < self.HANG_TIME: - return - self._hangcheck.stop() - try: - if self.fetcher.is_alive(): - error_dialog(self, _('Could not find metadata'), - _('The metadata download seems to have stalled. ' - 'Try again later.')).exec_() - self.terminate() - return self.queue_reject.emit() - self.model = Matches(self.fetcher.results) - warnings = [(x[0], force_unicode(x[1])) for x in \ - self.fetcher.exceptions if x[1] is not None] - if warnings: - warnings='<br>'.join(['<b>%s</b>: %s'%(name, exc) for name,exc in warnings]) - self.warning.setText('<p><b>'+ _('Warning')+':</b>'+\ - _('Could not fetch metadata from:')+\ - '<br>'+warnings+'</p>') - self.warning.setVisible(True) - if self.model.rowCount() < 1: - info_dialog(self, _('No metadata found'), - _('No metadata found, try adjusting the title and author ' - 'and/or removing the ISBN.')).exec_() - self.reject() - return - - self.matches.setModel(self.model) - QObject.connect(self.matches.selectionModel(), - SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'), - self.show_summary) - self.model.reset() - self.matches.selectionModel().select(self.model.index(0, 0), - QItemSelectionModel.Select | QItemSelectionModel.Rows) - self.matches.setCurrentIndex(self.model.index(0, 0)) - finally: - self.fetch.setEnabled(True) - self.unsetCursor() - self.matches.resizeColumnsToContents() - self.pi.stop() - - def terminate(self): - if hasattr(self, 'fetcher') and self.fetcher.is_alive(): - _hung_fetchers.add(self.fetcher) - if hasattr(self, '_hangcheck') and self._hangcheck.isActive(): - self._hangcheck.stop() - # Save value of auto_download_cover, since this is the only place it can - # be set. The values of the other options can be set in - # Preferences->Behavior and should not be set here as they affect bulk - # downloading as well. - if self.opt_auto_download_cover.isChecked() != config['auto_download_cover']: - config.set('auto_download_cover', self.opt_auto_download_cover.isChecked()) - - def __enter__(self, *args): - return self - - def __exit__(self, *args): - self.terminate() - - def selected_book(self): - try: - return self.matches.model().matches[self.matches.currentIndex().row()] - except: - return None - - def chosen(self, index): - self.matches.setCurrentIndex(index) - self.accept() - - def show_sort_indicator(self, *args): - self.matches.horizontalHeader().setSortIndicatorShown(True) - diff --git a/src/calibre/gui2/dialogs/fetch_metadata.ui b/src/calibre/gui2/dialogs/fetch_metadata.ui deleted file mode 100644 index b140fa158d..0000000000 --- a/src/calibre/gui2/dialogs/fetch_metadata.ui +++ /dev/null @@ -1,179 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<ui version="4.0"> - <class>FetchMetadata</class> - <widget class="QDialog" name="FetchMetadata"> - <property name="windowModality"> - <enum>Qt::WindowModal</enum> - </property> - <property name="geometry"> - <rect> - <x>0</x> - <y>0</y> - <width>890</width> - <height>642</height> - </rect> - </property> - <property name="windowTitle"> - <string>Fetch metadata</string> - </property> - <property name="windowIcon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/metadata.png</normaloff>:/images/metadata.png</iconset> - </property> - <layout class="QVBoxLayout"> - <item> - <widget class="QLabel" name="tlabel"> - <property name="text"> - <string><p>calibre can find metadata for your books from two locations: <b>Google Books</b> and <b>isbndb.com</b>. <p>To use isbndb.com you must sign up for a <a href="http://www.isbndb.com">free account</a> and enter your access key below.</string> - </property> - <property name="alignment"> - <set>Qt::AlignCenter</set> - </property> - <property name="wordWrap"> - <bool>true</bool> - </property> - <property name="openExternalLinks"> - <bool>true</bool> - </property> - </widget> - </item> - <item> - <layout class="QHBoxLayout"> - <item> - <widget class="QLabel" name="label_2"> - <property name="text"> - <string>&Access Key:</string> - </property> - <property name="buddy"> - <cstring>key</cstring> - </property> - </widget> - </item> - <item> - <widget class="QLineEdit" name="key"/> - </item> - <item> - <widget class="QPushButton" name="fetch"> - <property name="text"> - <string>Fetch</string> - </property> - </widget> - </item> - </layout> - </item> - <item> - <widget class="QLabel" name="warning"> - <property name="text"> - <string/> - </property> - <property name="wordWrap"> - <bool>true</bool> - </property> - </widget> - </item> - <item> - <widget class="QGroupBox" name="groupBox"> - <property name="title"> - <string>Matches</string> - </property> - <layout class="QVBoxLayout"> - <item> - <widget class="QLabel" name="label_3"> - <property name="text"> - <string>Select the book that most closely matches your copy from the list below</string> - </property> - </widget> - </item> - <item> - <widget class="QTableView" name="matches"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Expanding" vsizetype="Expanding"> - <horstretch>0</horstretch> - <verstretch>1</verstretch> - </sizepolicy> - </property> - <property name="alternatingRowColors"> - <bool>true</bool> - </property> - <property name="selectionMode"> - <enum>QAbstractItemView::SingleSelection</enum> - </property> - <property name="selectionBehavior"> - <enum>QAbstractItemView::SelectRows</enum> - </property> - </widget> - </item> - <item> - <widget class="QTextBrowser" name="summary"/> - </item> - </layout> - </widget> - </item> - <item> - <widget class="QCheckBox" name="opt_overwrite_author_title_metadata"> - <property name="text"> - <string>Overwrite author and title with author and title of selected book</string> - </property> - </widget> - </item> - <item> - <widget class="QCheckBox" name="opt_get_social_metadata"> - <property name="text"> - <string>Download &social metadata (tags/rating/etc.) for the selected book</string> - </property> - </widget> - </item> - <item> - <widget class="QCheckBox" name="opt_auto_download_cover"> - <property name="text"> - <string>Automatically download the cover, if available</string> - </property> - </widget> - </item> - <item> - <widget class="QDialogButtonBox" name="buttonBox"> - <property name="standardButtons"> - <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set> - </property> - </widget> - </item> - </layout> - </widget> - <resources> - <include location="../../../../resources/images.qrc"/> - </resources> - <connections> - <connection> - <sender>buttonBox</sender> - <signal>accepted()</signal> - <receiver>FetchMetadata</receiver> - <slot>accept()</slot> - <hints> - <hint type="sourcelabel"> - <x>460</x> - <y>599</y> - </hint> - <hint type="destinationlabel"> - <x>657</x> - <y>530</y> - </hint> - </hints> - </connection> - <connection> - <sender>buttonBox</sender> - <signal>rejected()</signal> - <receiver>FetchMetadata</receiver> - <slot>reject()</slot> - <hints> - <hint type="sourcelabel"> - <x>417</x> - <y>599</y> - </hint> - <hint type="destinationlabel"> - <x>0</x> - <y>491</y> - </hint> - </hints> - </connection> - </connections> -</ui> diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py deleted file mode 100644 index 4776562c29..0000000000 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ /dev/null @@ -1,1031 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' - -''' -The dialog used to edit meta information for a book as well as -add/remove formats -''' - -import os, re, time, traceback, textwrap -from functools import partial -from threading import Thread - -from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QDate, \ - QPixmap, QListWidgetItem, QDialog, pyqtSignal, QIcon, \ - QPushButton, QKeySequence - -from calibre.gui2 import error_dialog, file_icon_provider, dynamic, \ - choose_files, choose_images, ResizableDialog, \ - warning_dialog, question_dialog, UNDEFINED_QDATE -from calibre.gui2.dialogs.metadata_single_ui import Ui_MetadataSingleDialog -from calibre.gui2.dialogs.fetch_metadata import FetchMetadata -from calibre.gui2.dialogs.tag_editor import TagEditor -from calibre.gui2.widgets import ProgressIndicator -from calibre.ebooks import BOOK_EXTENSIONS -from calibre.ebooks.metadata import string_to_authors, \ - authors_to_string, check_isbn, title_sort -from calibre.ebooks.metadata.covers import download_cover -from calibre.ebooks.metadata import MetaInformation -from calibre.utils.config import prefs, tweaks -from calibre.utils.date import qt_to_dt, local_tz, utcfromtimestamp -from calibre.utils.icu import sort_key -from calibre.customize.ui import run_plugins_on_import, get_isbndb_key -from calibre.gui2.preferences.social import SocialMetadata -from calibre.gui2.custom_column_widgets import populate_metadata_page -from calibre import strftime -from calibre.library.comments import comments_to_html - -class CoverFetcher(Thread): # {{{ - - def __init__(self, username, password, isbn, timeout, title, author): - Thread.__init__(self) - self.daemon = True - - self.username = username.strip() if username else username - self.password = password.strip() if password else password - self.timeout = timeout - self.isbn = isbn - self.title = title - self.needs_isbn = False - self.author = author - self.exception = self.traceback = self.cover_data = self.errors = None - - def run(self): - try: - au = self.author if self.author else None - mi = MetaInformation(self.title, [au]) - if not self.isbn: - from calibre.ebooks.metadata.fetch import search - if not self.title: - self.needs_isbn = True - return - key = get_isbndb_key() - if not key: - key = None - results = search(title=self.title, author=au, - isbndb_key=key)[0] - results = sorted([x.isbn for x in results if x.isbn], - cmp=lambda x,y:cmp(len(x),len(y)), reverse=True) - if not results: - self.needs_isbn = True - return - self.isbn = results[0] - - mi.isbn = self.isbn - - self.cover_data, self.errors = download_cover(mi, - timeout=self.timeout) - except Exception as e: - self.exception = e - self.traceback = traceback.format_exc() - print self.traceback - -# }}} - -class Format(QListWidgetItem): # {{{ - - def __init__(self, parent, ext, size, path=None, timestamp=None): - self.path = path - self.ext = ext - self.size = float(size)/(1024*1024) - text = '%s (%.2f MB)'%(self.ext.upper(), self.size) - QListWidgetItem.__init__(self, file_icon_provider().icon_from_ext(ext), - text, parent, QListWidgetItem.UserType) - if timestamp is not None: - ts = timestamp.astimezone(local_tz) - t = strftime('%a, %d %b %Y [%H:%M:%S]', ts.timetuple()) - text = _('Last modified: %s')%t - self.setToolTip(text) - self.setStatusTip(text) - -# }}} - - -class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): - - COVER_FETCH_TIMEOUT = 240 # seconds - view_format = pyqtSignal(object) - - # Cover processing {{{ - - def set_cover(self): - mi, ext = self.get_selected_format_metadata() - if mi is None: - return - cdata = None - if mi.cover and os.access(mi.cover, os.R_OK): - cdata = open(mi.cover).read() - elif mi.cover_data[1] is not None: - cdata = mi.cover_data[1] - if cdata is None: - error_dialog(self, _('Could not read cover'), - _('Could not read cover from %s format')%ext).exec_() - return - pix = QPixmap() - pix.loadFromData(cdata) - if pix.isNull(): - error_dialog(self, _('Could not read cover'), - _('The cover in the %s format is invalid')%ext).exec_() - return - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - self.cover_data = cdata - - def trim_cover(self, *args): - from calibre.utils.magick import Image - cdata = self.cover_data - if not cdata: - return - im = Image() - im.load(cdata) - im.trim(10) - cdata = im.export('png') - pix = QPixmap() - pix.loadFromData(cdata) - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - self.cover_data = cdata - - - - def update_cover_tooltip(self): - p = self.cover.pixmap() - self.cover.setToolTip(_('Cover size: %dx%d pixels') % - (p.width(), p.height())) - - - def do_reset_cover(self, *args): - pix = QPixmap(I('default_cover.png')) - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cover_data = None - - def select_cover(self, checked): - files = choose_images(self, 'change cover dialog', - _('Choose cover for ') + unicode(self.title.text())) - if not files: - return - _file = files[0] - if _file: - _file = os.path.abspath(_file) - if not os.access(_file, os.R_OK): - d = error_dialog(self, _('Cannot read'), - _('You do not have permission to read the file: ') + _file) - d.exec_() - return - cf, cover = None, None - try: - cf = open(_file, "rb") - cover = cf.read() - except IOError as e: - d = error_dialog(self, _('Error reading file'), - _("<p>There was an error reading from file: <br /><b>") + _file + "</b></p><br />"+str(e)) - d.exec_() - if cover: - pix = QPixmap() - pix.loadFromData(cover) - if pix.isNull(): - d = error_dialog(self, - _("Not a valid picture"), - _file + _(" is not a valid picture")) - d.exec_() - else: - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - self.cover_data = cover - - def generate_cover(self, *args): - from calibre.ebooks import calibre_cover - from calibre.ebooks.metadata import fmt_sidx - from calibre.gui2 import config - title = unicode(self.title.text()).strip() - author = unicode(self.authors.text()).strip() - if author.endswith('&'): - author = author[:-1].strip() - if not title or not author: - return error_dialog(self, _('Specify title and author'), - _('You must specify a title and author before generating ' - 'a cover'), show=True) - series = unicode(self.series.text()).strip() - series_string = None - if series: - series_string = _('Book %s of %s')%( - fmt_sidx(self.series_index.value(), - use_roman=config['use_roman_numerals_for_series_number']), series) - self.cover_data = calibre_cover(title, author, - series_string=series_string) - pix = QPixmap() - pix.loadFromData(self.cover_data) - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - - def cover_dropped(self, cover_data): - self.cover_changed = True - self.cover_data = cover_data - self.update_cover_tooltip() - - def fetch_cover(self): - isbn = re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text())).strip() - self.fetch_cover_button.setEnabled(False) - self.setCursor(Qt.WaitCursor) - title, author = map(unicode, (self.title.text(), self.authors.text())) - self.cover_fetcher = CoverFetcher(None, None, isbn, - self.timeout, title, author) - self.cover_fetcher.start() - self.cf_start_time = time.time() - self.pi.start(_('Downloading cover...')) - QTimer.singleShot(100, self.hangcheck) - - def hangcheck(self): - cf = self.cover_fetcher - if cf is None: - # Called after dialog closed - return - - if cf.is_alive() and \ - time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT: - QTimer.singleShot(100, self.hangcheck) - return - - try: - if cf.is_alive(): - error_dialog(self, _('Cannot fetch cover'), - _('<b>Could not fetch cover.</b><br/>')+ - _('The download timed out.')).exec_() - return - if cf.needs_isbn: - error_dialog(self, _('Cannot fetch cover'), - _('Could not find cover for this book. Try ' - 'specifying the ISBN first.')).exec_() - return - if cf.exception is not None: - err = cf.exception - error_dialog(self, _('Cannot fetch cover'), - _('<b>Could not fetch cover.</b><br/>')+unicode(err)).exec_() - return - if cf.errors and cf.cover_data is None: - details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in cf.errors]) - error_dialog(self, _('Cannot fetch cover'), - _('<b>Could not fetch cover.</b><br/>') + - _('For the error message from each cover source, ' - 'click Show details below.'), det_msg=details, show=True) - return - - pix = QPixmap() - pix.loadFromData(cf.cover_data) - if pix.isNull(): - error_dialog(self, _('Bad cover'), - _('The cover is not a valid picture')).exec_() - else: - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - self.cover_data = cf.cover_data - finally: - self.fetch_cover_button.setEnabled(True) - self.unsetCursor() - if self.pi is not None: - self.pi.stop() - - - # }}} - - # Formats processing {{{ - def add_format(self, x): - files = choose_files(self, 'add formats dialog', - _("Choose formats for ") + unicode((self.title.text())), - [(_('Books'), BOOK_EXTENSIONS)]) - self._add_formats(files) - - def _add_formats(self, paths): - added = False - if not paths: - return added - bad_perms = [] - for _file in paths: - _file = os.path.abspath(_file) - if not os.access(_file, os.R_OK): - bad_perms.append(_file) - continue - - nfile = run_plugins_on_import(_file) - if nfile is not None: - _file = nfile - stat = os.stat(_file) - size = stat.st_size - ext = os.path.splitext(_file)[1].lower().replace('.', '') - timestamp = utcfromtimestamp(stat.st_mtime) - for row in range(self.formats.count()): - fmt = self.formats.item(row) - if fmt.ext.lower() == ext: - self.formats.takeItem(row) - break - Format(self.formats, ext, size, path=_file, timestamp=timestamp) - self.formats_changed = True - added = True - if bad_perms: - error_dialog(self, _('No permission'), - _('You do not have ' - 'permission to read the following files:'), - det_msg='\n'.join(bad_perms), show=True) - - return added - - def formats_dropped(self, event, paths): - if self._add_formats(paths): - event.accept() - - def remove_format(self, *args): - rows = self.formats.selectionModel().selectedRows(0) - for row in rows: - self.formats.takeItem(row.row()) - self.formats_changed = True - - def get_selected_format_metadata(self): - from calibre.ebooks.metadata.meta import get_metadata - old = prefs['read_file_metadata'] - if not old: - prefs['read_file_metadata'] = True - try: - row = self.formats.currentRow() - fmt = self.formats.item(row) - if fmt is None: - if self.formats.count() == 1: - fmt = self.formats.item(0) - if fmt is None: - error_dialog(self, _('No format selected'), - _('No format selected')).exec_() - return None, None - ext = fmt.ext.lower() - if fmt.path is None: - stream = self.db.format(self.row, ext, as_file=True) - else: - stream = open(fmt.path, 'r+b') - try: - mi = get_metadata(stream, ext) - return mi, ext - except: - error_dialog(self, _('Could not read metadata'), - _('Could not read metadata from %s format')%ext).exec_() - return None, None - finally: - if old != prefs['read_file_metadata']: - prefs['read_file_metadata'] = old - - def set_metadata_from_format(self): - mi, ext = self.get_selected_format_metadata() - if mi is None: - return - if mi.title: - self.title.setText(mi.title) - if mi.authors: - self.authors.setEditText(authors_to_string(mi.authors)) - if mi.author_sort: - self.author_sort.setText(mi.author_sort) - if mi.rating is not None: - try: - self.rating.setValue(mi.rating) - except: - pass - if mi.publisher: - self.publisher.setEditText(mi.publisher) - if mi.tags: - self.tags.setText(', '.join(mi.tags)) - if mi.isbn: - self.isbn.setText(mi.isbn) - if mi.pubdate: - self.pubdate.setDate(QDate(mi.pubdate.year, mi.pubdate.month, - mi.pubdate.day)) - if mi.series and mi.series.strip(): - self.series.setEditText(mi.series) - if mi.series_index is not None: - self.series_index.setValue(float(mi.series_index)) - if mi.comments and mi.comments.strip(): - comments = comments_to_html(mi.comments) - self.comments.html = comments - - - def sync_formats(self): - old_extensions, new_extensions, paths = set(), set(), {} - for row in range(self.formats.count()): - fmt = self.formats.item(row) - ext, path = fmt.ext.lower(), fmt.path - if 'unknown' in ext.lower(): - ext = None - if path: - new_extensions.add(ext) - paths[ext] = path - else: - old_extensions.add(ext) - for ext in new_extensions: - self.db.add_format(self.row, ext, open(paths[ext], 'rb'), notify=False) - dbfmts = self.db.formats(self.row) - db_extensions = set([f.lower() for f in (dbfmts.split(',') if dbfmts - else [])]) - extensions = new_extensions.union(old_extensions) - for ext in db_extensions: - if ext not in extensions and ext in self.original_formats: - self.db.remove_format(self.row, ext, notify=False) - - def show_format(self, item, *args): - fmt = item.ext - self.view_format.emit(fmt) - - # }}} - - def __init__(self, window, row, db, prev=None, - next_=None): - ResizableDialog.__init__(self, window) - self.cover_fetcher = None - self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter) - base = unicode(self.author_sort.toolTip()) - ok_tooltip = '<p>' + textwrap.fill(base+'<br><br>'+ - _(' The green color indicates that the current ' - 'author sort matches the current author')) - bad_tooltip = '<p>'+textwrap.fill(base + '<br><br>'+ - _(' The red color indicates that the current ' - 'author sort does not match the current author. ' - 'No action is required if this is what you want.')) - self.aus_tooltips = (ok_tooltip, bad_tooltip) - - base = unicode(self.title_sort.toolTip()) - ok_tooltip = '<p>' + textwrap.fill(base+'<br><br>'+ - _(' The green color indicates that the current ' - 'title sort matches the current title')) - bad_tooltip = '<p>'+textwrap.fill(base + '<br><br>'+ - _(' The red color warns that the current ' - 'title sort does not match the current title. ' - 'No action is required if this is what you want.')) - self.ts_tooltips = (ok_tooltip, bad_tooltip) - self.row_delta = 0 - if prev: - self.prev_button = QPushButton(QIcon(I('back.png')), _('Previous'), - self) - self.button_box.addButton(self.prev_button, self.button_box.ActionRole) - tip = (_('Save changes and edit the metadata of %s')+' [Alt+Left]')%prev - self.prev_button.setToolTip(tip) - self.prev_button.clicked.connect(partial(self.next_triggered, - -1)) - self.prev_button.setShortcut(QKeySequence('Alt+Left')) - if next_: - self.next_button = QPushButton(QIcon(I('forward.png')), _('Next'), - self) - self.button_box.addButton(self.next_button, self.button_box.ActionRole) - tip = (_('Save changes and edit the metadata of %s')+' [Alt+Right]')%next_ - self.next_button.setToolTip(tip) - self.next_button.clicked.connect(partial(self.next_triggered, 1)) - self.next_button.setShortcut(QKeySequence('Alt+Right')) - - self.splitter.setStretchFactor(100, 1) - self.read_state() - self.db = db - self.pi = ProgressIndicator(self) - self.id = db.id(row) - self.row = row - self.cover_data = None - self.formats_changed = False - self.formats.setAcceptDrops(True) - self.cover_changed = False - self.cpixmap = None - self.pubdate.setMinimumDate(UNDEFINED_QDATE) - pubdate_format = tweaks['gui_pubdate_display_format'] - if pubdate_format is not None: - self.pubdate.setDisplayFormat(pubdate_format) - self.date.setMinimumDate(UNDEFINED_QDATE) - self.pubdate.setSpecialValueText(_('Undefined')) - self.date.setSpecialValueText(_('Undefined')) - self.clear_pubdate_button.clicked.connect(self.clear_pubdate) - - - self.connect(self.cover, SIGNAL('cover_changed(PyQt_PyObject)'), self.cover_dropped) - QObject.connect(self.cover_button, SIGNAL("clicked(bool)"), \ - self.select_cover) - QObject.connect(self.add_format_button, SIGNAL("clicked(bool)"), \ - self.add_format) - self.connect(self.formats, - SIGNAL('formats_dropped(PyQt_PyObject,PyQt_PyObject)'), - self.formats_dropped) - QObject.connect(self.remove_format_button, SIGNAL("clicked(bool)"), \ - self.remove_format) - QObject.connect(self.fetch_metadata_button, SIGNAL('clicked()'), - self.fetch_metadata) - - QObject.connect(self.fetch_cover_button, SIGNAL('clicked()'), - self.fetch_cover) - QObject.connect(self.tag_editor_button, SIGNAL('clicked()'), - self.edit_tags) - QObject.connect(self.remove_series_button, SIGNAL('clicked()'), - self.remove_unused_series) - QObject.connect(self.auto_author_sort, SIGNAL('clicked()'), - self.deduce_author_sort) - QObject.connect(self.auto_title_sort, SIGNAL('clicked()'), - self.deduce_title_sort) - self.trim_cover_button.clicked.connect(self.trim_cover) - self.connect(self.title_sort, SIGNAL('textChanged(const QString&)'), - self.title_sort_box_changed) - self.connect(self.title, SIGNAL('textChanged(const QString&)'), - self.title_box_changed) - self.connect(self.author_sort, SIGNAL('textChanged(const QString&)'), - self.author_sort_box_changed) - self.connect(self.authors, SIGNAL('editTextChanged(const QString&)'), - self.authors_box_changed) - self.connect(self.formats, SIGNAL('itemDoubleClicked(QListWidgetItem*)'), - self.show_format) - self.connect(self.formats, SIGNAL('delete_format()'), self.remove_format) - self.connect(self.button_set_cover, SIGNAL('clicked()'), self.set_cover) - self.connect(self.button_set_metadata, SIGNAL('clicked()'), - self.set_metadata_from_format) - self.connect(self.reset_cover, SIGNAL('clicked()'), self.do_reset_cover) - self.connect(self.swap_button, SIGNAL('clicked()'), self.swap_title_author) - self.timeout = float(prefs['network_timeout']) - - - self.title.setText(db.title(row)) - self.title_sort.setText(db.title_sort(row)) - isbn = db.isbn(self.id, index_is_id=True) - if not isbn: - isbn = '' - self.isbn.textChanged.connect(self.validate_isbn) - self.isbn.setText(isbn) - aus = self.db.author_sort(row) - self.author_sort.setText(aus if aus else '') - tags = self.db.tags(row) - self.original_tags = ', '.join(tags.split(',')) if tags else '' - self.tags.setText(self.original_tags) - self.tags.update_items_cache(self.db.all_tags()) - rating = self.db.rating(row) - if rating > 0: - self.rating.setValue(int(rating/2.)) - comments = self.db.comments(row) - if comments and comments.strip(): - comments = comments_to_html(comments) - self.comments.html = comments - cover = self.db.cover(row) - pubdate = db.pubdate(self.id, index_is_id=True) - self.pubdate.setDate(QDate(pubdate.year, pubdate.month, - pubdate.day)) - timestamp = db.timestamp(self.id, index_is_id=True) - self.date.setDate(QDate(timestamp.year, timestamp.month, - timestamp.day)) - self.orig_date = qt_to_dt(self.date.date()) - - exts = self.db.formats(row) - self.original_formats = [] - if exts: - exts = exts.split(',') - for ext in exts: - if not ext: - ext = '' - size = self.db.sizeof_format(row, ext) - timestamp = self.db.format_last_modified(self.id, ext) - if size is None: - continue - Format(self.formats, ext, size, timestamp=timestamp) - self.original_formats.append(ext.lower()) - - - self.initialize_combos() - si = self.db.series_index(row) - if si is None: - si = 1.0 - try: - self.series_index.setValue(float(si)) - except: - self.series_index.setValue(1.0) - QObject.connect(self.series, SIGNAL('currentIndexChanged(int)'), self.enable_series_index) - QObject.connect(self.series, SIGNAL('editTextChanged(QString)'), self.enable_series_index) - self.series.lineEdit().editingFinished.connect(self.increment_series_index) - - pm = QPixmap() - if cover: - pm.loadFromData(cover) - if pm.isNull(): - pm = QPixmap(I('default_cover.png')) - else: - self.cover_data = cover - self.cover.setPixmap(pm) - self.update_cover_tooltip() - self.original_series_name = unicode(self.series.text()).strip() - if len(db.custom_column_label_map) == 0: - self.central_widget.tabBar().setVisible(False) - self.central_widget.setTabEnabled(1, False) - else: - self.create_custom_column_editors() - self.generate_cover_button.clicked.connect(self.generate_cover) - - self.original_author = unicode(self.authors.text()).strip() - self.original_title = unicode(self.title.text()).strip() - self.books_to_refresh = set() - - self.show() - - def clear_pubdate(self, *args): - self.pubdate.setDate(UNDEFINED_QDATE) - - def create_custom_column_editors(self): - w = self.central_widget.widget(1) - layout = w.layout() - self.custom_column_widgets, self.__cc_spacers = \ - populate_metadata_page(layout, self.db, self.id, parent=w, bulk=False, - two_column=tweaks['metadata_single_use_2_cols_for_custom_fields']) - self.__custom_col_layouts = [layout] - ans = self.custom_column_widgets - for i in range(len(ans)-1): - if len(ans[i+1].widgets) == 2: - w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[1]) - else: - w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[0]) - for c in range(2, len(ans[i].widgets), 2): - w.setTabOrder(ans[i].widgets[c-1], ans[i].widgets[c+1]) - - def title_box_changed(self, txt): - ts = unicode(txt) - ts = title_sort(ts) - self.mark_box_as_ok(control = self.title_sort, tt=self.ts_tooltips, - normal=(unicode(self.title_sort.text()) == ts)) - - def title_sort_box_changed(self, txt): - ts = unicode(txt) - self.mark_box_as_ok(control = self.title_sort, tt=self.ts_tooltips, - normal=(title_sort(unicode(self.title.text())) == ts)) - - def authors_box_changed(self, txt): - aus = unicode(txt) - aus = re.sub(r'\s+et al\.$', '', aus) - aus = self.db.author_sort_from_authors(string_to_authors(aus)) - self.mark_box_as_ok(control = self.author_sort, tt=self.aus_tooltips, - normal=(unicode(self.author_sort.text()) == aus)) - - def author_sort_box_changed(self, txt): - au = unicode(self.authors.text()) - au = re.sub(r'\s+et al\.$', '', au) - au = self.db.author_sort_from_authors(string_to_authors(au)) - self.mark_box_as_ok(control = self.author_sort, tt=self.aus_tooltips, - normal=(au == txt)) - - def mark_box_as_ok(self, control, tt, normal=True): - if normal: - col = 'rgb(0, 255, 0, 20%)' - else: - col = 'rgb(255, 0, 0, 20%)' - control.setStyleSheet('QLineEdit { color: black; ' - 'background-color: %s; }'%col) - tt = tt[0] if normal else tt[1] - control.setToolTip(tt) - - def validate_isbn(self, isbn): - isbn = unicode(isbn).strip() - if not isbn: - self.isbn.setStyleSheet('QLineEdit { background-color: rgba(0,255,0,0%) }') - self.isbn.setToolTip(_('This ISBN number is valid')) - return - - if check_isbn(isbn): - self.isbn.setStyleSheet('QLineEdit { background-color: rgba(0,255,0,20%) }') - self.isbn.setToolTip(_('This ISBN number is valid')) - else: - self.isbn.setStyleSheet('QLineEdit { background-color: rgba(255,0,0,20%) }') - self.isbn.setToolTip(_('This ISBN number is invalid')) - - def deduce_author_sort(self): - au = unicode(self.authors.text()) - au = re.sub(r'\s+et al\.$', '', au) - authors = string_to_authors(au) - self.author_sort.setText(self.db.author_sort_from_authors(authors)) - - def deduce_title_sort(self): - ts = unicode(self.title.text()) - self.title_sort.setText(title_sort(ts)) - - def swap_title_author(self): - title = self.title.text() - self.title.setText(self.authors.text()) - self.authors.setText(title) - self.deduce_author_sort() - self.deduce_title_sort() - - def initialize_combos(self): - self.initalize_authors() - self.initialize_series() - self.initialize_publisher() - - self.layout().activate() - - def initalize_authors(self): - all_authors = self.db.all_authors() - all_authors.sort(key=lambda x : sort_key(x[1])) - for i in all_authors: - id, name = i - name = [name.strip().replace('|', ',') for n in name.split(',')] - self.authors.addItem(authors_to_string(name)) - - au = self.db.authors(self.row) - if not au: - au = _('Unknown') - au = ' & '.join([a.strip().replace('|', ',') for a in au.split(',')]) - self.authors.setEditText(au) - - self.authors.set_separator('&') - self.authors.set_space_before_sep(True) - self.authors.set_add_separator(tweaks['authors_completer_append_separator']) - self.authors.update_items_cache(self.db.all_author_names()) - - def initialize_series(self): - self.series.setSizeAdjustPolicy(self.series.AdjustToContentsOnFirstShow) - all_series = self.db.all_series() - all_series.sort(key=lambda x : sort_key(x[1])) - self.series.set_separator(None) - self.series.update_items_cache([x[1] for x in all_series]) - series_id = self.db.series_id(self.row) - idx, c = None, 0 - for i in all_series: - id, name = i - if id == series_id: - idx = c - self.series.addItem(name) - c += 1 - - self.series.lineEdit().setText('') - if idx is not None: - self.series.setCurrentIndex(idx) - self.enable_series_index() - - def initialize_publisher(self): - all_publishers = self.db.all_publishers() - all_publishers.sort(key=lambda x : sort_key(x[1])) - self.publisher.set_separator(None) - self.publisher.update_items_cache([x[1] for x in all_publishers]) - publisher_id = self.db.publisher_id(self.row) - idx, c = None, 0 - for i in all_publishers: - id, name = i - if id == publisher_id: - idx = c - self.publisher.addItem(name) - c += 1 - - self.publisher.setEditText('') - if idx is not None: - self.publisher.setCurrentIndex(idx) - - def edit_tags(self): - if self.tags.text() != self.original_tags: - if question_dialog(self, _('Tags changed'), - _('You have changed the tags. In order to use the tags' - ' editor, you must either discard or apply these ' - 'changes. Apply changes?'), show_copy_button=False): - self.books_to_refresh |= self.apply_tags(commit=True, - notify=True) - self.original_tags = unicode(self.tags.text()) - else: - self.tags.setText(self.original_tags) - d = TagEditor(self, self.db, self.id) - d.exec_() - if d.result() == QDialog.Accepted: - tag_string = ', '.join(d.tags) - self.tags.setText(tag_string) - self.tags.update_items_cache(self.db.all_tags()) - - - def fetch_metadata(self): - isbn = re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text())) - title = unicode(self.title.text()) - try: - author = string_to_authors(unicode(self.authors.text()))[0] - except: - author = '' - publisher = unicode(self.publisher.currentText()) - if isbn or title or author or publisher: - d = FetchMetadata(self, isbn, title, author, publisher, self.timeout) - self._fetch_metadata_scope = d - with d: - if d.exec_() == QDialog.Accepted: - book = d.selected_book() - if book: - if d.opt_get_social_metadata.isChecked(): - d2 = SocialMetadata(book, self) - d2.exec_() - if d2.timed_out: - warning_dialog(self, _('Timed out'), - _('The download of social' - ' metadata timed out, the servers are' - ' probably busy. Try again later.'), - show=True) - elif d2.exceptions: - det = '\n'.join([x[0]+'\n\n'+x[-1]+'\n\n\n' for - x in d2.exceptions]) - warning_dialog(self, _('There were errors'), - _('There were errors downloading social metadata'), - det_msg=det, show=True) - else: - book.tags = [] - if d.opt_overwrite_author_title_metadata.isChecked(): - self.title.setText(book.title) - self.authors.setText(authors_to_string(book.authors)) - if book.author_sort: self.author_sort.setText(book.author_sort) - if book.publisher: self.publisher.setEditText(book.publisher) - if book.isbn: self.isbn.setText(book.isbn) - if book.pubdate: - dt = book.pubdate - self.pubdate.setDate(QDate(dt.year, dt.month, dt.day)) - summ = book.comments - if summ: - prefix = self.comments.html - if prefix: - prefix += '\n' - self.comments.html = prefix + comments_to_html(summ) - if book.rating is not None: - self.rating.setValue(int(book.rating)) - if book.tags: - self.tags.setText(', '.join(book.tags)) - if book.series is not None: - if self.series.text() is None or self.series.text() == '': - self.series.setText(book.series) - if book.series_index is not None: - self.series_index.setValue(book.series_index) - if book.has_cover: - if d.opt_auto_download_cover.isChecked(): - self.fetch_cover() - else: - self.fetch_cover_button.setFocus(Qt.OtherFocusReason) - else: - error_dialog(self, _('Cannot fetch metadata'), - _('You must specify at least one of ISBN, Title, ' - 'Authors or Publisher'), show=True) - self.title.setFocus(Qt.OtherFocusReason) - - def enable_series_index(self, *args): - self.series_index.setEnabled(True) - - def increment_series_index(self): - if self.db is not None: - try: - series = unicode(self.series.text()).strip() - if series and series != self.original_series_name: - ns = 1 - if tweaks['series_index_auto_increment'] != 'const': - ns = self.db.get_next_series_num_for(series) - self.series_index.setValue(ns) - self.original_series_name = series - except: - traceback.print_exc() - - def remove_unused_series(self): - self.db.remove_unused_series() - idx = unicode(self.series.currentText()) - self.series.clear() - self.initialize_series() - if idx: - for i in range(self.series.count()): - if unicode(self.series.itemText(i)) == idx: - self.series.setCurrentIndex(i) - break - - def apply_tags(self, commit=False, notify=False): - return self.db.set_tags(self.id, [x.strip() for x in - unicode(self.tags.text()).split(',')], - notify=notify, commit=commit, allow_case_change=True) - - def next_triggered(self, row_delta, *args): - self.row_delta = row_delta - self.accept() - - def accept(self): - try: - if self.formats_changed: - self.sync_formats() - title = unicode(self.title.text()).strip() - if title != self.original_title: - self.db.set_title(self.id, title, notify=False) - # This must be after setting the title because of the DB update trigger - ts = unicode(self.title_sort.text()).strip() - if ts: - self.db.set_title_sort(self.id, ts, notify=False, commit=False) - au = unicode(self.authors.text()).strip() - if au and au != self.original_author: - self.books_to_refresh |= self.db.set_authors(self.id, - string_to_authors(au), - notify=False, - allow_case_change=True) - aus = unicode(self.author_sort.text()).strip() - if aus: - self.db.set_author_sort(self.id, aus, notify=False, commit=False) - self.db.set_isbn(self.id, - re.sub(r'[^0-9a-zA-Z]', '', - unicode(self.isbn.text()).strip()), - notify=False, commit=False) - self.db.set_rating(self.id, 2*self.rating.value(), notify=False, - commit=False) - self.books_to_refresh |= self.apply_tags() - self.books_to_refresh |= self.db.set_publisher(self.id, - unicode(self.publisher.currentText()).strip(), - notify=False, commit=False, allow_case_change=True) - self.books_to_refresh |= self.db.set_series(self.id, - unicode(self.series.currentText()).strip(), notify=False, - commit=False, allow_case_change=True) - self.db.set_series_index(self.id, self.series_index.value(), - notify=False, commit=False) - self.db.set_comment(self.id, - self.comments.html, - notify=False, commit=False) - d = self.pubdate.date() - d = qt_to_dt(d) - self.db.set_pubdate(self.id, d, notify=False, commit=False) - d = self.date.date() - d = qt_to_dt(d) - if d != self.orig_date: - self.db.set_timestamp(self.id, d, notify=False, commit=False) - self.db.commit() - - if self.cover_changed: - if self.cover_data is not None: - self.db.set_cover(self.id, self.cover_data) - else: - self.db.remove_cover(self.id) - for w in getattr(self, 'custom_column_widgets', []): - self.books_to_refresh |= w.commit(self.id) - self.db.commit() - except (IOError, OSError) as err: - if getattr(err, 'errno', -1) == 13: # Permission denied - fname = err.filename if err.filename else 'file' - return error_dialog(self, _('Permission denied'), - _('Could not open %s. Is it being used by another' - ' program?')%fname, det_msg=traceback.format_exc(), - show=True) - raise - self.save_state() - self.cover_fetcher = None - QDialog.accept(self) - - def reject(self, *args): - self.save_state() - self.cover_fetcher = None - QDialog.reject(self, *args) - - def read_state(self): - wg = dynamic.get('metasingle_window_geometry2', None) - ss = dynamic.get('metasingle_splitter_state2', None) - if wg is not None: - self.restoreGeometry(wg) - if ss is not None: - self.splitter.restoreState(ss) - - def save_state(self): - dynamic.set('metasingle_window_geometry2', bytes(self.saveGeometry())) - dynamic.set('metasingle_splitter_state2', - bytes(self.splitter.saveState())) - - def break_cycles(self): - # Break any reference cycles that could prevent python - # from garbage collecting this dialog - def disconnect(signal): - try: - signal.disconnect() - except: - pass # Fails if view format was never connected - disconnect(self.view_format) - for b in ('next_button', 'prev_button'): - x = getattr(self, b, None) - if x is not None: - disconnect(x.clicked) - -if __name__ == '__main__': - from calibre.library import db - from PyQt4.Qt import QApplication - from calibre.utils.mem import memory - import gc - - - app = QApplication([]) - db = db() - - # Initialize all Qt Objects once - d = MetadataSingleDialog(None, 4, db) - d.break_cycles() - d.reject() - del d - - for i in range(5): - gc.collect() - before = memory() - - d = MetadataSingleDialog(None, 4, db) - d.reject() - d.break_cycles() - del d - - for i in range(5): - gc.collect() - print 'Used memory:', memory(before)/1024.**2, 'MB' - - diff --git a/src/calibre/gui2/dialogs/metadata_single.ui b/src/calibre/gui2/dialogs/metadata_single.ui deleted file mode 100644 index ced5030f94..0000000000 --- a/src/calibre/gui2/dialogs/metadata_single.ui +++ /dev/null @@ -1,937 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<ui version="4.0"> - <class>MetadataSingleDialog</class> - <widget class="QDialog" name="MetadataSingleDialog"> - <property name="geometry"> - <rect> - <x>0</x> - <y>0</y> - <width>994</width> - <height>716</height> - </rect> - </property> - <property name="sizePolicy"> - <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="windowTitle"> - <string>Edit Meta Information</string> - </property> - <property name="windowIcon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset> - </property> - <property name="sizeGripEnabled"> - <bool>true</bool> - </property> - <property name="modal"> - <bool>true</bool> - </property> - <layout class="QVBoxLayout" name="verticalLayout_6"> - <item> - <widget class="QScrollArea" name="scrollArea"> - <property name="frameShape"> - <enum>QFrame::NoFrame</enum> - </property> - <property name="widgetResizable"> - <bool>true</bool> - </property> - <widget class="QWidget" name="scrollAreaWidgetContents"> - <property name="geometry"> - <rect> - <x>0</x> - <y>0</y> - <width>986</width> - <height>677</height> - </rect> - </property> - <layout class="QVBoxLayout" name="verticalLayout_5"> - <property name="margin"> - <number>0</number> - </property> - <item> - <widget class="QTabWidget" name="central_widget"> - <property name="minimumSize"> - <size> - <width>800</width> - <height>665</height> - </size> - </property> - <property name="currentIndex"> - <number>0</number> - </property> - <widget class="QWidget" name="central_tabWidgetPage1"> - <attribute name="title"> - <string>&Basic metadata</string> - </attribute> - <layout class="QGridLayout" name="gridLayout_5"> - <item row="0" column="0"> - <widget class="QSplitter" name="splitter"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <widget class="QWidget" name="layoutWidget"> - <layout class="QVBoxLayout"> - <item> - <widget class="QGroupBox" name="meta_box"> - <property name="title"> - <string>Meta information</string> - </property> - <layout class="QGridLayout" name="gridLayout_3"> - <item row="0" column="0"> - <widget class="QLabel" name="label"> - <property name="text"> - <string>&Title: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>title</cstring> - </property> - </widget> - </item> - <item row="0" column="1"> - <widget class="EnLineEdit" name="title"> - <property name="toolTip"> - <string>Change the title of this book</string> - </property> - </widget> - </item> - <item row="0" column="2" rowspan="4"> - <layout class="QVBoxLayout" name="verticalLayout_7"> - <item> - <spacer name="verticalSpacer_3"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - <item> - <widget class="QToolButton" name="auto_title_sort"> - <property name="toolTip"> - <string>Automatically create the title sort entry based on the current title entry. -Using this button to create title sort will change title sort from red to green.</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/auto_author_sort.png</normaloff>:/images/auto_author_sort.png</iconset> - </property> - </widget> - </item> - <item> - <spacer name="verticalSpacer"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - <item> - <widget class="QToolButton" name="swap_button"> - <property name="toolTip"> - <string>Swap the author and title</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/swap.png</normaloff>:/images/swap.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>16</width> - <height>16</height> - </size> - </property> - </widget> - </item> - <item> - <spacer name="verticalSpacer_2"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - <item> - <widget class="QToolButton" name="auto_author_sort"> - <property name="toolTip"> - <string>Automatically create the author sort entry based on the current author entry. -Using this button to create author sort will change author sort from red to green.</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/auto_author_sort.png</normaloff>:/images/auto_author_sort.png</iconset> - </property> - </widget> - </item> - <item> - <spacer name="verticalSpacer_4"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - </layout> - </item> - <item row="1" column="0"> - <widget class="QLabel" name="label"> - <property name="text"> - <string>Title &sort: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>title_sort</cstring> - </property> - </widget> - </item> - <item row="1" column="1"> - <widget class="EnLineEdit" name="title_sort"> - <property name="toolTip"> - <string>Specify how this book should be sorted when by title. For example, The Exorcist might be sorted as Exorcist, The.</string> - </property> - </widget> - </item> - <item row="2" column="0"> - <widget class="QLabel" name="label_2"> - <property name="text"> - <string>&Author(s): </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>authors</cstring> - </property> - </widget> - </item> - <item row="2" column="1"> - <widget class="MultiCompleteComboBox" name="authors"> - <property name="editable"> - <bool>true</bool> - </property> - </widget> - </item> - <item row="3" column="0"> - <widget class="QLabel" name="label_8"> - <property name="text"> - <string>Author S&ort: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>author_sort</cstring> - </property> - </widget> - </item> - <item row="3" column="1"> - <widget class="EnLineEdit" name="author_sort"> - <property name="toolTip"> - <string>Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles. -If the box is colored green, then text matches the individual author's sort strings. If it is colored red, then the authors and this text do not match.</string> - </property> - </widget> - </item> - <item row="4" column="0"> - <widget class="QLabel" name="label_6"> - <property name="text"> - <string>&Rating:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>rating</cstring> - </property> - </widget> - </item> - <item row="4" column="1" colspan="2"> - <widget class="QSpinBox" name="rating"> - <property name="toolTip"> - <string>Rating of this book. 0-5 stars</string> - </property> - <property name="whatsThis"> - <string>Rating of this book. 0-5 stars</string> - </property> - <property name="buttonSymbols"> - <enum>QAbstractSpinBox::PlusMinus</enum> - </property> - <property name="suffix"> - <string> stars</string> - </property> - <property name="maximum"> - <number>5</number> - </property> - </widget> - </item> - <item row="5" column="0"> - <widget class="QLabel" name="label_3"> - <property name="text"> - <string>&Publisher: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>publisher</cstring> - </property> - </widget> - </item> - <item row="5" column="1" colspan="2"> - <widget class="MultiCompleteComboBox" name="publisher"> - <property name="editable"> - <bool>true</bool> - </property> - </widget> - </item> - <item row="6" column="0"> - <widget class="QLabel" name="label_4"> - <property name="text"> - <string>Ta&gs: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>tags</cstring> - </property> - </widget> - </item> - <item row="6" column="1"> - <layout class="QHBoxLayout" name="_2"> - <item> - <widget class="MultiCompleteLineEdit" name="tags"> - <property name="toolTip"> - <string>Tags categorize the book. This is particularly useful while searching. <br><br>They can be any words or phrases, separated by commas.</string> - </property> - </widget> - </item> - </layout> - </item> - <item row="6" column="2"> - <widget class="QToolButton" name="tag_editor_button"> - <property name="toolTip"> - <string>Open Tag Editor</string> - </property> - <property name="text"> - <string>Open Tag Editor</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/chapters.png</normaloff>:/images/chapters.png</iconset> - </property> - </widget> - </item> - <item row="7" column="0"> - <widget class="QLabel" name="label_7"> - <property name="text"> - <string>&Series:</string> - </property> - <property name="textFormat"> - <enum>Qt::PlainText</enum> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>series</cstring> - </property> - </widget> - </item> - <item row="7" column="1"> - <layout class="QHBoxLayout" name="_3"> - <property name="spacing"> - <number>5</number> - </property> - <item> - <widget class="MultiCompleteComboBox" name="series"> - <property name="toolTip"> - <string>List of known series. You can add new series.</string> - </property> - <property name="whatsThis"> - <string>List of known series. You can add new series.</string> - </property> - <property name="editable"> - <bool>true</bool> - </property> - <property name="insertPolicy"> - <enum>QComboBox::InsertAlphabetically</enum> - </property> - </widget> - </item> - </layout> - </item> - <item row="7" column="2"> - <widget class="QToolButton" name="remove_series_button"> - <property name="toolTip"> - <string>Remove unused series (Series that have no books)</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> - </property> - </widget> - </item> - <item row="8" column="1" colspan="2"> - <widget class="QDoubleSpinBox" name="series_index"> - <property name="enabled"> - <bool>false</bool> - </property> - <property name="prefix"> - <string>Book </string> - </property> - <property name="maximum"> - <double>99999999.989999994635582</double> - </property> - </widget> - </item> - <item row="9" column="0"> - <widget class="QLabel" name="label_9"> - <property name="text"> - <string>IS&BN:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>isbn</cstring> - </property> - </widget> - </item> - <item row="9" column="1" colspan="2"> - <widget class="QLineEdit" name="isbn"/> - </item> - <item row="10" column="0"> - <widget class="QLabel" name="label_11"> - <property name="text"> - <string>&Date:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>date</cstring> - </property> - </widget> - </item> - <item row="10" column="1" colspan="2"> - <widget class="QDateEdit" name="date"> - <property name="displayFormat"> - <string>dd MMM yyyy</string> - </property> - <property name="calendarPopup"> - <bool>true</bool> - </property> - </widget> - </item> - <item row="11" column="0"> - <widget class="QLabel" name="label_10"> - <property name="text"> - <string>Publishe&d:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>pubdate</cstring> - </property> - </widget> - </item> - <item row="11" column="1"> - <widget class="QDateEdit" name="pubdate"> - <property name="displayFormat"> - <string>MMM yyyy</string> - </property> - <property name="calendarPopup"> - <bool>true</bool> - </property> - </widget> - </item> - <item row="11" column="2"> - <widget class="QToolButton" name="clear_pubdate_button"> - <property name="toolTip"> - <string>Clear published date</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> - </property> - </widget> - </item> - </layout> - </widget> - </item> - <item> - <widget class="QPushButton" name="fetch_metadata_button"> - <property name="text"> - <string>&Fetch metadata from server</string> - </property> - </widget> - </item> - <item> - <spacer name="verticalSpacer_5"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeType"> - <enum>QSizePolicy::Fixed</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - </layout> - </widget> - <widget class="QWidget" name="layoutWidget_2"> - <layout class="QVBoxLayout" name="verticalLayout_2"> - <item> - <widget class="QGroupBox" name="bc_box"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Preferred" vsizetype="Expanding"> - <horstretch>0</horstretch> - <verstretch>10</verstretch> - </sizepolicy> - </property> - <property name="title"> - <string>Book Cover</string> - </property> - <layout class="QVBoxLayout" name="verticalLayout_4"> - <item> - <widget class="ImageView" name="cover" native="true"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Expanding" vsizetype="Expanding"> - <horstretch>0</horstretch> - <verstretch>100</verstretch> - </sizepolicy> - </property> - </widget> - </item> - <item> - <layout class="QVBoxLayout" name="_4"> - <property name="spacing"> - <number>6</number> - </property> - <property name="sizeConstraint"> - <enum>QLayout::SetMaximumSize</enum> - </property> - <property name="margin"> - <number>0</number> - </property> - <item> - <widget class="QLabel" name="label_5"> - <property name="text"> - <string>Change &cover image:</string> - </property> - <property name="buddy"> - <cstring>cover_button</cstring> - </property> - </widget> - </item> - <item> - <layout class="QHBoxLayout" name="_5"> - <property name="spacing"> - <number>6</number> - </property> - <property name="margin"> - <number>0</number> - </property> - <item> - <widget class="QPushButton" name="cover_button"> - <property name="text"> - <string>&Browse</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="trim_cover_button"> - <property name="toolTip"> - <string>Remove border (if any) from cover</string> - </property> - <property name="text"> - <string>T&rim</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trim.png</normaloff>:/images/trim.png</iconset> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="reset_cover"> - <property name="toolTip"> - <string>Reset cover to default</string> - </property> - <property name="text"> - <string>&Remove</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> - </property> - </widget> - </item> - </layout> - </item> - </layout> - </item> - <item> - <layout class="QHBoxLayout" name="_6"> - <item> - <widget class="QPushButton" name="fetch_cover_button"> - <property name="text"> - <string>Download co&ver</string> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="generate_cover_button"> - <property name="toolTip"> - <string>Generate a default cover based on the title and author</string> - </property> - <property name="text"> - <string>&Generate cover</string> - </property> - </widget> - </item> - </layout> - </item> - </layout> - </widget> - </item> - </layout> - </widget> - <widget class="QWidget" name="layoutWidget"> - <layout class="QVBoxLayout" name="verticalLayout_3"> - <item> - <widget class="QGroupBox" name="af_group_box"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Preferred" vsizetype="Minimum"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="title"> - <string>Available Formats</string> - </property> - <layout class="QVBoxLayout" name="verticalLayout"> - <item> - <layout class="QGridLayout" name="gridLayout"> - <item row="0" column="1" rowspan="3"> - <widget class="FormatList" name="formats"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Minimum" vsizetype="Minimum"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="maximumSize"> - <size> - <width>16777215</width> - <height>140</height> - </size> - </property> - <property name="baseSize"> - <size> - <width>100</width> - <height>0</height> - </size> - </property> - <property name="dragDropMode"> - <enum>QAbstractItemView::DropOnly</enum> - </property> - <property name="iconSize"> - <size> - <width>64</width> - <height>64</height> - </size> - </property> - </widget> - </item> - <item row="0" column="2"> - <widget class="QToolButton" name="add_format_button"> - <property name="toolTip"> - <string>Add a new format for this book to the database</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/add_book.png</normaloff>:/images/add_book.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>32</width> - <height>32</height> - </size> - </property> - </widget> - </item> - <item row="2" column="2"> - <widget class="QToolButton" name="remove_format_button"> - <property name="toolTip"> - <string>Remove the selected formats for this book from the database.</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>32</width> - <height>32</height> - </size> - </property> - </widget> - </item> - <item row="0" column="0"> - <widget class="QToolButton" name="button_set_cover"> - <property name="toolTip"> - <string>Set the cover for the book from the selected format</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/book.png</normaloff>:/images/book.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>32</width> - <height>32</height> - </size> - </property> - </widget> - </item> - <item row="2" column="0"> - <widget class="QToolButton" name="button_set_metadata"> - <property name="toolTip"> - <string>Update metadata from the metadata in the selected format</string> - </property> - <property name="text"> - <string/> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>32</width> - <height>32</height> - </size> - </property> - </widget> - </item> - </layout> - </item> - </layout> - <zorder></zorder> - </widget> - </item> - <item> - <widget class="QGroupBox" name="groupBox"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Preferred" vsizetype="Preferred"> - <horstretch>0</horstretch> - <verstretch>10</verstretch> - </sizepolicy> - </property> - <property name="title"> - <string>&Comments</string> - </property> - <layout class="QVBoxLayout" name="verticalLayout_8"> - <property name="margin"> - <number>0</number> - </property> - <item> - <widget class="Editor" name="comments" native="true"/> - </item> - </layout> - </widget> - </item> - </layout> - </widget> - </widget> - </item> - </layout> - </widget> - <widget class="QWidget" name="tab"> - <attribute name="title"> - <string>&Custom metadata</string> - </attribute> - <layout class="QGridLayout" name="gridLayout_2"/> - </widget> - </widget> - </item> - </layout> - </widget> - </widget> - </item> - <item> - <widget class="QDialogButtonBox" name="button_box"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="standardButtons"> - <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set> - </property> - </widget> - </item> - </layout> - </widget> - <customwidgets> - <customwidget> - <class>EnLineEdit</class> - <extends>QLineEdit</extends> - <header>widgets.h</header> - </customwidget> - <customwidget> - <class>MultiCompleteLineEdit</class> - <extends>QLineEdit</extends> - <header>calibre/gui2/complete.h</header> - </customwidget> - <customwidget> - <class>MultiCompleteComboBox</class> - <extends>QComboBox</extends> - <header>calibre/gui2/complete.h</header> - </customwidget> - <customwidget> - <class>FormatList</class> - <extends>QListWidget</extends> - <header location="global">calibre/gui2/widgets.h</header> - </customwidget> - <customwidget> - <class>ImageView</class> - <extends>QWidget</extends> - <header>calibre/gui2/widgets.h</header> - <container>1</container> - </customwidget> - <customwidget> - <class>Editor</class> - <extends>QWidget</extends> - <header location="global">calibre/gui2/comments_editor.h</header> - <container>1</container> - </customwidget> - </customwidgets> - <tabstops> - <tabstop>title</tabstop> - <tabstop>auto_title_sort</tabstop> - <tabstop>title_sort</tabstop> - <tabstop>swap_button</tabstop> - <tabstop>authors</tabstop> - <tabstop>auto_author_sort</tabstop> - <tabstop>author_sort</tabstop> - <tabstop>rating</tabstop> - <tabstop>publisher</tabstop> - <tabstop>tags</tabstop> - <tabstop>tag_editor_button</tabstop> - <tabstop>series</tabstop> - <tabstop>remove_series_button</tabstop> - <tabstop>series_index</tabstop> - <tabstop>isbn</tabstop> - <tabstop>date</tabstop> - <tabstop>pubdate</tabstop> - <tabstop>fetch_metadata_button</tabstop> - <tabstop>button_set_cover</tabstop> - <tabstop>button_set_metadata</tabstop> - <tabstop>formats</tabstop> - <tabstop>add_format_button</tabstop> - <tabstop>remove_format_button</tabstop> - <tabstop>cover_button</tabstop> - <tabstop>trim_cover_button</tabstop> - <tabstop>reset_cover</tabstop> - <tabstop>fetch_cover_button</tabstop> - <tabstop>generate_cover_button</tabstop> - <tabstop>button_box</tabstop> - <tabstop>scrollArea</tabstop> - <tabstop>central_widget</tabstop> - </tabstops> - <resources> - <include location="../../../../resources/images.qrc"/> - </resources> - <connections> - <connection> - <sender>button_box</sender> - <signal>accepted()</signal> - <receiver>MetadataSingleDialog</receiver> - <slot>accept()</slot> - <hints> - <hint type="sourcelabel"> - <x>261</x> - <y>710</y> - </hint> - <hint type="destinationlabel"> - <x>157</x> - <y>274</y> - </hint> - </hints> - </connection> - <connection> - <sender>button_box</sender> - <signal>rejected()</signal> - <receiver>MetadataSingleDialog</receiver> - <slot>reject()</slot> - <hints> - <hint type="sourcelabel"> - <x>329</x> - <y>710</y> - </hint> - <hint type="destinationlabel"> - <x>286</x> - <y>274</y> - </hint> - </hints> - </connection> - </connections> -</ui> diff --git a/src/calibre/gui2/metadata/bulk_download.py b/src/calibre/gui2/metadata/bulk_download.py index 7a7f49dabf..2a307fc902 100644 --- a/src/calibre/gui2/metadata/bulk_download.py +++ b/src/calibre/gui2/metadata/bulk_download.py @@ -1,308 +1,195 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement +from __future__ import (unicode_literals, division, absolute_import, + print_function) __license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' +__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' -import traceback -from threading import Thread -from Queue import Queue, Empty from functools import partial +from itertools import izip +from threading import Event -from PyQt4.Qt import QObject, QTimer, QDialog, \ - QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox +from PyQt4.Qt import (QIcon, QDialog, + QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt) -from calibre.ebooks.metadata.fetch import search, get_social_metadata -from calibre.gui2 import config, error_dialog -from calibre.gui2.dialogs.progress import ProgressDialog -from calibre.ebooks.metadata.covers import download_cover -from calibre.customize.ui import get_isbndb_key +from calibre.gui2.threaded_jobs import ThreadedJob +from calibre.ebooks.metadata.sources.identify import identify, msprefs +from calibre.ebooks.metadata.sources.covers import download_cover +from calibre.ebooks.metadata.book.base import Metadata +from calibre.customize.ui import metadata_plugins +from calibre.ptempfile import PersistentTemporaryFile -class Worker(Thread): - 'Cover downloader' +# Start download {{{ +def show_config(gui, parent): + from calibre.gui2.preferences import show_config_widget + show_config_widget('Sharing', 'Metadata download', parent=parent, + gui=gui, never_shutdown=True) - def __init__(self): - Thread.__init__(self) - self.daemon = True - self.jobs = Queue() - self.results = Queue() +class ConfirmDialog(QDialog): - def run(self): - while True: - id, mi = self.jobs.get() - if not getattr(mi, 'isbn', False): - break + def __init__(self, ids, parent): + QDialog.__init__(self, parent) + self.setWindowTitle(_('Schedule download?')) + self.setWindowIcon(QIcon(I('dialog_question.png'))) + + l = self.l = QGridLayout() + self.setLayout(l) + + i = QLabel(self) + i.setPixmap(QPixmap(I('dialog_question.png'))) + l.addWidget(i, 0, 0) + + t = QLabel( + '<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will' + ' run in the background. Proceed?')%len(ids) + + '<p>'+_('You can monitor the progress of the download ' + 'by clicking the rotating spinner in the bottom right ' + 'corner.') + + '<p>'+_('When the download completes you will be asked for' + ' confirmation before calibre applies the downloaded metadata.') + ) + t.setWordWrap(True) + l.addWidget(t, 0, 1) + l.setColumnStretch(0, 1) + l.setColumnStretch(1, 100) + + self.identify = self.covers = True + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) + self.bb.rejected.connect(self.reject) + b = self.bb.addButton(_('Download only &metadata'), + self.bb.AcceptRole) + b.clicked.connect(self.only_metadata) + b.setIcon(QIcon(I('edit_input.png'))) + b = self.bb.addButton(_('Download only &covers'), + self.bb.AcceptRole) + b.clicked.connect(self.only_covers) + b.setIcon(QIcon(I('default_cover.png'))) + b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole) + b.setIcon(QIcon(I('config.png'))) + b.clicked.connect(partial(show_config, parent, self)) + l.addWidget(self.bb, 1, 0, 1, 2) + b = self.bb.addButton(_('Download &both'), + self.bb.AcceptRole) + b.clicked.connect(self.accept) + b.setDefault(True) + b.setAutoDefault(True) + b.setIcon(QIcon(I('ok.png'))) + + self.resize(self.sizeHint()) + b.setFocus(Qt.OtherFocusReason) + + def only_metadata(self): + self.covers = False + self.accept() + + def only_covers(self): + self.identify = False + self.accept() + +def start_download(gui, ids, callback): + d = ConfirmDialog(ids, gui) + ret = d.exec_() + d.b.clicked.disconnect() + if ret != d.Accepted: + return + + job = ThreadedJob('metadata bulk download', + _('Download metadata for %d books')%len(ids), + download, (ids, gui.current_db, d.identify, d.covers), {}, callback) + gui.job_manager.run_threaded_job(job) + gui.status_bar.show_message(_('Metadata download started'), 3000) +# }}} + +def get_job_details(job): + id_map, failed_ids, failed_covers, title_map, all_failed = job.result + det_msg = [] + for i in failed_ids | failed_covers: + title = title_map[i] + if i in failed_ids: + title += (' ' + _('(Failed metadata)')) + if i in failed_covers: + title += (' ' + _('(Failed cover)')) + det_msg.append(title) + det_msg = '\n'.join(det_msg) + return id_map, failed_ids, failed_covers, all_failed, det_msg + +def merge_result(oldmi, newmi): + dummy = Metadata(_('Unknown')) + for f in msprefs['ignore_fields']: + if ':' not in f: + setattr(newmi, f, getattr(dummy, f)) + fields = set() + for plugin in metadata_plugins(['identify']): + fields |= plugin.touched_fields + + for f in fields: + # Optimize so that set_metadata does not have to do extra work later + if not f.startswith('identifier:'): + if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): + setattr(newmi, f, getattr(dummy, f)) + + newmi.last_modified = oldmi.last_modified + + return newmi + +def download(ids, db, do_identify, covers, + log=None, abort=None, notifications=None): + ids = list(ids) + metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) + for i in ids] + failed_ids = set() + failed_covers = set() + title_map = {} + ans = {} + count = 0 + all_failed = True + ''' + # Test apply dialog + all_failed = do_identify = covers = False + ''' + for i, mi in izip(ids, metadata): + if abort.is_set(): + log.error('Aborting...') + break + title, authors, identifiers = mi.title, mi.authors, mi.identifiers + title_map[i] = title + if do_identify: + results = [] try: - cdata, errors = download_cover(mi) - if cdata: - self.results.put((id, mi, True, cdata)) - else: - msg = [] - for e in errors: - if not e[0]: - msg.append(e[-1] + ' - ' + e[1]) - self.results.put((id, mi, False, '\n'.join(msg))) + results = identify(log, Event(), title=title, authors=authors, + identifiers=identifiers) except: - self.results.put((id, mi, False, traceback.format_exc())) - - def __enter__(self): - self.start() - return self - - def __exit__(self, *args): - self.jobs.put((False, False)) - - -class DownloadMetadata(Thread): - 'Metadata downloader' - - def __init__(self, db, ids, get_covers, set_metadata=True, - get_social_metadata=True): - Thread.__init__(self) - self.daemon = True - self.metadata = {} - self.covers = {} - self.set_metadata = set_metadata - self.get_social_metadata = get_social_metadata - self.social_metadata_exceptions = [] - self.db = db - self.updated = set([]) - self.get_covers = get_covers - self.worker = Worker() - self.results = Queue() - self.keep_going = True - for id in ids: - self.metadata[id] = db.get_metadata(id, index_is_id=True) - self.metadata[id].rating = None - self.total = len(ids) - if self.get_covers: - self.total += len(ids) - self.fetched_metadata = {} - self.fetched_covers = {} - self.failures = {} - self.cover_failures = {} - self.exception = self.tb = None - - def run(self): - try: - self._run() - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - def _run(self): - self.key = get_isbndb_key() - if not self.key: - self.key = None - with self.worker: - for id, mi in self.metadata.items(): - if not self.keep_going: - break - args = {} - if mi.isbn: - args['isbn'] = mi.isbn - else: - if mi.is_null('title'): - self.failures[id] = \ - _('Book has neither title nor ISBN') - continue - args['title'] = mi.title - if mi.authors and mi.authors[0] != _('Unknown'): - args['author'] = mi.authors[0] - if self.key: - args['isbndb_key'] = self.key - results, exceptions = search(**args) - if results: - fmi = results[0] - self.fetched_metadata[id] = fmi - if self.get_covers: - if fmi.isbn: - self.worker.jobs.put((id, fmi)) - else: - self.results.put((id, 'cover', False, mi.title)) - if (not config['overwrite_author_title_metadata']): - fmi.authors = mi.authors - fmi.author_sort = mi.author_sort - fmi.title = mi.title - mi.smart_update(fmi) - if mi.isbn and self.get_social_metadata: - self.social_metadata_exceptions = get_social_metadata(mi) - if mi.rating: - mi.rating *= 2 - if not self.get_social_metadata: - mi.tags = [] - self.results.put((id, 'metadata', True, mi.title)) - else: - self.failures[id] = _('No matches found for this book') - self.results.put((id, 'metadata', False, mi.title)) - self.results.put((id, 'cover', False, mi.title)) - self.commit_covers() - - self.commit_covers(True) - - def commit_covers(self, all=False): - if all: - self.worker.jobs.put((False, False)) - while True: - try: - id, fmi, ok, cdata = self.worker.results.get_nowait() - if ok: - self.fetched_covers[id] = cdata - self.results.put((id, 'cover', ok, fmi.title)) - else: - self.results.put((id, 'cover', ok, fmi.title)) - try: - self.cover_failures[id] = unicode(cdata) - except: - self.cover_failures[id] = repr(cdata) - except Empty: - if not all or not self.worker.is_alive(): - return - -class DoDownload(QObject): - - def __init__(self, parent, title, db, ids, get_covers, set_metadata=True, - get_social_metadata=True): - QObject.__init__(self, parent) - self.pd = ProgressDialog(title, min=0, max=0, parent=parent) - self.pd.canceled_signal.connect(self.cancel) - self.downloader = None - self.create = partial(DownloadMetadata, db, ids, get_covers, - set_metadata=set_metadata, - get_social_metadata=get_social_metadata) - self.get_covers = get_covers - self.db = db - self.updated = set([]) - self.total = len(ids) - self.keep_going = True - - def exec_(self): - QTimer.singleShot(50, self.do_one) - ret = self.pd.exec_() - if getattr(self.downloader, 'exception', None) is not None and \ - ret == self.pd.Accepted: - error_dialog(self.parent(), _('Failed'), - _('Failed to download metadata'), show=True) - else: - self.show_report() - return ret - - def cancel(self, *args): - self.keep_going = False - self.downloader.keep_going = False - self.pd.reject() - - def do_one(self): - try: - if not self.keep_going: - return - if self.downloader is None: - self.downloader = self.create() - self.downloader.start() - self.pd.set_min(0) - self.pd.set_max(self.downloader.total) - try: - r = self.downloader.results.get_nowait() - self.handle_result(r) - except Empty: pass - if not self.downloader.is_alive(): - while True: - try: - r = self.downloader.results.get_nowait() - self.handle_result(r) - except Empty: - break - self.pd.accept() - return - except: - self.cancel() - raise - QTimer.singleShot(50, self.do_one) - - def handle_result(self, r): - id_, typ, ok, title = r - what = _('cover') if typ == 'cover' else _('metadata') - which = _('Downloaded') if ok else _('Failed to get') - if self.get_covers or typ != 'cover' or ok: - # Do not show message when cover fetch fails if user didn't ask to - # download covers - self.pd.set_msg(_('%s %s for: %s') % (which, what, title)) - self.pd.value += 1 - if ok: - self.updated.add(id_) - if typ == 'cover': - try: - self.db.set_cover(id_, - self.downloader.fetched_covers.pop(id_)) - except: - self.downloader.cover_failures[id_] = \ - traceback.format_exc() + if results: + all_failed = False + mi = merge_result(mi, results[0]) + identifiers = mi.identifiers + if not mi.is_null('rating'): + # set_metadata expects a rating out of 10 + mi.rating *= 2 else: - try: - self.set_metadata(id_) - except: - self.downloader.failures[id_] = \ - traceback.format_exc() - - def set_metadata(self, id_): - mi = self.downloader.metadata[id_] - if self.downloader.set_metadata: - self.db.set_metadata(id_, mi) - if not self.downloader.set_metadata and self.downloader.get_social_metadata: - if mi.rating: - self.db.set_rating(id_, mi.rating) - if mi.tags: - self.db.set_tags(id_, mi.tags) - if mi.comments: - self.db.set_comment(id_, mi.comments) - if mi.series: - self.db.set_series(id_, mi.series) - if mi.series_index is not None: - self.db.set_series_index(id_, mi.series_index) - - def show_report(self): - f, cf = self.downloader.failures, self.downloader.cover_failures - report = [] - if f: - report.append( - '<h3>Failed to download metadata for the following:</h3><ol>') - for id_, err in f.items(): - mi = self.downloader.metadata[id_] - report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title, - unicode(err))) - report.append('</ol>') - if cf: - report.append( - '<h3>Failed to download cover for the following:</h3><ol>') - for id_, err in cf.items(): - mi = self.downloader.metadata[id_] - report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title, - unicode(err))) - report.append('</ol>') - - if len(self.updated) != self.total or report: - d = QDialog(self.parent()) - bb = QDialogButtonBox(QDialogButtonBox.Ok, parent=d) - v1 = QVBoxLayout() - d.setLayout(v1) - d.setWindowTitle(_('Done')) - v1.addWidget(QLabel(_('Successfully downloaded metadata for %d out of %d books') % - (len(self.updated), self.total))) - gb = QGroupBox(_('Details'), self.parent()) - v2 = QVBoxLayout() - gb.setLayout(v2) - b = QTextBrowser(self.parent()) - v2.addWidget(b) - b.setHtml('\n'.join(report)) - v1.addWidget(gb) - v1.addWidget(bb) - bb.accepted.connect(d.accept) - d.resize(800, 600) - d.exec_() - + log.error('Failed to download metadata for', title) + failed_ids.add(i) + # We don't want set_metadata operating on anything but covers + mi = merge_result(mi, mi) + if covers: + cdata = download_cover(log, title=title, authors=authors, + identifiers=identifiers) + if cdata is not None: + with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: + f.write(cdata[-1]) + mi.cover = f.name + all_failed = False + else: + failed_covers.add(i) + ans[i] = mi + count += 1 + notifications.put((count/len(ids), + _('Downloaded %d of %d')%(count, len(ids)))) + log('Download complete, with %d failures'%len(failed_ids)) + return (ans, failed_ids, failed_covers, title_map, all_failed) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py deleted file mode 100644 index 2a307fc902..0000000000 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import (unicode_literals, division, absolute_import, - print_function) - -__license__ = 'GPL v3' -__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -from functools import partial -from itertools import izip -from threading import Event - -from PyQt4.Qt import (QIcon, QDialog, - QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt) - -from calibre.gui2.threaded_jobs import ThreadedJob -from calibre.ebooks.metadata.sources.identify import identify, msprefs -from calibre.ebooks.metadata.sources.covers import download_cover -from calibre.ebooks.metadata.book.base import Metadata -from calibre.customize.ui import metadata_plugins -from calibre.ptempfile import PersistentTemporaryFile - -# Start download {{{ -def show_config(gui, parent): - from calibre.gui2.preferences import show_config_widget - show_config_widget('Sharing', 'Metadata download', parent=parent, - gui=gui, never_shutdown=True) - -class ConfirmDialog(QDialog): - - def __init__(self, ids, parent): - QDialog.__init__(self, parent) - self.setWindowTitle(_('Schedule download?')) - self.setWindowIcon(QIcon(I('dialog_question.png'))) - - l = self.l = QGridLayout() - self.setLayout(l) - - i = QLabel(self) - i.setPixmap(QPixmap(I('dialog_question.png'))) - l.addWidget(i, 0, 0) - - t = QLabel( - '<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will' - ' run in the background. Proceed?')%len(ids) + - '<p>'+_('You can monitor the progress of the download ' - 'by clicking the rotating spinner in the bottom right ' - 'corner.') + - '<p>'+_('When the download completes you will be asked for' - ' confirmation before calibre applies the downloaded metadata.') - ) - t.setWordWrap(True) - l.addWidget(t, 0, 1) - l.setColumnStretch(0, 1) - l.setColumnStretch(1, 100) - - self.identify = self.covers = True - self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) - self.bb.rejected.connect(self.reject) - b = self.bb.addButton(_('Download only &metadata'), - self.bb.AcceptRole) - b.clicked.connect(self.only_metadata) - b.setIcon(QIcon(I('edit_input.png'))) - b = self.bb.addButton(_('Download only &covers'), - self.bb.AcceptRole) - b.clicked.connect(self.only_covers) - b.setIcon(QIcon(I('default_cover.png'))) - b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole) - b.setIcon(QIcon(I('config.png'))) - b.clicked.connect(partial(show_config, parent, self)) - l.addWidget(self.bb, 1, 0, 1, 2) - b = self.bb.addButton(_('Download &both'), - self.bb.AcceptRole) - b.clicked.connect(self.accept) - b.setDefault(True) - b.setAutoDefault(True) - b.setIcon(QIcon(I('ok.png'))) - - self.resize(self.sizeHint()) - b.setFocus(Qt.OtherFocusReason) - - def only_metadata(self): - self.covers = False - self.accept() - - def only_covers(self): - self.identify = False - self.accept() - -def start_download(gui, ids, callback): - d = ConfirmDialog(ids, gui) - ret = d.exec_() - d.b.clicked.disconnect() - if ret != d.Accepted: - return - - job = ThreadedJob('metadata bulk download', - _('Download metadata for %d books')%len(ids), - download, (ids, gui.current_db, d.identify, d.covers), {}, callback) - gui.job_manager.run_threaded_job(job) - gui.status_bar.show_message(_('Metadata download started'), 3000) -# }}} - -def get_job_details(job): - id_map, failed_ids, failed_covers, title_map, all_failed = job.result - det_msg = [] - for i in failed_ids | failed_covers: - title = title_map[i] - if i in failed_ids: - title += (' ' + _('(Failed metadata)')) - if i in failed_covers: - title += (' ' + _('(Failed cover)')) - det_msg.append(title) - det_msg = '\n'.join(det_msg) - return id_map, failed_ids, failed_covers, all_failed, det_msg - -def merge_result(oldmi, newmi): - dummy = Metadata(_('Unknown')) - for f in msprefs['ignore_fields']: - if ':' not in f: - setattr(newmi, f, getattr(dummy, f)) - fields = set() - for plugin in metadata_plugins(['identify']): - fields |= plugin.touched_fields - - for f in fields: - # Optimize so that set_metadata does not have to do extra work later - if not f.startswith('identifier:'): - if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): - setattr(newmi, f, getattr(dummy, f)) - - newmi.last_modified = oldmi.last_modified - - return newmi - -def download(ids, db, do_identify, covers, - log=None, abort=None, notifications=None): - ids = list(ids) - metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) - for i in ids] - failed_ids = set() - failed_covers = set() - title_map = {} - ans = {} - count = 0 - all_failed = True - ''' - # Test apply dialog - all_failed = do_identify = covers = False - ''' - for i, mi in izip(ids, metadata): - if abort.is_set(): - log.error('Aborting...') - break - title, authors, identifiers = mi.title, mi.authors, mi.identifiers - title_map[i] = title - if do_identify: - results = [] - try: - results = identify(log, Event(), title=title, authors=authors, - identifiers=identifiers) - except: - pass - if results: - all_failed = False - mi = merge_result(mi, results[0]) - identifiers = mi.identifiers - if not mi.is_null('rating'): - # set_metadata expects a rating out of 10 - mi.rating *= 2 - else: - log.error('Failed to download metadata for', title) - failed_ids.add(i) - # We don't want set_metadata operating on anything but covers - mi = merge_result(mi, mi) - if covers: - cdata = download_cover(log, title=title, authors=authors, - identifiers=identifiers) - if cdata is not None: - with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: - f.write(cdata[-1]) - mi.cover = f.name - all_failed = False - else: - failed_covers.add(i) - ans[i] = mi - count += 1 - notifications.put((count/len(ids), - _('Downloaded %d of %d')%(count, len(ids)))) - log('Download complete, with %d failures'%len(failed_ids)) - return (ans, failed_ids, failed_covers, title_map, all_failed) - - - diff --git a/src/calibre/gui2/preferences/behavior.py b/src/calibre/gui2/preferences/behavior.py index b376d067bc..e062ae2662 100644 --- a/src/calibre/gui2/preferences/behavior.py +++ b/src/calibre/gui2/preferences/behavior.py @@ -19,7 +19,6 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.oeb.iterator import is_supported from calibre.constants import iswindows from calibre.utils.icu import sort_key -from calibre.utils.config import test_eight_code class OutputFormatSetting(Setting): @@ -40,12 +39,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('network_timeout', prefs) - - r('overwrite_author_title_metadata', config) - r('get_social_metadata', config) - if test_eight_code: - self.opt_overwrite_author_title_metadata.setVisible(False) - self.opt_get_social_metadata.setVisible(False) r('new_version_notification', config) r('upload_news_to_device', config) r('delete_news_from_library_on_upload', config) @@ -67,13 +60,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): signal.connect(self.internally_viewed_formats_changed) r('bools_are_tristate', db.prefs, restart_required=True) - if test_eight_code: - r = self.register - choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')] - r('edit_metadata_single_layout', gprefs, choices=choices) - else: - self.opt_edit_metadata_single_layout.setVisible(False) - self.edit_metadata_single_label.setVisible(False) + r = self.register + choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')] + r('edit_metadata_single_layout', gprefs, choices=choices) def initialize(self): ConfigWidgetBase.initialize(self) diff --git a/src/calibre/gui2/preferences/behavior.ui b/src/calibre/gui2/preferences/behavior.ui index 69ebce6acf..ffd59d72bb 100644 --- a/src/calibre/gui2/preferences/behavior.ui +++ b/src/calibre/gui2/preferences/behavior.ui @@ -14,41 +14,14 @@ <string>Form</string> </property> <layout class="QGridLayout" name="gridLayout"> - <item row="0" column="1"> - <spacer> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>10</width> - <height>0</height> - </size> - </property> - </spacer> - </item> - <item row="0" column="0"> - <widget class="QCheckBox" name="opt_overwrite_author_title_metadata"> - <property name="text"> - <string>&Overwrite author and title by default when fetching metadata</string> - </property> - </widget> - </item> - <item row="0" column="2"> - <widget class="QCheckBox" name="opt_get_social_metadata"> - <property name="text"> - <string>Download &social metadata (tags/ratings/etc.) by default</string> - </property> - </widget> - </item> - <item row="2" column="0"> + <item row="1" column="0"> <widget class="QCheckBox" name="opt_new_version_notification"> <property name="text"> <string>Show notification when &new version is available</string> </property> </widget> </item> - <item row="2" column="2"> + <item row="1" column="1"> <widget class="QCheckBox" name="opt_bools_are_tristate"> <property name="toolTip"> <string>If checked, Yes/No custom columns values can be Yes, No, or Unknown. @@ -59,21 +32,21 @@ If not checked, the values can be Yes or No.</string> </property> </widget> </item> - <item row="4" column="0"> + <item row="3" column="0"> <widget class="QCheckBox" name="opt_upload_news_to_device"> <property name="text"> <string>Automatically send downloaded &news to ebook reader</string> </property> </widget> </item> - <item row="4" column="2"> + <item row="3" column="1"> <widget class="QCheckBox" name="opt_delete_news_from_library_on_upload"> <property name="text"> <string>&Delete news from library when it is automatically sent to reader</string> </property> </widget> </item> - <item row="6" column="0"> + <item row="5" column="0"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="label_23"> @@ -97,7 +70,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="6" column="2"> + <item row="5" column="1"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="label_2"> @@ -130,7 +103,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="8" column="0"> + <item row="7" column="0"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="priority_label"> @@ -169,7 +142,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="8" column="2"> + <item row="7" column="1"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="label_170"> @@ -202,7 +175,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="9" column="0"> + <item row="8" column="0"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="edit_metadata_single_label"> @@ -223,7 +196,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="20" column="0"> + <item row="19" column="0"> <widget class="QGroupBox" name="groupBox_5"> <property name="title"> <string>Preferred &input format order:</string> @@ -285,7 +258,7 @@ If not checked, the values can be Yes or No.</string> </layout> </widget> </item> - <item row="20" column="2"> + <item row="19" column="1"> <widget class="QGroupBox" name="groupBox_3"> <property name="title"> <string>Use internal &viewer for:</string> @@ -304,7 +277,7 @@ If not checked, the values can be Yes or No.</string> </layout> </widget> </item> - <item row="9" column="2"> + <item row="8" column="1"> <widget class="QPushButton" name="reset_confirmation_button"> <property name="text"> <string>Reset all disabled &confirmation dialogs</string> diff --git a/src/calibre/gui2/preferences/social.py b/src/calibre/gui2/preferences/social.py deleted file mode 100644 index a22bcce091..0000000000 --- a/src/calibre/gui2/preferences/social.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement - -__license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -import time -from threading import Thread - -from PyQt4.Qt import QDialog, QDialogButtonBox, Qt, QLabel, QVBoxLayout, \ - QTimer - -from calibre.ebooks.metadata import MetaInformation - -class Worker(Thread): - - def __init__(self, mi): - Thread.__init__(self) - self.daemon = True - self.mi = MetaInformation(mi) - self.exceptions = [] - - def run(self): - from calibre.ebooks.metadata.fetch import get_social_metadata - self.exceptions = get_social_metadata(self.mi) - -class SocialMetadata(QDialog): - - TIMEOUT = 300 # seconds - - def __init__(self, mi, parent): - QDialog.__init__(self, parent) - - self.bbox = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self) - self.mi = mi - self.layout = QVBoxLayout(self) - self.label = QLabel(_('Downloading social metadata, please wait...'), self) - self.label.setWordWrap(True) - self.layout.addWidget(self.label) - self.layout.addWidget(self.bbox) - - self.worker = Worker(mi) - self.bbox.rejected.connect(self.reject) - self.worker.start() - self.start_time = time.time() - self.timed_out = False - self.rejected = False - QTimer.singleShot(50, self.update) - - def reject(self): - self.rejected = True - QDialog.reject(self) - - def update(self): - if self.rejected: - return - if time.time() - self.start_time > self.TIMEOUT: - self.timed_out = True - self.reject() - return - if not self.worker.is_alive(): - self.accept() - return - QTimer.singleShot(50, self.update) - - def accept(self): - self.mi.tags = self.worker.mi.tags - self.mi.rating = self.worker.mi.rating - self.mi.comments = self.worker.mi.comments - if self.worker.mi.series: - self.mi.series = self.worker.mi.series - self.mi.series_index = self.worker.mi.series_index - QDialog.accept(self) - - @property - def exceptions(self): - return self.worker.exceptions diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 8b23cf3071..0fcd047619 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -24,8 +24,6 @@ if False: OptionSet, ConfigInterface, read_tweaks, write_tweaks read_raw_tweaks, tweaks, plugin_dir -test_eight_code = tweaks.get('test_eight_code', False) - def check_config_write_access(): return os.access(config_dir, os.W_OK) and os.access(config_dir, os.X_OK)