Cutlery/calibre
1
0
Fork 0
mirror of https://github.com/kovidgoyal/calibre.git synced 2025-07-09 03:04:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Get rid of the test_eight_code tweak. That means the next release will be 0.8.0!

Browse Source
This commit is contained in:
Kovid Goyal 2011-04-30 13:14:23 -06:00
parent 373adf3cf4
commit 571f50be43
25 changed files with 222 additions and 6283 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/calibre/customize/builtins.py
View File

@ -9,7 +9,6 @@ from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \
from calibre.constants import numeric_version
from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata
from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.utils.config import test_eight_code
# To archive plugins {{{
class HTML2ZIP(FileTypePlugin):
@ -621,28 +620,16 @@ from calibre.ebooks.epub.fix.epubcheck import Epubcheck
plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested,
Epubcheck, ]
if test_eight_code:
# New metadata download plugins {{{
from calibre.ebooks.metadata.sources.google import GoogleBooks
from calibre.ebooks.metadata.sources.amazon import Amazon
from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary
from calibre.ebooks.metadata.sources.isbndb import ISBNDB
from calibre.ebooks.metadata.sources.overdrive import OverDrive
from calibre.ebooks.metadata.sources.google import GoogleBooks
from calibre.ebooks.metadata.sources.amazon import Amazon
from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary
from calibre.ebooks.metadata.sources.isbndb import ISBNDB
from calibre.ebooks.metadata.sources.overdrive import OverDrive
plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive]
plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive]
# }}}
else:
from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \
KentDistrictLibrary
from calibre.ebooks.metadata.douban import DoubanBooks
from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers
from calibre.ebooks.metadata.covers import OpenLibraryCovers, \
AmazonCovers, DoubanCovers
plugins += [GoogleBooks, ISBNDB, Amazon,
OpenLibraryCovers, AmazonCovers, DoubanCovers,
NiceBooksCovers, KentDistrictLibrary, DoubanBooks, NiceBooks]
plugins += [
ComicInput,
@ -867,10 +854,7 @@ plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog,
ActionRestart, ActionOpenFolder, ActionConnectShare,
ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks,
ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary,
ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch]
if test_eight_code:
plugins += [ActionStore]
ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore]
# }}}
@ -1096,10 +1080,8 @@ class Misc(PreferencesPlugin):
plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions,
CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard,
Email, Server, Plugins, Tweaks, Misc, TemplateFunctions]
if test_eight_code:
plugins.append(MetadataSources)
Email, Server, Plugins, Tweaks, Misc, TemplateFunctions,
MetadataSources]
#}}}

44
src/calibre/customize/ui.py
View File

@ -15,10 +15,8 @@ from calibre.customize.profiles import InputProfile, OutputProfile
from calibre.customize.builtins import plugins as builtin_plugins
from calibre.devices.interface import DevicePlugin
from calibre.ebooks.metadata import MetaInformation
from calibre.ebooks.metadata.covers import CoverDownload
from calibre.ebooks.metadata.fetch import MetadataSource
from calibre.utils.config import make_config_dir, Config, ConfigProxy, \
plugin_dir, OptionParser, prefs
from calibre.utils.config import (make_config_dir, Config, ConfigProxy,
plugin_dir, OptionParser)
from calibre.ebooks.epub.fix import ePubFixer
from calibre.ebooks.metadata.sources.base import Source
@ -190,44 +188,6 @@ def output_profiles():
yield plugin
# }}}
# Metadata sources {{{
def metadata_sources(metadata_type='basic', customize=True, isbndb_key=None):
for plugin in _initialized_plugins:
if isinstance(plugin, MetadataSource) and \
plugin.metadata_type == metadata_type:
if is_disabled(plugin):
continue
if customize:
customization = config['plugin_customization']
plugin.site_customization = customization.get(plugin.name, None)
if plugin.name == 'IsbnDB' and isbndb_key is not None:
plugin.site_customization = isbndb_key
yield plugin
def get_isbndb_key():
return config['plugin_customization'].get('IsbnDB', None)
def set_isbndb_key(key):
for plugin in _initialized_plugins:
if plugin.name == 'IsbnDB':
return customize_plugin(plugin, key)
def migrate_isbndb_key():
key = prefs['isbndb_com_key']
if key:
prefs.set('isbndb_com_key', '')
set_isbndb_key(key)
def cover_sources():
customization = config['plugin_customization']
for plugin in _initialized_plugins:
if isinstance(plugin, CoverDownload):
if not is_disabled(plugin):
plugin.site_customization = customization.get(plugin.name, '')
yield plugin
# }}}
# Interface Actions # {{{
def interface_actions():

224
src/calibre/ebooks/metadata/amazon.py
View File

@ -1,224 +0,0 @@
#!/usr/bin/env python
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net'
__docformat__ = 'restructuredtext en'
'''
Fetch metadata using Amazon AWS
'''
import sys, re
from threading import RLock
from lxml import html
from lxml.html import soupparser
from calibre import browser
from calibre.ebooks.metadata import check_isbn
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.chardet import xml_to_unicode
from calibre.library.comments import sanitize_comments_html
asin_cache = {}
cover_url_cache = {}
cache_lock = RLock()
def find_asin(br, isbn):
q = 'http://www.amazon.com/s/?search-alias=aps&field-keywords='+isbn
res = br.open_novisit(q)
raw = res.read()
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
root = html.fromstring(raw)
revs = root.xpath('//*[@class="asinReviewsSummary" and @name]')
revs = [x.get('name') for x in revs]
if revs:
return revs[0]
def to_asin(br, isbn):
with cache_lock:
ans = asin_cache.get(isbn, None)
if ans:
return ans
if ans is False:
return None
if len(isbn) == 13:
try:
asin = find_asin(br, isbn)
except:
import traceback
traceback.print_exc()
asin = None
else:
asin = isbn
with cache_lock:
asin_cache[isbn] = asin if asin else False
return asin
def get_social_metadata(title, authors, publisher, isbn):
mi = Metadata(title, authors)
if not isbn:
return mi
isbn = check_isbn(isbn)
if not isbn:
return mi
br = browser()
asin = to_asin(br, isbn)
if asin and get_metadata(br, asin, mi):
return mi
from calibre.ebooks.metadata.xisbn import xisbn
for i in xisbn.get_associated_isbns(isbn):
asin = to_asin(br, i)
if asin and get_metadata(br, asin, mi):
return mi
return mi
def get_cover_url(isbn, br):
isbn = check_isbn(isbn)
if not isbn:
return None
with cache_lock:
ans = cover_url_cache.get(isbn, None)
if ans:
return ans
if ans is False:
return None
asin = to_asin(br, isbn)
if asin:
ans = _get_cover_url(br, asin)
if ans:
with cache_lock:
cover_url_cache[isbn] = ans
return ans
from calibre.ebooks.metadata.xisbn import xisbn
for i in xisbn.get_associated_isbns(isbn):
asin = to_asin(br, i)
if asin:
ans = _get_cover_url(br, asin)
if ans:
with cache_lock:
cover_url_cache[isbn] = ans
cover_url_cache[i] = ans
return ans
with cache_lock:
cover_url_cache[isbn] = False
return None
def _get_cover_url(br, asin):
q = 'http://amzn.com/'+asin
try:
raw = br.open_novisit(q).read()
except Exception as e:
if callable(getattr(e, 'getcode', None)) and \
e.getcode() == 404:
return None
raise
if '<title>404 - ' in raw:
return None
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
root = soupparser.fromstring(raw)
except:
return False
imgs = root.xpath('//img[@id="prodImage" and @src]')
if imgs:
src = imgs[0].get('src')
parts = src.split('/')
if len(parts) > 3:
bn = parts[-1]
sparts = bn.split('_')
if len(sparts) > 2:
bn = sparts[0] + sparts[-1]
return ('/'.join(parts[:-1]))+'/'+bn
return None
def get_metadata(br, asin, mi):
q = 'http://amzn.com/'+asin
try:
raw = br.open_novisit(q).read()
except Exception as e:
if callable(getattr(e, 'getcode', None)) and \
e.getcode() == 404:
return False
raise
if '<title>404 - ' in raw:
return False
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
root = soupparser.fromstring(raw)
except:
return False
if root.xpath('//*[@id="errorMessage"]'):
return False
ratings = root.xpath('//div[@class="jumpBar"]/descendant::span[@class="asinReviewsSummary"]')
pat = re.compile(r'([0-9.]+) out of (\d+) stars')
if ratings:
for elem in ratings[0].xpath('descendant::*[@title]'):
t = elem.get('title').strip()
m = pat.match(t)
if m is not None:
try:
mi.rating = float(m.group(1))/float(m.group(2)) * 5
except:
pass
desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]')
if desc:
desc = desc[0]
for c in desc.xpath('descendant::*[@class="seeAll" or'
' @class="emptyClear" or @href]'):
c.getparent().remove(c)
desc = html.tostring(desc, method='html', encoding=unicode).strip()
# remove all attributes from tags
desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc)
# Collapse whitespace
#desc = re.sub('\n+', '\n', desc)
#desc = re.sub(' +', ' ', desc)
# Remove the notice about text referring to out of print editions
desc = re.sub(r'(?s)<em>--This text ref.*?</em>', '', desc)
# Remove comments
desc = re.sub(r'(?s)<!--.*?-->', '', desc)
mi.comments = sanitize_comments_html(desc)
return True
def main(args=sys.argv):
import tempfile, os
tdir = tempfile.gettempdir()
br = browser()
for title, isbn in [
('The Heroes', '9780316044981'), # Test find_asin
('Learning Python', '8324616489'), # Test xisbn
('Angels & Demons', '9781416580829'), # Test sophisticated comment formatting
# Random tests
('Star Trek: Destiny: Mere Mortals', '9781416551720'),
('The Great Gatsby', '0743273567'),
]:
cpath = os.path.join(tdir, title+'.jpg')
curl = get_cover_url(isbn, br)
if curl is None:
print 'No cover found for', title
else:
open(cpath, 'wb').write(br.open_novisit(curl).read())
print 'Cover for', title, 'saved to', cpath
#import time
#st = time.time()
mi = get_social_metadata(title, None, None, isbn)
if not mi.comments:
print 'Failed to downlaod social metadata for', title
return 1
#print '\n\n', time.time() - st, '\n\n'
print mi
print '\n'
return 0
if __name__ == '__main__':
sys.exit(main())

516
src/calibre/ebooks/metadata/amazonfr.py
View File

@ -1,516 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL 3'
__copyright__ = '2010, sengian <sengian1@gmail.com>'
import sys, textwrap, re, traceback
from urllib import urlencode
from math import ceil
from lxml import html
from lxml.html import soupparser
from calibre.utils.date import parse_date, utcnow, replace_months
from calibre.utils.cleantext import clean_ascii_chars
from calibre import browser, preferred_encoding
from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.metadata import MetaInformation, check_isbn, \
authors_to_sort_string
from calibre.ebooks.metadata.fetch import MetadataSource
from calibre.utils.config import OptionParser
from calibre.library.comments import sanitize_comments_html
class AmazonFr(MetadataSource):
name = 'Amazon French'
description = _('Downloads metadata from amazon.fr')
supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian'
version = (1, 0, 0)
has_html_comments = True
def fetch(self):
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10, verbose=self.verbose, lang='fr')
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
class AmazonEs(MetadataSource):
name = 'Amazon Spanish'
description = _('Downloads metadata from amazon.com in spanish')
supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian'
version = (1, 0, 0)
has_html_comments = True
def fetch(self):
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10, verbose=self.verbose, lang='es')
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
class AmazonEn(MetadataSource):
name = 'Amazon English'
description = _('Downloads metadata from amazon.com in english')
supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian'
version = (1, 0, 0)
has_html_comments = True
def fetch(self):
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10, verbose=self.verbose, lang='en')
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
class AmazonDe(MetadataSource):
name = 'Amazon German'
description = _('Downloads metadata from amazon.de')
supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian'
version = (1, 0, 0)
has_html_comments = True
def fetch(self):
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10, verbose=self.verbose, lang='de')
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
class Amazon(MetadataSource):
name = 'Amazon'
description = _('Downloads metadata from amazon.com')
supported_platforms = ['windows', 'osx', 'linux']
author = 'Kovid Goyal & Sengian'
version = (1, 1, 0)
has_html_comments = True
def fetch(self):
# if not self.site_customization:
# return
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10, verbose=self.verbose, lang='all')
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
# @property
# def string_customization_help(self):
# return _('You can select here the language for metadata search with amazon.com')
def report(verbose):
if verbose:
traceback.print_exc()
class Query(object):
BASE_URL_ALL = 'http://www.amazon.com'
BASE_URL_FR = 'http://www.amazon.fr'
BASE_URL_DE = 'http://www.amazon.de'
def __init__(self, title=None, author=None, publisher=None, isbn=None, keywords=None,
max_results=20, rlang='all'):
assert not(title is None and author is None and publisher is None \
and isbn is None and keywords is None)
assert (max_results < 21)
self.max_results = int(max_results)
self.renbres = re.compile(u'\s*(\d+)\s*')
q = { 'search-alias' : 'stripbooks' ,
'unfiltered' : '1',
'field-keywords' : '',
'field-author' : '',
'field-title' : '',
'field-isbn' : '',
'field-publisher' : ''
#get to amazon detailed search page to get all options
# 'node' : '',
# 'field-binding' : '',
#before, during, after
# 'field-dateop' : '',
#month as number
# 'field-datemod' : '',
# 'field-dateyear' : '',
#french only
# 'field-collection' : '',
#many options available
}
if rlang =='all':
q['sort'] = 'relevanceexprank'
self.urldata = self.BASE_URL_ALL
elif rlang =='es':
q['sort'] = 'relevanceexprank'
q['field-language'] = 'Spanish'
self.urldata = self.BASE_URL_ALL
elif rlang =='en':
q['sort'] = 'relevanceexprank'
q['field-language'] = 'English'
self.urldata = self.BASE_URL_ALL
elif rlang =='fr':
q['sort'] = 'relevancerank'
self.urldata = self.BASE_URL_FR
elif rlang =='de':
q['sort'] = 'relevancerank'
self.urldata = self.BASE_URL_DE
self.baseurl = self.urldata
if isbn is not None:
q['field-isbn'] = isbn.replace('-', '')
else:
if title is not None:
q['field-title'] = title
if author is not None:
q['field-author'] = author
if publisher is not None:
q['field-publisher'] = publisher
if keywords is not None:
q['field-keywords'] = keywords
if isinstance(q, unicode):
q = q.encode('utf-8')
self.urldata += '/gp/search/ref=sr_adv_b/?' + urlencode(q)
def __call__(self, browser, verbose, timeout = 5.):
if verbose:
print 'Query:', self.urldata
try:
raw = browser.open_novisit(self.urldata, timeout=timeout).read()
except Exception as e:
report(verbose)
if callable(getattr(e, 'getcode', None)) and \
e.getcode() == 404:
return
raise
if '<title>404 - ' in raw:
return
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
feed = soupparser.fromstring(raw)
except:
try:
#remove ASCII invalid chars
return soupparser.fromstring(clean_ascii_chars(raw))
except:
return None, self.urldata
#nb of page
try:
nbresults = self.renbres.findall(feed.xpath("//*[@class='resultCount']")[0].text)
except:
return None, self.urldata
pages =[feed]
if len(nbresults) > 1:
nbpagetoquery = int(ceil(float(min(int(nbresults[2]), self.max_results))/ int(nbresults[1])))
for i in xrange(2, nbpagetoquery + 1):
try:
urldata = self.urldata + '&page=' + str(i)
raw = browser.open_novisit(urldata, timeout=timeout).read()
except Exception as e:
continue
if '<title>404 - ' in raw:
continue
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
feed = soupparser.fromstring(raw)
except:
try:
#remove ASCII invalid chars
return soupparser.fromstring(clean_ascii_chars(raw))
except:
continue
pages.append(feed)
results = []
for x in pages:
results.extend([i.getparent().get('href') \
for i in x.xpath("//a/span[@class='srTitle']")])
return results[:self.max_results], self.baseurl
class ResultList(list):
def __init__(self, baseurl, lang = 'all'):
self.baseurl = baseurl
self.lang = lang
self.repub = re.compile(u'\((.*)\)')
self.rerat = re.compile(u'([0-9.]+)')
self.reattr = re.compile(r'<([a-zA-Z0-9]+)\s[^>]+>')
self.reoutp = re.compile(r'(?s)<em>--This text ref.*?</em>')
self.recom = re.compile(r'(?s)<!--.*?-->')
self.republi = re.compile(u'(Editeur|Publisher|Verlag)', re.I)
self.reisbn = re.compile(u'(ISBN-10|ISBN-10|ASIN)', re.I)
self.relang = re.compile(u'(Language|Langue|Sprache)', re.I)
self.reratelt = re.compile(u'(Average\s*Customer\s*Review|Moyenne\s*des\s*commentaires\s*client|Durchschnittliche\s*Kundenbewertung)', re.I)
self.reprod = re.compile(u'(Product\s*Details|D.tails\s*sur\s*le\s*produit|Produktinformation)', re.I)
def strip_tags_etree(self, etreeobj, invalid_tags):
for (itag, rmv) in invalid_tags.iteritems():
if rmv:
for elts in etreeobj.getiterator(itag):
elts.drop_tree()
else:
for elts in etreeobj.getiterator(itag):
elts.drop_tag()
def clean_entry(self, entry, invalid_tags = {'script': True},
invalid_id = (), invalid_class=()):
#invalid_tags: remove tag and keep content if False else remove
#remove tags
if invalid_tags:
self.strip_tags_etree(entry, invalid_tags)
#remove id
if invalid_id:
for eltid in invalid_id:
elt = entry.get_element_by_id(eltid)
if elt is not None:
elt.drop_tree()
#remove class
if invalid_class:
for eltclass in invalid_class:
elts = entry.find_class(eltclass)
if elts is not None:
for elt in elts:
elt.drop_tree()
def get_title(self, entry):
title = entry.get_element_by_id('btAsinTitle')
if title is not None:
title = title.text
return unicode(title.replace('\n', '').strip())
def get_authors(self, entry):
author = entry.get_element_by_id('btAsinTitle')
while author.getparent().tag != 'div':
author = author.getparent()
author = author.getparent()
authortext = []
for x in author.getiterator('a'):
authortext.append(unicode(x.text_content().strip()))
return authortext
def get_description(self, entry, verbose):
try:
description = entry.get_element_by_id("productDescription").find("div[@class='content']")
inv_class = ('seeAll', 'emptyClear')
inv_tags ={'img': True, 'a': False}
self.clean_entry(description, invalid_tags=inv_tags, invalid_class=inv_class)
description = html.tostring(description, method='html', encoding=unicode).strip()
# remove all attributes from tags
description = self.reattr.sub(r'<\1>', description)
# Remove the notice about text referring to out of print editions
description = self.reoutp.sub('', description)
# Remove comments
description = self.recom.sub('', description)
return unicode(sanitize_comments_html(description))
except:
report(verbose)
return None
def get_tags(self, entry, browser, verbose):
try:
tags = entry.get_element_by_id('tagContentHolder')
testptag = tags.find_class('see-all')
if testptag:
for x in testptag:
alink = x.xpath('descendant-or-self::a')
if alink:
if alink[0].get('class') == 'tgJsActive':
continue
link = self.baseurl + alink[0].get('href')
entry = self.get_individual_metadata(browser, link, verbose)
tags = entry.get_element_by_id('tagContentHolder')
break
tags = [a.text for a in tags.getiterator('a') if a.get('rel') == 'tag']
except:
report(verbose)
tags = []
return tags
def get_book_info(self, entry, mi, verbose):
try:
entry = entry.get_element_by_id('SalesRank').getparent()
except:
try:
for z in entry.getiterator('h2'):
if self.reprod.search(z.text_content()):
entry = z.getparent().find("div[@class='content']/ul")
break
except:
report(verbose)
return mi
elts = entry.findall('li')
#pub & date
elt = filter(lambda x: self.republi.search(x.find('b').text), elts)
if elt:
pub = elt[0].find('b').tail
mi.publisher = unicode(self.repub.sub('', pub).strip())
d = self.repub.search(pub)
if d is not None:
d = d.group(1)
try:
default = utcnow().replace(day=15)
if self.lang != 'all':
d = replace_months(d, self.lang)
d = parse_date(d, assume_utc=True, default=default)
mi.pubdate = d
except:
report(verbose)
#ISBN
elt = filter(lambda x: self.reisbn.search(x.find('b').text), elts)
if elt:
isbn = elt[0].find('b').tail.replace('-', '').strip()
if check_isbn(isbn):
mi.isbn = unicode(isbn)
elif len(elt) > 1:
isbn = elt[1].find('b').tail.replace('-', '').strip()
if check_isbn(isbn):
mi.isbn = unicode(isbn)
#Langue
elt = filter(lambda x: self.relang.search(x.find('b').text), elts)
if elt:
langue = elt[0].find('b').tail.strip()
if langue:
mi.language = unicode(langue)
#ratings
elt = filter(lambda x: self.reratelt.search(x.find('b').text), elts)
if elt:
ratings = elt[0].find_class('swSprite')
if ratings:
ratings = self.rerat.findall(ratings[0].get('title'))
if len(ratings) == 2:
mi.rating = float(ratings[0])/float(ratings[1]) * 5
return mi
def fill_MI(self, entry, title, authors, browser, verbose):
mi = MetaInformation(title, authors)
mi.author_sort = authors_to_sort_string(authors)
mi.comments = self.get_description(entry, verbose)
mi = self.get_book_info(entry, mi, verbose)
mi.tags = self.get_tags(entry, browser, verbose)
return mi
def get_individual_metadata(self, browser, linkdata, verbose):
try:
raw = browser.open_novisit(linkdata).read()
except Exception as e:
report(verbose)
if callable(getattr(e, 'getcode', None)) and \
e.getcode() == 404:
return
raise
if '<title>404 - ' in raw:
report(verbose)
return
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
return soupparser.fromstring(raw)
except:
try:
#remove ASCII invalid chars
return soupparser.fromstring(clean_ascii_chars(raw))
except:
report(verbose)
return
def populate(self, entries, browser, verbose=False):
for x in entries:
try:
entry = self.get_individual_metadata(browser, x, verbose)
# clean results
# inv_ids = ('divsinglecolumnminwidth', 'sims.purchase', 'AutoBuyXGetY', 'A9AdsMiddleBoxTop')
# inv_class = ('buyingDetailsGrid', 'productImageGrid')
# inv_tags ={'script': True, 'style': True, 'form': False}
# self.clean_entry(entry, invalid_id=inv_ids)
title = self.get_title(entry)
authors = self.get_authors(entry)
except Exception as e:
if verbose:
print 'Failed to get all details for an entry'
print e
print 'URL who failed:', x
report(verbose)
continue
self.append(self.fill_MI(entry, title, authors, browser, verbose))
def search(title=None, author=None, publisher=None, isbn=None,
max_results=5, verbose=False, keywords=None, lang='all'):
br = browser()
entries, baseurl = Query(title=title, author=author, isbn=isbn, publisher=publisher,
keywords=keywords, max_results=max_results,rlang=lang)(br, verbose)
if entries is None or len(entries) == 0:
return
#List of entry
ans = ResultList(baseurl, lang)
ans.populate(entries, br, verbose)
return ans
def option_parser():
parser = OptionParser(textwrap.dedent(\
_('''\
%prog [options]
Fetch book metadata from Amazon. You must specify one of title, author,
ISBN, publisher or keywords. Will fetch a maximum of 10 matches,
so you should make your query as specific as possible.
You can chose the language for metadata retrieval:
All & english & french & german & spanish
'''
)))
parser.add_option('-t', '--title', help='Book title')
parser.add_option('-a', '--author', help='Book author(s)')
parser.add_option('-p', '--publisher', help='Book publisher')
parser.add_option('-i', '--isbn', help='Book ISBN')
parser.add_option('-k', '--keywords', help='Keywords')
parser.add_option('-m', '--max-results', default=10,
help='Maximum number of results to fetch')
parser.add_option('-l', '--lang', default='all',
help='Chosen language for metadata search (all, en, fr, es, de)')
parser.add_option('-v', '--verbose', default=0, action='count',
help='Be more verbose about errors')
return parser
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
try:
results = search(opts.title, opts.author, isbn=opts.isbn, publisher=opts.publisher,
keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results,
lang=opts.lang)
except AssertionError:
report(True)
parser.print_help()
return 1
if results is None or len(results) == 0:
print 'No result found for this search!'
return 0
for result in results:
print unicode(result).encode(preferred_encoding, 'replace')
print
if __name__ == '__main__':
sys.exit(main())

317
src/calibre/ebooks/metadata/covers.py
View File

@ -1,317 +0,0 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
__license__ = 'GPL v3'
__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import traceback, socket, sys
from functools import partial
from threading import Thread, Event
from Queue import Queue, Empty
from lxml import etree
import mechanize
from calibre.customize import Plugin
from calibre import browser, prints
from calibre.constants import preferred_encoding, DEBUG
class CoverDownload(Plugin):
'''
These plugins are used to download covers for books.
'''
supported_platforms = ['windows', 'osx', 'linux']
author = 'Kovid Goyal'
type = _('Cover download')
def has_cover(self, mi, ans, timeout=5.):
'''
Check if the book described by mi has a cover. Call ans.set() if it
does. Do nothing if it doesn't.
:param mi: MetaInformation object
:param timeout: timeout in seconds
:param ans: A threading.Event object
'''
raise NotImplementedError()
def get_covers(self, mi, result_queue, abort, timeout=5.):
'''
Download covers for books described by the mi object. Downloaded covers
must be put into the result_queue. If more than one cover is available,
the plugin should continue downloading them and putting them into
result_queue until abort.is_set() returns True.
:param mi: MetaInformation object
:param result_queue: A multithreaded Queue
:param abort: A threading.Event object
:param timeout: timeout in seconds
'''
raise NotImplementedError()
def exception_to_string(self, ex):
try:
return unicode(ex)
except:
try:
return str(ex).decode(preferred_encoding, 'replace')
except:
return repr(ex)
def debug(self, *args, **kwargs):
if DEBUG:
prints('\t'+self.name+':', *args, **kwargs)
class HeadRequest(mechanize.Request):
def get_method(self):
return 'HEAD'
class OpenLibraryCovers(CoverDownload): # {{{
'Download covers from openlibrary.org'
# See http://openlibrary.org/dev/docs/api/covers
OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false'
name = 'openlibrary.org covers'
description = _('Download covers from openlibrary.org')
author = 'Kovid Goyal'
def has_cover(self, mi, ans, timeout=5.):
if not mi.isbn:
return False
from calibre.ebooks.metadata.library_thing import get_browser
br = get_browser()
br.set_handle_redirect(False)
try:
br.open_novisit(HeadRequest(self.OPENLIBRARY%mi.isbn), timeout=timeout)
self.debug('cover for', mi.isbn, 'found')
ans.set()
except Exception as e:
if callable(getattr(e, 'getcode', None)) and e.getcode() == 302:
self.debug('cover for', mi.isbn, 'found')
ans.set()
else:
self.debug(e)
def get_covers(self, mi, result_queue, abort, timeout=5.):
if not mi.isbn:
return
from calibre.ebooks.metadata.library_thing import get_browser
br = get_browser()
try:
ans = br.open(self.OPENLIBRARY%mi.isbn, timeout=timeout).read()
result_queue.put((True, ans, 'jpg', self.name))
except Exception as e:
if callable(getattr(e, 'getcode', None)) and e.getcode() == 404:
result_queue.put((False, _('ISBN: %s not found')%mi.isbn, '', self.name))
else:
result_queue.put((False, self.exception_to_string(e),
traceback.format_exc(), self.name))
# }}}
class AmazonCovers(CoverDownload): # {{{
name = 'amazon.com covers'
description = _('Download covers from amazon.com')
author = 'Kovid Goyal'
def has_cover(self, mi, ans, timeout=5.):
if not mi.isbn:
return False
from calibre.ebooks.metadata.amazon import get_cover_url
br = browser()
try:
get_cover_url(mi.isbn, br)
self.debug('cover for', mi.isbn, 'found')
ans.set()
except Exception as e:
self.debug(e)
def get_covers(self, mi, result_queue, abort, timeout=5.):
if not mi.isbn:
return
from calibre.ebooks.metadata.amazon import get_cover_url
br = browser()
try:
url = get_cover_url(mi.isbn, br)
if url is None:
raise ValueError('No cover found for ISBN: %s'%mi.isbn)
cover_data = br.open_novisit(url).read()
result_queue.put((True, cover_data, 'jpg', self.name))
except Exception as e:
result_queue.put((False, self.exception_to_string(e),
traceback.format_exc(), self.name))
# }}}
def check_for_cover(mi, timeout=5.): # {{{
from calibre.customize.ui import cover_sources
ans = Event()
checkers = [partial(p.has_cover, mi, ans, timeout=timeout) for p in
cover_sources()]
workers = [Thread(target=c) for c in checkers]
for w in workers:
w.daemon = True
w.start()
while not ans.is_set():
ans.wait(0.1)
if sum([int(w.is_alive()) for w in workers]) == 0:
break
return ans.is_set()
# }}}
def download_covers(mi, result_queue, max_covers=50, timeout=5.): # {{{
from calibre.customize.ui import cover_sources
abort = Event()
temp = Queue()
getters = [partial(p.get_covers, mi, temp, abort, timeout=timeout) for p in
cover_sources()]
workers = [Thread(target=c) for c in getters]
for w in workers:
w.daemon = True
w.start()
count = 0
while count < max_covers:
try:
result = temp.get_nowait()
if result[0]:
count += 1
result_queue.put(result)
except Empty:
pass
if sum([int(w.is_alive()) for w in workers]) == 0:
break
abort.set()
while True:
try:
result = temp.get_nowait()
count += 1
result_queue.put(result)
except Empty:
break
# }}}
class DoubanCovers(CoverDownload): # {{{
'Download covers from Douban.com'
DOUBAN_ISBN_URL = 'http://api.douban.com/book/subject/isbn/'
CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d'
name = 'Douban.com covers'
description = _('Download covers from Douban.com')
author = 'Li Fanxi'
def get_cover_url(self, isbn, br, timeout=5.):
try:
url = self.DOUBAN_ISBN_URL + isbn + "?apikey=" + self.CALIBRE_DOUBAN_API_KEY
src = br.open(url, timeout=timeout).read()
except Exception as err:
if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
err = Exception(_('Douban.com API timed out. Try again later.'))
raise err
else:
feed = etree.fromstring(src)
NAMESPACES = {
'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
'atom' : 'http://www.w3.org/2005/Atom',
'db': 'http://www.douban.com/xmlns/'
}
XPath = partial(etree.XPath, namespaces=NAMESPACES)
entries = XPath('//atom:entry')(feed)
if len(entries) < 1:
return None
try:
cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href")
u = cover_url(entries[0])[0].replace('/spic/', '/lpic/');
# If URL contains "book-default", the book doesn't have a cover
if u.find('book-default') != -1:
return None
except:
return None
return u
def has_cover(self, mi, ans, timeout=5.):
if not mi.isbn:
return False
br = browser()
try:
if self.get_cover_url(mi.isbn, br, timeout=timeout) != None:
self.debug('cover for', mi.isbn, 'found')
ans.set()
except Exception as e:
self.debug(e)
def get_covers(self, mi, result_queue, abort, timeout=5.):
if not mi.isbn:
return
br = browser()
try:
url = self.get_cover_url(mi.isbn, br, timeout=timeout)
cover_data = br.open_novisit(url).read()
result_queue.put((True, cover_data, 'jpg', self.name))
except Exception as e:
result_queue.put((False, self.exception_to_string(e),
traceback.format_exc(), self.name))
# }}}
def download_cover(mi, timeout=5.): # {{{
results = Queue()
download_covers(mi, results, max_covers=1, timeout=timeout)
errors, ans = [], None
while True:
try:
x = results.get_nowait()
if x[0]:
ans = x[1]
else:
errors.append(x)
except Empty:
break
return ans, errors
# }}}
def test(isbns): # {{{
from calibre.ebooks.metadata import MetaInformation
mi = MetaInformation('test', ['test'])
for isbn in isbns:
prints('Testing ISBN:', isbn)
mi.isbn = isbn
found = check_for_cover(mi)
prints('Has cover:', found)
ans, errors = download_cover(mi)
if ans is not None:
prints('Cover downloaded')
else:
prints('Download failed:')
for err in errors:
prints('\t', err[-1]+':', err[1])
print '\n'
# }}}
if __name__ == '__main__':
isbns = sys.argv[1:] + ['9781591025412', '9780307272119']
#test(isbns)
from calibre.ebooks.metadata import MetaInformation
oc = OpenLibraryCovers(None)
for isbn in isbns:
mi = MetaInformation('xx', ['yy'])
mi.isbn = isbn
rq = Queue()
oc.get_covers(mi, rq, Event())
result = rq.get_nowait()
if not result[0]:
print 'Failed for ISBN:', isbn
print result

263
src/calibre/ebooks/metadata/douban.py
View File

@ -1,263 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL 3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>; 2010, Li Fanxi <lifanxi@freemindworld.com>'
__docformat__ = 'restructuredtext en'
import sys, textwrap
import traceback
from urllib import urlencode
from functools import partial
from lxml import etree
from calibre import browser, preferred_encoding
from calibre.ebooks.metadata import MetaInformation
from calibre.utils.config import OptionParser
from calibre.ebooks.metadata.fetch import MetadataSource
from calibre.utils.date import parse_date, utcnow
NAMESPACES = {
'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
'atom' : 'http://www.w3.org/2005/Atom',
'db': 'http://www.douban.com/xmlns/'
}
XPath = partial(etree.XPath, namespaces=NAMESPACES)
total_results = XPath('//openSearch:totalResults')
start_index = XPath('//openSearch:startIndex')
items_per_page = XPath('//openSearch:itemsPerPage')
entry = XPath('//atom:entry')
entry_id = XPath('descendant::atom:id')
title = XPath('descendant::atom:title')
description = XPath('descendant::atom:summary')
publisher = XPath("descendant::db:attribute[@name='publisher']")
isbn = XPath("descendant::db:attribute[@name='isbn13']")
date = XPath("descendant::db:attribute[@name='pubdate']")
creator = XPath("descendant::db:attribute[@name='author']")
tag = XPath("descendant::db:tag")
CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d'
class DoubanBooks(MetadataSource):
name = 'Douban Books'
description = _('Downloads metadata from Douban.com')
supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on
author = 'Li Fanxi <lifanxi@freemindworld.com>' # The author of this plugin
version = (1, 0, 1) # The version number of this plugin
def fetch(self):
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10,
verbose=self.verbose)
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
def report(verbose):
if verbose:
import traceback
traceback.print_exc()
class Query(object):
SEARCH_URL = 'http://api.douban.com/book/subjects?'
ISBN_URL = 'http://api.douban.com/book/subject/isbn/'
type = "search"
def __init__(self, title=None, author=None, publisher=None, isbn=None,
max_results=20, start_index=1, api_key=''):
assert not(title is None and author is None and publisher is None and \
isbn is None)
assert (int(max_results) < 21)
q = ''
if isbn is not None:
q = isbn
self.type = 'isbn'
else:
def build_term(parts):
return ' '.join(x for x in parts)
if title is not None:
q += build_term(title.split())
if author is not None:
q += (' ' if q else '') + build_term(author.split())
if publisher is not None:
q += (' ' if q else '') + build_term(publisher.split())
self.type = 'search'
if isinstance(q, unicode):
q = q.encode('utf-8')
if self.type == "isbn":
self.url = self.ISBN_URL + q
if api_key != '':
self.url = self.url + "?apikey=" + api_key
else:
self.url = self.SEARCH_URL+urlencode({
'q':q,
'max-results':max_results,
'start-index':start_index,
})
if api_key != '':
self.url = self.url + "&apikey=" + api_key
def __call__(self, browser, verbose):
if verbose:
print 'Query:', self.url
if self.type == "search":
feed = etree.fromstring(browser.open(self.url).read())
total = int(total_results(feed)[0].text)
start = int(start_index(feed)[0].text)
entries = entry(feed)
new_start = start + len(entries)
if new_start > total:
new_start = 0
return entries, new_start
elif self.type == "isbn":
feed = etree.fromstring(browser.open(self.url).read())
entries = entry(feed)
return entries, 0
class ResultList(list):
def get_description(self, entry, verbose):
try:
desc = description(entry)
if desc:
return 'SUMMARY:\n'+desc[0].text
except:
report(verbose)
def get_title(self, entry):
candidates = [x.text for x in title(entry)]
return ': '.join(candidates)
def get_authors(self, entry):
m = creator(entry)
if not m:
m = []
m = [x.text for x in m]
return m
def get_tags(self, entry, verbose):
try:
btags = [x.attrib["name"] for x in tag(entry)]
tags = []
for t in btags:
tags.extend([y.strip() for y in t.split('/')])
tags = list(sorted(list(set(tags))))
except:
report(verbose)
tags = []
return [x.replace(',', ';') for x in tags]
def get_publisher(self, entry, verbose):
try:
pub = publisher(entry)[0].text
except:
pub = None
return pub
def get_isbn(self, entry, verbose):
try:
isbn13 = isbn(entry)[0].text
except Exception:
isbn13 = None
return isbn13
def get_date(self, entry, verbose):
try:
d = date(entry)
if d:
default = utcnow().replace(day=15)
d = parse_date(d[0].text, assume_utc=True, default=default)
else:
d = None
except:
report(verbose)
d = None
return d
def populate(self, entries, browser, verbose=False, api_key=''):
for x in entries:
try:
id_url = entry_id(x)[0].text
title = self.get_title(x)
except:
report(verbose)
mi = MetaInformation(title, self.get_authors(x))
try:
if api_key != '':
id_url = id_url + "?apikey=" + api_key
raw = browser.open(id_url).read()
feed = etree.fromstring(raw)
x = entry(feed)[0]
except Exception as e:
if verbose:
print 'Failed to get all details for an entry'
print e
mi.comments = self.get_description(x, verbose)
mi.tags = self.get_tags(x, verbose)
mi.isbn = self.get_isbn(x, verbose)
mi.publisher = self.get_publisher(x, verbose)
mi.pubdate = self.get_date(x, verbose)
self.append(mi)
def search(title=None, author=None, publisher=None, isbn=None,
verbose=False, max_results=40, api_key=None):
br = browser()
start, entries = 1, []
if api_key is None:
api_key = CALIBRE_DOUBAN_API_KEY
while start > 0 and len(entries) <= max_results:
new, start = Query(title=title, author=author, publisher=publisher,
isbn=isbn, max_results=max_results, start_index=start, api_key=api_key)(br, verbose)
if not new:
break
entries.extend(new)
entries = entries[:max_results]
ans = ResultList()
ans.populate(entries, br, verbose, api_key)
return ans
def option_parser():
parser = OptionParser(textwrap.dedent(
'''\
%prog [options]
Fetch book metadata from Douban. You must specify one of title, author,
publisher or ISBN. If you specify ISBN the others are ignored. Will
fetch a maximum of 100 matches, so you should make your query as
specific as possible.
'''
))
parser.add_option('-t', '--title', help='Book title')
parser.add_option('-a', '--author', help='Book author(s)')
parser.add_option('-p', '--publisher', help='Book publisher')
parser.add_option('-i', '--isbn', help='Book ISBN')
parser.add_option('-m', '--max-results', default=10,
help='Maximum number of results to fetch')
parser.add_option('-v', '--verbose', default=0, action='count',
help='Be more verbose about errors')
return parser
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
try:
results = search(opts.title, opts.author, opts.publisher, opts.isbn,
verbose=opts.verbose, max_results=int(opts.max_results))
except AssertionError:
report(True)
parser.print_help()
return 1
for result in results:
print unicode(result).encode(preferred_encoding)
print
if __name__ == '__main__':
sys.exit(main())

523
src/calibre/ebooks/metadata/fetch.py
View File

@ -1,523 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL 3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import traceback, sys, textwrap, re
from threading import Thread
from calibre import prints
from calibre.utils.config import OptionParser
from calibre.utils.logging import default_log
from calibre.utils.titlecase import titlecase
from calibre.customize import Plugin
from calibre.ebooks.metadata.covers import check_for_cover
from calibre.utils.html2text import html2text
metadata_config = None
class MetadataSource(Plugin): # {{{
'''
Represents a source to query for metadata. Subclasses must implement
at least the fetch method.
When :meth:`fetch` is called, the `self` object will have the following
useful attributes (each of which may be None)::
title, book_author, publisher, isbn, log, verbose and extra
Use these attributes to construct the search query. extra is reserved for
future use.
The fetch method must store the results in `self.results` as a list of
:class:`Metadata` objects. If there is an error, it should be stored
in `self.exception` and `self.tb` (for the traceback).
'''
author = 'Kovid Goyal'
supported_platforms = ['windows', 'osx', 'linux']
#: The type of metadata fetched. 'basic' means basic metadata like
#: title/author/isbn/etc. 'social' means social metadata like
#: tags/rating/reviews/etc.
metadata_type = 'basic'
#: If not None, the customization dialog will allow for string
#: based customization as well the default customization. The
#: string customization will be saved in the site_customization
#: member.
string_customization_help = None
#: Set this to true if your plugin returns HTML markup in comments.
#: Then if the user disables HTML, calibre will automagically convert
#: the HTML to Markdown.
has_html_comments = False
type = _('Metadata download')
def __call__(self, title, author, publisher, isbn, verbose, log=None,
extra=None):
self.worker = Thread(target=self._fetch)
self.worker.daemon = True
self.title = title
self.verbose = verbose
self.book_author = author
self.publisher = publisher
self.isbn = isbn
self.log = log if log is not None else default_log
self.extra = extra
self.exception, self.tb, self.results = None, None, []
self.worker.start()
def _fetch(self):
try:
self.fetch()
if self.results:
c = self.config_store().get(self.name, {})
res = self.results
if hasattr(res, 'authors'):
res = [res]
for mi in res:
if not c.get('rating', True):
mi.rating = None
if not c.get('comments', True):
mi.comments = None
if not c.get('tags', True):
mi.tags = []
if self.has_html_comments and mi.comments and \
c.get('textcomments', False):
try:
mi.comments = html2text(mi.comments)
except:
traceback.print_exc()
mi.comments = None
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
def fetch(self):
'''
All the actual work is done here.
'''
raise NotImplementedError
def join(self):
return self.worker.join()
def is_alive(self):
return self.worker.is_alive()
def is_customizable(self):
return True
def config_store(self):
global metadata_config
if metadata_config is None:
from calibre.utils.config import XMLConfig
metadata_config = XMLConfig('plugins/metadata_download')
return metadata_config
def config_widget(self):
from PyQt4.Qt import QWidget, QVBoxLayout, QLabel, Qt, QLineEdit, \
QCheckBox
from calibre.customize.ui import config
w = QWidget()
w._layout = QVBoxLayout(w)
w.setLayout(w._layout)
if self.string_customization_help is not None:
w._sc_label = QLabel(self.string_customization_help, w)
w._layout.addWidget(w._sc_label)
customization = config['plugin_customization']
def_sc = customization.get(self.name, '')
if not def_sc:
def_sc = ''
w._sc = QLineEdit(def_sc, w)
w._layout.addWidget(w._sc)
w._sc_label.setWordWrap(True)
w._sc_label.setTextInteractionFlags(Qt.LinksAccessibleByMouse
| Qt.LinksAccessibleByKeyboard)
w._sc_label.setOpenExternalLinks(True)
c = self.config_store()
c = c.get(self.name, {})
for x, l in {'rating':_('ratings'), 'tags':_('tags'),
'comments':_('description/reviews')}.items():
cb = QCheckBox(_('Download %s from %s')%(l,
self.name))
setattr(w, '_'+x, cb)
cb.setChecked(c.get(x, True))
w._layout.addWidget(cb)
if self.has_html_comments:
cb = QCheckBox(_('Convert comments downloaded from %s to plain text')%(self.name))
setattr(w, '_textcomments', cb)
cb.setChecked(c.get('textcomments', False))
w._layout.addWidget(cb)
return w
def save_settings(self, w):
dl_settings = {}
for x in ('rating', 'tags', 'comments'):
dl_settings[x] = getattr(w, '_'+x).isChecked()
if self.has_html_comments:
dl_settings['textcomments'] = getattr(w, '_textcomments').isChecked()
c = self.config_store()
c.set(self.name, dl_settings)
if hasattr(w, '_sc'):
sc = unicode(w._sc.text()).strip()
from calibre.customize.ui import customize_plugin
customize_plugin(self, sc)
def customization_help(self):
return 'This plugin can only be customized using the GUI'
# }}}
class GoogleBooks(MetadataSource): # {{{
name = 'Google Books'
description = _('Downloads metadata from Google Books')
def fetch(self):
from calibre.ebooks.metadata.google_books import search
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10,
verbose=self.verbose)
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
# }}}
class ISBNDB(MetadataSource): # {{{
name = 'IsbnDB'
description = _('Downloads metadata from isbndb.com')
def fetch(self):
if not self.site_customization:
return
from calibre.ebooks.metadata.isbndb import option_parser, create_books
args = ['isbndb']
if self.isbn:
args.extend(['--isbn', self.isbn])
else:
if self.title:
args.extend(['--title', self.title])
if self.book_author:
args.extend(['--author', self.book_author])
if self.publisher:
args.extend(['--publisher', self.publisher])
if self.verbose:
args.extend(['--verbose'])
args.append(self.site_customization) # IsbnDb key
try:
opts, args = option_parser().parse_args(args)
self.results = create_books(opts, args)
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
@property
def string_customization_help(self):
ans = _('To use isbndb.com you must sign up for a %sfree account%s '
'and enter your access key below.')
return '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>')
# }}}
class Amazon(MetadataSource): # {{{
name = 'Amazon'
metadata_type = 'social'
description = _('Downloads social metadata from amazon.com')
has_html_comments = True
def fetch(self):
if not self.isbn:
return
from calibre.ebooks.metadata.amazon import get_social_metadata
try:
self.results = get_social_metadata(self.title, self.book_author,
self.publisher, self.isbn)
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
# }}}
class KentDistrictLibrary(MetadataSource): # {{{
name = 'Kent District Library'
metadata_type = 'social'
description = _('Downloads series information from ww2.kdl.org. '
'This website cannot handle large numbers of queries, '
'so the plugin is disabled by default.')
def fetch(self):
if not self.title or not self.book_author:
return
from calibre.ebooks.metadata.kdl import get_series
try:
self.results = get_series(self.title, self.book_author)
except Exception as e:
import traceback
traceback.print_exc()
self.exception = e
self.tb = traceback.format_exc()
# }}}
def result_index(source, result):
if not result.isbn:
return -1
for i, x in enumerate(source):
if x.isbn == result.isbn:
return i
return -1
def merge_results(one, two):
if two is not None and one is not None:
for x in two:
idx = result_index(one, x)
if idx < 0:
one.append(x)
else:
one[idx].smart_update(x)
class MetadataSources(object):
def __init__(self, sources):
self.sources = sources
def __enter__(self):
for s in self.sources:
s.__enter__()
return self
def __exit__(self, *args):
for s in self.sources:
s.__exit__()
def __call__(self, *args, **kwargs):
for s in self.sources:
s(*args, **kwargs)
def join(self):
for s in self.sources:
s.join()
def filter_metadata_results(item):
keywords = ["audio", "tape", "cassette", "abridged", "playaway"]
for keyword in keywords:
if item.publisher and keyword in item.publisher.lower():
return False
return True
def do_cover_check(item):
item.has_cover = False
try:
item.has_cover = check_for_cover(item)
except:
pass # Cover not found
def check_for_covers(items):
threads = [Thread(target=do_cover_check, args=(item,)) for item in items]
for t in threads: t.start()
for t in threads: t.join()
def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None,
verbose=0):
assert not(title is None and author is None and publisher is None and \
isbn is None)
from calibre.customize.ui import metadata_sources, migrate_isbndb_key
migrate_isbndb_key()
if isbn is not None:
isbn = re.sub(r'[^a-zA-Z0-9]', '', isbn).upper()
fetchers = list(metadata_sources(isbndb_key=isbndb_key))
with MetadataSources(fetchers) as manager:
manager(title, author, publisher, isbn, verbose)
manager.join()
results = list(fetchers[0].results) if fetchers else []
for fetcher in fetchers[1:]:
merge_results(results, fetcher.results)
results = list(filter(filter_metadata_results, results))
check_for_covers(results)
words = ("the", "a", "an", "of", "and")
prefix_pat = re.compile(r'^(%s)\s+'%("|".join(words)))
trailing_paren_pat = re.compile(r'\(.*\)$')
whitespace_pat = re.compile(r'\s+')
def sort_func(x, y):
def cleanup_title(s):
if s is None:
s = _('Unknown')
s = s.strip().lower()
s = prefix_pat.sub(' ', s)
s = trailing_paren_pat.sub('', s)
s = whitespace_pat.sub(' ', s)
return s.strip()
t = cleanup_title(title)
x_title = cleanup_title(x.title)
y_title = cleanup_title(y.title)
# prefer titles that start with the search title
tx = cmp(t, x_title)
ty = cmp(t, y_title)
result = 0 if abs(tx) == abs(ty) else abs(tx) - abs(ty)
# then prefer titles that have a cover image
if result == 0:
result = -cmp(x.has_cover, y.has_cover)
# then prefer titles with the longest comment, with in 10%
if result == 0:
cx = len(x.comments.strip() if x.comments else '')
cy = len(y.comments.strip() if y.comments else '')
t = (cx + cy) / 20
result = cy - cx
if abs(result) < t:
result = 0
return result
results = sorted(results, cmp=sort_func)
# if for some reason there is no comment in the top selection, go looking for one
if len(results) > 1:
if not results[0].comments or len(results[0].comments) == 0:
for r in results[1:]:
try:
if title and title.lower() == r.title[:len(title)].lower() \
and r.comments and len(r.comments):
results[0].comments = r.comments
break
except:
pass
# Find a pubdate
pubdate = None
for r in results:
if r.pubdate is not None:
pubdate = r.pubdate
break
if pubdate is not None:
for r in results:
if r.pubdate is None:
r.pubdate = pubdate
def fix_case(x):
if x:
x = titlecase(x)
return x
for r in results:
r.title = fix_case(r.title)
if r.authors:
r.authors = list(map(fix_case, r.authors))
return results, [(x.name, x.exception, x.tb) for x in fetchers]
def get_social_metadata(mi, verbose=0):
from calibre.customize.ui import metadata_sources
fetchers = list(metadata_sources(metadata_type='social'))
with MetadataSources(fetchers) as manager:
manager(mi.title, mi.authors, mi.publisher, mi.isbn, verbose)
manager.join()
ratings, tags, comments, series, series_index = [], set([]), set([]), None, None
for fetcher in fetchers:
if fetcher.results:
dmi = fetcher.results
if dmi.rating is not None:
ratings.append(dmi.rating)
if dmi.tags:
for t in dmi.tags:
tags.add(t)
if mi.pubdate is None and dmi.pubdate is not None:
mi.pubdate = dmi.pubdate
if dmi.comments:
comments.add(dmi.comments)
if dmi.series is not None:
series = dmi.series
if dmi.series_index is not None:
series_index = dmi.series_index
if ratings:
rating = sum(ratings)/float(len(ratings))
if mi.rating is None or mi.rating < 0.1:
mi.rating = rating
else:
mi.rating = (mi.rating + rating)/2.0
if tags:
if not mi.tags:
mi.tags = []
mi.tags += list(tags)
mi.tags = list(sorted(list(set(mi.tags))))
if comments:
if not mi.comments or len(mi.comments)+20 < len(' '.join(comments)):
mi.comments = ''
for x in comments:
mi.comments += x+'\n\n'
if series and series_index is not None:
mi.series = series
mi.series_index = series_index
return [(x.name, x.exception, x.tb) for x in fetchers if x.exception is not
None]
def option_parser():
parser = OptionParser(textwrap.dedent(
'''\
%prog [options]
Fetch book metadata from online sources. You must specify at least one
of title, author, publisher or ISBN. If you specify ISBN, the others
are ignored.
'''
))
parser.add_option('-t', '--title', help='Book title')
parser.add_option('-a', '--author', help='Book author(s)')
parser.add_option('-p', '--publisher', help='Book publisher')
parser.add_option('-i', '--isbn', help='Book ISBN')
parser.add_option('-m', '--max-results', default=10,
help='Maximum number of results to fetch')
parser.add_option('-k', '--isbndb-key',
help=('The access key for your ISBNDB.com account. '
'Only needed if you want to search isbndb.com '
'and you haven\'t customized the IsbnDB plugin.'))
parser.add_option('-v', '--verbose', default=0, action='count',
help='Be more verbose about errors')
return parser
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
results, exceptions = search(opts.title, opts.author, opts.publisher,
opts.isbn, opts.isbndb_key, opts.verbose)
social_exceptions = []
for result in results:
social_exceptions.extend(get_social_metadata(result, opts.verbose))
prints(unicode(result))
print
for name, exception, tb in exceptions+social_exceptions:
if exception is not None:
print 'WARNING: Fetching from', name, 'failed with error:'
print exception
print tb
return 0
if __name__ == '__main__':
sys.exit(main())

390
src/calibre/ebooks/metadata/fictionwise.py
View File

@ -1,390 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL 3'
__copyright__ = '2010, sengian <sengian1@gmail.com>'
__docformat__ = 'restructuredtext en'
import sys, textwrap, re, traceback, socket
from urllib import urlencode
from lxml.html import soupparser, tostring
from calibre import browser, preferred_encoding
from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.metadata import MetaInformation, check_isbn, \
authors_to_sort_string
from calibre.library.comments import sanitize_comments_html
from calibre.ebooks.metadata.fetch import MetadataSource
from calibre.utils.config import OptionParser
from calibre.utils.date import parse_date, utcnow
from calibre.utils.cleantext import clean_ascii_chars
class Fictionwise(MetadataSource): # {{{
author = 'Sengian'
name = 'Fictionwise'
description = _('Downloads metadata from Fictionwise')
has_html_comments = True
def fetch(self):
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10, verbose=self.verbose)
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
# }}}
class FictionwiseError(Exception):
pass
def report(verbose):
if verbose:
traceback.print_exc()
class Query(object):
BASE_URL = 'http://www.fictionwise.com/servlet/mw'
def __init__(self, title=None, author=None, publisher=None, keywords=None, max_results=20):
assert not(title is None and author is None and publisher is None and keywords is None)
assert (max_results < 21)
self.max_results = int(max_results)
q = { 'template' : 'searchresults_adv.htm' ,
'searchtitle' : '',
'searchauthor' : '',
'searchpublisher' : '',
'searchkeyword' : '',
#possibilities startoflast, fullname, lastfirst
'searchauthortype' : 'startoflast',
'searchcategory' : '',
'searchcategory2' : '',
'searchprice_s' : '0',
'searchprice_e' : 'ANY',
'searchformat' : '',
'searchgeo' : 'US',
'searchfwdatetype' : '',
#maybe use dates fields if needed?
#'sortorder' : 'DESC',
#many options available: b.SortTitle, a.SortName,
#b.DateFirstPublished, b.FWPublishDate
'sortby' : 'b.SortTitle'
}
if title is not None:
q['searchtitle'] = title
if author is not None:
q['searchauthor'] = author
if publisher is not None:
q['searchpublisher'] = publisher
if keywords is not None:
q['searchkeyword'] = keywords
if isinstance(q, unicode):
q = q.encode('utf-8')
self.urldata = urlencode(q)
def __call__(self, browser, verbose, timeout = 5.):
if verbose:
print _('Query: %s') % self.BASE_URL+self.urldata
try:
raw = browser.open_novisit(self.BASE_URL, self.urldata, timeout=timeout).read()
except Exception as e:
report(verbose)
if callable(getattr(e, 'getcode', None)) and \
e.getcode() == 404:
return
if isinstance(getattr(e, 'args', [None])[0], socket.timeout):
raise FictionwiseError(_('Fictionwise timed out. Try again later.'))
raise FictionwiseError(_('Fictionwise encountered an error.'))
if '<title>404 - ' in raw:
return
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
feed = soupparser.fromstring(raw)
except:
try:
#remove ASCII invalid chars
feed = soupparser.fromstring(clean_ascii_chars(raw))
except:
return None
# get list of results as links
results = feed.xpath("//table[3]/tr/td[2]/table/tr/td/p/table[2]/tr[@valign]")
results = results[:self.max_results]
results = [i.xpath('descendant-or-self::a')[0].get('href') for i in results]
#return feed if no links ie normally a single book or nothing
if not results:
results = [feed]
return results
class ResultList(list):
BASE_URL = 'http://www.fictionwise.com'
COLOR_VALUES = {'BLUE': 4, 'GREEN': 3, 'YELLOW': 2, 'RED': 1, 'NA': 0}
def __init__(self):
self.retitle = re.compile(r'\[[^\[\]]+\]')
self.rechkauth = re.compile(r'.*book\s*by', re.I)
self.redesc = re.compile(r'book\s*description\s*:\s*(<br[^>]+>)*(?P<desc>.*)<br[^>]*>.{,15}publisher\s*:', re.I)
self.repub = re.compile(r'.*publisher\s*:\s*', re.I)
self.redate = re.compile(r'.*release\s*date\s*:\s*', re.I)
self.retag = re.compile(r'.*book\s*category\s*:\s*', re.I)
self.resplitbr = re.compile(r'<br[^>]*>', re.I)
self.recomment = re.compile(r'(?s)<!--.*?-->')
self.reimg = re.compile(r'<img[^>]*>', re.I)
self.resanitize = re.compile(r'\[HTML_REMOVED\]\s*', re.I)
self.renbcom = re.compile('(?P<nbcom>\d+)\s*Reader Ratings:')
self.recolor = re.compile('(?P<ncolor>[^/]+).gif')
self.resplitbrdiv = re.compile(r'(<br[^>]+>|</?div[^>]*>)', re.I)
self.reisbn = re.compile(r'.*ISBN\s*:\s*', re.I)
def strip_tags_etree(self, etreeobj, invalid_tags):
for (itag, rmv) in invalid_tags.iteritems():
if rmv:
for elts in etreeobj.getiterator(itag):
elts.drop_tree()
else:
for elts in etreeobj.getiterator(itag):
elts.drop_tag()
def clean_entry(self, entry, invalid_tags = {'script': True},
invalid_id = (), invalid_class=(), invalid_xpath = ()):
#invalid_tags: remove tag and keep content if False else remove
#remove tags
if invalid_tags:
self.strip_tags_etree(entry, invalid_tags)
#remove xpath
if invalid_xpath:
for eltid in invalid_xpath:
elt = entry.xpath(eltid)
for el in elt:
el.drop_tree()
#remove id
if invalid_id:
for eltid in invalid_id:
elt = entry.get_element_by_id(eltid)
if elt is not None:
elt.drop_tree()
#remove class
if invalid_class:
for eltclass in invalid_class:
elts = entry.find_class(eltclass)
if elts is not None:
for elt in elts:
elt.drop_tree()
def output_entry(self, entry, prettyout = True, htmlrm="\d+"):
out = tostring(entry, pretty_print=prettyout)
#try to work around tostring to remove this encoding for exemle
reclean = re.compile('(\n+|\t+|\r+|&#'+htmlrm+';)')
return reclean.sub('', out)
def get_title(self, entry):
title = entry.findtext('./')
return self.retitle.sub('', title).strip()
def get_authors(self, entry):
authortext = entry.find('./br').tail
if not self.rechkauth.search(authortext):
return []
authortext = self.rechkauth.sub('', authortext)
return [a.strip() for a in authortext.split('&')]
def get_rating(self, entrytable, verbose):
nbcomment = tostring(entrytable.getprevious())
try:
nbcomment = self.renbcom.search(nbcomment).group("nbcom")
except:
report(verbose)
return None
hval = dict((self.COLOR_VALUES[self.recolor.search(image.get('src', default='NA.gif')).group("ncolor")],
float(image.get('height', default=0))) \
for image in entrytable.getiterator('img'))
#ratings as x/5
return float(1.25*sum(k*v for (k, v) in hval.iteritems())/sum(hval.itervalues()))
def get_description(self, entry):
description = self.output_entry(entry.xpath('./p')[1],htmlrm="")
description = self.redesc.search(description)
if not description or not description.group("desc"):
return None
#remove invalid tags
description = self.reimg.sub('', description.group("desc"))
description = self.recomment.sub('', description)
description = self.resanitize.sub('', sanitize_comments_html(description))
return _('SUMMARY:\n %s') % re.sub(r'\n\s+</p>','\n</p>', description)
def get_publisher(self, entry):
publisher = self.output_entry(entry.xpath('./p')[1])
publisher = filter(lambda x: self.repub.search(x) is not None,
self.resplitbr.split(publisher))
if not len(publisher):
return None
publisher = self.repub.sub('', publisher[0])
return publisher.split(',')[0].strip()
def get_tags(self, entry):
tag = self.output_entry(entry.xpath('./p')[1])
tag = filter(lambda x: self.retag.search(x) is not None,
self.resplitbr.split(tag))
if not len(tag):
return []
return map(lambda x: x.strip(), self.retag.sub('', tag[0]).split('/'))
def get_date(self, entry, verbose):
date = self.output_entry(entry.xpath('./p')[1])
date = filter(lambda x: self.redate.search(x) is not None,
self.resplitbr.split(date))
if not len(date):
return None
try:
d = self.redate.sub('', date[0])
if d:
default = utcnow().replace(day=15)
d = parse_date(d, assume_utc=True, default=default)
else:
d = None
except:
report(verbose)
d = None
return d
def get_ISBN(self, entry):
isbns = self.output_entry(entry.xpath('./p')[2])
isbns = filter(lambda x: self.reisbn.search(x) is not None,
self.resplitbrdiv.split(isbns))
if not len(isbns):
return None
isbns = [self.reisbn.sub('', x) for x in isbns if check_isbn(self.reisbn.sub('', x))]
return sorted(isbns, cmp=lambda x,y:cmp(len(x), len(y)))[-1]
def fill_MI(self, entry, title, authors, ratings, verbose):
mi = MetaInformation(title, authors)
mi.rating = ratings
mi.comments = self.get_description(entry)
mi.publisher = self.get_publisher(entry)
mi.tags = self.get_tags(entry)
mi.pubdate = self.get_date(entry, verbose)
mi.isbn = self.get_ISBN(entry)
mi.author_sort = authors_to_sort_string(authors)
return mi
def get_individual_metadata(self, browser, linkdata, verbose):
try:
raw = browser.open_novisit(self.BASE_URL + linkdata).read()
except Exception as e:
report(verbose)
if callable(getattr(e, 'getcode', None)) and \
e.getcode() == 404:
return
if isinstance(getattr(e, 'args', [None])[0], socket.timeout):
raise FictionwiseError(_('Fictionwise timed out. Try again later.'))
raise FictionwiseError(_('Fictionwise encountered an error.'))
if '<title>404 - ' in raw:
report(verbose)
return
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
return soupparser.fromstring(raw)
except:
try:
#remove ASCII invalid chars
return soupparser.fromstring(clean_ascii_chars(raw))
except:
return None
def populate(self, entries, browser, verbose=False):
inv_tags ={'script': True, 'a': False, 'font': False, 'strong': False, 'b': False,
'ul': False, 'span': False}
inv_xpath =('./table',)
#single entry
if len(entries) == 1 and not isinstance(entries[0], str):
try:
entry = entries.xpath("//table[3]/tr/td[2]/table[1]/tr/td/font/table/tr/td")
self.clean_entry(entry, invalid_tags=inv_tags, invalid_xpath=inv_xpath)
title = self.get_title(entry)
#maybe strenghten the search
ratings = self.get_rating(entry.xpath("./p/table")[1], verbose)
authors = self.get_authors(entry)
except Exception as e:
if verbose:
print _('Failed to get all details for an entry')
print e
return
self.append(self.fill_MI(entry, title, authors, ratings, verbose))
else:
#multiple entries
for x in entries:
try:
entry = self.get_individual_metadata(browser, x, verbose)
entry = entry.xpath("//table[3]/tr/td[2]/table[1]/tr/td/font/table/tr/td")[0]
self.clean_entry(entry, invalid_tags=inv_tags, invalid_xpath=inv_xpath)
title = self.get_title(entry)
#maybe strenghten the search
ratings = self.get_rating(entry.xpath("./p/table")[1], verbose)
authors = self.get_authors(entry)
except Exception as e:
if verbose:
print _('Failed to get all details for an entry')
print e
continue
self.append(self.fill_MI(entry, title, authors, ratings, verbose))
def search(title=None, author=None, publisher=None, isbn=None,
min_viewability='none', verbose=False, max_results=5,
keywords=None):
br = browser()
entries = Query(title=title, author=author, publisher=publisher,
keywords=keywords, max_results=max_results)(br, verbose, timeout = 15.)
#List of entry
ans = ResultList()
ans.populate(entries, br, verbose)
return ans
def option_parser():
parser = OptionParser(textwrap.dedent(\
_('''\
%prog [options]
Fetch book metadata from Fictionwise. You must specify one of title, author,
or keywords. No ISBN specification possible. Will fetch a maximum of 20 matches,
so you should make your query as specific as possible.
''')
))
parser.add_option('-t', '--title', help=_('Book title'))
parser.add_option('-a', '--author', help=_('Book author(s)'))
parser.add_option('-p', '--publisher', help=_('Book publisher'))
parser.add_option('-k', '--keywords', help=_('Keywords'))
parser.add_option('-m', '--max-results', default=20,
help=_('Maximum number of results to fetch'))
parser.add_option('-v', '--verbose', default=0, action='count',
help=_('Be more verbose about errors'))
return parser
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
try:
results = search(opts.title, opts.author, publisher=opts.publisher,
keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results)
except AssertionError:
report(True)
parser.print_help()
return 1
if results is None or len(results) == 0:
print _('No result found for this search!')
return 0
for result in results:
print unicode(result).encode(preferred_encoding, 'replace')
print
if __name__ == '__main__':
sys.exit(main())

247
src/calibre/ebooks/metadata/google_books.py
View File

@ -1,247 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL 3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import sys, textwrap
from urllib import urlencode
from functools import partial
from lxml import etree
from calibre import browser, preferred_encoding
from calibre.ebooks.metadata import MetaInformation
from calibre.utils.config import OptionParser
from calibre.utils.date import parse_date, utcnow
NAMESPACES = {
'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/',
'atom' : 'http://www.w3.org/2005/Atom',
'dc': 'http://purl.org/dc/terms'
}
XPath = partial(etree.XPath, namespaces=NAMESPACES)
total_results = XPath('//openSearch:totalResults')
start_index = XPath('//openSearch:startIndex')
items_per_page = XPath('//openSearch:itemsPerPage')
entry = XPath('//atom:entry')
entry_id = XPath('descendant::atom:id')
creator = XPath('descendant::dc:creator')
identifier = XPath('descendant::dc:identifier')
title = XPath('descendant::dc:title')
date = XPath('descendant::dc:date')
publisher = XPath('descendant::dc:publisher')
subject = XPath('descendant::dc:subject')
description = XPath('descendant::dc:description')
language = XPath('descendant::dc:language')
def report(verbose):
if verbose:
import traceback
traceback.print_exc()
class Query(object):
BASE_URL = 'http://books.google.com/books/feeds/volumes?'
def __init__(self, title=None, author=None, publisher=None, isbn=None,
max_results=20, min_viewability='none', start_index=1):
assert not(title is None and author is None and publisher is None and \
isbn is None)
assert (max_results < 21)
assert (min_viewability in ('none', 'partial', 'full'))
q = ''
if isbn is not None:
q += 'isbn:'+isbn
else:
def build_term(prefix, parts):
return ' '.join('in'+prefix + ':' + x for x in parts)
if title is not None:
q += build_term('title', title.split())
if author is not None:
q += ('+' if q else '')+build_term('author', author.split())
if publisher is not None:
q += ('+' if q else '')+build_term('publisher', publisher.split())
if isinstance(q, unicode):
q = q.encode('utf-8')
self.url = self.BASE_URL+urlencode({
'q':q,
'max-results':max_results,
'start-index':start_index,
'min-viewability':min_viewability,
})
def __call__(self, browser, verbose):
if verbose:
print 'Query:', self.url
feed = etree.fromstring(browser.open(self.url).read())
#print etree.tostring(feed, pretty_print=True)
total = int(total_results(feed)[0].text)
start = int(start_index(feed)[0].text)
entries = entry(feed)
new_start = start + len(entries)
if new_start > total:
new_start = 0
return entries, new_start
class ResultList(list):
def get_description(self, entry, verbose):
try:
desc = description(entry)
if desc:
return 'SUMMARY:\n'+desc[0].text
except:
report(verbose)
def get_language(self, entry, verbose):
try:
l = language(entry)
if l:
return l[0].text
except:
report(verbose)
def get_title(self, entry):
candidates = [x.text for x in title(entry)]
return ': '.join(candidates)
def get_authors(self, entry):
m = creator(entry)
if not m:
m = []
m = [x.text for x in m]
return m
def get_author_sort(self, entry, verbose):
for x in creator(entry):
for key, val in x.attrib.items():
if key.endswith('file-as'):
return val
def get_identifiers(self, entry, mi):
isbns = []
for x in identifier(entry):
t = str(x.text).strip()
if t[:5].upper() in ('ISBN:', 'LCCN:', 'OCLC:'):
if t[:5].upper() == 'ISBN:':
isbns.append(t[5:])
if isbns:
mi.isbn = sorted(isbns, cmp=lambda x,y:cmp(len(x), len(y)))[-1]
def get_tags(self, entry, verbose):
try:
btags = [x.text for x in subject(entry)]
tags = []
for t in btags:
tags.extend([y.strip() for y in t.split('/')])
tags = list(sorted(list(set(tags))))
except:
report(verbose)
tags = []
return [x.replace(',', ';') for x in tags]
def get_publisher(self, entry, verbose):
try:
pub = publisher(entry)[0].text
except:
pub = None
return pub
def get_date(self, entry, verbose):
try:
d = date(entry)
if d:
default = utcnow().replace(day=15)
d = parse_date(d[0].text, assume_utc=True, default=default)
else:
d = None
except:
report(verbose)
d = None
return d
def populate(self, entries, browser, verbose=False):
for x in entries:
try:
id_url = entry_id(x)[0].text
title = self.get_title(x)
except:
report(verbose)
mi = MetaInformation(title, self.get_authors(x))
try:
raw = browser.open(id_url).read()
feed = etree.fromstring(raw)
x = entry(feed)[0]
except Exception as e:
if verbose:
print 'Failed to get all details for an entry'
print e
mi.author_sort = self.get_author_sort(x, verbose)
mi.comments = self.get_description(x, verbose)
self.get_identifiers(x, mi)
mi.tags = self.get_tags(x, verbose)
mi.publisher = self.get_publisher(x, verbose)
mi.pubdate = self.get_date(x, verbose)
mi.language = self.get_language(x, verbose)
self.append(mi)
def search(title=None, author=None, publisher=None, isbn=None,
min_viewability='none', verbose=False, max_results=40):
br = browser()
br.set_handle_gzip(True)
start, entries = 1, []
while start > 0 and len(entries) <= max_results:
new, start = Query(title=title, author=author, publisher=publisher,
isbn=isbn, min_viewability=min_viewability)(br, verbose)
if not new:
break
entries.extend(new)
entries = entries[:max_results]
ans = ResultList()
ans.populate(entries, br, verbose)
return ans
def option_parser():
parser = OptionParser(textwrap.dedent(
'''\
%prog [options]
Fetch book metadata from Google. You must specify one of title, author,
publisher or ISBN. If you specify ISBN the others are ignored. Will
fetch a maximum of 100 matches, so you should make your query as
specific as possible.
'''
))
parser.add_option('-t', '--title', help='Book title')
parser.add_option('-a', '--author', help='Book author(s)')
parser.add_option('-p', '--publisher', help='Book publisher')
parser.add_option('-i', '--isbn', help='Book ISBN')
parser.add_option('-m', '--max-results', default=10,
help='Maximum number of results to fetch')
parser.add_option('-v', '--verbose', default=0, action='count',
help='Be more verbose about errors')
return parser
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
try:
results = search(opts.title, opts.author, opts.publisher, opts.isbn,
verbose=opts.verbose, max_results=opts.max_results)
except AssertionError:
report(True)
parser.print_help()
return 1
for result in results:
print unicode(result).encode(preferred_encoding)
print
if __name__ == '__main__':
sys.exit(main())

159
src/calibre/ebooks/metadata/isbndb.py
View File

@ -1,159 +0,0 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
'''
Interface to isbndb.com. My key HLLXQX2A.
'''
import sys, re
from urllib import quote
from calibre.utils.config import OptionParser
from calibre.ebooks.metadata.book.base import Metadata
from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup
from calibre import browser
BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%(key)s&page_number=1&results=subjects,authors,texts&'
class ISBNDBError(Exception):
pass
def fetch_metadata(url, max=3, timeout=5.):
books = []
page_number = 1
total_results = 31
br = browser()
while len(books) < total_results and max > 0:
try:
raw = br.open(url, timeout=timeout).read()
except Exception as err:
raise ISBNDBError('Could not fetch ISBNDB metadata. Error: '+str(err))
soup = BeautifulStoneSoup(raw,
convertEntities=BeautifulStoneSoup.XML_ENTITIES)
book_list = soup.find('booklist')
if book_list is None:
errmsg = soup.find('errormessage').string
raise ISBNDBError('Error fetching metadata: '+errmsg)
total_results = int(book_list['total_results'])
page_number += 1
np = '&page_number=%s&'%page_number
url = re.sub(r'\&page_number=\d+\&', np, url)
books.extend(book_list.findAll('bookdata'))
max -= 1
return books
class ISBNDBMetadata(Metadata):
def __init__(self, book):
Metadata.__init__(self, None)
def tostring(e):
if not hasattr(e, 'string'):
return None
ans = e.string
if ans is not None:
ans = unicode(ans).strip()
if not ans:
ans = None
return ans
self.isbn = unicode(book.get('isbn13', book.get('isbn')))
title = tostring(book.find('titlelong'))
if not title:
title = tostring(book.find('title'))
self.title = title
self.title = unicode(self.title).strip()
authors = []
au = tostring(book.find('authorstext'))
if au:
au = au.strip()
temp = au.split(',')
for au in temp:
if not au: continue
authors.extend([a.strip() for a in au.split('&amp;')])
if authors:
self.authors = authors
try:
self.author_sort = tostring(book.find('authors').find('person'))
if self.authors and self.author_sort == self.authors[0]:
self.author_sort = None
except:
pass
self.publisher = tostring(book.find('publishertext'))
summ = tostring(book.find('summary'))
if summ:
self.comments = 'SUMMARY:\n'+summ
def build_isbn(base_url, opts):
return base_url + 'index1=isbn&value1='+opts.isbn
def build_combined(base_url, opts):
query = ' '.join([e for e in (opts.title, opts.author, opts.publisher) \
if e is not None ])
query = query.strip()
if len(query) == 0:
raise ISBNDBError('You must specify at least one of --author, --title or --publisher')
query = re.sub(r'\s+', '+', query)
if isinstance(query, unicode):
query = query.encode('utf-8')
return base_url+'index1=combined&value1='+quote(query, '+')
def option_parser():
parser = OptionParser(usage=\
_('''
%prog [options] key
Fetch metadata for books from isndb.com. You can specify either the
books ISBN ID or its title and author. If you specify the title and author,
then more than one book may be returned.
key is the account key you generate after signing up for a free account from isbndb.com.
'''))
parser.add_option('-i', '--isbn', default=None, dest='isbn',
help=_('The ISBN ID of the book you want metadata for.'))
parser.add_option('-a', '--author', dest='author',
default=None, help=_('The author whose book to search for.'))
parser.add_option('-t', '--title', dest='title',
default=None, help=_('The title of the book to search for.'))
parser.add_option('-p', '--publisher', default=None, dest='publisher',
help=_('The publisher of the book to search for.'))
parser.add_option('-v', '--verbose', default=False,
action='store_true', help=_('Verbose processing'))
return parser
def create_books(opts, args, timeout=5.):
base_url = BASE_URL%dict(key=args[1])
if opts.isbn is not None:
url = build_isbn(base_url, opts)
else:
url = build_combined(base_url, opts)
if opts.verbose:
print ('ISBNDB query: '+url)
tans = [ISBNDBMetadata(book) for book in fetch_metadata(url, timeout=timeout)]
#remove duplicates ISBN
return list(dict((book.isbn, book) for book in tans).values())
def main(args=sys.argv):
parser = option_parser()
opts, args = parser.parse_args(args)
if len(args) != 2:
parser.print_help()
print ('You must supply the isbndb.com key')
return 1
for book in create_books(opts, args):
print unicode(book).encode('utf-8')
return 0
if __name__ == '__main__':
sys.exit(main())

411
src/calibre/ebooks/metadata/nicebooks.py
View File

@ -1,411 +0,0 @@
from __future__ import with_statement
__license__ = 'GPL 3'
__copyright__ = '2010, sengian <sengian1@gmail.com>'
__docformat__ = 'restructuredtext en'
import sys, textwrap, re, traceback, socket
from urllib import urlencode
from math import ceil
from copy import deepcopy
from lxml.html import soupparser
from calibre.utils.date import parse_date, utcnow, replace_months
from calibre.utils.cleantext import clean_ascii_chars
from calibre import browser, preferred_encoding
from calibre.ebooks.chardet import xml_to_unicode
from calibre.ebooks.metadata import MetaInformation, check_isbn, \
authors_to_sort_string
from calibre.ebooks.metadata.fetch import MetadataSource
from calibre.ebooks.metadata.covers import CoverDownload
from calibre.utils.config import OptionParser
class NiceBooks(MetadataSource):
name = 'Nicebooks'
description = _('Downloads metadata from french Nicebooks')
supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian'
version = (1, 0, 0)
def fetch(self):
try:
self.results = search(self.title, self.book_author, self.publisher,
self.isbn, max_results=10, verbose=self.verbose)
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
class NiceBooksCovers(CoverDownload):
name = 'Nicebooks covers'
description = _('Downloads covers from french Nicebooks')
supported_platforms = ['windows', 'osx', 'linux']
author = 'Sengian'
type = _('Cover download')
version = (1, 0, 0)
def has_cover(self, mi, ans, timeout=5.):
if not mi.isbn:
return False
br = browser()
try:
entry = Query(isbn=mi.isbn, max_results=1)(br, False, timeout)[0]
if Covers(mi.isbn)(entry).check_cover():
self.debug('cover for', mi.isbn, 'found')
ans.set()
except Exception as e:
self.debug(e)
def get_covers(self, mi, result_queue, abort, timeout=5.):
if not mi.isbn:
return
br = browser()
try:
entry = Query(isbn=mi.isbn, max_results=1)(br, False, timeout)[0]
cover_data, ext = Covers(mi.isbn)(entry).get_cover(br, timeout)
if not ext:
ext = 'jpg'
result_queue.put((True, cover_data, ext, self.name))
except Exception as e:
result_queue.put((False, self.exception_to_string(e),
traceback.format_exc(), self.name))
class NiceBooksError(Exception):
pass
class ISBNNotFound(NiceBooksError):
pass
def report(verbose):
if verbose:
traceback.print_exc()
class Query(object):
BASE_URL = 'http://fr.nicebooks.com/'
def __init__(self, title=None, author=None, publisher=None, isbn=None, keywords=None, max_results=20):
assert not(title is None and author is None and publisher is None \
and isbn is None and keywords is None)
assert (max_results < 21)
self.max_results = int(max_results)
if isbn is not None:
q = isbn
else:
q = ' '.join([i for i in (title, author, publisher, keywords) \
if i is not None])
if isinstance(q, unicode):
q = q.encode('utf-8')
self.urldata = 'search?' + urlencode({'q':q,'s':'Rechercher'})
def __call__(self, browser, verbose, timeout = 5.):
if verbose:
print _('Query: %s') % self.BASE_URL+self.urldata
try:
raw = browser.open_novisit(self.BASE_URL+self.urldata, timeout=timeout).read()
except Exception as e:
report(verbose)
if callable(getattr(e, 'getcode', None)) and \
e.getcode() == 404:
return
if isinstance(getattr(e, 'args', [None])[0], socket.timeout):
raise NiceBooksError(_('Nicebooks timed out. Try again later.'))
raise NiceBooksError(_('Nicebooks encountered an error.'))
if '<title>404 - ' in raw:
return
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
feed = soupparser.fromstring(raw)
except:
try:
#remove ASCII invalid chars
feed = soupparser.fromstring(clean_ascii_chars(raw))
except:
return None
#nb of page to call
try:
nbresults = int(feed.xpath("//div[@id='topbar']/b")[0].text)
except:
#direct hit
return [feed]
nbpagetoquery = int(ceil(float(min(nbresults, self.max_results))/10))
pages =[feed]
if nbpagetoquery > 1:
for i in xrange(2, nbpagetoquery + 1):
try:
urldata = self.urldata + '&p=' + str(i)
raw = browser.open_novisit(self.BASE_URL+urldata, timeout=timeout).read()
except Exception as e:
continue
if '<title>404 - ' in raw:
continue
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
feed = soupparser.fromstring(raw)
except:
try:
#remove ASCII invalid chars
feed = soupparser.fromstring(clean_ascii_chars(raw))
except:
continue
pages.append(feed)
results = []
for x in pages:
results.extend([i.find_class('title')[0].get('href') \
for i in x.xpath("//ul[@id='results']/li")])
return results[:self.max_results]
class ResultList(list):
BASE_URL = 'http://fr.nicebooks.com'
def __init__(self):
self.repub = re.compile(u'\s*.diteur\s*', re.I)
self.reauteur = re.compile(u'\s*auteur.*', re.I)
self.reautclean = re.compile(u'\s*\(.*\)\s*')
def get_title(self, entry):
title = deepcopy(entry)
title.remove(title.find("dl[@title='Informations sur le livre']"))
title = ' '.join([i.text_content() for i in title.iterchildren()])
return unicode(title.replace('\n', ''))
def get_authors(self, entry):
author = entry.find("dl[@title='Informations sur le livre']")
authortext = []
for x in author.getiterator('dt'):
if self.reauteur.match(x.text):
elt = x.getnext()
while elt.tag == 'dd':
authortext.append(unicode(elt.text_content()))
elt = elt.getnext()
break
if len(authortext) == 1:
authortext = [self.reautclean.sub('', authortext[0])]
return authortext
def get_description(self, entry, verbose):
try:
return u'RESUME:\n' + unicode(entry.getparent().xpath("//p[@id='book-description']")[0].text)
except:
report(verbose)
return None
def get_book_info(self, entry, mi, verbose):
entry = entry.find("dl[@title='Informations sur le livre']")
for x in entry.getiterator('dt'):
if x.text == 'ISBN':
isbntext = x.getnext().text_content().replace('-', '')
if check_isbn(isbntext):
mi.isbn = unicode(isbntext)
elif self.repub.match(x.text):
mi.publisher = unicode(x.getnext().text_content())
elif x.text == 'Langue':
mi.language = unicode(x.getnext().text_content())
elif x.text == 'Date de parution':
d = x.getnext().text_content()
try:
default = utcnow().replace(day=15)
d = replace_months(d, 'fr')
d = parse_date(d, assume_utc=True, default=default)
mi.pubdate = d
except:
report(verbose)
return mi
def fill_MI(self, entry, title, authors, verbose):
mi = MetaInformation(title, authors)
mi.author_sort = authors_to_sort_string(authors)
mi.comments = self.get_description(entry, verbose)
return self.get_book_info(entry, mi, verbose)
def get_individual_metadata(self, browser, linkdata, verbose):
try:
raw = browser.open_novisit(self.BASE_URL + linkdata).read()
except Exception as e:
report(verbose)
if callable(getattr(e, 'getcode', None)) and \
e.getcode() == 404:
return
if isinstance(getattr(e, 'args', [None])[0], socket.timeout):
raise NiceBooksError(_('Nicebooks timed out. Try again later.'))
raise NiceBooksError(_('Nicebooks encountered an error.'))
if '<title>404 - ' in raw:
report(verbose)
return
raw = xml_to_unicode(raw, strip_encoding_pats=True,
resolve_entities=True)[0]
try:
feed = soupparser.fromstring(raw)
except:
try:
#remove ASCII invalid chars
feed = soupparser.fromstring(clean_ascii_chars(raw))
except:
return None
# get results
return feed.xpath("//div[@id='container']")[0]
def populate(self, entries, browser, verbose=False):
#single entry
if len(entries) == 1 and not isinstance(entries[0], str):
try:
entry = entries[0].xpath("//div[@id='container']")[0]
entry = entry.find("div[@id='book-info']")
title = self.get_title(entry)
authors = self.get_authors(entry)
except Exception as e:
if verbose:
print 'Failed to get all details for an entry'
print e
return
self.append(self.fill_MI(entry, title, authors, verbose))
else:
#multiple entries
for x in entries:
try:
entry = self.get_individual_metadata(browser, x, verbose)
entry = entry.find("div[@id='book-info']")
title = self.get_title(entry)
authors = self.get_authors(entry)
except Exception as e:
if verbose:
print 'Failed to get all details for an entry'
print e
continue
self.append(self.fill_MI(entry, title, authors, verbose))
class Covers(object):
def __init__(self, isbn = None):
assert isbn is not None
self.urlimg = ''
self.isbn = isbn
self.isbnf = False
def __call__(self, entry = None):
try:
self.urlimg = entry.xpath("//div[@id='book-picture']/a")[0].get('href')
except:
return self
isbno = entry.get_element_by_id('book-info').find("dl[@title='Informations sur le livre']")
for x in isbno.getiterator('dt'):
if x.text == 'ISBN' and check_isbn(x.getnext().text_content()):
self.isbnf = True
break
return self
def check_cover(self):
return True if self.urlimg else False
def get_cover(self, browser, timeout = 5.):
try:
cover, ext = browser.open_novisit(self.urlimg, timeout=timeout).read(), \
self.urlimg.rpartition('.')[-1]
return cover, ext if ext else 'jpg'
except Exception as err:
if isinstance(getattr(err, 'args', [None])[0], socket.timeout):
raise NiceBooksError(_('Nicebooks timed out. Try again later.'))
if not len(self.urlimg):
if not self.isbnf:
raise ISBNNotFound(_('ISBN: %s not found.') % self.isbn)
raise NiceBooksError(_('An errror occured with Nicebooks cover fetcher'))
def search(title=None, author=None, publisher=None, isbn=None,
max_results=5, verbose=False, keywords=None):
br = browser()
entries = Query(title=title, author=author, isbn=isbn, publisher=publisher,
keywords=keywords, max_results=max_results)(br, verbose,timeout = 10.)
if entries is None or len(entries) == 0:
return None
#List of entry
ans = ResultList()
ans.populate(entries, br, verbose)
return ans
def check_for_cover(isbn):
br = browser()
entry = Query(isbn=isbn, max_results=1)(br, False)[0]
return Covers(isbn)(entry).check_cover()
def cover_from_isbn(isbn, timeout = 5.):
br = browser()
entry = Query(isbn=isbn, max_results=1)(br, False, timeout)[0]
return Covers(isbn)(entry).get_cover(br, timeout)
def option_parser():
parser = OptionParser(textwrap.dedent(\
_('''\
%prog [options]
Fetch book metadata from Nicebooks. You must specify one of title, author,
ISBN, publisher or keywords. Will fetch a maximum of 20 matches,
so you should make your query as specific as possible.
It can also get covers if the option is activated.
''')
))
parser.add_option('-t', '--title', help=_('Book title'))
parser.add_option('-a', '--author', help=_('Book author(s)'))
parser.add_option('-p', '--publisher', help=_('Book publisher'))
parser.add_option('-i', '--isbn', help=_('Book ISBN'))
parser.add_option('-k', '--keywords', help=_('Keywords'))
parser.add_option('-c', '--covers', default=0,
help=_('Covers: 1-Check/ 2-Download'))
parser.add_option('-p', '--coverspath', default='',
help=_('Covers files path'))
parser.add_option('-m', '--max-results', default=20,
help=_('Maximum number of results to fetch'))
parser.add_option('-v', '--verbose', default=0, action='count',
help=_('Be more verbose about errors'))
return parser
def main(args=sys.argv):
import os
parser = option_parser()
opts, args = parser.parse_args(args)
try:
results = search(opts.title, opts.author, isbn=opts.isbn, publisher=opts.publisher,
keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results)
except AssertionError:
report(True)
parser.print_help()
return 1
if results is None or len(results) == 0:
print _('No result found for this search!')
return 0
for result in results:
print unicode(result).encode(preferred_encoding, 'replace')
covact = int(opts.covers)
if covact == 1:
textcover = _('No cover found!')
if check_for_cover(result.isbn):
textcover = _('A cover was found for this book')
print textcover
elif covact == 2:
cover_data, ext = cover_from_isbn(result.isbn)
cpath = result.isbn
if len(opts.coverspath):
cpath = os.path.normpath(opts.coverspath + '/' + result.isbn)
oname = os.path.abspath(cpath+'.'+ext)
open(oname, 'wb').write(cover_data)
print _('Cover saved to file '), oname
print
if __name__ == '__main__':
sys.exit(main())

2
src/calibre/ebooks/metadata/sources/base.py
View File

@ -307,7 +307,7 @@ class Source(Plugin):
title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in
[
# Remove things like: (2010) (Omnibus) etc.
(r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|mass\s*market|edition|ed\.)[\])}]', ''),
(r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|turtleback|mass\s*market|edition|ed\.)[\])}]', ''),
# Remove any strings that contain the substring edition inside
# parentheses
(r'(?i)[({\[].*?(edition|ed.).*?[\]})]', ''),

8
src/calibre/ebooks/metadata/sources/cli.py
View File

@ -19,13 +19,8 @@ from calibre.ebooks.metadata.opf2 import metadata_to_opf
from calibre.ebooks.metadata.sources.base import create_log
from calibre.ebooks.metadata.sources.identify import identify
from calibre.ebooks.metadata.sources.covers import download_cover
from calibre.utils.config import test_eight_code
def option_parser():
if not test_eight_code:
from calibre.ebooks.metadata.fetch import option_parser
return option_parser()
parser = OptionParser(textwrap.dedent(
'''\
%prog [options]
@ -48,9 +43,6 @@ def option_parser():
return parser
def main(args=sys.argv):
if not test_eight_code:
from calibre.ebooks.metadata.fetch import main
return main(args)
parser = option_parser()
opts, args = parser.parse_args(args)

34
src/calibre/gui2/actions/add.py
View File

@ -20,9 +20,8 @@ from calibre.ebooks import BOOK_EXTENSIONS
from calibre.utils.filenames import ascii_filename
from calibre.constants import preferred_encoding, filesystem_encoding
from calibre.gui2.actions import InterfaceAction
from calibre.gui2 import config, question_dialog
from calibre.gui2 import question_dialog
from calibre.ebooks.metadata import MetaInformation
from calibre.utils.config import test_eight_code
from calibre.ebooks.metadata.sources.base import msprefs
def get_filters():
@ -180,26 +179,17 @@ class AddAction(InterfaceAction):
except IndexError:
self.gui.library_view.model().books_added(self.isbn_add_dialog.value)
self.isbn_add_dialog.accept()
if test_eight_code:
orig = msprefs['ignore_fields']
new = list(orig)
for x in ('title', 'authors'):
if x in new:
new.remove(x)
msprefs['ignore_fields'] = new
try:
self.gui.iactions['Edit Metadata'].download_metadata(
ids=self.add_by_isbn_ids)
finally:
msprefs['ignore_fields'] = orig
else:
orig = config['overwrite_author_title_metadata']
config['overwrite_author_title_metadata'] = True
try:
self.gui.iactions['Edit Metadata'].do_download_metadata(
self.add_by_isbn_ids)
finally:
config['overwrite_author_title_metadata'] = orig
orig = msprefs['ignore_fields']
new = list(orig)
for x in ('title', 'authors'):
if x in new:
new.remove(x)
msprefs['ignore_fields'] = new
try:
self.gui.iactions['Edit Metadata'].download_metadata(
ids=self.add_by_isbn_ids)
finally:
msprefs['ignore_fields'] = orig
return

104
src/calibre/gui2/actions/edit_metadata.py
View File

@ -10,15 +10,13 @@ from functools import partial
from PyQt4.Qt import Qt, QMenu, QModelIndex, QTimer
from calibre.gui2 import error_dialog, config, Dispatcher, question_dialog
from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog
from calibre.gui2 import error_dialog, Dispatcher, question_dialog
from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog
from calibre.gui2.dialogs.confirm_delete import confirm
from calibre.gui2.dialogs.tag_list_editor import TagListEditor
from calibre.gui2.actions import InterfaceAction
from calibre.ebooks.metadata import authors_to_string
from calibre.utils.icu import sort_key
from calibre.utils.config import test_eight_code
class EditMetadataAction(InterfaceAction):
@ -36,22 +34,8 @@ class EditMetadataAction(InterfaceAction):
md.addAction(_('Edit metadata in bulk'),
partial(self.edit_metadata, False, bulk=True))
md.addSeparator()
if test_eight_code:
dall = self.download_metadata
else:
dall = partial(self.download_metadata_old, False, covers=True)
dident = partial(self.download_metadata_old, False, covers=False)
dcovers = partial(self.download_metadata_old, False, covers=True,
set_metadata=False, set_social_metadata=False)
md.addAction(_('Download metadata and covers'), dall,
md.addAction(_('Download metadata and covers'), self.download_metadata,
Qt.ControlModifier+Qt.Key_D)
if not test_eight_code:
md.addAction(_('Download only metadata'), dident)
md.addAction(_('Download only covers'), dcovers)
md.addAction(_('Download only social metadata'),
partial(self.download_metadata_old, False, covers=False,
set_metadata=False, set_social_metadata=True))
self.metadata_menu = md
mb = QMenu()
@ -88,7 +72,7 @@ class EditMetadataAction(InterfaceAction):
_('No books selected'), show=True)
db = self.gui.library_view.model().db
ids = [db.id(row.row()) for row in rows]
from calibre.gui2.metadata.bulk_download2 import start_download
from calibre.gui2.metadata.bulk_download import start_download
start_download(self.gui, ids,
Dispatcher(self.metadata_downloaded))
@ -96,7 +80,7 @@ class EditMetadataAction(InterfaceAction):
if job.failed:
self.gui.job_exception(job, dialog_title=_('Failed to download metadata'))
return
from calibre.gui2.metadata.bulk_download2 import get_job_details
from calibre.gui2.metadata.bulk_download import get_job_details
id_map, failed_ids, failed_covers, all_failed, det_msg = \
get_job_details(job)
if all_failed:
@ -112,8 +96,9 @@ class EditMetadataAction(InterfaceAction):
show_copy_button = False
if failed_ids or failed_covers:
show_copy_button = True
num = len(failed_ids.union(failed_covers))
msg += '<p>'+_('Could not download metadata and/or covers for %d of the books. Click'
' "Show details" to see which books.')%len(failed_ids)
' "Show details" to see which books.')%num
payload = (id_map, failed_ids, failed_covers)
from calibre.gui2.dialogs.message_box import ProceedNotification
@ -158,49 +143,6 @@ class EditMetadataAction(InterfaceAction):
self.apply_metadata_changes(id_map)
def download_metadata_old(self, checked, covers=True, set_metadata=True,
set_social_metadata=None):
rows = self.gui.library_view.selectionModel().selectedRows()
if not rows or len(rows) == 0:
d = error_dialog(self.gui, _('Cannot download metadata'),
_('No books selected'))
d.exec_()
return
db = self.gui.library_view.model().db
ids = [db.id(row.row()) for row in rows]
self.do_download_metadata(ids, covers=covers,
set_metadata=set_metadata,
set_social_metadata=set_social_metadata)
def do_download_metadata(self, ids, covers=True, set_metadata=True,
set_social_metadata=None):
m = self.gui.library_view.model()
db = m.db
if set_social_metadata is None:
get_social_metadata = config['get_social_metadata']
else:
get_social_metadata = set_social_metadata
from calibre.gui2.metadata.bulk_download import DoDownload
if set_social_metadata is not None and set_social_metadata:
x = _('social metadata')
else:
x = _('covers') if covers and not set_metadata else _('metadata')
title = _('Downloading {0} for {1} book(s)').format(x, len(ids))
self._download_book_metadata = DoDownload(self.gui, title, db, ids,
get_covers=covers, set_metadata=set_metadata,
get_social_metadata=get_social_metadata)
m.stop_metadata_backup()
try:
self._download_book_metadata.exec_()
finally:
m.start_metadata_backup()
cr = self.gui.library_view.currentIndex().row()
x = self._download_book_metadata
if x.updated:
self.gui.library_view.model().refresh_ids(
x.updated, cr)
if self.gui.cover_flow:
self.gui.cover_flow.dataChanged()
# }}}
def edit_metadata(self, checked, bulk=None):
@ -227,9 +169,7 @@ class EditMetadataAction(InterfaceAction):
list(range(self.gui.library_view.model().rowCount(QModelIndex())))
current_row = row_list.index(cr)
func = (self.do_edit_metadata if test_eight_code else
self.do_edit_metadata_old)
changed, rows_to_refresh = func(row_list, current_row)
changed, rows_to_refresh = self.do_edit_metadata(row_list, current_row)
m = self.gui.library_view.model()
@ -244,36 +184,6 @@ class EditMetadataAction(InterfaceAction):
m.current_changed(current, previous)
self.gui.tags_view.recount()
def do_edit_metadata_old(self, row_list, current_row):
changed = set([])
db = self.gui.library_view.model().db
while True:
prev = next_ = None
if current_row > 0:
prev = db.title(row_list[current_row-1])
if current_row < len(row_list) - 1:
next_ = db.title(row_list[current_row+1])
d = MetadataSingleDialog(self.gui, row_list[current_row], db,
prev=prev, next_=next_)
d.view_format.connect(lambda
fmt:self.gui.iactions['View'].view_format(row_list[current_row],
fmt))
ret = d.exec_()
d.break_cycles()
if ret != d.Accepted:
break
changed.add(d.id)
self.gui.library_view.model().refresh_ids(list(d.books_to_refresh))
if d.row_delta == 0:
break
current_row += d.row_delta
self.gui.library_view.set_current_row(current_row)
self.gui.library_view.scroll_to_row(current_row)
return changed, set()
def do_edit_metadata(self, row_list, current_row):
from calibre.gui2.metadata.single import edit_metadata
db = self.gui.library_view.model().db

271
src/calibre/gui2/dialogs/fetch_metadata.py
View File

@ -1,271 +0,0 @@
__license__ = 'GPL v3'
__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>'
'''
GUI for fetching metadata from servers.
'''
import time
from threading import Thread
from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, pyqtSignal, \
QAbstractTableModel, QCoreApplication, QTimer
from PyQt4.QtGui import QDialog, QItemSelectionModel, QIcon
from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata
from calibre.gui2 import error_dialog, NONE, info_dialog, config
from calibre.gui2.widgets import ProgressIndicator
from calibre import strftime, force_unicode
from calibre.customize.ui import get_isbndb_key, set_isbndb_key
from calibre.utils.icu import sort_key
_hung_fetchers = set([])
class Fetcher(Thread):
def __init__(self, title, author, publisher, isbn, key):
Thread.__init__(self)
self.daemon = True
self.title = title
self.author = author
self.publisher = publisher
self.isbn = isbn
self.key = key
self.results, self.exceptions = [], []
def run(self):
from calibre.ebooks.metadata.fetch import search
self.results, self.exceptions = search(self.title, self.author,
self.publisher, self.isbn,
self.key if self.key else None)
class Matches(QAbstractTableModel):
def __init__(self, matches):
self.matches = matches
self.yes_icon = QVariant(QIcon(I('ok.png')))
QAbstractTableModel.__init__(self)
def rowCount(self, *args):
return len(self.matches)
def columnCount(self, *args):
return 8
def headerData(self, section, orientation, role):
if role != Qt.DisplayRole:
return NONE
text = ""
if orientation == Qt.Horizontal:
if section == 0: text = _("Title")
elif section == 1: text = _("Author(s)")
elif section == 2: text = _("Author Sort")
elif section == 3: text = _("Publisher")
elif section == 4: text = _("ISBN")
elif section == 5: text = _("Published")
elif section == 6: text = _("Has Cover")
elif section == 7: text = _("Has Summary")
return QVariant(text)
else:
return QVariant(section+1)
def summary(self, row):
return self.matches[row].comments
def data_as_text(self, book, col):
if col == 0 and book.title is not None:
return book.title
elif col == 1:
return ', '.join(book.authors)
elif col == 2 and book.author_sort is not None:
return book.author_sort
elif col == 3 and book.publisher is not None:
return book.publisher
elif col == 4 and book.isbn is not None:
return book.isbn
elif col == 5 and hasattr(book.pubdate, 'timetuple'):
return strftime('%b %Y', book.pubdate.timetuple())
elif col == 6 and book.has_cover:
return 'y'
elif col == 7 and book.comments:
return 'y'
return ''
def data(self, index, role):
row, col = index.row(), index.column()
book = self.matches[row]
if role == Qt.DisplayRole:
res = self.data_as_text(book, col)
if col <= 5 and res:
return QVariant(res)
return NONE
elif role == Qt.DecorationRole:
if col == 6 and book.has_cover:
return self.yes_icon
if col == 7 and book.comments:
return self.yes_icon
return NONE
def sort(self, col, order, reset=True):
if not self.matches:
return
descending = order == Qt.DescendingOrder
self.matches.sort(None,
lambda x: sort_key(unicode(force_unicode(self.data_as_text(x, col)))),
descending)
if reset:
self.reset()
class FetchMetadata(QDialog, Ui_FetchMetadata):
HANG_TIME = 75 #seconds
queue_reject = pyqtSignal()
def __init__(self, parent, isbn, title, author, publisher, timeout):
QDialog.__init__(self, parent)
Ui_FetchMetadata.__init__(self)
self.setupUi(self)
for fetcher in list(_hung_fetchers):
if not fetcher.is_alive():
_hung_fetchers.remove(fetcher)
self.pi = ProgressIndicator(self)
self.timeout = timeout
QObject.connect(self.fetch, SIGNAL('clicked()'), self.fetch_metadata)
self.queue_reject.connect(self.reject, Qt.QueuedConnection)
isbndb_key = get_isbndb_key()
if not isbndb_key:
isbndb_key = ''
self.key.setText(isbndb_key)
self.setWindowTitle(title if title else _('Unknown'))
self.isbn = isbn
self.title = title
self.author = author.strip()
self.publisher = publisher
self.previous_row = None
self.warning.setVisible(False)
self.connect(self.matches, SIGNAL('activated(QModelIndex)'), self.chosen)
self.connect(self.matches, SIGNAL('entered(QModelIndex)'),
self.show_summary)
self.matches.setMouseTracking(True)
# Enabling sorting and setting a sort column will not change the initial
# order of the results, as they are filled in later
self.matches.setSortingEnabled(True)
self.matches.horizontalHeader().sectionClicked.connect(self.show_sort_indicator)
self.matches.horizontalHeader().setSortIndicatorShown(False)
self.fetch_metadata()
self.opt_get_social_metadata.setChecked(config['get_social_metadata'])
self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata'])
self.opt_auto_download_cover.setChecked(config['auto_download_cover'])
def show_summary(self, current, *args):
row = current.row()
if row != self.previous_row:
summ = self.model.summary(row)
self.summary.setText(summ if summ else '')
self.previous_row = row
def fetch_metadata(self):
self.warning.setVisible(False)
key = str(self.key.text())
if key:
set_isbndb_key(key)
else:
key = None
title = author = publisher = isbn = None
if self.isbn:
isbn = self.isbn
if self.title:
title = self.title
if self.author and not self.author == _('Unknown'):
author = self.author
self.fetch.setEnabled(False)
self.setCursor(Qt.WaitCursor)
QCoreApplication.instance().processEvents()
self.fetcher = Fetcher(title, author, publisher, isbn, key)
self.fetcher.start()
self.pi.start(_('Finding metadata...'))
self._hangcheck = QTimer(self)
self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck,
Qt.QueuedConnection)
self.start_time = time.time()
self._hangcheck.start(100)
def hangcheck(self):
if self.fetcher.is_alive() and \
time.time() - self.start_time < self.HANG_TIME:
return
self._hangcheck.stop()
try:
if self.fetcher.is_alive():
error_dialog(self, _('Could not find metadata'),
_('The metadata download seems to have stalled. '
'Try again later.')).exec_()
self.terminate()
return self.queue_reject.emit()
self.model = Matches(self.fetcher.results)
warnings = [(x[0], force_unicode(x[1])) for x in \
self.fetcher.exceptions if x[1] is not None]
if warnings:
warnings='<br>'.join(['<b>%s</b>: %s'%(name, exc) for name,exc in warnings])
self.warning.setText('<p><b>'+ _('Warning')+':</b>'+\
_('Could not fetch metadata from:')+\
'<br>'+warnings+'</p>')
self.warning.setVisible(True)
if self.model.rowCount() < 1:
info_dialog(self, _('No metadata found'),
_('No metadata found, try adjusting the title and author '
'and/or removing the ISBN.')).exec_()
self.reject()
return
self.matches.setModel(self.model)
QObject.connect(self.matches.selectionModel(),
SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'),
self.show_summary)
self.model.reset()
self.matches.selectionModel().select(self.model.index(0, 0),
QItemSelectionModel.Select | QItemSelectionModel.Rows)
self.matches.setCurrentIndex(self.model.index(0, 0))
finally:
self.fetch.setEnabled(True)
self.unsetCursor()
self.matches.resizeColumnsToContents()
self.pi.stop()
def terminate(self):
if hasattr(self, 'fetcher') and self.fetcher.is_alive():
_hung_fetchers.add(self.fetcher)
if hasattr(self, '_hangcheck') and self._hangcheck.isActive():
self._hangcheck.stop()
# Save value of auto_download_cover, since this is the only place it can
# be set. The values of the other options can be set in
# Preferences->Behavior and should not be set here as they affect bulk
# downloading as well.
if self.opt_auto_download_cover.isChecked() != config['auto_download_cover']:
config.set('auto_download_cover', self.opt_auto_download_cover.isChecked())
def __enter__(self, *args):
return self
def __exit__(self, *args):
self.terminate()
def selected_book(self):
try:
return self.matches.model().matches[self.matches.currentIndex().row()]
except:
return None
def chosen(self, index):
self.matches.setCurrentIndex(index)
self.accept()
def show_sort_indicator(self, *args):
self.matches.horizontalHeader().setSortIndicatorShown(True)

179
src/calibre/gui2/dialogs/fetch_metadata.ui
View File

@ -1,179 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>FetchMetadata</class>
<widget class="QDialog" name="FetchMetadata">
<property name="windowModality">
<enum>Qt::WindowModal</enum>
</property>
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>890</width>
<height>642</height>
</rect>
</property>
<property name="windowTitle">
<string>Fetch metadata</string>
</property>
<property name="windowIcon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/metadata.png</normaloff>:/images/metadata.png</iconset>
</property>
<layout class="QVBoxLayout">
<item>
<widget class="QLabel" name="tlabel">
<property name="text">
<string>&lt;p&gt;calibre can find metadata for your books from two locations: &lt;b&gt;Google Books&lt;/b&gt; and &lt;b&gt;isbndb.com&lt;/b&gt;. &lt;p&gt;To use isbndb.com you must sign up for a &lt;a href=&quot;http://www.isbndb.com&quot;&gt;free account&lt;/a&gt; and enter your access key below.</string>
</property>
<property name="alignment">
<set>Qt::AlignCenter</set>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
<property name="openExternalLinks">
<bool>true</bool>
</property>
</widget>
</item>
<item>
<layout class="QHBoxLayout">
<item>
<widget class="QLabel" name="label_2">
<property name="text">
<string>&amp;Access Key:</string>
</property>
<property name="buddy">
<cstring>key</cstring>
</property>
</widget>
</item>
<item>
<widget class="QLineEdit" name="key"/>
</item>
<item>
<widget class="QPushButton" name="fetch">
<property name="text">
<string>Fetch</string>
</property>
</widget>
</item>
</layout>
</item>
<item>
<widget class="QLabel" name="warning">
<property name="text">
<string/>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>
<item>
<widget class="QGroupBox" name="groupBox">
<property name="title">
<string>Matches</string>
</property>
<layout class="QVBoxLayout">
<item>
<widget class="QLabel" name="label_3">
<property name="text">
<string>Select the book that most closely matches your copy from the list below</string>
</property>
</widget>
</item>
<item>
<widget class="QTableView" name="matches">
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Expanding">
<horstretch>0</horstretch>
<verstretch>1</verstretch>
</sizepolicy>
</property>
<property name="alternatingRowColors">
<bool>true</bool>
</property>
<property name="selectionMode">
<enum>QAbstractItemView::SingleSelection</enum>
</property>
<property name="selectionBehavior">
<enum>QAbstractItemView::SelectRows</enum>
</property>
</widget>
</item>
<item>
<widget class="QTextBrowser" name="summary"/>
</item>
</layout>
</widget>
</item>
<item>
<widget class="QCheckBox" name="opt_overwrite_author_title_metadata">
<property name="text">
<string>Overwrite author and title with author and title of selected book</string>
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="opt_get_social_metadata">
<property name="text">
<string>Download &amp;social metadata (tags/rating/etc.) for the selected book</string>
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="opt_auto_download_cover">
<property name="text">
<string>Automatically download the cover, if available</string>
</property>
</widget>
</item>
<item>
<widget class="QDialogButtonBox" name="buttonBox">
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
</layout>
</widget>
<resources>
<include location="../../../../resources/images.qrc"/>
</resources>
<connections>
<connection>
<sender>buttonBox</sender>
<signal>accepted()</signal>
<receiver>FetchMetadata</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>460</x>
<y>599</y>
</hint>
<hint type="destinationlabel">
<x>657</x>
<y>530</y>
</hint>
</hints>
</connection>
<connection>
<sender>buttonBox</sender>
<signal>rejected()</signal>
<receiver>FetchMetadata</receiver>
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>417</x>
<y>599</y>
</hint>
<hint type="destinationlabel">
<x>0</x>
<y>491</y>
</hint>
</hints>
</connection>
</connections>
</ui>

1031
src/calibre/gui2/dialogs/metadata_single.py
View File

File diff suppressed because it is too large Load Diff

937
src/calibre/gui2/dialogs/metadata_single.ui
View File

@ -1,937 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>MetadataSingleDialog</class>
<widget class="QDialog" name="MetadataSingleDialog">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>994</width>
<height>716</height>
</rect>
</property>
<property name="sizePolicy">
<sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="windowTitle">
<string>Edit Meta Information</string>
</property>
<property name="windowIcon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset>
</property>
<property name="sizeGripEnabled">
<bool>true</bool>
</property>
<property name="modal">
<bool>true</bool>
</property>
<layout class="QVBoxLayout" name="verticalLayout_6">
<item>
<widget class="QScrollArea" name="scrollArea">
<property name="frameShape">
<enum>QFrame::NoFrame</enum>
</property>
<property name="widgetResizable">
<bool>true</bool>
</property>
<widget class="QWidget" name="scrollAreaWidgetContents">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>986</width>
<height>677</height>
</rect>
</property>
<layout class="QVBoxLayout" name="verticalLayout_5">
<property name="margin">
<number>0</number>
</property>
<item>
<widget class="QTabWidget" name="central_widget">
<property name="minimumSize">
<size>
<width>800</width>
<height>665</height>
</size>
</property>
<property name="currentIndex">
<number>0</number>
</property>
<widget class="QWidget" name="central_tabWidgetPage1">
<attribute name="title">
<string>&amp;Basic metadata</string>
</attribute>
<layout class="QGridLayout" name="gridLayout_5">
<item row="0" column="0">
<widget class="QSplitter" name="splitter">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<widget class="QWidget" name="layoutWidget">
<layout class="QVBoxLayout">
<item>
<widget class="QGroupBox" name="meta_box">
<property name="title">
<string>Meta information</string>
</property>
<layout class="QGridLayout" name="gridLayout_3">
<item row="0" column="0">
<widget class="QLabel" name="label">
<property name="text">
<string>&amp;Title: </string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>title</cstring>
</property>
</widget>
</item>
<item row="0" column="1">
<widget class="EnLineEdit" name="title">
<property name="toolTip">
<string>Change the title of this book</string>
</property>
</widget>
</item>
<item row="0" column="2" rowspan="4">
<layout class="QVBoxLayout" name="verticalLayout_7">
<item>
<spacer name="verticalSpacer_3">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QToolButton" name="auto_title_sort">
<property name="toolTip">
<string>Automatically create the title sort entry based on the current title entry.
Using this button to create title sort will change title sort from red to green.</string>
</property>
<property name="text">
<string>...</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/auto_author_sort.png</normaloff>:/images/auto_author_sort.png</iconset>
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QToolButton" name="swap_button">
<property name="toolTip">
<string>Swap the author and title</string>
</property>
<property name="text">
<string>...</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/swap.png</normaloff>:/images/swap.png</iconset>
</property>
<property name="iconSize">
<size>
<width>16</width>
<height>16</height>
</size>
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer_2">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
<item>
<widget class="QToolButton" name="auto_author_sort">
<property name="toolTip">
<string>Automatically create the author sort entry based on the current author entry.
Using this button to create author sort will change author sort from red to green.</string>
</property>
<property name="text">
<string>...</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/auto_author_sort.png</normaloff>:/images/auto_author_sort.png</iconset>
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer_4">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
</layout>
</item>
<item row="1" column="0">
<widget class="QLabel" name="label">
<property name="text">
<string>Title &amp;sort: </string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>title_sort</cstring>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="EnLineEdit" name="title_sort">
<property name="toolTip">
<string>Specify how this book should be sorted when by title. For example, The Exorcist might be sorted as Exorcist, The.</string>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="label_2">
<property name="text">
<string>&amp;Author(s): </string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>authors</cstring>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="MultiCompleteComboBox" name="authors">
<property name="editable">
<bool>true</bool>
</property>
</widget>
</item>
<item row="3" column="0">
<widget class="QLabel" name="label_8">
<property name="text">
<string>Author S&amp;ort: </string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>author_sort</cstring>
</property>
</widget>
</item>
<item row="3" column="1">
<widget class="EnLineEdit" name="author_sort">
<property name="toolTip">
<string>Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles.
If the box is colored green, then text matches the individual author's sort strings. If it is colored red, then the authors and this text do not match.</string>
</property>
</widget>
</item>
<item row="4" column="0">
<widget class="QLabel" name="label_6">
<property name="text">
<string>&amp;Rating:</string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>rating</cstring>
</property>
</widget>
</item>
<item row="4" column="1" colspan="2">
<widget class="QSpinBox" name="rating">
<property name="toolTip">
<string>Rating of this book. 0-5 stars</string>
</property>
<property name="whatsThis">
<string>Rating of this book. 0-5 stars</string>
</property>
<property name="buttonSymbols">
<enum>QAbstractSpinBox::PlusMinus</enum>
</property>
<property name="suffix">
<string> stars</string>
</property>
<property name="maximum">
<number>5</number>
</property>
</widget>
</item>
<item row="5" column="0">
<widget class="QLabel" name="label_3">
<property name="text">
<string>&amp;Publisher: </string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>publisher</cstring>
</property>
</widget>
</item>
<item row="5" column="1" colspan="2">
<widget class="MultiCompleteComboBox" name="publisher">
<property name="editable">
<bool>true</bool>
</property>
</widget>
</item>
<item row="6" column="0">
<widget class="QLabel" name="label_4">
<property name="text">
<string>Ta&amp;gs: </string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>tags</cstring>
</property>
</widget>
</item>
<item row="6" column="1">
<layout class="QHBoxLayout" name="_2">
<item>
<widget class="MultiCompleteLineEdit" name="tags">
<property name="toolTip">
<string>Tags categorize the book. This is particularly useful while searching. &lt;br&gt;&lt;br&gt;They can be any words or phrases, separated by commas.</string>
</property>
</widget>
</item>
</layout>
</item>
<item row="6" column="2">
<widget class="QToolButton" name="tag_editor_button">
<property name="toolTip">
<string>Open Tag Editor</string>
</property>
<property name="text">
<string>Open Tag Editor</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/chapters.png</normaloff>:/images/chapters.png</iconset>
</property>
</widget>
</item>
<item row="7" column="0">
<widget class="QLabel" name="label_7">
<property name="text">
<string>&amp;Series:</string>
</property>
<property name="textFormat">
<enum>Qt::PlainText</enum>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>series</cstring>
</property>
</widget>
</item>
<item row="7" column="1">
<layout class="QHBoxLayout" name="_3">
<property name="spacing">
<number>5</number>
</property>
<item>
<widget class="MultiCompleteComboBox" name="series">
<property name="toolTip">
<string>List of known series. You can add new series.</string>
</property>
<property name="whatsThis">
<string>List of known series. You can add new series.</string>
</property>
<property name="editable">
<bool>true</bool>
</property>
<property name="insertPolicy">
<enum>QComboBox::InsertAlphabetically</enum>
</property>
</widget>
</item>
</layout>
</item>
<item row="7" column="2">
<widget class="QToolButton" name="remove_series_button">
<property name="toolTip">
<string>Remove unused series (Series that have no books)</string>
</property>
<property name="text">
<string>...</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
</property>
</widget>
</item>
<item row="8" column="1" colspan="2">
<widget class="QDoubleSpinBox" name="series_index">
<property name="enabled">
<bool>false</bool>
</property>
<property name="prefix">
<string>Book </string>
</property>
<property name="maximum">
<double>99999999.989999994635582</double>
</property>
</widget>
</item>
<item row="9" column="0">
<widget class="QLabel" name="label_9">
<property name="text">
<string>IS&amp;BN:</string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>isbn</cstring>
</property>
</widget>
</item>
<item row="9" column="1" colspan="2">
<widget class="QLineEdit" name="isbn"/>
</item>
<item row="10" column="0">
<widget class="QLabel" name="label_11">
<property name="text">
<string>&amp;Date:</string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>date</cstring>
</property>
</widget>
</item>
<item row="10" column="1" colspan="2">
<widget class="QDateEdit" name="date">
<property name="displayFormat">
<string>dd MMM yyyy</string>
</property>
<property name="calendarPopup">
<bool>true</bool>
</property>
</widget>
</item>
<item row="11" column="0">
<widget class="QLabel" name="label_10">
<property name="text">
<string>Publishe&amp;d:</string>
</property>
<property name="alignment">
<set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
</property>
<property name="buddy">
<cstring>pubdate</cstring>
</property>
</widget>
</item>
<item row="11" column="1">
<widget class="QDateEdit" name="pubdate">
<property name="displayFormat">
<string>MMM yyyy</string>
</property>
<property name="calendarPopup">
<bool>true</bool>
</property>
</widget>
</item>
<item row="11" column="2">
<widget class="QToolButton" name="clear_pubdate_button">
<property name="toolTip">
<string>Clear published date</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
</property>
</widget>
</item>
</layout>
</widget>
</item>
<item>
<widget class="QPushButton" name="fetch_metadata_button">
<property name="text">
<string>&amp;Fetch metadata from server</string>
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer_5">
<property name="orientation">
<enum>Qt::Vertical</enum>
</property>
<property name="sizeType">
<enum>QSizePolicy::Fixed</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>20</width>
<height>40</height>
</size>
</property>
</spacer>
</item>
</layout>
</widget>
<widget class="QWidget" name="layoutWidget_2">
<layout class="QVBoxLayout" name="verticalLayout_2">
<item>
<widget class="QGroupBox" name="bc_box">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Expanding">
<horstretch>0</horstretch>
<verstretch>10</verstretch>
</sizepolicy>
</property>
<property name="title">
<string>Book Cover</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout_4">
<item>
<widget class="ImageView" name="cover" native="true">
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Expanding">
<horstretch>0</horstretch>
<verstretch>100</verstretch>
</sizepolicy>
</property>
</widget>
</item>
<item>
<layout class="QVBoxLayout" name="_4">
<property name="spacing">
<number>6</number>
</property>
<property name="sizeConstraint">
<enum>QLayout::SetMaximumSize</enum>
</property>
<property name="margin">
<number>0</number>
</property>
<item>
<widget class="QLabel" name="label_5">
<property name="text">
<string>Change &amp;cover image:</string>
</property>
<property name="buddy">
<cstring>cover_button</cstring>
</property>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="_5">
<property name="spacing">
<number>6</number>
</property>
<property name="margin">
<number>0</number>
</property>
<item>
<widget class="QPushButton" name="cover_button">
<property name="text">
<string>&amp;Browse</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="trim_cover_button">
<property name="toolTip">
<string>Remove border (if any) from cover</string>
</property>
<property name="text">
<string>T&amp;rim</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/trim.png</normaloff>:/images/trim.png</iconset>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="reset_cover">
<property name="toolTip">
<string>Reset cover to default</string>
</property>
<property name="text">
<string>&amp;Remove</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
</property>
</widget>
</item>
</layout>
</item>
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="_6">
<item>
<widget class="QPushButton" name="fetch_cover_button">
<property name="text">
<string>Download co&amp;ver</string>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="generate_cover_button">
<property name="toolTip">
<string>Generate a default cover based on the title and author</string>
</property>
<property name="text">
<string>&amp;Generate cover</string>
</property>
</widget>
</item>
</layout>
</item>
</layout>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="layoutWidget">
<layout class="QVBoxLayout" name="verticalLayout_3">
<item>
<widget class="QGroupBox" name="af_group_box">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Minimum">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="title">
<string>Available Formats</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="1" rowspan="3">
<widget class="FormatList" name="formats">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Minimum">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="maximumSize">
<size>
<width>16777215</width>
<height>140</height>
</size>
</property>
<property name="baseSize">
<size>
<width>100</width>
<height>0</height>
</size>
</property>
<property name="dragDropMode">
<enum>QAbstractItemView::DropOnly</enum>
</property>
<property name="iconSize">
<size>
<width>64</width>
<height>64</height>
</size>
</property>
</widget>
</item>
<item row="0" column="2">
<widget class="QToolButton" name="add_format_button">
<property name="toolTip">
<string>Add a new format for this book to the database</string>
</property>
<property name="text">
<string>...</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/add_book.png</normaloff>:/images/add_book.png</iconset>
</property>
<property name="iconSize">
<size>
<width>32</width>
<height>32</height>
</size>
</property>
</widget>
</item>
<item row="2" column="2">
<widget class="QToolButton" name="remove_format_button">
<property name="toolTip">
<string>Remove the selected formats for this book from the database.</string>
</property>
<property name="text">
<string>...</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset>
</property>
<property name="iconSize">
<size>
<width>32</width>
<height>32</height>
</size>
</property>
</widget>
</item>
<item row="0" column="0">
<widget class="QToolButton" name="button_set_cover">
<property name="toolTip">
<string>Set the cover for the book from the selected format</string>
</property>
<property name="text">
<string>...</string>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/book.png</normaloff>:/images/book.png</iconset>
</property>
<property name="iconSize">
<size>
<width>32</width>
<height>32</height>
</size>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QToolButton" name="button_set_metadata">
<property name="toolTip">
<string>Update metadata from the metadata in the selected format</string>
</property>
<property name="text">
<string/>
</property>
<property name="icon">
<iconset resource="../../../../resources/images.qrc">
<normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset>
</property>
<property name="iconSize">
<size>
<width>32</width>
<height>32</height>
</size>
</property>
</widget>
</item>
</layout>
</item>
</layout>
<zorder></zorder>
</widget>
</item>
<item>
<widget class="QGroupBox" name="groupBox">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
<horstretch>0</horstretch>
<verstretch>10</verstretch>
</sizepolicy>
</property>
<property name="title">
<string>&amp;Comments</string>
</property>
<layout class="QVBoxLayout" name="verticalLayout_8">
<property name="margin">
<number>0</number>
</property>
<item>
<widget class="Editor" name="comments" native="true"/>
</item>
</layout>
</widget>
</item>
</layout>
</widget>
</widget>
</item>
</layout>
</widget>
<widget class="QWidget" name="tab">
<attribute name="title">
<string>&amp;Custom metadata</string>
</attribute>
<layout class="QGridLayout" name="gridLayout_2"/>
</widget>
</widget>
</item>
</layout>
</widget>
</widget>
</item>
<item>
<widget class="QDialogButtonBox" name="button_box">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
</layout>
</widget>
<customwidgets>
<customwidget>
<class>EnLineEdit</class>
<extends>QLineEdit</extends>
<header>widgets.h</header>
</customwidget>
<customwidget>
<class>MultiCompleteLineEdit</class>
<extends>QLineEdit</extends>
<header>calibre/gui2/complete.h</header>
</customwidget>
<customwidget>
<class>MultiCompleteComboBox</class>
<extends>QComboBox</extends>
<header>calibre/gui2/complete.h</header>
</customwidget>
<customwidget>
<class>FormatList</class>
<extends>QListWidget</extends>
<header location="global">calibre/gui2/widgets.h</header>
</customwidget>
<customwidget>
<class>ImageView</class>
<extends>QWidget</extends>
<header>calibre/gui2/widgets.h</header>
<container>1</container>
</customwidget>
<customwidget>
<class>Editor</class>
<extends>QWidget</extends>
<header location="global">calibre/gui2/comments_editor.h</header>
<container>1</container>
</customwidget>
</customwidgets>
<tabstops>
<tabstop>title</tabstop>
<tabstop>auto_title_sort</tabstop>
<tabstop>title_sort</tabstop>
<tabstop>swap_button</tabstop>
<tabstop>authors</tabstop>
<tabstop>auto_author_sort</tabstop>
<tabstop>author_sort</tabstop>
<tabstop>rating</tabstop>
<tabstop>publisher</tabstop>
<tabstop>tags</tabstop>
<tabstop>tag_editor_button</tabstop>
<tabstop>series</tabstop>
<tabstop>remove_series_button</tabstop>
<tabstop>series_index</tabstop>
<tabstop>isbn</tabstop>
<tabstop>date</tabstop>
<tabstop>pubdate</tabstop>
<tabstop>fetch_metadata_button</tabstop>
<tabstop>button_set_cover</tabstop>
<tabstop>button_set_metadata</tabstop>
<tabstop>formats</tabstop>
<tabstop>add_format_button</tabstop>
<tabstop>remove_format_button</tabstop>
<tabstop>cover_button</tabstop>
<tabstop>trim_cover_button</tabstop>
<tabstop>reset_cover</tabstop>
<tabstop>fetch_cover_button</tabstop>
<tabstop>generate_cover_button</tabstop>
<tabstop>button_box</tabstop>
<tabstop>scrollArea</tabstop>
<tabstop>central_widget</tabstop>
</tabstops>
<resources>
<include location="../../../../resources/images.qrc"/>
</resources>
<connections>
<connection>
<sender>button_box</sender>
<signal>accepted()</signal>
<receiver>MetadataSingleDialog</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>261</x>
<y>710</y>
</hint>
<hint type="destinationlabel">
<x>157</x>
<y>274</y>
</hint>
</hints>
</connection>
<connection>
<sender>button_box</sender>
<signal>rejected()</signal>
<receiver>MetadataSingleDialog</receiver>
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>329</x>
<y>710</y>
</hint>
<hint type="destinationlabel">
<x>286</x>
<y>274</y>
</hint>
</hints>
</connection>
</connections>
</ui>

465
src/calibre/gui2/metadata/bulk_download.py
View File

@ -1,308 +1,195 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import with_statement
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import traceback
from threading import Thread
from Queue import Queue, Empty
from functools import partial
from itertools import izip
from threading import Event
from PyQt4.Qt import QObject, QTimer, QDialog, \
QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox
from PyQt4.Qt import (QIcon, QDialog,
QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt)
from calibre.ebooks.metadata.fetch import search, get_social_metadata
from calibre.gui2 import config, error_dialog
from calibre.gui2.dialogs.progress import ProgressDialog
from calibre.ebooks.metadata.covers import download_cover
from calibre.customize.ui import get_isbndb_key
from calibre.gui2.threaded_jobs import ThreadedJob
from calibre.ebooks.metadata.sources.identify import identify, msprefs
from calibre.ebooks.metadata.sources.covers import download_cover
from calibre.ebooks.metadata.book.base import Metadata
from calibre.customize.ui import metadata_plugins
from calibre.ptempfile import PersistentTemporaryFile
class Worker(Thread):
'Cover downloader'
# Start download {{{
def show_config(gui, parent):
from calibre.gui2.preferences import show_config_widget
show_config_widget('Sharing', 'Metadata download', parent=parent,
gui=gui, never_shutdown=True)
def __init__(self):
Thread.__init__(self)
self.daemon = True
self.jobs = Queue()
self.results = Queue()
class ConfirmDialog(QDialog):
def run(self):
while True:
id, mi = self.jobs.get()
if not getattr(mi, 'isbn', False):
break
def __init__(self, ids, parent):
QDialog.__init__(self, parent)
self.setWindowTitle(_('Schedule download?'))
self.setWindowIcon(QIcon(I('dialog_question.png')))
l = self.l = QGridLayout()
self.setLayout(l)
i = QLabel(self)
i.setPixmap(QPixmap(I('dialog_question.png')))
l.addWidget(i, 0, 0)
t = QLabel(
'<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will'
' run in the background. Proceed?')%len(ids) +
'<p>'+_('You can monitor the progress of the download '
'by clicking the rotating spinner in the bottom right '
'corner.') +
'<p>'+_('When the download completes you will be asked for'
' confirmation before calibre applies the downloaded metadata.')
)
t.setWordWrap(True)
l.addWidget(t, 0, 1)
l.setColumnStretch(0, 1)
l.setColumnStretch(1, 100)
self.identify = self.covers = True
self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
self.bb.rejected.connect(self.reject)
b = self.bb.addButton(_('Download only &metadata'),
self.bb.AcceptRole)
b.clicked.connect(self.only_metadata)
b.setIcon(QIcon(I('edit_input.png')))
b = self.bb.addButton(_('Download only &covers'),
self.bb.AcceptRole)
b.clicked.connect(self.only_covers)
b.setIcon(QIcon(I('default_cover.png')))
b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole)
b.setIcon(QIcon(I('config.png')))
b.clicked.connect(partial(show_config, parent, self))
l.addWidget(self.bb, 1, 0, 1, 2)
b = self.bb.addButton(_('Download &both'),
self.bb.AcceptRole)
b.clicked.connect(self.accept)
b.setDefault(True)
b.setAutoDefault(True)
b.setIcon(QIcon(I('ok.png')))
self.resize(self.sizeHint())
b.setFocus(Qt.OtherFocusReason)
def only_metadata(self):
self.covers = False
self.accept()
def only_covers(self):
self.identify = False
self.accept()
def start_download(gui, ids, callback):
d = ConfirmDialog(ids, gui)
ret = d.exec_()
d.b.clicked.disconnect()
if ret != d.Accepted:
return
job = ThreadedJob('metadata bulk download',
_('Download metadata for %d books')%len(ids),
download, (ids, gui.current_db, d.identify, d.covers), {}, callback)
gui.job_manager.run_threaded_job(job)
gui.status_bar.show_message(_('Metadata download started'), 3000)
# }}}
def get_job_details(job):
id_map, failed_ids, failed_covers, title_map, all_failed = job.result
det_msg = []
for i in failed_ids | failed_covers:
title = title_map[i]
if i in failed_ids:
title += (' ' + _('(Failed metadata)'))
if i in failed_covers:
title += (' ' + _('(Failed cover)'))
det_msg.append(title)
det_msg = '\n'.join(det_msg)
return id_map, failed_ids, failed_covers, all_failed, det_msg
def merge_result(oldmi, newmi):
dummy = Metadata(_('Unknown'))
for f in msprefs['ignore_fields']:
if ':' not in f:
setattr(newmi, f, getattr(dummy, f))
fields = set()
for plugin in metadata_plugins(['identify']):
fields |= plugin.touched_fields
for f in fields:
# Optimize so that set_metadata does not have to do extra work later
if not f.startswith('identifier:'):
if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)):
setattr(newmi, f, getattr(dummy, f))
newmi.last_modified = oldmi.last_modified
return newmi
def download(ids, db, do_identify, covers,
log=None, abort=None, notifications=None):
ids = list(ids)
metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False)
for i in ids]
failed_ids = set()
failed_covers = set()
title_map = {}
ans = {}
count = 0
all_failed = True
'''
# Test apply dialog
all_failed = do_identify = covers = False
'''
for i, mi in izip(ids, metadata):
if abort.is_set():
log.error('Aborting...')
break
title, authors, identifiers = mi.title, mi.authors, mi.identifiers
title_map[i] = title
if do_identify:
results = []
try:
cdata, errors = download_cover(mi)
if cdata:
self.results.put((id, mi, True, cdata))
else:
msg = []
for e in errors:
if not e[0]:
msg.append(e[-1] + ' - ' + e[1])
self.results.put((id, mi, False, '\n'.join(msg)))
results = identify(log, Event(), title=title, authors=authors,
identifiers=identifiers)
except:
self.results.put((id, mi, False, traceback.format_exc()))
def __enter__(self):
self.start()
return self
def __exit__(self, *args):
self.jobs.put((False, False))
class DownloadMetadata(Thread):
'Metadata downloader'
def __init__(self, db, ids, get_covers, set_metadata=True,
get_social_metadata=True):
Thread.__init__(self)
self.daemon = True
self.metadata = {}
self.covers = {}
self.set_metadata = set_metadata
self.get_social_metadata = get_social_metadata
self.social_metadata_exceptions = []
self.db = db
self.updated = set([])
self.get_covers = get_covers
self.worker = Worker()
self.results = Queue()
self.keep_going = True
for id in ids:
self.metadata[id] = db.get_metadata(id, index_is_id=True)
self.metadata[id].rating = None
self.total = len(ids)
if self.get_covers:
self.total += len(ids)
self.fetched_metadata = {}
self.fetched_covers = {}
self.failures = {}
self.cover_failures = {}
self.exception = self.tb = None
def run(self):
try:
self._run()
except Exception as e:
self.exception = e
self.tb = traceback.format_exc()
def _run(self):
self.key = get_isbndb_key()
if not self.key:
self.key = None
with self.worker:
for id, mi in self.metadata.items():
if not self.keep_going:
break
args = {}
if mi.isbn:
args['isbn'] = mi.isbn
else:
if mi.is_null('title'):
self.failures[id] = \
_('Book has neither title nor ISBN')
continue
args['title'] = mi.title
if mi.authors and mi.authors[0] != _('Unknown'):
args['author'] = mi.authors[0]
if self.key:
args['isbndb_key'] = self.key
results, exceptions = search(**args)
if results:
fmi = results[0]
self.fetched_metadata[id] = fmi
if self.get_covers:
if fmi.isbn:
self.worker.jobs.put((id, fmi))
else:
self.results.put((id, 'cover', False, mi.title))
if (not config['overwrite_author_title_metadata']):
fmi.authors = mi.authors
fmi.author_sort = mi.author_sort
fmi.title = mi.title
mi.smart_update(fmi)
if mi.isbn and self.get_social_metadata:
self.social_metadata_exceptions = get_social_metadata(mi)
if mi.rating:
mi.rating *= 2
if not self.get_social_metadata:
mi.tags = []
self.results.put((id, 'metadata', True, mi.title))
else:
self.failures[id] = _('No matches found for this book')
self.results.put((id, 'metadata', False, mi.title))
self.results.put((id, 'cover', False, mi.title))
self.commit_covers()
self.commit_covers(True)
def commit_covers(self, all=False):
if all:
self.worker.jobs.put((False, False))
while True:
try:
id, fmi, ok, cdata = self.worker.results.get_nowait()
if ok:
self.fetched_covers[id] = cdata
self.results.put((id, 'cover', ok, fmi.title))
else:
self.results.put((id, 'cover', ok, fmi.title))
try:
self.cover_failures[id] = unicode(cdata)
except:
self.cover_failures[id] = repr(cdata)
except Empty:
if not all or not self.worker.is_alive():
return
class DoDownload(QObject):
def __init__(self, parent, title, db, ids, get_covers, set_metadata=True,
get_social_metadata=True):
QObject.__init__(self, parent)
self.pd = ProgressDialog(title, min=0, max=0, parent=parent)
self.pd.canceled_signal.connect(self.cancel)
self.downloader = None
self.create = partial(DownloadMetadata, db, ids, get_covers,
set_metadata=set_metadata,
get_social_metadata=get_social_metadata)
self.get_covers = get_covers
self.db = db
self.updated = set([])
self.total = len(ids)
self.keep_going = True
def exec_(self):
QTimer.singleShot(50, self.do_one)
ret = self.pd.exec_()
if getattr(self.downloader, 'exception', None) is not None and \
ret == self.pd.Accepted:
error_dialog(self.parent(), _('Failed'),
_('Failed to download metadata'), show=True)
else:
self.show_report()
return ret
def cancel(self, *args):
self.keep_going = False
self.downloader.keep_going = False
self.pd.reject()
def do_one(self):
try:
if not self.keep_going:
return
if self.downloader is None:
self.downloader = self.create()
self.downloader.start()
self.pd.set_min(0)
self.pd.set_max(self.downloader.total)
try:
r = self.downloader.results.get_nowait()
self.handle_result(r)
except Empty:
pass
if not self.downloader.is_alive():
while True:
try:
r = self.downloader.results.get_nowait()
self.handle_result(r)
except Empty:
break
self.pd.accept()
return
except:
self.cancel()
raise
QTimer.singleShot(50, self.do_one)
def handle_result(self, r):
id_, typ, ok, title = r
what = _('cover') if typ == 'cover' else _('metadata')
which = _('Downloaded') if ok else _('Failed to get')
if self.get_covers or typ != 'cover' or ok:
# Do not show message when cover fetch fails if user didn't ask to
# download covers
self.pd.set_msg(_('%s %s for: %s') % (which, what, title))
self.pd.value += 1
if ok:
self.updated.add(id_)
if typ == 'cover':
try:
self.db.set_cover(id_,
self.downloader.fetched_covers.pop(id_))
except:
self.downloader.cover_failures[id_] = \
traceback.format_exc()
if results:
all_failed = False
mi = merge_result(mi, results[0])
identifiers = mi.identifiers
if not mi.is_null('rating'):
# set_metadata expects a rating out of 10
mi.rating *= 2
else:
try:
self.set_metadata(id_)
except:
self.downloader.failures[id_] = \
traceback.format_exc()
def set_metadata(self, id_):
mi = self.downloader.metadata[id_]
if self.downloader.set_metadata:
self.db.set_metadata(id_, mi)
if not self.downloader.set_metadata and self.downloader.get_social_metadata:
if mi.rating:
self.db.set_rating(id_, mi.rating)
if mi.tags:
self.db.set_tags(id_, mi.tags)
if mi.comments:
self.db.set_comment(id_, mi.comments)
if mi.series:
self.db.set_series(id_, mi.series)
if mi.series_index is not None:
self.db.set_series_index(id_, mi.series_index)
def show_report(self):
f, cf = self.downloader.failures, self.downloader.cover_failures
report = []
if f:
report.append(
'<h3>Failed to download metadata for the following:</h3><ol>')
for id_, err in f.items():
mi = self.downloader.metadata[id_]
report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
unicode(err)))
report.append('</ol>')
if cf:
report.append(
'<h3>Failed to download cover for the following:</h3><ol>')
for id_, err in cf.items():
mi = self.downloader.metadata[id_]
report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title,
unicode(err)))
report.append('</ol>')
if len(self.updated) != self.total or report:
d = QDialog(self.parent())
bb = QDialogButtonBox(QDialogButtonBox.Ok, parent=d)
v1 = QVBoxLayout()
d.setLayout(v1)
d.setWindowTitle(_('Done'))
v1.addWidget(QLabel(_('Successfully downloaded metadata for %d out of %d books') %
(len(self.updated), self.total)))
gb = QGroupBox(_('Details'), self.parent())
v2 = QVBoxLayout()
gb.setLayout(v2)
b = QTextBrowser(self.parent())
v2.addWidget(b)
b.setHtml('\n'.join(report))
v1.addWidget(gb)
v1.addWidget(bb)
bb.accepted.connect(d.accept)
d.resize(800, 600)
d.exec_()
log.error('Failed to download metadata for', title)
failed_ids.add(i)
# We don't want set_metadata operating on anything but covers
mi = merge_result(mi, mi)
if covers:
cdata = download_cover(log, title=title, authors=authors,
identifiers=identifiers)
if cdata is not None:
with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f:
f.write(cdata[-1])
mi.cover = f.name
all_failed = False
else:
failed_covers.add(i)
ans[i] = mi
count += 1
notifications.put((count/len(ids),
_('Downloaded %d of %d')%(count, len(ids))))
log('Download complete, with %d failures'%len(failed_ids))
return (ans, failed_ids, failed_covers, title_map, all_failed)

195
src/calibre/gui2/metadata/bulk_download2.py
View File

@ -1,195 +0,0 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import (unicode_literals, division, absolute_import,
print_function)
__license__ = 'GPL v3'
__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
from functools import partial
from itertools import izip
from threading import Event
from PyQt4.Qt import (QIcon, QDialog,
QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt)
from calibre.gui2.threaded_jobs import ThreadedJob
from calibre.ebooks.metadata.sources.identify import identify, msprefs
from calibre.ebooks.metadata.sources.covers import download_cover
from calibre.ebooks.metadata.book.base import Metadata
from calibre.customize.ui import metadata_plugins
from calibre.ptempfile import PersistentTemporaryFile
# Start download {{{
def show_config(gui, parent):
from calibre.gui2.preferences import show_config_widget
show_config_widget('Sharing', 'Metadata download', parent=parent,
gui=gui, never_shutdown=True)
class ConfirmDialog(QDialog):
def __init__(self, ids, parent):
QDialog.__init__(self, parent)
self.setWindowTitle(_('Schedule download?'))
self.setWindowIcon(QIcon(I('dialog_question.png')))
l = self.l = QGridLayout()
self.setLayout(l)
i = QLabel(self)
i.setPixmap(QPixmap(I('dialog_question.png')))
l.addWidget(i, 0, 0)
t = QLabel(
'<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will'
' run in the background. Proceed?')%len(ids) +
'<p>'+_('You can monitor the progress of the download '
'by clicking the rotating spinner in the bottom right '
'corner.') +
'<p>'+_('When the download completes you will be asked for'
' confirmation before calibre applies the downloaded metadata.')
)
t.setWordWrap(True)
l.addWidget(t, 0, 1)
l.setColumnStretch(0, 1)
l.setColumnStretch(1, 100)
self.identify = self.covers = True
self.bb = QDialogButtonBox(QDialogButtonBox.Cancel)
self.bb.rejected.connect(self.reject)
b = self.bb.addButton(_('Download only &metadata'),
self.bb.AcceptRole)
b.clicked.connect(self.only_metadata)
b.setIcon(QIcon(I('edit_input.png')))
b = self.bb.addButton(_('Download only &covers'),
self.bb.AcceptRole)
b.clicked.connect(self.only_covers)
b.setIcon(QIcon(I('default_cover.png')))
b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole)
b.setIcon(QIcon(I('config.png')))
b.clicked.connect(partial(show_config, parent, self))
l.addWidget(self.bb, 1, 0, 1, 2)
b = self.bb.addButton(_('Download &both'),
self.bb.AcceptRole)
b.clicked.connect(self.accept)
b.setDefault(True)
b.setAutoDefault(True)
b.setIcon(QIcon(I('ok.png')))
self.resize(self.sizeHint())
b.setFocus(Qt.OtherFocusReason)
def only_metadata(self):
self.covers = False
self.accept()
def only_covers(self):
self.identify = False
self.accept()
def start_download(gui, ids, callback):
d = ConfirmDialog(ids, gui)
ret = d.exec_()
d.b.clicked.disconnect()
if ret != d.Accepted:
return
job = ThreadedJob('metadata bulk download',
_('Download metadata for %d books')%len(ids),
download, (ids, gui.current_db, d.identify, d.covers), {}, callback)
gui.job_manager.run_threaded_job(job)
gui.status_bar.show_message(_('Metadata download started'), 3000)
# }}}
def get_job_details(job):
id_map, failed_ids, failed_covers, title_map, all_failed = job.result
det_msg = []
for i in failed_ids | failed_covers:
title = title_map[i]
if i in failed_ids:
title += (' ' + _('(Failed metadata)'))
if i in failed_covers:
title += (' ' + _('(Failed cover)'))
det_msg.append(title)
det_msg = '\n'.join(det_msg)
return id_map, failed_ids, failed_covers, all_failed, det_msg
def merge_result(oldmi, newmi):
dummy = Metadata(_('Unknown'))
for f in msprefs['ignore_fields']:
if ':' not in f:
setattr(newmi, f, getattr(dummy, f))
fields = set()
for plugin in metadata_plugins(['identify']):
fields |= plugin.touched_fields
for f in fields:
# Optimize so that set_metadata does not have to do extra work later
if not f.startswith('identifier:'):
if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)):
setattr(newmi, f, getattr(dummy, f))
newmi.last_modified = oldmi.last_modified
return newmi
def download(ids, db, do_identify, covers,
log=None, abort=None, notifications=None):
ids = list(ids)
metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False)
for i in ids]
failed_ids = set()
failed_covers = set()
title_map = {}
ans = {}
count = 0
all_failed = True
'''
# Test apply dialog
all_failed = do_identify = covers = False
'''
for i, mi in izip(ids, metadata):
if abort.is_set():
log.error('Aborting...')
break
title, authors, identifiers = mi.title, mi.authors, mi.identifiers
title_map[i] = title
if do_identify:
results = []
try:
results = identify(log, Event(), title=title, authors=authors,
identifiers=identifiers)
except:
pass
if results:
all_failed = False
mi = merge_result(mi, results[0])
identifiers = mi.identifiers
if not mi.is_null('rating'):
# set_metadata expects a rating out of 10
mi.rating *= 2
else:
log.error('Failed to download metadata for', title)
failed_ids.add(i)
# We don't want set_metadata operating on anything but covers
mi = merge_result(mi, mi)
if covers:
cdata = download_cover(log, title=title, authors=authors,
identifiers=identifiers)
if cdata is not None:
with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f:
f.write(cdata[-1])
mi.cover = f.name
all_failed = False
else:
failed_covers.add(i)
ans[i] = mi
count += 1
notifications.put((count/len(ids),
_('Downloaded %d of %d')%(count, len(ids))))
log('Download complete, with %d failures'%len(failed_ids))
return (ans, failed_ids, failed_covers, title_map, all_failed)

17
src/calibre/gui2/preferences/behavior.py
View File

@ -19,7 +19,6 @@ from calibre.ebooks import BOOK_EXTENSIONS
from calibre.ebooks.oeb.iterator import is_supported
from calibre.constants import iswindows
from calibre.utils.icu import sort_key
from calibre.utils.config import test_eight_code
class OutputFormatSetting(Setting):
@ -40,12 +39,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
r('network_timeout', prefs)
r('overwrite_author_title_metadata', config)
r('get_social_metadata', config)
if test_eight_code:
self.opt_overwrite_author_title_metadata.setVisible(False)
self.opt_get_social_metadata.setVisible(False)
r('new_version_notification', config)
r('upload_news_to_device', config)
r('delete_news_from_library_on_upload', config)
@ -67,13 +60,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form):
signal.connect(self.internally_viewed_formats_changed)
r('bools_are_tristate', db.prefs, restart_required=True)
if test_eight_code:
r = self.register
choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')]
r('edit_metadata_single_layout', gprefs, choices=choices)
else:
self.opt_edit_metadata_single_layout.setVisible(False)
self.edit_metadata_single_label.setVisible(False)
r = self.register
choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')]
r('edit_metadata_single_layout', gprefs, choices=choices)
def initialize(self):
ConfigWidgetBase.initialize(self)

51
src/calibre/gui2/preferences/behavior.ui
View File

@ -14,41 +14,14 @@
<string>Form</string>
</property>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="1">
<spacer>
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="sizeHint" stdset="0">
<size>
<width>10</width>
<height>0</height>
</size>
</property>
</spacer>
</item>
<item row="0" column="0">
<widget class="QCheckBox" name="opt_overwrite_author_title_metadata">
<property name="text">
<string>&amp;Overwrite author and title by default when fetching metadata</string>
</property>
</widget>
</item>
<item row="0" column="2">
<widget class="QCheckBox" name="opt_get_social_metadata">
<property name="text">
<string>Download &amp;social metadata (tags/ratings/etc.) by default</string>
</property>
</widget>
</item>
<item row="2" column="0">
<item row="1" column="0">
<widget class="QCheckBox" name="opt_new_version_notification">
<property name="text">
<string>Show notification when &amp;new version is available</string>
</property>
</widget>
</item>
<item row="2" column="2">
<item row="1" column="1">
<widget class="QCheckBox" name="opt_bools_are_tristate">
<property name="toolTip">
<string>If checked, Yes/No custom columns values can be Yes, No, or Unknown.
@ -59,21 +32,21 @@ If not checked, the values can be Yes or No.</string>
</property>
</widget>
</item>
<item row="4" column="0">
<item row="3" column="0">
<widget class="QCheckBox" name="opt_upload_news_to_device">
<property name="text">
<string>Automatically send downloaded &amp;news to ebook reader</string>
</property>
</widget>
</item>
<item row="4" column="2">
<item row="3" column="1">
<widget class="QCheckBox" name="opt_delete_news_from_library_on_upload">
<property name="text">
<string>&amp;Delete news from library when it is automatically sent to reader</string>
</property>
</widget>
</item>
<item row="6" column="0">
<item row="5" column="0">
<layout class="QHBoxLayout">
<item>
<widget class="QLabel" name="label_23">
@ -97,7 +70,7 @@ If not checked, the values can be Yes or No.</string>
</item>
</layout>
</item>
<item row="6" column="2">
<item row="5" column="1">
<layout class="QHBoxLayout">
<item>
<widget class="QLabel" name="label_2">
@ -130,7 +103,7 @@ If not checked, the values can be Yes or No.</string>
</item>
</layout>
</item>
<item row="8" column="0">
<item row="7" column="0">
<layout class="QHBoxLayout">
<item>
<widget class="QLabel" name="priority_label">
@ -169,7 +142,7 @@ If not checked, the values can be Yes or No.</string>
</item>
</layout>
</item>
<item row="8" column="2">
<item row="7" column="1">
<layout class="QHBoxLayout">
<item>
<widget class="QLabel" name="label_170">
@ -202,7 +175,7 @@ If not checked, the values can be Yes or No.</string>
</item>
</layout>
</item>
<item row="9" column="0">
<item row="8" column="0">
<layout class="QHBoxLayout">
<item>
<widget class="QLabel" name="edit_metadata_single_label">
@ -223,7 +196,7 @@ If not checked, the values can be Yes or No.</string>
</item>
</layout>
</item>
<item row="20" column="0">
<item row="19" column="0">
<widget class="QGroupBox" name="groupBox_5">
<property name="title">
<string>Preferred &amp;input format order:</string>
@ -285,7 +258,7 @@ If not checked, the values can be Yes or No.</string>
</layout>
</widget>
</item>
<item row="20" column="2">
<item row="19" column="1">
<widget class="QGroupBox" name="groupBox_3">
<property name="title">
<string>Use internal &amp;viewer for:</string>
@ -304,7 +277,7 @@ If not checked, the values can be Yes or No.</string>
</layout>
</widget>
</item>
<item row="9" column="2">
<item row="8" column="1">
<widget class="QPushButton" name="reset_confirmation_button">
<property name="text">
<string>Reset all disabled &amp;confirmation dialogs</string>

79
src/calibre/gui2/preferences/social.py
View File

@ -1,79 +0,0 @@
#!/usr/bin/env python
# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai
from __future__ import with_statement
__license__ = 'GPL v3'
__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>'
__docformat__ = 'restructuredtext en'
import time
from threading import Thread
from PyQt4.Qt import QDialog, QDialogButtonBox, Qt, QLabel, QVBoxLayout, \
QTimer
from calibre.ebooks.metadata import MetaInformation
class Worker(Thread):
def __init__(self, mi):
Thread.__init__(self)
self.daemon = True
self.mi = MetaInformation(mi)
self.exceptions = []
def run(self):
from calibre.ebooks.metadata.fetch import get_social_metadata
self.exceptions = get_social_metadata(self.mi)
class SocialMetadata(QDialog):
TIMEOUT = 300 # seconds
def __init__(self, mi, parent):
QDialog.__init__(self, parent)
self.bbox = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self)
self.mi = mi
self.layout = QVBoxLayout(self)
self.label = QLabel(_('Downloading social metadata, please wait...'), self)
self.label.setWordWrap(True)
self.layout.addWidget(self.label)
self.layout.addWidget(self.bbox)
self.worker = Worker(mi)
self.bbox.rejected.connect(self.reject)
self.worker.start()
self.start_time = time.time()
self.timed_out = False
self.rejected = False
QTimer.singleShot(50, self.update)
def reject(self):
self.rejected = True
QDialog.reject(self)
def update(self):
if self.rejected:
return
if time.time() - self.start_time > self.TIMEOUT:
self.timed_out = True
self.reject()
return
if not self.worker.is_alive():
self.accept()
return
QTimer.singleShot(50, self.update)
def accept(self):
self.mi.tags = self.worker.mi.tags
self.mi.rating = self.worker.mi.rating
self.mi.comments = self.worker.mi.comments
if self.worker.mi.series:
self.mi.series = self.worker.mi.series
self.mi.series_index = self.worker.mi.series_index
QDialog.accept(self)
@property
def exceptions(self):
return self.worker.exceptions

2
src/calibre/utils/config.py
View File

@ -24,8 +24,6 @@ if False:
OptionSet, ConfigInterface, read_tweaks, write_tweaks
read_raw_tweaks, tweaks, plugin_dir
test_eight_code = tweaks.get('test_eight_code', False)
def check_config_write_access():
return os.access(config_dir, os.W_OK) and os.access(config_dir, os.X_OK)