From 571f50be431f69504a0f72366be298ec84e5dc90 Mon Sep 17 00:00:00 2001 From: Kovid Goyal Date: Sat, 30 Apr 2011 13:14:23 -0600 Subject: [PATCH 01/39] Get rid of the test_eight_code tweak. That means the next release will be 0.8.0! --- src/calibre/customize/builtins.py | 36 +- src/calibre/customize/ui.py | 44 +- src/calibre/ebooks/metadata/amazon.py | 224 ---- src/calibre/ebooks/metadata/amazonfr.py | 516 ---------- src/calibre/ebooks/metadata/covers.py | 317 ------ src/calibre/ebooks/metadata/douban.py | 263 ----- src/calibre/ebooks/metadata/fetch.py | 523 ---------- src/calibre/ebooks/metadata/fictionwise.py | 390 ------- src/calibre/ebooks/metadata/google_books.py | 247 ----- src/calibre/ebooks/metadata/isbndb.py | 159 --- src/calibre/ebooks/metadata/nicebooks.py | 411 -------- src/calibre/ebooks/metadata/sources/base.py | 2 +- src/calibre/ebooks/metadata/sources/cli.py | 8 - src/calibre/gui2/actions/add.py | 34 +- src/calibre/gui2/actions/edit_metadata.py | 104 +- src/calibre/gui2/dialogs/fetch_metadata.py | 271 ----- src/calibre/gui2/dialogs/fetch_metadata.ui | 179 ---- src/calibre/gui2/dialogs/metadata_single.py | 1031 ------------------- src/calibre/gui2/dialogs/metadata_single.ui | 937 ----------------- src/calibre/gui2/metadata/bulk_download.py | 465 ++++----- src/calibre/gui2/metadata/bulk_download2.py | 195 ---- src/calibre/gui2/preferences/behavior.py | 17 +- src/calibre/gui2/preferences/behavior.ui | 51 +- src/calibre/gui2/preferences/social.py | 79 -- src/calibre/utils/config.py | 2 - 25 files changed, 222 insertions(+), 6283 deletions(-) delete mode 100644 src/calibre/ebooks/metadata/amazon.py delete mode 100644 src/calibre/ebooks/metadata/amazonfr.py delete mode 100644 src/calibre/ebooks/metadata/covers.py delete mode 100644 src/calibre/ebooks/metadata/douban.py delete mode 100644 src/calibre/ebooks/metadata/fetch.py delete mode 100644 src/calibre/ebooks/metadata/fictionwise.py delete mode 100644 src/calibre/ebooks/metadata/google_books.py delete mode 100644 src/calibre/ebooks/metadata/isbndb.py delete mode 100644 src/calibre/ebooks/metadata/nicebooks.py delete mode 100644 src/calibre/gui2/dialogs/fetch_metadata.py delete mode 100644 src/calibre/gui2/dialogs/fetch_metadata.ui delete mode 100644 src/calibre/gui2/dialogs/metadata_single.py delete mode 100644 src/calibre/gui2/dialogs/metadata_single.ui delete mode 100644 src/calibre/gui2/metadata/bulk_download2.py delete mode 100644 src/calibre/gui2/preferences/social.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index b3bca01c5b..36bcbdbfe2 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -9,7 +9,6 @@ from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \ from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf -from calibre.utils.config import test_eight_code # To archive plugins {{{ class HTML2ZIP(FileTypePlugin): @@ -621,28 +620,16 @@ from calibre.ebooks.epub.fix.epubcheck import Epubcheck plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested, Epubcheck, ] -if test_eight_code: # New metadata download plugins {{{ - from calibre.ebooks.metadata.sources.google import GoogleBooks - from calibre.ebooks.metadata.sources.amazon import Amazon - from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary - from calibre.ebooks.metadata.sources.isbndb import ISBNDB - from calibre.ebooks.metadata.sources.overdrive import OverDrive +from calibre.ebooks.metadata.sources.google import GoogleBooks +from calibre.ebooks.metadata.sources.amazon import Amazon +from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary +from calibre.ebooks.metadata.sources.isbndb import ISBNDB +from calibre.ebooks.metadata.sources.overdrive import OverDrive - plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive] +plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive] # }}} -else: - from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \ - KentDistrictLibrary - from calibre.ebooks.metadata.douban import DoubanBooks - from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers - from calibre.ebooks.metadata.covers import OpenLibraryCovers, \ - AmazonCovers, DoubanCovers - - plugins += [GoogleBooks, ISBNDB, Amazon, - OpenLibraryCovers, AmazonCovers, DoubanCovers, - NiceBooksCovers, KentDistrictLibrary, DoubanBooks, NiceBooks] plugins += [ ComicInput, @@ -867,10 +854,7 @@ plugins += [ActionAdd, ActionFetchAnnotations, ActionGenerateCatalog, ActionRestart, ActionOpenFolder, ActionConnectShare, ActionSendToDevice, ActionHelp, ActionPreferences, ActionSimilarBooks, ActionAddToLibrary, ActionEditCollections, ActionChooseLibrary, - ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch] - -if test_eight_code: - plugins += [ActionStore] + ActionCopyToLibrary, ActionTweakEpub, ActionNextMatch, ActionStore] # }}} @@ -1096,10 +1080,8 @@ class Misc(PreferencesPlugin): plugins += [LookAndFeel, Behavior, Columns, Toolbar, Search, InputOptions, CommonOptions, OutputOptions, Adding, Saving, Sending, Plugboard, - Email, Server, Plugins, Tweaks, Misc, TemplateFunctions] - -if test_eight_code: - plugins.append(MetadataSources) + Email, Server, Plugins, Tweaks, Misc, TemplateFunctions, + MetadataSources] #}}} diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index d3ecab7f16..151235cef9 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -15,10 +15,8 @@ from calibre.customize.profiles import InputProfile, OutputProfile from calibre.customize.builtins import plugins as builtin_plugins from calibre.devices.interface import DevicePlugin from calibre.ebooks.metadata import MetaInformation -from calibre.ebooks.metadata.covers import CoverDownload -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.utils.config import make_config_dir, Config, ConfigProxy, \ - plugin_dir, OptionParser, prefs +from calibre.utils.config import (make_config_dir, Config, ConfigProxy, + plugin_dir, OptionParser) from calibre.ebooks.epub.fix import ePubFixer from calibre.ebooks.metadata.sources.base import Source @@ -190,44 +188,6 @@ def output_profiles(): yield plugin # }}} -# Metadata sources {{{ -def metadata_sources(metadata_type='basic', customize=True, isbndb_key=None): - for plugin in _initialized_plugins: - if isinstance(plugin, MetadataSource) and \ - plugin.metadata_type == metadata_type: - if is_disabled(plugin): - continue - if customize: - customization = config['plugin_customization'] - plugin.site_customization = customization.get(plugin.name, None) - if plugin.name == 'IsbnDB' and isbndb_key is not None: - plugin.site_customization = isbndb_key - yield plugin - -def get_isbndb_key(): - return config['plugin_customization'].get('IsbnDB', None) - -def set_isbndb_key(key): - for plugin in _initialized_plugins: - if plugin.name == 'IsbnDB': - return customize_plugin(plugin, key) - -def migrate_isbndb_key(): - key = prefs['isbndb_com_key'] - if key: - prefs.set('isbndb_com_key', '') - set_isbndb_key(key) - -def cover_sources(): - customization = config['plugin_customization'] - for plugin in _initialized_plugins: - if isinstance(plugin, CoverDownload): - if not is_disabled(plugin): - plugin.site_customization = customization.get(plugin.name, '') - yield plugin - -# }}} - # Interface Actions # {{{ def interface_actions(): diff --git a/src/calibre/ebooks/metadata/amazon.py b/src/calibre/ebooks/metadata/amazon.py deleted file mode 100644 index 4100439feb..0000000000 --- a/src/calibre/ebooks/metadata/amazon.py +++ /dev/null @@ -1,224 +0,0 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Fetch metadata using Amazon AWS -''' -import sys, re -from threading import RLock - -from lxml import html -from lxml.html import soupparser - -from calibre import browser -from calibre.ebooks.metadata import check_isbn -from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.chardet import xml_to_unicode -from calibre.library.comments import sanitize_comments_html - -asin_cache = {} -cover_url_cache = {} -cache_lock = RLock() - -def find_asin(br, isbn): - q = 'http://www.amazon.com/s/?search-alias=aps&field-keywords='+isbn - res = br.open_novisit(q) - raw = res.read() - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - root = html.fromstring(raw) - revs = root.xpath('//*[@class="asinReviewsSummary" and @name]') - revs = [x.get('name') for x in revs] - if revs: - return revs[0] - -def to_asin(br, isbn): - with cache_lock: - ans = asin_cache.get(isbn, None) - if ans: - return ans - if ans is False: - return None - if len(isbn) == 13: - try: - asin = find_asin(br, isbn) - except: - import traceback - traceback.print_exc() - asin = None - else: - asin = isbn - with cache_lock: - asin_cache[isbn] = asin if asin else False - return asin - - -def get_social_metadata(title, authors, publisher, isbn): - mi = Metadata(title, authors) - if not isbn: - return mi - isbn = check_isbn(isbn) - if not isbn: - return mi - br = browser() - asin = to_asin(br, isbn) - if asin and get_metadata(br, asin, mi): - return mi - from calibre.ebooks.metadata.xisbn import xisbn - for i in xisbn.get_associated_isbns(isbn): - asin = to_asin(br, i) - if asin and get_metadata(br, asin, mi): - return mi - return mi - -def get_cover_url(isbn, br): - isbn = check_isbn(isbn) - if not isbn: - return None - with cache_lock: - ans = cover_url_cache.get(isbn, None) - if ans: - return ans - if ans is False: - return None - asin = to_asin(br, isbn) - if asin: - ans = _get_cover_url(br, asin) - if ans: - with cache_lock: - cover_url_cache[isbn] = ans - return ans - from calibre.ebooks.metadata.xisbn import xisbn - for i in xisbn.get_associated_isbns(isbn): - asin = to_asin(br, i) - if asin: - ans = _get_cover_url(br, asin) - if ans: - with cache_lock: - cover_url_cache[isbn] = ans - cover_url_cache[i] = ans - return ans - with cache_lock: - cover_url_cache[isbn] = False - return None - -def _get_cover_url(br, asin): - q = 'http://amzn.com/'+asin - try: - raw = br.open_novisit(q).read() - except Exception as e: - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return None - raise - if '404 - ' in raw: - return None - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - root = soupparser.fromstring(raw) - except: - return False - - imgs = root.xpath('//img[@id="prodImage" and @src]') - if imgs: - src = imgs[0].get('src') - parts = src.split('/') - if len(parts) > 3: - bn = parts[-1] - sparts = bn.split('_') - if len(sparts) > 2: - bn = sparts[0] + sparts[-1] - return ('/'.join(parts[:-1]))+'/'+bn - return None - - -def get_metadata(br, asin, mi): - q = 'http://amzn.com/'+asin - try: - raw = br.open_novisit(q).read() - except Exception as e: - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return False - raise - if '<title>404 - ' in raw: - return False - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - root = soupparser.fromstring(raw) - except: - return False - if root.xpath('//*[@id="errorMessage"]'): - return False - - ratings = root.xpath('//div[@class="jumpBar"]/descendant::span[@class="asinReviewsSummary"]') - pat = re.compile(r'([0-9.]+) out of (\d+) stars') - if ratings: - for elem in ratings[0].xpath('descendant::*[@title]'): - t = elem.get('title').strip() - m = pat.match(t) - if m is not None: - try: - mi.rating = float(m.group(1))/float(m.group(2)) * 5 - except: - pass - - desc = root.xpath('//div[@id="productDescription"]/*[@class="content"]') - if desc: - desc = desc[0] - for c in desc.xpath('descendant::*[@class="seeAll" or' - ' @class="emptyClear" or @href]'): - c.getparent().remove(c) - desc = html.tostring(desc, method='html', encoding=unicode).strip() - # remove all attributes from tags - desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc) - # Collapse whitespace - #desc = re.sub('\n+', '\n', desc) - #desc = re.sub(' +', ' ', desc) - # Remove the notice about text referring to out of print editions - desc = re.sub(r'(?s)<em>--This text ref.*?</em>', '', desc) - # Remove comments - desc = re.sub(r'(?s)<!--.*?-->', '', desc) - mi.comments = sanitize_comments_html(desc) - - return True - - -def main(args=sys.argv): - import tempfile, os - tdir = tempfile.gettempdir() - br = browser() - for title, isbn in [ - ('The Heroes', '9780316044981'), # Test find_asin - ('Learning Python', '8324616489'), # Test xisbn - ('Angels & Demons', '9781416580829'), # Test sophisticated comment formatting - # Random tests - ('Star Trek: Destiny: Mere Mortals', '9781416551720'), - ('The Great Gatsby', '0743273567'), - ]: - cpath = os.path.join(tdir, title+'.jpg') - curl = get_cover_url(isbn, br) - if curl is None: - print 'No cover found for', title - else: - open(cpath, 'wb').write(br.open_novisit(curl).read()) - print 'Cover for', title, 'saved to', cpath - - #import time - #st = time.time() - mi = get_social_metadata(title, None, None, isbn) - if not mi.comments: - print 'Failed to downlaod social metadata for', title - return 1 - #print '\n\n', time.time() - st, '\n\n' - print mi - print '\n' - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/amazonfr.py b/src/calibre/ebooks/metadata/amazonfr.py deleted file mode 100644 index 248c8d9ed0..0000000000 --- a/src/calibre/ebooks/metadata/amazonfr.py +++ /dev/null @@ -1,516 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2010, sengian <sengian1@gmail.com>' - -import sys, textwrap, re, traceback -from urllib import urlencode -from math import ceil - -from lxml import html -from lxml.html import soupparser - -from calibre.utils.date import parse_date, utcnow, replace_months -from calibre.utils.cleantext import clean_ascii_chars -from calibre import browser, preferred_encoding -from calibre.ebooks.chardet import xml_to_unicode -from calibre.ebooks.metadata import MetaInformation, check_isbn, \ - authors_to_sort_string -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.utils.config import OptionParser -from calibre.library.comments import sanitize_comments_html - - -class AmazonFr(MetadataSource): - - name = 'Amazon French' - description = _('Downloads metadata from amazon.fr') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='fr') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class AmazonEs(MetadataSource): - - name = 'Amazon Spanish' - description = _('Downloads metadata from amazon.com in spanish') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='es') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class AmazonEn(MetadataSource): - - name = 'Amazon English' - description = _('Downloads metadata from amazon.com in english') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='en') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class AmazonDe(MetadataSource): - - name = 'Amazon German' - description = _('Downloads metadata from amazon.de') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='de') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class Amazon(MetadataSource): - - name = 'Amazon' - description = _('Downloads metadata from amazon.com') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Kovid Goyal & Sengian' - version = (1, 1, 0) - has_html_comments = True - - def fetch(self): - # if not self.site_customization: - # return - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose, lang='all') - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - # @property - # def string_customization_help(self): - # return _('You can select here the language for metadata search with amazon.com') - - -def report(verbose): - if verbose: - traceback.print_exc() - - -class Query(object): - - BASE_URL_ALL = 'http://www.amazon.com' - BASE_URL_FR = 'http://www.amazon.fr' - BASE_URL_DE = 'http://www.amazon.de' - - def __init__(self, title=None, author=None, publisher=None, isbn=None, keywords=None, - max_results=20, rlang='all'): - assert not(title is None and author is None and publisher is None \ - and isbn is None and keywords is None) - assert (max_results < 21) - - self.max_results = int(max_results) - self.renbres = re.compile(u'\s*(\d+)\s*') - - q = { 'search-alias' : 'stripbooks' , - 'unfiltered' : '1', - 'field-keywords' : '', - 'field-author' : '', - 'field-title' : '', - 'field-isbn' : '', - 'field-publisher' : '' - #get to amazon detailed search page to get all options - # 'node' : '', - # 'field-binding' : '', - #before, during, after - # 'field-dateop' : '', - #month as number - # 'field-datemod' : '', - # 'field-dateyear' : '', - #french only - # 'field-collection' : '', - #many options available - } - - if rlang =='all': - q['sort'] = 'relevanceexprank' - self.urldata = self.BASE_URL_ALL - elif rlang =='es': - q['sort'] = 'relevanceexprank' - q['field-language'] = 'Spanish' - self.urldata = self.BASE_URL_ALL - elif rlang =='en': - q['sort'] = 'relevanceexprank' - q['field-language'] = 'English' - self.urldata = self.BASE_URL_ALL - elif rlang =='fr': - q['sort'] = 'relevancerank' - self.urldata = self.BASE_URL_FR - elif rlang =='de': - q['sort'] = 'relevancerank' - self.urldata = self.BASE_URL_DE - self.baseurl = self.urldata - - if isbn is not None: - q['field-isbn'] = isbn.replace('-', '') - else: - if title is not None: - q['field-title'] = title - if author is not None: - q['field-author'] = author - if publisher is not None: - q['field-publisher'] = publisher - if keywords is not None: - q['field-keywords'] = keywords - - if isinstance(q, unicode): - q = q.encode('utf-8') - self.urldata += '/gp/search/ref=sr_adv_b/?' + urlencode(q) - - def __call__(self, browser, verbose, timeout = 5.): - if verbose: - print 'Query:', self.urldata - - try: - raw = browser.open_novisit(self.urldata, timeout=timeout).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - raise - if '<title>404 - ' in raw: - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None, self.urldata - - #nb of page - try: - nbresults = self.renbres.findall(feed.xpath("//*[@class='resultCount']")[0].text) - except: - return None, self.urldata - - pages =[feed] - if len(nbresults) > 1: - nbpagetoquery = int(ceil(float(min(int(nbresults[2]), self.max_results))/ int(nbresults[1]))) - for i in xrange(2, nbpagetoquery + 1): - try: - urldata = self.urldata + '&page=' + str(i) - raw = browser.open_novisit(urldata, timeout=timeout).read() - except Exception as e: - continue - if '<title>404 - ' in raw: - continue - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_chars(raw)) - except: - continue - pages.append(feed) - - results = [] - for x in pages: - results.extend([i.getparent().get('href') \ - for i in x.xpath("//a/span[@class='srTitle']")]) - return results[:self.max_results], self.baseurl - -class ResultList(list): - - def __init__(self, baseurl, lang = 'all'): - self.baseurl = baseurl - self.lang = lang - self.repub = re.compile(u'\((.*)\)') - self.rerat = re.compile(u'([0-9.]+)') - self.reattr = re.compile(r'<([a-zA-Z0-9]+)\s[^>]+>') - self.reoutp = re.compile(r'(?s)<em>--This text ref.*?</em>') - self.recom = re.compile(r'(?s)<!--.*?-->') - self.republi = re.compile(u'(Editeur|Publisher|Verlag)', re.I) - self.reisbn = re.compile(u'(ISBN-10|ISBN-10|ASIN)', re.I) - self.relang = re.compile(u'(Language|Langue|Sprache)', re.I) - self.reratelt = re.compile(u'(Average\s*Customer\s*Review|Moyenne\s*des\s*commentaires\s*client|Durchschnittliche\s*Kundenbewertung)', re.I) - self.reprod = re.compile(u'(Product\s*Details|D.tails\s*sur\s*le\s*produit|Produktinformation)', re.I) - - def strip_tags_etree(self, etreeobj, invalid_tags): - for (itag, rmv) in invalid_tags.iteritems(): - if rmv: - for elts in etreeobj.getiterator(itag): - elts.drop_tree() - else: - for elts in etreeobj.getiterator(itag): - elts.drop_tag() - - def clean_entry(self, entry, invalid_tags = {'script': True}, - invalid_id = (), invalid_class=()): - #invalid_tags: remove tag and keep content if False else remove - #remove tags - if invalid_tags: - self.strip_tags_etree(entry, invalid_tags) - #remove id - if invalid_id: - for eltid in invalid_id: - elt = entry.get_element_by_id(eltid) - if elt is not None: - elt.drop_tree() - #remove class - if invalid_class: - for eltclass in invalid_class: - elts = entry.find_class(eltclass) - if elts is not None: - for elt in elts: - elt.drop_tree() - - def get_title(self, entry): - title = entry.get_element_by_id('btAsinTitle') - if title is not None: - title = title.text - return unicode(title.replace('\n', '').strip()) - - def get_authors(self, entry): - author = entry.get_element_by_id('btAsinTitle') - while author.getparent().tag != 'div': - author = author.getparent() - author = author.getparent() - authortext = [] - for x in author.getiterator('a'): - authortext.append(unicode(x.text_content().strip())) - return authortext - - def get_description(self, entry, verbose): - try: - description = entry.get_element_by_id("productDescription").find("div[@class='content']") - inv_class = ('seeAll', 'emptyClear') - inv_tags ={'img': True, 'a': False} - self.clean_entry(description, invalid_tags=inv_tags, invalid_class=inv_class) - description = html.tostring(description, method='html', encoding=unicode).strip() - # remove all attributes from tags - description = self.reattr.sub(r'<\1>', description) - # Remove the notice about text referring to out of print editions - description = self.reoutp.sub('', description) - # Remove comments - description = self.recom.sub('', description) - return unicode(sanitize_comments_html(description)) - except: - report(verbose) - return None - - def get_tags(self, entry, browser, verbose): - try: - tags = entry.get_element_by_id('tagContentHolder') - testptag = tags.find_class('see-all') - if testptag: - for x in testptag: - alink = x.xpath('descendant-or-self::a') - if alink: - if alink[0].get('class') == 'tgJsActive': - continue - link = self.baseurl + alink[0].get('href') - entry = self.get_individual_metadata(browser, link, verbose) - tags = entry.get_element_by_id('tagContentHolder') - break - tags = [a.text for a in tags.getiterator('a') if a.get('rel') == 'tag'] - except: - report(verbose) - tags = [] - return tags - - def get_book_info(self, entry, mi, verbose): - try: - entry = entry.get_element_by_id('SalesRank').getparent() - except: - try: - for z in entry.getiterator('h2'): - if self.reprod.search(z.text_content()): - entry = z.getparent().find("div[@class='content']/ul") - break - except: - report(verbose) - return mi - elts = entry.findall('li') - #pub & date - elt = filter(lambda x: self.republi.search(x.find('b').text), elts) - if elt: - pub = elt[0].find('b').tail - mi.publisher = unicode(self.repub.sub('', pub).strip()) - d = self.repub.search(pub) - if d is not None: - d = d.group(1) - try: - default = utcnow().replace(day=15) - if self.lang != 'all': - d = replace_months(d, self.lang) - d = parse_date(d, assume_utc=True, default=default) - mi.pubdate = d - except: - report(verbose) - #ISBN - elt = filter(lambda x: self.reisbn.search(x.find('b').text), elts) - if elt: - isbn = elt[0].find('b').tail.replace('-', '').strip() - if check_isbn(isbn): - mi.isbn = unicode(isbn) - elif len(elt) > 1: - isbn = elt[1].find('b').tail.replace('-', '').strip() - if check_isbn(isbn): - mi.isbn = unicode(isbn) - #Langue - elt = filter(lambda x: self.relang.search(x.find('b').text), elts) - if elt: - langue = elt[0].find('b').tail.strip() - if langue: - mi.language = unicode(langue) - #ratings - elt = filter(lambda x: self.reratelt.search(x.find('b').text), elts) - if elt: - ratings = elt[0].find_class('swSprite') - if ratings: - ratings = self.rerat.findall(ratings[0].get('title')) - if len(ratings) == 2: - mi.rating = float(ratings[0])/float(ratings[1]) * 5 - return mi - - def fill_MI(self, entry, title, authors, browser, verbose): - mi = MetaInformation(title, authors) - mi.author_sort = authors_to_sort_string(authors) - mi.comments = self.get_description(entry, verbose) - mi = self.get_book_info(entry, mi, verbose) - mi.tags = self.get_tags(entry, browser, verbose) - return mi - - def get_individual_metadata(self, browser, linkdata, verbose): - try: - raw = browser.open_novisit(linkdata).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - raise - if '<title>404 - ' in raw: - report(verbose) - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - return soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_chars(raw)) - except: - report(verbose) - return - - def populate(self, entries, browser, verbose=False): - for x in entries: - try: - entry = self.get_individual_metadata(browser, x, verbose) - # clean results - # inv_ids = ('divsinglecolumnminwidth', 'sims.purchase', 'AutoBuyXGetY', 'A9AdsMiddleBoxTop') - # inv_class = ('buyingDetailsGrid', 'productImageGrid') - # inv_tags ={'script': True, 'style': True, 'form': False} - # self.clean_entry(entry, invalid_id=inv_ids) - title = self.get_title(entry) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - print 'URL who failed:', x - report(verbose) - continue - self.append(self.fill_MI(entry, title, authors, browser, verbose)) - - -def search(title=None, author=None, publisher=None, isbn=None, - max_results=5, verbose=False, keywords=None, lang='all'): - br = browser() - entries, baseurl = Query(title=title, author=author, isbn=isbn, publisher=publisher, - keywords=keywords, max_results=max_results,rlang=lang)(br, verbose) - - if entries is None or len(entries) == 0: - return - - #List of entry - ans = ResultList(baseurl, lang) - ans.populate(entries, br, verbose) - return ans - -def option_parser(): - parser = OptionParser(textwrap.dedent(\ - _('''\ - %prog [options] - - Fetch book metadata from Amazon. You must specify one of title, author, - ISBN, publisher or keywords. Will fetch a maximum of 10 matches, - so you should make your query as specific as possible. - You can chose the language for metadata retrieval: - All & english & french & german & spanish - ''' - ))) - parser.add_option('-t', '--title', help='Book title') - parser.add_option('-a', '--author', help='Book author(s)') - parser.add_option('-p', '--publisher', help='Book publisher') - parser.add_option('-i', '--isbn', help='Book ISBN') - parser.add_option('-k', '--keywords', help='Keywords') - parser.add_option('-m', '--max-results', default=10, - help='Maximum number of results to fetch') - parser.add_option('-l', '--lang', default='all', - help='Chosen language for metadata search (all, en, fr, es, de)') - parser.add_option('-v', '--verbose', default=0, action='count', - help='Be more verbose about errors') - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, isbn=opts.isbn, publisher=opts.publisher, - keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results, - lang=opts.lang) - except AssertionError: - report(True) - parser.print_help() - return 1 - if results is None or len(results) == 0: - print 'No result found for this search!' - return 0 - for result in results: - print unicode(result).encode(preferred_encoding, 'replace') - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py deleted file mode 100644 index 10acff4e61..0000000000 --- a/src/calibre/ebooks/metadata/covers.py +++ /dev/null @@ -1,317 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai - -__license__ = 'GPL v3' -__copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -import traceback, socket, sys -from functools import partial -from threading import Thread, Event -from Queue import Queue, Empty -from lxml import etree - -import mechanize - -from calibre.customize import Plugin -from calibre import browser, prints -from calibre.constants import preferred_encoding, DEBUG - -class CoverDownload(Plugin): - ''' - These plugins are used to download covers for books. - ''' - - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Kovid Goyal' - type = _('Cover download') - - def has_cover(self, mi, ans, timeout=5.): - ''' - Check if the book described by mi has a cover. Call ans.set() if it - does. Do nothing if it doesn't. - - :param mi: MetaInformation object - :param timeout: timeout in seconds - :param ans: A threading.Event object - ''' - raise NotImplementedError() - - def get_covers(self, mi, result_queue, abort, timeout=5.): - ''' - Download covers for books described by the mi object. Downloaded covers - must be put into the result_queue. If more than one cover is available, - the plugin should continue downloading them and putting them into - result_queue until abort.is_set() returns True. - - :param mi: MetaInformation object - :param result_queue: A multithreaded Queue - :param abort: A threading.Event object - :param timeout: timeout in seconds - ''' - raise NotImplementedError() - - def exception_to_string(self, ex): - try: - return unicode(ex) - except: - try: - return str(ex).decode(preferred_encoding, 'replace') - except: - return repr(ex) - - def debug(self, *args, **kwargs): - if DEBUG: - prints('\t'+self.name+':', *args, **kwargs) - - - -class HeadRequest(mechanize.Request): - - def get_method(self): - return 'HEAD' - -class OpenLibraryCovers(CoverDownload): # {{{ - 'Download covers from openlibrary.org' - - # See http://openlibrary.org/dev/docs/api/covers - - OPENLIBRARY = 'http://covers.openlibrary.org/b/isbn/%s-L.jpg?default=false' - name = 'openlibrary.org covers' - description = _('Download covers from openlibrary.org') - author = 'Kovid Goyal' - - def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: - return False - from calibre.ebooks.metadata.library_thing import get_browser - br = get_browser() - br.set_handle_redirect(False) - try: - br.open_novisit(HeadRequest(self.OPENLIBRARY%mi.isbn), timeout=timeout) - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception as e: - if callable(getattr(e, 'getcode', None)) and e.getcode() == 302: - self.debug('cover for', mi.isbn, 'found') - ans.set() - else: - self.debug(e) - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - from calibre.ebooks.metadata.library_thing import get_browser - br = get_browser() - try: - ans = br.open(self.OPENLIBRARY%mi.isbn, timeout=timeout).read() - result_queue.put((True, ans, 'jpg', self.name)) - except Exception as e: - if callable(getattr(e, 'getcode', None)) and e.getcode() == 404: - result_queue.put((False, _('ISBN: %s not found')%mi.isbn, '', self.name)) - else: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) - -# }}} - -class AmazonCovers(CoverDownload): # {{{ - - name = 'amazon.com covers' - description = _('Download covers from amazon.com') - author = 'Kovid Goyal' - - - def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: - return False - from calibre.ebooks.metadata.amazon import get_cover_url - br = browser() - try: - get_cover_url(mi.isbn, br) - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception as e: - self.debug(e) - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - from calibre.ebooks.metadata.amazon import get_cover_url - br = browser() - try: - url = get_cover_url(mi.isbn, br) - if url is None: - raise ValueError('No cover found for ISBN: %s'%mi.isbn) - cover_data = br.open_novisit(url).read() - result_queue.put((True, cover_data, 'jpg', self.name)) - except Exception as e: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) - -# }}} - -def check_for_cover(mi, timeout=5.): # {{{ - from calibre.customize.ui import cover_sources - ans = Event() - checkers = [partial(p.has_cover, mi, ans, timeout=timeout) for p in - cover_sources()] - workers = [Thread(target=c) for c in checkers] - for w in workers: - w.daemon = True - w.start() - while not ans.is_set(): - ans.wait(0.1) - if sum([int(w.is_alive()) for w in workers]) == 0: - break - return ans.is_set() - -# }}} - -def download_covers(mi, result_queue, max_covers=50, timeout=5.): # {{{ - from calibre.customize.ui import cover_sources - abort = Event() - temp = Queue() - getters = [partial(p.get_covers, mi, temp, abort, timeout=timeout) for p in - cover_sources()] - workers = [Thread(target=c) for c in getters] - for w in workers: - w.daemon = True - w.start() - count = 0 - while count < max_covers: - try: - result = temp.get_nowait() - if result[0]: - count += 1 - result_queue.put(result) - except Empty: - pass - if sum([int(w.is_alive()) for w in workers]) == 0: - break - - abort.set() - - while True: - try: - result = temp.get_nowait() - count += 1 - result_queue.put(result) - except Empty: - break - -# }}} - -class DoubanCovers(CoverDownload): # {{{ - 'Download covers from Douban.com' - - DOUBAN_ISBN_URL = 'http://api.douban.com/book/subject/isbn/' - CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d' - name = 'Douban.com covers' - description = _('Download covers from Douban.com') - author = 'Li Fanxi' - - def get_cover_url(self, isbn, br, timeout=5.): - try: - url = self.DOUBAN_ISBN_URL + isbn + "?apikey=" + self.CALIBRE_DOUBAN_API_KEY - src = br.open(url, timeout=timeout).read() - except Exception as err: - if isinstance(getattr(err, 'args', [None])[0], socket.timeout): - err = Exception(_('Douban.com API timed out. Try again later.')) - raise err - else: - feed = etree.fromstring(src) - NAMESPACES = { - 'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/', - 'atom' : 'http://www.w3.org/2005/Atom', - 'db': 'http://www.douban.com/xmlns/' - } - XPath = partial(etree.XPath, namespaces=NAMESPACES) - entries = XPath('//atom:entry')(feed) - if len(entries) < 1: - return None - try: - cover_url = XPath("descendant::atom:link[@rel='image']/attribute::href") - u = cover_url(entries[0])[0].replace('/spic/', '/lpic/'); - # If URL contains "book-default", the book doesn't have a cover - if u.find('book-default') != -1: - return None - except: - return None - return u - - def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: - return False - br = browser() - try: - if self.get_cover_url(mi.isbn, br, timeout=timeout) != None: - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception as e: - self.debug(e) - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - br = browser() - try: - url = self.get_cover_url(mi.isbn, br, timeout=timeout) - cover_data = br.open_novisit(url).read() - result_queue.put((True, cover_data, 'jpg', self.name)) - except Exception as e: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) -# }}} - -def download_cover(mi, timeout=5.): # {{{ - results = Queue() - download_covers(mi, results, max_covers=1, timeout=timeout) - errors, ans = [], None - while True: - try: - x = results.get_nowait() - if x[0]: - ans = x[1] - else: - errors.append(x) - except Empty: - break - return ans, errors - -# }}} - -def test(isbns): # {{{ - from calibre.ebooks.metadata import MetaInformation - mi = MetaInformation('test', ['test']) - for isbn in isbns: - prints('Testing ISBN:', isbn) - mi.isbn = isbn - found = check_for_cover(mi) - prints('Has cover:', found) - ans, errors = download_cover(mi) - if ans is not None: - prints('Cover downloaded') - else: - prints('Download failed:') - for err in errors: - prints('\t', err[-1]+':', err[1]) - print '\n' - -# }}} - -if __name__ == '__main__': - isbns = sys.argv[1:] + ['9781591025412', '9780307272119'] - #test(isbns) - - from calibre.ebooks.metadata import MetaInformation - oc = OpenLibraryCovers(None) - for isbn in isbns: - mi = MetaInformation('xx', ['yy']) - mi.isbn = isbn - rq = Queue() - oc.get_covers(mi, rq, Event()) - result = rq.get_nowait() - if not result[0]: - print 'Failed for ISBN:', isbn - print result diff --git a/src/calibre/ebooks/metadata/douban.py b/src/calibre/ebooks/metadata/douban.py deleted file mode 100644 index 98a51f69d1..0000000000 --- a/src/calibre/ebooks/metadata/douban.py +++ /dev/null @@ -1,263 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>; 2010, Li Fanxi <lifanxi@freemindworld.com>' -__docformat__ = 'restructuredtext en' - -import sys, textwrap -import traceback -from urllib import urlencode -from functools import partial -from lxml import etree - -from calibre import browser, preferred_encoding -from calibre.ebooks.metadata import MetaInformation -from calibre.utils.config import OptionParser -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.utils.date import parse_date, utcnow - -NAMESPACES = { - 'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/', - 'atom' : 'http://www.w3.org/2005/Atom', - 'db': 'http://www.douban.com/xmlns/' - } -XPath = partial(etree.XPath, namespaces=NAMESPACES) -total_results = XPath('//openSearch:totalResults') -start_index = XPath('//openSearch:startIndex') -items_per_page = XPath('//openSearch:itemsPerPage') -entry = XPath('//atom:entry') -entry_id = XPath('descendant::atom:id') -title = XPath('descendant::atom:title') -description = XPath('descendant::atom:summary') -publisher = XPath("descendant::db:attribute[@name='publisher']") -isbn = XPath("descendant::db:attribute[@name='isbn13']") -date = XPath("descendant::db:attribute[@name='pubdate']") -creator = XPath("descendant::db:attribute[@name='author']") -tag = XPath("descendant::db:tag") - -CALIBRE_DOUBAN_API_KEY = '0bd1672394eb1ebf2374356abec15c3d' - -class DoubanBooks(MetadataSource): - - name = 'Douban Books' - description = _('Downloads metadata from Douban.com') - supported_platforms = ['windows', 'osx', 'linux'] # Platforms this plugin will run on - author = 'Li Fanxi <lifanxi@freemindworld.com>' # The author of this plugin - version = (1, 0, 1) # The version number of this plugin - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, - verbose=self.verbose) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -def report(verbose): - if verbose: - import traceback - traceback.print_exc() - -class Query(object): - - SEARCH_URL = 'http://api.douban.com/book/subjects?' - ISBN_URL = 'http://api.douban.com/book/subject/isbn/' - - type = "search" - - def __init__(self, title=None, author=None, publisher=None, isbn=None, - max_results=20, start_index=1, api_key=''): - assert not(title is None and author is None and publisher is None and \ - isbn is None) - assert (int(max_results) < 21) - q = '' - if isbn is not None: - q = isbn - self.type = 'isbn' - else: - def build_term(parts): - return ' '.join(x for x in parts) - if title is not None: - q += build_term(title.split()) - if author is not None: - q += (' ' if q else '') + build_term(author.split()) - if publisher is not None: - q += (' ' if q else '') + build_term(publisher.split()) - self.type = 'search' - - if isinstance(q, unicode): - q = q.encode('utf-8') - - if self.type == "isbn": - self.url = self.ISBN_URL + q - if api_key != '': - self.url = self.url + "?apikey=" + api_key - else: - self.url = self.SEARCH_URL+urlencode({ - 'q':q, - 'max-results':max_results, - 'start-index':start_index, - }) - if api_key != '': - self.url = self.url + "&apikey=" + api_key - - def __call__(self, browser, verbose): - if verbose: - print 'Query:', self.url - if self.type == "search": - feed = etree.fromstring(browser.open(self.url).read()) - total = int(total_results(feed)[0].text) - start = int(start_index(feed)[0].text) - entries = entry(feed) - new_start = start + len(entries) - if new_start > total: - new_start = 0 - return entries, new_start - elif self.type == "isbn": - feed = etree.fromstring(browser.open(self.url).read()) - entries = entry(feed) - return entries, 0 - -class ResultList(list): - - def get_description(self, entry, verbose): - try: - desc = description(entry) - if desc: - return 'SUMMARY:\n'+desc[0].text - except: - report(verbose) - - def get_title(self, entry): - candidates = [x.text for x in title(entry)] - return ': '.join(candidates) - - def get_authors(self, entry): - m = creator(entry) - if not m: - m = [] - m = [x.text for x in m] - return m - - def get_tags(self, entry, verbose): - try: - btags = [x.attrib["name"] for x in tag(entry)] - tags = [] - for t in btags: - tags.extend([y.strip() for y in t.split('/')]) - tags = list(sorted(list(set(tags)))) - except: - report(verbose) - tags = [] - return [x.replace(',', ';') for x in tags] - - def get_publisher(self, entry, verbose): - try: - pub = publisher(entry)[0].text - except: - pub = None - return pub - - def get_isbn(self, entry, verbose): - try: - isbn13 = isbn(entry)[0].text - except Exception: - isbn13 = None - return isbn13 - - def get_date(self, entry, verbose): - try: - d = date(entry) - if d: - default = utcnow().replace(day=15) - d = parse_date(d[0].text, assume_utc=True, default=default) - else: - d = None - except: - report(verbose) - d = None - return d - - def populate(self, entries, browser, verbose=False, api_key=''): - for x in entries: - try: - id_url = entry_id(x)[0].text - title = self.get_title(x) - except: - report(verbose) - mi = MetaInformation(title, self.get_authors(x)) - try: - if api_key != '': - id_url = id_url + "?apikey=" + api_key - raw = browser.open(id_url).read() - feed = etree.fromstring(raw) - x = entry(feed)[0] - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - mi.comments = self.get_description(x, verbose) - mi.tags = self.get_tags(x, verbose) - mi.isbn = self.get_isbn(x, verbose) - mi.publisher = self.get_publisher(x, verbose) - mi.pubdate = self.get_date(x, verbose) - self.append(mi) - -def search(title=None, author=None, publisher=None, isbn=None, - verbose=False, max_results=40, api_key=None): - br = browser() - start, entries = 1, [] - - if api_key is None: - api_key = CALIBRE_DOUBAN_API_KEY - - while start > 0 and len(entries) <= max_results: - new, start = Query(title=title, author=author, publisher=publisher, - isbn=isbn, max_results=max_results, start_index=start, api_key=api_key)(br, verbose) - if not new: - break - entries.extend(new) - - entries = entries[:max_results] - - ans = ResultList() - ans.populate(entries, br, verbose, api_key) - return ans - -def option_parser(): - parser = OptionParser(textwrap.dedent( - '''\ - %prog [options] - - Fetch book metadata from Douban. You must specify one of title, author, - publisher or ISBN. If you specify ISBN the others are ignored. Will - fetch a maximum of 100 matches, so you should make your query as - specific as possible. - ''' - )) - parser.add_option('-t', '--title', help='Book title') - parser.add_option('-a', '--author', help='Book author(s)') - parser.add_option('-p', '--publisher', help='Book publisher') - parser.add_option('-i', '--isbn', help='Book ISBN') - parser.add_option('-m', '--max-results', default=10, - help='Maximum number of results to fetch') - parser.add_option('-v', '--verbose', default=0, action='count', - help='Be more verbose about errors') - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, opts.publisher, opts.isbn, - verbose=opts.verbose, max_results=int(opts.max_results)) - except AssertionError: - report(True) - parser.print_help() - return 1 - for result in results: - print unicode(result).encode(preferred_encoding) - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py deleted file mode 100644 index e1fac50d16..0000000000 --- a/src/calibre/ebooks/metadata/fetch.py +++ /dev/null @@ -1,523 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -import traceback, sys, textwrap, re -from threading import Thread - -from calibre import prints -from calibre.utils.config import OptionParser -from calibre.utils.logging import default_log -from calibre.utils.titlecase import titlecase -from calibre.customize import Plugin -from calibre.ebooks.metadata.covers import check_for_cover -from calibre.utils.html2text import html2text - -metadata_config = None - -class MetadataSource(Plugin): # {{{ - ''' - Represents a source to query for metadata. Subclasses must implement - at least the fetch method. - - When :meth:`fetch` is called, the `self` object will have the following - useful attributes (each of which may be None):: - - title, book_author, publisher, isbn, log, verbose and extra - - Use these attributes to construct the search query. extra is reserved for - future use. - - The fetch method must store the results in `self.results` as a list of - :class:`Metadata` objects. If there is an error, it should be stored - in `self.exception` and `self.tb` (for the traceback). - ''' - - author = 'Kovid Goyal' - - supported_platforms = ['windows', 'osx', 'linux'] - - #: The type of metadata fetched. 'basic' means basic metadata like - #: title/author/isbn/etc. 'social' means social metadata like - #: tags/rating/reviews/etc. - metadata_type = 'basic' - - #: If not None, the customization dialog will allow for string - #: based customization as well the default customization. The - #: string customization will be saved in the site_customization - #: member. - string_customization_help = None - - #: Set this to true if your plugin returns HTML markup in comments. - #: Then if the user disables HTML, calibre will automagically convert - #: the HTML to Markdown. - has_html_comments = False - - type = _('Metadata download') - - def __call__(self, title, author, publisher, isbn, verbose, log=None, - extra=None): - self.worker = Thread(target=self._fetch) - self.worker.daemon = True - self.title = title - self.verbose = verbose - self.book_author = author - self.publisher = publisher - self.isbn = isbn - self.log = log if log is not None else default_log - self.extra = extra - self.exception, self.tb, self.results = None, None, [] - self.worker.start() - - def _fetch(self): - try: - self.fetch() - if self.results: - c = self.config_store().get(self.name, {}) - res = self.results - if hasattr(res, 'authors'): - res = [res] - for mi in res: - if not c.get('rating', True): - mi.rating = None - if not c.get('comments', True): - mi.comments = None - if not c.get('tags', True): - mi.tags = [] - if self.has_html_comments and mi.comments and \ - c.get('textcomments', False): - try: - mi.comments = html2text(mi.comments) - except: - traceback.print_exc() - mi.comments = None - - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - def fetch(self): - ''' - All the actual work is done here. - ''' - raise NotImplementedError - - def join(self): - return self.worker.join() - - def is_alive(self): - return self.worker.is_alive() - - def is_customizable(self): - return True - - def config_store(self): - global metadata_config - if metadata_config is None: - from calibre.utils.config import XMLConfig - metadata_config = XMLConfig('plugins/metadata_download') - return metadata_config - - def config_widget(self): - from PyQt4.Qt import QWidget, QVBoxLayout, QLabel, Qt, QLineEdit, \ - QCheckBox - from calibre.customize.ui import config - w = QWidget() - w._layout = QVBoxLayout(w) - w.setLayout(w._layout) - if self.string_customization_help is not None: - w._sc_label = QLabel(self.string_customization_help, w) - w._layout.addWidget(w._sc_label) - customization = config['plugin_customization'] - def_sc = customization.get(self.name, '') - if not def_sc: - def_sc = '' - w._sc = QLineEdit(def_sc, w) - w._layout.addWidget(w._sc) - w._sc_label.setWordWrap(True) - w._sc_label.setTextInteractionFlags(Qt.LinksAccessibleByMouse - | Qt.LinksAccessibleByKeyboard) - w._sc_label.setOpenExternalLinks(True) - c = self.config_store() - c = c.get(self.name, {}) - for x, l in {'rating':_('ratings'), 'tags':_('tags'), - 'comments':_('description/reviews')}.items(): - cb = QCheckBox(_('Download %s from %s')%(l, - self.name)) - setattr(w, '_'+x, cb) - cb.setChecked(c.get(x, True)) - w._layout.addWidget(cb) - - if self.has_html_comments: - cb = QCheckBox(_('Convert comments downloaded from %s to plain text')%(self.name)) - setattr(w, '_textcomments', cb) - cb.setChecked(c.get('textcomments', False)) - w._layout.addWidget(cb) - - return w - - def save_settings(self, w): - dl_settings = {} - for x in ('rating', 'tags', 'comments'): - dl_settings[x] = getattr(w, '_'+x).isChecked() - if self.has_html_comments: - dl_settings['textcomments'] = getattr(w, '_textcomments').isChecked() - c = self.config_store() - c.set(self.name, dl_settings) - if hasattr(w, '_sc'): - sc = unicode(w._sc.text()).strip() - from calibre.customize.ui import customize_plugin - customize_plugin(self, sc) - - def customization_help(self): - return 'This plugin can only be customized using the GUI' - - # }}} - -class GoogleBooks(MetadataSource): # {{{ - - name = 'Google Books' - description = _('Downloads metadata from Google Books') - - def fetch(self): - from calibre.ebooks.metadata.google_books import search - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, - verbose=self.verbose) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - # }}} - -class ISBNDB(MetadataSource): # {{{ - - name = 'IsbnDB' - description = _('Downloads metadata from isbndb.com') - - def fetch(self): - if not self.site_customization: - return - from calibre.ebooks.metadata.isbndb import option_parser, create_books - args = ['isbndb'] - if self.isbn: - args.extend(['--isbn', self.isbn]) - else: - if self.title: - args.extend(['--title', self.title]) - if self.book_author: - args.extend(['--author', self.book_author]) - if self.publisher: - args.extend(['--publisher', self.publisher]) - if self.verbose: - args.extend(['--verbose']) - args.append(self.site_customization) # IsbnDb key - try: - opts, args = option_parser().parse_args(args) - self.results = create_books(opts, args) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - @property - def string_customization_help(self): - ans = _('To use isbndb.com you must sign up for a %sfree account%s ' - 'and enter your access key below.') - return '<p>'+ans%('<a href="http://www.isbndb.com">', '</a>') - - # }}} - -class Amazon(MetadataSource): # {{{ - - name = 'Amazon' - metadata_type = 'social' - description = _('Downloads social metadata from amazon.com') - - has_html_comments = True - - def fetch(self): - if not self.isbn: - return - from calibre.ebooks.metadata.amazon import get_social_metadata - try: - self.results = get_social_metadata(self.title, self.book_author, - self.publisher, self.isbn) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - # }}} - -class KentDistrictLibrary(MetadataSource): # {{{ - - name = 'Kent District Library' - metadata_type = 'social' - description = _('Downloads series information from ww2.kdl.org. ' - 'This website cannot handle large numbers of queries, ' - 'so the plugin is disabled by default.') - - def fetch(self): - if not self.title or not self.book_author: - return - from calibre.ebooks.metadata.kdl import get_series - try: - self.results = get_series(self.title, self.book_author) - except Exception as e: - import traceback - traceback.print_exc() - self.exception = e - self.tb = traceback.format_exc() - - # }}} - - -def result_index(source, result): - if not result.isbn: - return -1 - for i, x in enumerate(source): - if x.isbn == result.isbn: - return i - return -1 - -def merge_results(one, two): - if two is not None and one is not None: - for x in two: - idx = result_index(one, x) - if idx < 0: - one.append(x) - else: - one[idx].smart_update(x) - -class MetadataSources(object): - - def __init__(self, sources): - self.sources = sources - - def __enter__(self): - for s in self.sources: - s.__enter__() - return self - - def __exit__(self, *args): - for s in self.sources: - s.__exit__() - - def __call__(self, *args, **kwargs): - for s in self.sources: - s(*args, **kwargs) - - def join(self): - for s in self.sources: - s.join() - -def filter_metadata_results(item): - keywords = ["audio", "tape", "cassette", "abridged", "playaway"] - for keyword in keywords: - if item.publisher and keyword in item.publisher.lower(): - return False - return True - -def do_cover_check(item): - item.has_cover = False - try: - item.has_cover = check_for_cover(item) - except: - pass # Cover not found - -def check_for_covers(items): - threads = [Thread(target=do_cover_check, args=(item,)) for item in items] - for t in threads: t.start() - for t in threads: t.join() - -def search(title=None, author=None, publisher=None, isbn=None, isbndb_key=None, - verbose=0): - assert not(title is None and author is None and publisher is None and \ - isbn is None) - from calibre.customize.ui import metadata_sources, migrate_isbndb_key - migrate_isbndb_key() - if isbn is not None: - isbn = re.sub(r'[^a-zA-Z0-9]', '', isbn).upper() - fetchers = list(metadata_sources(isbndb_key=isbndb_key)) - with MetadataSources(fetchers) as manager: - manager(title, author, publisher, isbn, verbose) - manager.join() - - results = list(fetchers[0].results) if fetchers else [] - for fetcher in fetchers[1:]: - merge_results(results, fetcher.results) - - results = list(filter(filter_metadata_results, results)) - - check_for_covers(results) - - words = ("the", "a", "an", "of", "and") - prefix_pat = re.compile(r'^(%s)\s+'%("|".join(words))) - trailing_paren_pat = re.compile(r'\(.*\)$') - whitespace_pat = re.compile(r'\s+') - - def sort_func(x, y): - - def cleanup_title(s): - if s is None: - s = _('Unknown') - s = s.strip().lower() - s = prefix_pat.sub(' ', s) - s = trailing_paren_pat.sub('', s) - s = whitespace_pat.sub(' ', s) - return s.strip() - - t = cleanup_title(title) - x_title = cleanup_title(x.title) - y_title = cleanup_title(y.title) - - # prefer titles that start with the search title - tx = cmp(t, x_title) - ty = cmp(t, y_title) - result = 0 if abs(tx) == abs(ty) else abs(tx) - abs(ty) - - # then prefer titles that have a cover image - if result == 0: - result = -cmp(x.has_cover, y.has_cover) - - # then prefer titles with the longest comment, with in 10% - if result == 0: - cx = len(x.comments.strip() if x.comments else '') - cy = len(y.comments.strip() if y.comments else '') - t = (cx + cy) / 20 - result = cy - cx - if abs(result) < t: - result = 0 - - return result - - results = sorted(results, cmp=sort_func) - - # if for some reason there is no comment in the top selection, go looking for one - if len(results) > 1: - if not results[0].comments or len(results[0].comments) == 0: - for r in results[1:]: - try: - if title and title.lower() == r.title[:len(title)].lower() \ - and r.comments and len(r.comments): - results[0].comments = r.comments - break - except: - pass - # Find a pubdate - pubdate = None - for r in results: - if r.pubdate is not None: - pubdate = r.pubdate - break - if pubdate is not None: - for r in results: - if r.pubdate is None: - r.pubdate = pubdate - - def fix_case(x): - if x: - x = titlecase(x) - return x - - for r in results: - r.title = fix_case(r.title) - if r.authors: - r.authors = list(map(fix_case, r.authors)) - - return results, [(x.name, x.exception, x.tb) for x in fetchers] - -def get_social_metadata(mi, verbose=0): - from calibre.customize.ui import metadata_sources - fetchers = list(metadata_sources(metadata_type='social')) - with MetadataSources(fetchers) as manager: - manager(mi.title, mi.authors, mi.publisher, mi.isbn, verbose) - manager.join() - ratings, tags, comments, series, series_index = [], set([]), set([]), None, None - for fetcher in fetchers: - if fetcher.results: - dmi = fetcher.results - if dmi.rating is not None: - ratings.append(dmi.rating) - if dmi.tags: - for t in dmi.tags: - tags.add(t) - if mi.pubdate is None and dmi.pubdate is not None: - mi.pubdate = dmi.pubdate - if dmi.comments: - comments.add(dmi.comments) - if dmi.series is not None: - series = dmi.series - if dmi.series_index is not None: - series_index = dmi.series_index - if ratings: - rating = sum(ratings)/float(len(ratings)) - if mi.rating is None or mi.rating < 0.1: - mi.rating = rating - else: - mi.rating = (mi.rating + rating)/2.0 - if tags: - if not mi.tags: - mi.tags = [] - mi.tags += list(tags) - mi.tags = list(sorted(list(set(mi.tags)))) - if comments: - if not mi.comments or len(mi.comments)+20 < len(' '.join(comments)): - mi.comments = '' - for x in comments: - mi.comments += x+'\n\n' - if series and series_index is not None: - mi.series = series - mi.series_index = series_index - - return [(x.name, x.exception, x.tb) for x in fetchers if x.exception is not - None] - - - -def option_parser(): - parser = OptionParser(textwrap.dedent( - '''\ - %prog [options] - - Fetch book metadata from online sources. You must specify at least one - of title, author, publisher or ISBN. If you specify ISBN, the others - are ignored. - ''' - )) - parser.add_option('-t', '--title', help='Book title') - parser.add_option('-a', '--author', help='Book author(s)') - parser.add_option('-p', '--publisher', help='Book publisher') - parser.add_option('-i', '--isbn', help='Book ISBN') - parser.add_option('-m', '--max-results', default=10, - help='Maximum number of results to fetch') - parser.add_option('-k', '--isbndb-key', - help=('The access key for your ISBNDB.com account. ' - 'Only needed if you want to search isbndb.com ' - 'and you haven\'t customized the IsbnDB plugin.')) - parser.add_option('-v', '--verbose', default=0, action='count', - help='Be more verbose about errors') - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - results, exceptions = search(opts.title, opts.author, opts.publisher, - opts.isbn, opts.isbndb_key, opts.verbose) - social_exceptions = [] - for result in results: - social_exceptions.extend(get_social_metadata(result, opts.verbose)) - prints(unicode(result)) - print - - for name, exception, tb in exceptions+social_exceptions: - if exception is not None: - print 'WARNING: Fetching from', name, 'failed with error:' - print exception - print tb - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/fictionwise.py b/src/calibre/ebooks/metadata/fictionwise.py deleted file mode 100644 index 145e39768d..0000000000 --- a/src/calibre/ebooks/metadata/fictionwise.py +++ /dev/null @@ -1,390 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2010, sengian <sengian1@gmail.com>' -__docformat__ = 'restructuredtext en' - -import sys, textwrap, re, traceback, socket -from urllib import urlencode - -from lxml.html import soupparser, tostring - -from calibre import browser, preferred_encoding -from calibre.ebooks.chardet import xml_to_unicode -from calibre.ebooks.metadata import MetaInformation, check_isbn, \ - authors_to_sort_string -from calibre.library.comments import sanitize_comments_html -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.utils.config import OptionParser -from calibre.utils.date import parse_date, utcnow -from calibre.utils.cleantext import clean_ascii_chars - -class Fictionwise(MetadataSource): # {{{ - - author = 'Sengian' - name = 'Fictionwise' - description = _('Downloads metadata from Fictionwise') - - has_html_comments = True - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - # }}} - -class FictionwiseError(Exception): - pass - -def report(verbose): - if verbose: - traceback.print_exc() - -class Query(object): - - BASE_URL = 'http://www.fictionwise.com/servlet/mw' - - def __init__(self, title=None, author=None, publisher=None, keywords=None, max_results=20): - assert not(title is None and author is None and publisher is None and keywords is None) - assert (max_results < 21) - - self.max_results = int(max_results) - q = { 'template' : 'searchresults_adv.htm' , - 'searchtitle' : '', - 'searchauthor' : '', - 'searchpublisher' : '', - 'searchkeyword' : '', - #possibilities startoflast, fullname, lastfirst - 'searchauthortype' : 'startoflast', - 'searchcategory' : '', - 'searchcategory2' : '', - 'searchprice_s' : '0', - 'searchprice_e' : 'ANY', - 'searchformat' : '', - 'searchgeo' : 'US', - 'searchfwdatetype' : '', - #maybe use dates fields if needed? - #'sortorder' : 'DESC', - #many options available: b.SortTitle, a.SortName, - #b.DateFirstPublished, b.FWPublishDate - 'sortby' : 'b.SortTitle' - } - if title is not None: - q['searchtitle'] = title - if author is not None: - q['searchauthor'] = author - if publisher is not None: - q['searchpublisher'] = publisher - if keywords is not None: - q['searchkeyword'] = keywords - - if isinstance(q, unicode): - q = q.encode('utf-8') - self.urldata = urlencode(q) - - def __call__(self, browser, verbose, timeout = 5.): - if verbose: - print _('Query: %s') % self.BASE_URL+self.urldata - - try: - raw = browser.open_novisit(self.BASE_URL, self.urldata, timeout=timeout).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - if isinstance(getattr(e, 'args', [None])[0], socket.timeout): - raise FictionwiseError(_('Fictionwise timed out. Try again later.')) - raise FictionwiseError(_('Fictionwise encountered an error.')) - if '<title>404 - ' in raw: - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None - - # get list of results as links - results = feed.xpath("//table[3]/tr/td[2]/table/tr/td/p/table[2]/tr[@valign]") - results = results[:self.max_results] - results = [i.xpath('descendant-or-self::a')[0].get('href') for i in results] - #return feed if no links ie normally a single book or nothing - if not results: - results = [feed] - return results - -class ResultList(list): - - BASE_URL = 'http://www.fictionwise.com' - COLOR_VALUES = {'BLUE': 4, 'GREEN': 3, 'YELLOW': 2, 'RED': 1, 'NA': 0} - - def __init__(self): - self.retitle = re.compile(r'\[[^\[\]]+\]') - self.rechkauth = re.compile(r'.*book\s*by', re.I) - self.redesc = re.compile(r'book\s*description\s*:\s*(<br[^>]+>)*(?P<desc>.*)<br[^>]*>.{,15}publisher\s*:', re.I) - self.repub = re.compile(r'.*publisher\s*:\s*', re.I) - self.redate = re.compile(r'.*release\s*date\s*:\s*', re.I) - self.retag = re.compile(r'.*book\s*category\s*:\s*', re.I) - self.resplitbr = re.compile(r'<br[^>]*>', re.I) - self.recomment = re.compile(r'(?s)<!--.*?-->') - self.reimg = re.compile(r'<img[^>]*>', re.I) - self.resanitize = re.compile(r'\[HTML_REMOVED\]\s*', re.I) - self.renbcom = re.compile('(?P<nbcom>\d+)\s*Reader Ratings:') - self.recolor = re.compile('(?P<ncolor>[^/]+).gif') - self.resplitbrdiv = re.compile(r'(<br[^>]+>|</?div[^>]*>)', re.I) - self.reisbn = re.compile(r'.*ISBN\s*:\s*', re.I) - - def strip_tags_etree(self, etreeobj, invalid_tags): - for (itag, rmv) in invalid_tags.iteritems(): - if rmv: - for elts in etreeobj.getiterator(itag): - elts.drop_tree() - else: - for elts in etreeobj.getiterator(itag): - elts.drop_tag() - - def clean_entry(self, entry, invalid_tags = {'script': True}, - invalid_id = (), invalid_class=(), invalid_xpath = ()): - #invalid_tags: remove tag and keep content if False else remove - #remove tags - if invalid_tags: - self.strip_tags_etree(entry, invalid_tags) - #remove xpath - if invalid_xpath: - for eltid in invalid_xpath: - elt = entry.xpath(eltid) - for el in elt: - el.drop_tree() - #remove id - if invalid_id: - for eltid in invalid_id: - elt = entry.get_element_by_id(eltid) - if elt is not None: - elt.drop_tree() - #remove class - if invalid_class: - for eltclass in invalid_class: - elts = entry.find_class(eltclass) - if elts is not None: - for elt in elts: - elt.drop_tree() - - def output_entry(self, entry, prettyout = True, htmlrm="\d+"): - out = tostring(entry, pretty_print=prettyout) - #try to work around tostring to remove this encoding for exemle - reclean = re.compile('(\n+|\t+|\r+|&#'+htmlrm+';)') - return reclean.sub('', out) - - def get_title(self, entry): - title = entry.findtext('./') - return self.retitle.sub('', title).strip() - - def get_authors(self, entry): - authortext = entry.find('./br').tail - if not self.rechkauth.search(authortext): - return [] - authortext = self.rechkauth.sub('', authortext) - return [a.strip() for a in authortext.split('&')] - - def get_rating(self, entrytable, verbose): - nbcomment = tostring(entrytable.getprevious()) - try: - nbcomment = self.renbcom.search(nbcomment).group("nbcom") - except: - report(verbose) - return None - hval = dict((self.COLOR_VALUES[self.recolor.search(image.get('src', default='NA.gif')).group("ncolor")], - float(image.get('height', default=0))) \ - for image in entrytable.getiterator('img')) - #ratings as x/5 - return float(1.25*sum(k*v for (k, v) in hval.iteritems())/sum(hval.itervalues())) - - def get_description(self, entry): - description = self.output_entry(entry.xpath('./p')[1],htmlrm="") - description = self.redesc.search(description) - if not description or not description.group("desc"): - return None - #remove invalid tags - description = self.reimg.sub('', description.group("desc")) - description = self.recomment.sub('', description) - description = self.resanitize.sub('', sanitize_comments_html(description)) - return _('SUMMARY:\n %s') % re.sub(r'\n\s+</p>','\n</p>', description) - - def get_publisher(self, entry): - publisher = self.output_entry(entry.xpath('./p')[1]) - publisher = filter(lambda x: self.repub.search(x) is not None, - self.resplitbr.split(publisher)) - if not len(publisher): - return None - publisher = self.repub.sub('', publisher[0]) - return publisher.split(',')[0].strip() - - def get_tags(self, entry): - tag = self.output_entry(entry.xpath('./p')[1]) - tag = filter(lambda x: self.retag.search(x) is not None, - self.resplitbr.split(tag)) - if not len(tag): - return [] - return map(lambda x: x.strip(), self.retag.sub('', tag[0]).split('/')) - - def get_date(self, entry, verbose): - date = self.output_entry(entry.xpath('./p')[1]) - date = filter(lambda x: self.redate.search(x) is not None, - self.resplitbr.split(date)) - if not len(date): - return None - try: - d = self.redate.sub('', date[0]) - if d: - default = utcnow().replace(day=15) - d = parse_date(d, assume_utc=True, default=default) - else: - d = None - except: - report(verbose) - d = None - return d - - def get_ISBN(self, entry): - isbns = self.output_entry(entry.xpath('./p')[2]) - isbns = filter(lambda x: self.reisbn.search(x) is not None, - self.resplitbrdiv.split(isbns)) - if not len(isbns): - return None - isbns = [self.reisbn.sub('', x) for x in isbns if check_isbn(self.reisbn.sub('', x))] - return sorted(isbns, cmp=lambda x,y:cmp(len(x), len(y)))[-1] - - def fill_MI(self, entry, title, authors, ratings, verbose): - mi = MetaInformation(title, authors) - mi.rating = ratings - mi.comments = self.get_description(entry) - mi.publisher = self.get_publisher(entry) - mi.tags = self.get_tags(entry) - mi.pubdate = self.get_date(entry, verbose) - mi.isbn = self.get_ISBN(entry) - mi.author_sort = authors_to_sort_string(authors) - return mi - - def get_individual_metadata(self, browser, linkdata, verbose): - try: - raw = browser.open_novisit(self.BASE_URL + linkdata).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - if isinstance(getattr(e, 'args', [None])[0], socket.timeout): - raise FictionwiseError(_('Fictionwise timed out. Try again later.')) - raise FictionwiseError(_('Fictionwise encountered an error.')) - if '<title>404 - ' in raw: - report(verbose) - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - return soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - return soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None - - def populate(self, entries, browser, verbose=False): - inv_tags ={'script': True, 'a': False, 'font': False, 'strong': False, 'b': False, - 'ul': False, 'span': False} - inv_xpath =('./table',) - #single entry - if len(entries) == 1 and not isinstance(entries[0], str): - try: - entry = entries.xpath("//table[3]/tr/td[2]/table[1]/tr/td/font/table/tr/td") - self.clean_entry(entry, invalid_tags=inv_tags, invalid_xpath=inv_xpath) - title = self.get_title(entry) - #maybe strenghten the search - ratings = self.get_rating(entry.xpath("./p/table")[1], verbose) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print _('Failed to get all details for an entry') - print e - return - self.append(self.fill_MI(entry, title, authors, ratings, verbose)) - else: - #multiple entries - for x in entries: - try: - entry = self.get_individual_metadata(browser, x, verbose) - entry = entry.xpath("//table[3]/tr/td[2]/table[1]/tr/td/font/table/tr/td")[0] - self.clean_entry(entry, invalid_tags=inv_tags, invalid_xpath=inv_xpath) - title = self.get_title(entry) - #maybe strenghten the search - ratings = self.get_rating(entry.xpath("./p/table")[1], verbose) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print _('Failed to get all details for an entry') - print e - continue - self.append(self.fill_MI(entry, title, authors, ratings, verbose)) - - -def search(title=None, author=None, publisher=None, isbn=None, - min_viewability='none', verbose=False, max_results=5, - keywords=None): - br = browser() - entries = Query(title=title, author=author, publisher=publisher, - keywords=keywords, max_results=max_results)(br, verbose, timeout = 15.) - - #List of entry - ans = ResultList() - ans.populate(entries, br, verbose) - return ans - - -def option_parser(): - parser = OptionParser(textwrap.dedent(\ - _('''\ - %prog [options] - - Fetch book metadata from Fictionwise. You must specify one of title, author, - or keywords. No ISBN specification possible. Will fetch a maximum of 20 matches, - so you should make your query as specific as possible. - ''') - )) - parser.add_option('-t', '--title', help=_('Book title')) - parser.add_option('-a', '--author', help=_('Book author(s)')) - parser.add_option('-p', '--publisher', help=_('Book publisher')) - parser.add_option('-k', '--keywords', help=_('Keywords')) - parser.add_option('-m', '--max-results', default=20, - help=_('Maximum number of results to fetch')) - parser.add_option('-v', '--verbose', default=0, action='count', - help=_('Be more verbose about errors')) - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, publisher=opts.publisher, - keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results) - except AssertionError: - report(True) - parser.print_help() - return 1 - if results is None or len(results) == 0: - print _('No result found for this search!') - return 0 - for result in results: - print unicode(result).encode(preferred_encoding, 'replace') - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/google_books.py b/src/calibre/ebooks/metadata/google_books.py deleted file mode 100644 index 2e52bf020d..0000000000 --- a/src/calibre/ebooks/metadata/google_books.py +++ /dev/null @@ -1,247 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -import sys, textwrap -from urllib import urlencode -from functools import partial - -from lxml import etree - -from calibre import browser, preferred_encoding -from calibre.ebooks.metadata import MetaInformation -from calibre.utils.config import OptionParser -from calibre.utils.date import parse_date, utcnow - -NAMESPACES = { - 'openSearch':'http://a9.com/-/spec/opensearchrss/1.0/', - 'atom' : 'http://www.w3.org/2005/Atom', - 'dc': 'http://purl.org/dc/terms' - } -XPath = partial(etree.XPath, namespaces=NAMESPACES) - -total_results = XPath('//openSearch:totalResults') -start_index = XPath('//openSearch:startIndex') -items_per_page = XPath('//openSearch:itemsPerPage') -entry = XPath('//atom:entry') -entry_id = XPath('descendant::atom:id') -creator = XPath('descendant::dc:creator') -identifier = XPath('descendant::dc:identifier') -title = XPath('descendant::dc:title') -date = XPath('descendant::dc:date') -publisher = XPath('descendant::dc:publisher') -subject = XPath('descendant::dc:subject') -description = XPath('descendant::dc:description') -language = XPath('descendant::dc:language') - -def report(verbose): - if verbose: - import traceback - traceback.print_exc() - - -class Query(object): - - BASE_URL = 'http://books.google.com/books/feeds/volumes?' - - def __init__(self, title=None, author=None, publisher=None, isbn=None, - max_results=20, min_viewability='none', start_index=1): - assert not(title is None and author is None and publisher is None and \ - isbn is None) - assert (max_results < 21) - assert (min_viewability in ('none', 'partial', 'full')) - q = '' - if isbn is not None: - q += 'isbn:'+isbn - else: - def build_term(prefix, parts): - return ' '.join('in'+prefix + ':' + x for x in parts) - if title is not None: - q += build_term('title', title.split()) - if author is not None: - q += ('+' if q else '')+build_term('author', author.split()) - if publisher is not None: - q += ('+' if q else '')+build_term('publisher', publisher.split()) - - if isinstance(q, unicode): - q = q.encode('utf-8') - self.url = self.BASE_URL+urlencode({ - 'q':q, - 'max-results':max_results, - 'start-index':start_index, - 'min-viewability':min_viewability, - }) - - def __call__(self, browser, verbose): - if verbose: - print 'Query:', self.url - feed = etree.fromstring(browser.open(self.url).read()) - #print etree.tostring(feed, pretty_print=True) - total = int(total_results(feed)[0].text) - start = int(start_index(feed)[0].text) - entries = entry(feed) - new_start = start + len(entries) - if new_start > total: - new_start = 0 - return entries, new_start - - -class ResultList(list): - - def get_description(self, entry, verbose): - try: - desc = description(entry) - if desc: - return 'SUMMARY:\n'+desc[0].text - except: - report(verbose) - - def get_language(self, entry, verbose): - try: - l = language(entry) - if l: - return l[0].text - except: - report(verbose) - - def get_title(self, entry): - candidates = [x.text for x in title(entry)] - return ': '.join(candidates) - - def get_authors(self, entry): - m = creator(entry) - if not m: - m = [] - m = [x.text for x in m] - return m - - def get_author_sort(self, entry, verbose): - for x in creator(entry): - for key, val in x.attrib.items(): - if key.endswith('file-as'): - return val - - def get_identifiers(self, entry, mi): - isbns = [] - for x in identifier(entry): - t = str(x.text).strip() - if t[:5].upper() in ('ISBN:', 'LCCN:', 'OCLC:'): - if t[:5].upper() == 'ISBN:': - isbns.append(t[5:]) - if isbns: - mi.isbn = sorted(isbns, cmp=lambda x,y:cmp(len(x), len(y)))[-1] - - def get_tags(self, entry, verbose): - try: - btags = [x.text for x in subject(entry)] - tags = [] - for t in btags: - tags.extend([y.strip() for y in t.split('/')]) - tags = list(sorted(list(set(tags)))) - except: - report(verbose) - tags = [] - return [x.replace(',', ';') for x in tags] - - def get_publisher(self, entry, verbose): - try: - pub = publisher(entry)[0].text - except: - pub = None - return pub - - def get_date(self, entry, verbose): - try: - d = date(entry) - if d: - default = utcnow().replace(day=15) - d = parse_date(d[0].text, assume_utc=True, default=default) - else: - d = None - except: - report(verbose) - d = None - return d - - def populate(self, entries, browser, verbose=False): - for x in entries: - try: - id_url = entry_id(x)[0].text - title = self.get_title(x) - except: - report(verbose) - mi = MetaInformation(title, self.get_authors(x)) - try: - raw = browser.open(id_url).read() - feed = etree.fromstring(raw) - x = entry(feed)[0] - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - mi.author_sort = self.get_author_sort(x, verbose) - mi.comments = self.get_description(x, verbose) - self.get_identifiers(x, mi) - mi.tags = self.get_tags(x, verbose) - mi.publisher = self.get_publisher(x, verbose) - mi.pubdate = self.get_date(x, verbose) - mi.language = self.get_language(x, verbose) - self.append(mi) - - -def search(title=None, author=None, publisher=None, isbn=None, - min_viewability='none', verbose=False, max_results=40): - br = browser() - br.set_handle_gzip(True) - start, entries = 1, [] - while start > 0 and len(entries) <= max_results: - new, start = Query(title=title, author=author, publisher=publisher, - isbn=isbn, min_viewability=min_viewability)(br, verbose) - if not new: - break - entries.extend(new) - - entries = entries[:max_results] - - ans = ResultList() - ans.populate(entries, br, verbose) - return ans - -def option_parser(): - parser = OptionParser(textwrap.dedent( - '''\ - %prog [options] - - Fetch book metadata from Google. You must specify one of title, author, - publisher or ISBN. If you specify ISBN the others are ignored. Will - fetch a maximum of 100 matches, so you should make your query as - specific as possible. - ''' - )) - parser.add_option('-t', '--title', help='Book title') - parser.add_option('-a', '--author', help='Book author(s)') - parser.add_option('-p', '--publisher', help='Book publisher') - parser.add_option('-i', '--isbn', help='Book ISBN') - parser.add_option('-m', '--max-results', default=10, - help='Maximum number of results to fetch') - parser.add_option('-v', '--verbose', default=0, action='count', - help='Be more verbose about errors') - return parser - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, opts.publisher, opts.isbn, - verbose=opts.verbose, max_results=opts.max_results) - except AssertionError: - report(True) - parser.print_help() - return 1 - for result in results: - print unicode(result).encode(preferred_encoding) - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/isbndb.py b/src/calibre/ebooks/metadata/isbndb.py deleted file mode 100644 index 54cd403c62..0000000000 --- a/src/calibre/ebooks/metadata/isbndb.py +++ /dev/null @@ -1,159 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' -''' -Interface to isbndb.com. My key HLLXQX2A. -''' - -import sys, re -from urllib import quote - -from calibre.utils.config import OptionParser -from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.BeautifulSoup import BeautifulStoneSoup -from calibre import browser - -BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%(key)s&page_number=1&results=subjects,authors,texts&' - -class ISBNDBError(Exception): - pass - -def fetch_metadata(url, max=3, timeout=5.): - books = [] - page_number = 1 - total_results = 31 - br = browser() - while len(books) < total_results and max > 0: - try: - raw = br.open(url, timeout=timeout).read() - except Exception as err: - raise ISBNDBError('Could not fetch ISBNDB metadata. Error: '+str(err)) - soup = BeautifulStoneSoup(raw, - convertEntities=BeautifulStoneSoup.XML_ENTITIES) - book_list = soup.find('booklist') - if book_list is None: - errmsg = soup.find('errormessage').string - raise ISBNDBError('Error fetching metadata: '+errmsg) - total_results = int(book_list['total_results']) - page_number += 1 - np = '&page_number=%s&'%page_number - url = re.sub(r'\&page_number=\d+\&', np, url) - books.extend(book_list.findAll('bookdata')) - max -= 1 - return books - - -class ISBNDBMetadata(Metadata): - - def __init__(self, book): - Metadata.__init__(self, None) - - def tostring(e): - if not hasattr(e, 'string'): - return None - ans = e.string - if ans is not None: - ans = unicode(ans).strip() - if not ans: - ans = None - return ans - - self.isbn = unicode(book.get('isbn13', book.get('isbn'))) - title = tostring(book.find('titlelong')) - if not title: - title = tostring(book.find('title')) - self.title = title - self.title = unicode(self.title).strip() - authors = [] - au = tostring(book.find('authorstext')) - if au: - au = au.strip() - temp = au.split(',') - for au in temp: - if not au: continue - authors.extend([a.strip() for a in au.split('&')]) - if authors: - self.authors = authors - try: - self.author_sort = tostring(book.find('authors').find('person')) - if self.authors and self.author_sort == self.authors[0]: - self.author_sort = None - except: - pass - self.publisher = tostring(book.find('publishertext')) - - summ = tostring(book.find('summary')) - if summ: - self.comments = 'SUMMARY:\n'+summ - - -def build_isbn(base_url, opts): - return base_url + 'index1=isbn&value1='+opts.isbn - -def build_combined(base_url, opts): - query = ' '.join([e for e in (opts.title, opts.author, opts.publisher) \ - if e is not None ]) - query = query.strip() - if len(query) == 0: - raise ISBNDBError('You must specify at least one of --author, --title or --publisher') - - query = re.sub(r'\s+', '+', query) - if isinstance(query, unicode): - query = query.encode('utf-8') - return base_url+'index1=combined&value1='+quote(query, '+') - - -def option_parser(): - parser = OptionParser(usage=\ -_(''' -%prog [options] key - -Fetch metadata for books from isndb.com. You can specify either the -books ISBN ID or its title and author. If you specify the title and author, -then more than one book may be returned. - -key is the account key you generate after signing up for a free account from isbndb.com. - -''')) - parser.add_option('-i', '--isbn', default=None, dest='isbn', - help=_('The ISBN ID of the book you want metadata for.')) - parser.add_option('-a', '--author', dest='author', - default=None, help=_('The author whose book to search for.')) - parser.add_option('-t', '--title', dest='title', - default=None, help=_('The title of the book to search for.')) - parser.add_option('-p', '--publisher', default=None, dest='publisher', - help=_('The publisher of the book to search for.')) - parser.add_option('-v', '--verbose', default=False, - action='store_true', help=_('Verbose processing')) - - return parser - - -def create_books(opts, args, timeout=5.): - base_url = BASE_URL%dict(key=args[1]) - if opts.isbn is not None: - url = build_isbn(base_url, opts) - else: - url = build_combined(base_url, opts) - - if opts.verbose: - print ('ISBNDB query: '+url) - - tans = [ISBNDBMetadata(book) for book in fetch_metadata(url, timeout=timeout)] - #remove duplicates ISBN - return list(dict((book.isbn, book) for book in tans).values()) - -def main(args=sys.argv): - parser = option_parser() - opts, args = parser.parse_args(args) - if len(args) != 2: - parser.print_help() - print ('You must supply the isbndb.com key') - return 1 - - for book in create_books(opts, args): - print unicode(book).encode('utf-8') - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/nicebooks.py b/src/calibre/ebooks/metadata/nicebooks.py deleted file mode 100644 index 2afa6c018a..0000000000 --- a/src/calibre/ebooks/metadata/nicebooks.py +++ /dev/null @@ -1,411 +0,0 @@ -from __future__ import with_statement -__license__ = 'GPL 3' -__copyright__ = '2010, sengian <sengian1@gmail.com>' -__docformat__ = 'restructuredtext en' - -import sys, textwrap, re, traceback, socket -from urllib import urlencode -from math import ceil -from copy import deepcopy - -from lxml.html import soupparser - -from calibre.utils.date import parse_date, utcnow, replace_months -from calibre.utils.cleantext import clean_ascii_chars -from calibre import browser, preferred_encoding -from calibre.ebooks.chardet import xml_to_unicode -from calibre.ebooks.metadata import MetaInformation, check_isbn, \ - authors_to_sort_string -from calibre.ebooks.metadata.fetch import MetadataSource -from calibre.ebooks.metadata.covers import CoverDownload -from calibre.utils.config import OptionParser - -class NiceBooks(MetadataSource): - - name = 'Nicebooks' - description = _('Downloads metadata from french Nicebooks') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - version = (1, 0, 0) - - def fetch(self): - try: - self.results = search(self.title, self.book_author, self.publisher, - self.isbn, max_results=10, verbose=self.verbose) - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - -class NiceBooksCovers(CoverDownload): - - name = 'Nicebooks covers' - description = _('Downloads covers from french Nicebooks') - supported_platforms = ['windows', 'osx', 'linux'] - author = 'Sengian' - type = _('Cover download') - version = (1, 0, 0) - - def has_cover(self, mi, ans, timeout=5.): - if not mi.isbn: - return False - br = browser() - try: - entry = Query(isbn=mi.isbn, max_results=1)(br, False, timeout)[0] - if Covers(mi.isbn)(entry).check_cover(): - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception as e: - self.debug(e) - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - br = browser() - try: - entry = Query(isbn=mi.isbn, max_results=1)(br, False, timeout)[0] - cover_data, ext = Covers(mi.isbn)(entry).get_cover(br, timeout) - if not ext: - ext = 'jpg' - result_queue.put((True, cover_data, ext, self.name)) - except Exception as e: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) - - -class NiceBooksError(Exception): - pass - -class ISBNNotFound(NiceBooksError): - pass - -def report(verbose): - if verbose: - traceback.print_exc() - -class Query(object): - - BASE_URL = 'http://fr.nicebooks.com/' - - def __init__(self, title=None, author=None, publisher=None, isbn=None, keywords=None, max_results=20): - assert not(title is None and author is None and publisher is None \ - and isbn is None and keywords is None) - assert (max_results < 21) - - self.max_results = int(max_results) - - if isbn is not None: - q = isbn - else: - q = ' '.join([i for i in (title, author, publisher, keywords) \ - if i is not None]) - - if isinstance(q, unicode): - q = q.encode('utf-8') - self.urldata = 'search?' + urlencode({'q':q,'s':'Rechercher'}) - - def __call__(self, browser, verbose, timeout = 5.): - if verbose: - print _('Query: %s') % self.BASE_URL+self.urldata - - try: - raw = browser.open_novisit(self.BASE_URL+self.urldata, timeout=timeout).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - if isinstance(getattr(e, 'args', [None])[0], socket.timeout): - raise NiceBooksError(_('Nicebooks timed out. Try again later.')) - raise NiceBooksError(_('Nicebooks encountered an error.')) - if '<title>404 - ' in raw: - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None - - #nb of page to call - try: - nbresults = int(feed.xpath("//div[@id='topbar']/b")[0].text) - except: - #direct hit - return [feed] - - nbpagetoquery = int(ceil(float(min(nbresults, self.max_results))/10)) - pages =[feed] - if nbpagetoquery > 1: - for i in xrange(2, nbpagetoquery + 1): - try: - urldata = self.urldata + '&p=' + str(i) - raw = browser.open_novisit(self.BASE_URL+urldata, timeout=timeout).read() - except Exception as e: - continue - if '<title>404 - ' in raw: - continue - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_chars(raw)) - except: - continue - pages.append(feed) - - results = [] - for x in pages: - results.extend([i.find_class('title')[0].get('href') \ - for i in x.xpath("//ul[@id='results']/li")]) - return results[:self.max_results] - -class ResultList(list): - - BASE_URL = 'http://fr.nicebooks.com' - - def __init__(self): - self.repub = re.compile(u'\s*.diteur\s*', re.I) - self.reauteur = re.compile(u'\s*auteur.*', re.I) - self.reautclean = re.compile(u'\s*\(.*\)\s*') - - def get_title(self, entry): - title = deepcopy(entry) - title.remove(title.find("dl[@title='Informations sur le livre']")) - title = ' '.join([i.text_content() for i in title.iterchildren()]) - return unicode(title.replace('\n', '')) - - def get_authors(self, entry): - author = entry.find("dl[@title='Informations sur le livre']") - authortext = [] - for x in author.getiterator('dt'): - if self.reauteur.match(x.text): - elt = x.getnext() - while elt.tag == 'dd': - authortext.append(unicode(elt.text_content())) - elt = elt.getnext() - break - if len(authortext) == 1: - authortext = [self.reautclean.sub('', authortext[0])] - return authortext - - def get_description(self, entry, verbose): - try: - return u'RESUME:\n' + unicode(entry.getparent().xpath("//p[@id='book-description']")[0].text) - except: - report(verbose) - return None - - def get_book_info(self, entry, mi, verbose): - entry = entry.find("dl[@title='Informations sur le livre']") - for x in entry.getiterator('dt'): - if x.text == 'ISBN': - isbntext = x.getnext().text_content().replace('-', '') - if check_isbn(isbntext): - mi.isbn = unicode(isbntext) - elif self.repub.match(x.text): - mi.publisher = unicode(x.getnext().text_content()) - elif x.text == 'Langue': - mi.language = unicode(x.getnext().text_content()) - elif x.text == 'Date de parution': - d = x.getnext().text_content() - try: - default = utcnow().replace(day=15) - d = replace_months(d, 'fr') - d = parse_date(d, assume_utc=True, default=default) - mi.pubdate = d - except: - report(verbose) - return mi - - def fill_MI(self, entry, title, authors, verbose): - mi = MetaInformation(title, authors) - mi.author_sort = authors_to_sort_string(authors) - mi.comments = self.get_description(entry, verbose) - return self.get_book_info(entry, mi, verbose) - - def get_individual_metadata(self, browser, linkdata, verbose): - try: - raw = browser.open_novisit(self.BASE_URL + linkdata).read() - except Exception as e: - report(verbose) - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return - if isinstance(getattr(e, 'args', [None])[0], socket.timeout): - raise NiceBooksError(_('Nicebooks timed out. Try again later.')) - raise NiceBooksError(_('Nicebooks encountered an error.')) - if '<title>404 - ' in raw: - report(verbose) - return - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - feed = soupparser.fromstring(raw) - except: - try: - #remove ASCII invalid chars - feed = soupparser.fromstring(clean_ascii_chars(raw)) - except: - return None - - # get results - return feed.xpath("//div[@id='container']")[0] - - def populate(self, entries, browser, verbose=False): - #single entry - if len(entries) == 1 and not isinstance(entries[0], str): - try: - entry = entries[0].xpath("//div[@id='container']")[0] - entry = entry.find("div[@id='book-info']") - title = self.get_title(entry) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - return - self.append(self.fill_MI(entry, title, authors, verbose)) - else: - #multiple entries - for x in entries: - try: - entry = self.get_individual_metadata(browser, x, verbose) - entry = entry.find("div[@id='book-info']") - title = self.get_title(entry) - authors = self.get_authors(entry) - except Exception as e: - if verbose: - print 'Failed to get all details for an entry' - print e - continue - self.append(self.fill_MI(entry, title, authors, verbose)) - -class Covers(object): - - def __init__(self, isbn = None): - assert isbn is not None - self.urlimg = '' - self.isbn = isbn - self.isbnf = False - - def __call__(self, entry = None): - try: - self.urlimg = entry.xpath("//div[@id='book-picture']/a")[0].get('href') - except: - return self - isbno = entry.get_element_by_id('book-info').find("dl[@title='Informations sur le livre']") - for x in isbno.getiterator('dt'): - if x.text == 'ISBN' and check_isbn(x.getnext().text_content()): - self.isbnf = True - break - return self - - def check_cover(self): - return True if self.urlimg else False - - def get_cover(self, browser, timeout = 5.): - try: - cover, ext = browser.open_novisit(self.urlimg, timeout=timeout).read(), \ - self.urlimg.rpartition('.')[-1] - return cover, ext if ext else 'jpg' - except Exception as err: - if isinstance(getattr(err, 'args', [None])[0], socket.timeout): - raise NiceBooksError(_('Nicebooks timed out. Try again later.')) - if not len(self.urlimg): - if not self.isbnf: - raise ISBNNotFound(_('ISBN: %s not found.') % self.isbn) - raise NiceBooksError(_('An errror occured with Nicebooks cover fetcher')) - - -def search(title=None, author=None, publisher=None, isbn=None, - max_results=5, verbose=False, keywords=None): - br = browser() - entries = Query(title=title, author=author, isbn=isbn, publisher=publisher, - keywords=keywords, max_results=max_results)(br, verbose,timeout = 10.) - - if entries is None or len(entries) == 0: - return None - - #List of entry - ans = ResultList() - ans.populate(entries, br, verbose) - return ans - -def check_for_cover(isbn): - br = browser() - entry = Query(isbn=isbn, max_results=1)(br, False)[0] - return Covers(isbn)(entry).check_cover() - -def cover_from_isbn(isbn, timeout = 5.): - br = browser() - entry = Query(isbn=isbn, max_results=1)(br, False, timeout)[0] - return Covers(isbn)(entry).get_cover(br, timeout) - - -def option_parser(): - parser = OptionParser(textwrap.dedent(\ - _('''\ - %prog [options] - - Fetch book metadata from Nicebooks. You must specify one of title, author, - ISBN, publisher or keywords. Will fetch a maximum of 20 matches, - so you should make your query as specific as possible. - It can also get covers if the option is activated. - ''') - )) - parser.add_option('-t', '--title', help=_('Book title')) - parser.add_option('-a', '--author', help=_('Book author(s)')) - parser.add_option('-p', '--publisher', help=_('Book publisher')) - parser.add_option('-i', '--isbn', help=_('Book ISBN')) - parser.add_option('-k', '--keywords', help=_('Keywords')) - parser.add_option('-c', '--covers', default=0, - help=_('Covers: 1-Check/ 2-Download')) - parser.add_option('-p', '--coverspath', default='', - help=_('Covers files path')) - parser.add_option('-m', '--max-results', default=20, - help=_('Maximum number of results to fetch')) - parser.add_option('-v', '--verbose', default=0, action='count', - help=_('Be more verbose about errors')) - return parser - -def main(args=sys.argv): - import os - parser = option_parser() - opts, args = parser.parse_args(args) - try: - results = search(opts.title, opts.author, isbn=opts.isbn, publisher=opts.publisher, - keywords=opts.keywords, verbose=opts.verbose, max_results=opts.max_results) - except AssertionError: - report(True) - parser.print_help() - return 1 - if results is None or len(results) == 0: - print _('No result found for this search!') - return 0 - for result in results: - print unicode(result).encode(preferred_encoding, 'replace') - covact = int(opts.covers) - if covact == 1: - textcover = _('No cover found!') - if check_for_cover(result.isbn): - textcover = _('A cover was found for this book') - print textcover - elif covact == 2: - cover_data, ext = cover_from_isbn(result.isbn) - cpath = result.isbn - if len(opts.coverspath): - cpath = os.path.normpath(opts.coverspath + '/' + result.isbn) - oname = os.path.abspath(cpath+'.'+ext) - open(oname, 'wb').write(cover_data) - print _('Cover saved to file '), oname - print - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index e67b87efbd..3eff9b11b3 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -307,7 +307,7 @@ class Source(Plugin): title_patterns = [(re.compile(pat, re.IGNORECASE), repl) for pat, repl in [ # Remove things like: (2010) (Omnibus) etc. - (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|mass\s*market|edition|ed\.)[\])}]', ''), + (r'(?i)[({\[](\d{4}|omnibus|anthology|hardcover|paperback|turtleback|mass\s*market|edition|ed\.)[\])}]', ''), # Remove any strings that contain the substring edition inside # parentheses (r'(?i)[({\[].*?(edition|ed.).*?[\]})]', ''), diff --git a/src/calibre/ebooks/metadata/sources/cli.py b/src/calibre/ebooks/metadata/sources/cli.py index cb422f939d..f8b9c6b7a9 100644 --- a/src/calibre/ebooks/metadata/sources/cli.py +++ b/src/calibre/ebooks/metadata/sources/cli.py @@ -19,13 +19,8 @@ from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.metadata.sources.base import create_log from calibre.ebooks.metadata.sources.identify import identify from calibre.ebooks.metadata.sources.covers import download_cover -from calibre.utils.config import test_eight_code def option_parser(): - if not test_eight_code: - from calibre.ebooks.metadata.fetch import option_parser - return option_parser() - parser = OptionParser(textwrap.dedent( '''\ %prog [options] @@ -48,9 +43,6 @@ def option_parser(): return parser def main(args=sys.argv): - if not test_eight_code: - from calibre.ebooks.metadata.fetch import main - return main(args) parser = option_parser() opts, args = parser.parse_args(args) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index f8dd0693ea..737bf38a56 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -20,9 +20,8 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.utils.filenames import ascii_filename from calibre.constants import preferred_encoding, filesystem_encoding from calibre.gui2.actions import InterfaceAction -from calibre.gui2 import config, question_dialog +from calibre.gui2 import question_dialog from calibre.ebooks.metadata import MetaInformation -from calibre.utils.config import test_eight_code from calibre.ebooks.metadata.sources.base import msprefs def get_filters(): @@ -180,26 +179,17 @@ class AddAction(InterfaceAction): except IndexError: self.gui.library_view.model().books_added(self.isbn_add_dialog.value) self.isbn_add_dialog.accept() - if test_eight_code: - orig = msprefs['ignore_fields'] - new = list(orig) - for x in ('title', 'authors'): - if x in new: - new.remove(x) - msprefs['ignore_fields'] = new - try: - self.gui.iactions['Edit Metadata'].download_metadata( - ids=self.add_by_isbn_ids) - finally: - msprefs['ignore_fields'] = orig - else: - orig = config['overwrite_author_title_metadata'] - config['overwrite_author_title_metadata'] = True - try: - self.gui.iactions['Edit Metadata'].do_download_metadata( - self.add_by_isbn_ids) - finally: - config['overwrite_author_title_metadata'] = orig + orig = msprefs['ignore_fields'] + new = list(orig) + for x in ('title', 'authors'): + if x in new: + new.remove(x) + msprefs['ignore_fields'] = new + try: + self.gui.iactions['Edit Metadata'].download_metadata( + ids=self.add_by_isbn_ids) + finally: + msprefs['ignore_fields'] = orig return diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 3349be8d80..4ab4950179 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -10,15 +10,13 @@ from functools import partial from PyQt4.Qt import Qt, QMenu, QModelIndex, QTimer -from calibre.gui2 import error_dialog, config, Dispatcher, question_dialog -from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog +from calibre.gui2 import error_dialog, Dispatcher, question_dialog from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog from calibre.gui2.dialogs.confirm_delete import confirm from calibre.gui2.dialogs.tag_list_editor import TagListEditor from calibre.gui2.actions import InterfaceAction from calibre.ebooks.metadata import authors_to_string from calibre.utils.icu import sort_key -from calibre.utils.config import test_eight_code class EditMetadataAction(InterfaceAction): @@ -36,22 +34,8 @@ class EditMetadataAction(InterfaceAction): md.addAction(_('Edit metadata in bulk'), partial(self.edit_metadata, False, bulk=True)) md.addSeparator() - if test_eight_code: - dall = self.download_metadata - else: - dall = partial(self.download_metadata_old, False, covers=True) - dident = partial(self.download_metadata_old, False, covers=False) - dcovers = partial(self.download_metadata_old, False, covers=True, - set_metadata=False, set_social_metadata=False) - - md.addAction(_('Download metadata and covers'), dall, + md.addAction(_('Download metadata and covers'), self.download_metadata, Qt.ControlModifier+Qt.Key_D) - if not test_eight_code: - md.addAction(_('Download only metadata'), dident) - md.addAction(_('Download only covers'), dcovers) - md.addAction(_('Download only social metadata'), - partial(self.download_metadata_old, False, covers=False, - set_metadata=False, set_social_metadata=True)) self.metadata_menu = md mb = QMenu() @@ -88,7 +72,7 @@ class EditMetadataAction(InterfaceAction): _('No books selected'), show=True) db = self.gui.library_view.model().db ids = [db.id(row.row()) for row in rows] - from calibre.gui2.metadata.bulk_download2 import start_download + from calibre.gui2.metadata.bulk_download import start_download start_download(self.gui, ids, Dispatcher(self.metadata_downloaded)) @@ -96,7 +80,7 @@ class EditMetadataAction(InterfaceAction): if job.failed: self.gui.job_exception(job, dialog_title=_('Failed to download metadata')) return - from calibre.gui2.metadata.bulk_download2 import get_job_details + from calibre.gui2.metadata.bulk_download import get_job_details id_map, failed_ids, failed_covers, all_failed, det_msg = \ get_job_details(job) if all_failed: @@ -112,8 +96,9 @@ class EditMetadataAction(InterfaceAction): show_copy_button = False if failed_ids or failed_covers: show_copy_button = True + num = len(failed_ids.union(failed_covers)) msg += '<p>'+_('Could not download metadata and/or covers for %d of the books. Click' - ' "Show details" to see which books.')%len(failed_ids) + ' "Show details" to see which books.')%num payload = (id_map, failed_ids, failed_covers) from calibre.gui2.dialogs.message_box import ProceedNotification @@ -158,49 +143,6 @@ class EditMetadataAction(InterfaceAction): self.apply_metadata_changes(id_map) - def download_metadata_old(self, checked, covers=True, set_metadata=True, - set_social_metadata=None): - rows = self.gui.library_view.selectionModel().selectedRows() - if not rows or len(rows) == 0: - d = error_dialog(self.gui, _('Cannot download metadata'), - _('No books selected')) - d.exec_() - return - db = self.gui.library_view.model().db - ids = [db.id(row.row()) for row in rows] - self.do_download_metadata(ids, covers=covers, - set_metadata=set_metadata, - set_social_metadata=set_social_metadata) - - def do_download_metadata(self, ids, covers=True, set_metadata=True, - set_social_metadata=None): - m = self.gui.library_view.model() - db = m.db - if set_social_metadata is None: - get_social_metadata = config['get_social_metadata'] - else: - get_social_metadata = set_social_metadata - from calibre.gui2.metadata.bulk_download import DoDownload - if set_social_metadata is not None and set_social_metadata: - x = _('social metadata') - else: - x = _('covers') if covers and not set_metadata else _('metadata') - title = _('Downloading {0} for {1} book(s)').format(x, len(ids)) - self._download_book_metadata = DoDownload(self.gui, title, db, ids, - get_covers=covers, set_metadata=set_metadata, - get_social_metadata=get_social_metadata) - m.stop_metadata_backup() - try: - self._download_book_metadata.exec_() - finally: - m.start_metadata_backup() - cr = self.gui.library_view.currentIndex().row() - x = self._download_book_metadata - if x.updated: - self.gui.library_view.model().refresh_ids( - x.updated, cr) - if self.gui.cover_flow: - self.gui.cover_flow.dataChanged() # }}} def edit_metadata(self, checked, bulk=None): @@ -227,9 +169,7 @@ class EditMetadataAction(InterfaceAction): list(range(self.gui.library_view.model().rowCount(QModelIndex()))) current_row = row_list.index(cr) - func = (self.do_edit_metadata if test_eight_code else - self.do_edit_metadata_old) - changed, rows_to_refresh = func(row_list, current_row) + changed, rows_to_refresh = self.do_edit_metadata(row_list, current_row) m = self.gui.library_view.model() @@ -244,36 +184,6 @@ class EditMetadataAction(InterfaceAction): m.current_changed(current, previous) self.gui.tags_view.recount() - def do_edit_metadata_old(self, row_list, current_row): - changed = set([]) - db = self.gui.library_view.model().db - - while True: - prev = next_ = None - if current_row > 0: - prev = db.title(row_list[current_row-1]) - if current_row < len(row_list) - 1: - next_ = db.title(row_list[current_row+1]) - - d = MetadataSingleDialog(self.gui, row_list[current_row], db, - prev=prev, next_=next_) - d.view_format.connect(lambda - fmt:self.gui.iactions['View'].view_format(row_list[current_row], - fmt)) - ret = d.exec_() - d.break_cycles() - if ret != d.Accepted: - break - - changed.add(d.id) - self.gui.library_view.model().refresh_ids(list(d.books_to_refresh)) - if d.row_delta == 0: - break - current_row += d.row_delta - self.gui.library_view.set_current_row(current_row) - self.gui.library_view.scroll_to_row(current_row) - return changed, set() - def do_edit_metadata(self, row_list, current_row): from calibre.gui2.metadata.single import edit_metadata db = self.gui.library_view.model().db diff --git a/src/calibre/gui2/dialogs/fetch_metadata.py b/src/calibre/gui2/dialogs/fetch_metadata.py deleted file mode 100644 index 426c7b1d60..0000000000 --- a/src/calibre/gui2/dialogs/fetch_metadata.py +++ /dev/null @@ -1,271 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' -''' -GUI for fetching metadata from servers. -''' - -import time -from threading import Thread - -from PyQt4.QtCore import Qt, QObject, SIGNAL, QVariant, pyqtSignal, \ - QAbstractTableModel, QCoreApplication, QTimer -from PyQt4.QtGui import QDialog, QItemSelectionModel, QIcon - -from calibre.gui2.dialogs.fetch_metadata_ui import Ui_FetchMetadata -from calibre.gui2 import error_dialog, NONE, info_dialog, config -from calibre.gui2.widgets import ProgressIndicator -from calibre import strftime, force_unicode -from calibre.customize.ui import get_isbndb_key, set_isbndb_key -from calibre.utils.icu import sort_key - -_hung_fetchers = set([]) - -class Fetcher(Thread): - - def __init__(self, title, author, publisher, isbn, key): - Thread.__init__(self) - self.daemon = True - self.title = title - self.author = author - self.publisher = publisher - self.isbn = isbn - self.key = key - self.results, self.exceptions = [], [] - - def run(self): - from calibre.ebooks.metadata.fetch import search - self.results, self.exceptions = search(self.title, self.author, - self.publisher, self.isbn, - self.key if self.key else None) - - -class Matches(QAbstractTableModel): - - def __init__(self, matches): - self.matches = matches - self.yes_icon = QVariant(QIcon(I('ok.png'))) - QAbstractTableModel.__init__(self) - - def rowCount(self, *args): - return len(self.matches) - - def columnCount(self, *args): - return 8 - - def headerData(self, section, orientation, role): - if role != Qt.DisplayRole: - return NONE - text = "" - if orientation == Qt.Horizontal: - if section == 0: text = _("Title") - elif section == 1: text = _("Author(s)") - elif section == 2: text = _("Author Sort") - elif section == 3: text = _("Publisher") - elif section == 4: text = _("ISBN") - elif section == 5: text = _("Published") - elif section == 6: text = _("Has Cover") - elif section == 7: text = _("Has Summary") - - return QVariant(text) - else: - return QVariant(section+1) - - def summary(self, row): - return self.matches[row].comments - - def data_as_text(self, book, col): - if col == 0 and book.title is not None: - return book.title - elif col == 1: - return ', '.join(book.authors) - elif col == 2 and book.author_sort is not None: - return book.author_sort - elif col == 3 and book.publisher is not None: - return book.publisher - elif col == 4 and book.isbn is not None: - return book.isbn - elif col == 5 and hasattr(book.pubdate, 'timetuple'): - return strftime('%b %Y', book.pubdate.timetuple()) - elif col == 6 and book.has_cover: - return 'y' - elif col == 7 and book.comments: - return 'y' - return '' - - def data(self, index, role): - row, col = index.row(), index.column() - book = self.matches[row] - if role == Qt.DisplayRole: - res = self.data_as_text(book, col) - if col <= 5 and res: - return QVariant(res) - return NONE - elif role == Qt.DecorationRole: - if col == 6 and book.has_cover: - return self.yes_icon - if col == 7 and book.comments: - return self.yes_icon - return NONE - - def sort(self, col, order, reset=True): - if not self.matches: - return - descending = order == Qt.DescendingOrder - self.matches.sort(None, - lambda x: sort_key(unicode(force_unicode(self.data_as_text(x, col)))), - descending) - if reset: - self.reset() - -class FetchMetadata(QDialog, Ui_FetchMetadata): - - HANG_TIME = 75 #seconds - - queue_reject = pyqtSignal() - - def __init__(self, parent, isbn, title, author, publisher, timeout): - QDialog.__init__(self, parent) - Ui_FetchMetadata.__init__(self) - self.setupUi(self) - - for fetcher in list(_hung_fetchers): - if not fetcher.is_alive(): - _hung_fetchers.remove(fetcher) - - self.pi = ProgressIndicator(self) - self.timeout = timeout - QObject.connect(self.fetch, SIGNAL('clicked()'), self.fetch_metadata) - self.queue_reject.connect(self.reject, Qt.QueuedConnection) - - isbndb_key = get_isbndb_key() - if not isbndb_key: - isbndb_key = '' - self.key.setText(isbndb_key) - - self.setWindowTitle(title if title else _('Unknown')) - self.isbn = isbn - self.title = title - self.author = author.strip() - self.publisher = publisher - self.previous_row = None - self.warning.setVisible(False) - self.connect(self.matches, SIGNAL('activated(QModelIndex)'), self.chosen) - self.connect(self.matches, SIGNAL('entered(QModelIndex)'), - self.show_summary) - self.matches.setMouseTracking(True) - # Enabling sorting and setting a sort column will not change the initial - # order of the results, as they are filled in later - self.matches.setSortingEnabled(True) - self.matches.horizontalHeader().sectionClicked.connect(self.show_sort_indicator) - self.matches.horizontalHeader().setSortIndicatorShown(False) - self.fetch_metadata() - self.opt_get_social_metadata.setChecked(config['get_social_metadata']) - self.opt_overwrite_author_title_metadata.setChecked(config['overwrite_author_title_metadata']) - self.opt_auto_download_cover.setChecked(config['auto_download_cover']) - - def show_summary(self, current, *args): - row = current.row() - if row != self.previous_row: - summ = self.model.summary(row) - self.summary.setText(summ if summ else '') - self.previous_row = row - - def fetch_metadata(self): - self.warning.setVisible(False) - key = str(self.key.text()) - if key: - set_isbndb_key(key) - else: - key = None - title = author = publisher = isbn = None - if self.isbn: - isbn = self.isbn - if self.title: - title = self.title - if self.author and not self.author == _('Unknown'): - author = self.author - self.fetch.setEnabled(False) - self.setCursor(Qt.WaitCursor) - QCoreApplication.instance().processEvents() - self.fetcher = Fetcher(title, author, publisher, isbn, key) - self.fetcher.start() - self.pi.start(_('Finding metadata...')) - self._hangcheck = QTimer(self) - self.connect(self._hangcheck, SIGNAL('timeout()'), self.hangcheck, - Qt.QueuedConnection) - self.start_time = time.time() - self._hangcheck.start(100) - - def hangcheck(self): - if self.fetcher.is_alive() and \ - time.time() - self.start_time < self.HANG_TIME: - return - self._hangcheck.stop() - try: - if self.fetcher.is_alive(): - error_dialog(self, _('Could not find metadata'), - _('The metadata download seems to have stalled. ' - 'Try again later.')).exec_() - self.terminate() - return self.queue_reject.emit() - self.model = Matches(self.fetcher.results) - warnings = [(x[0], force_unicode(x[1])) for x in \ - self.fetcher.exceptions if x[1] is not None] - if warnings: - warnings='<br>'.join(['<b>%s</b>: %s'%(name, exc) for name,exc in warnings]) - self.warning.setText('<p><b>'+ _('Warning')+':</b>'+\ - _('Could not fetch metadata from:')+\ - '<br>'+warnings+'</p>') - self.warning.setVisible(True) - if self.model.rowCount() < 1: - info_dialog(self, _('No metadata found'), - _('No metadata found, try adjusting the title and author ' - 'and/or removing the ISBN.')).exec_() - self.reject() - return - - self.matches.setModel(self.model) - QObject.connect(self.matches.selectionModel(), - SIGNAL('currentRowChanged(QModelIndex, QModelIndex)'), - self.show_summary) - self.model.reset() - self.matches.selectionModel().select(self.model.index(0, 0), - QItemSelectionModel.Select | QItemSelectionModel.Rows) - self.matches.setCurrentIndex(self.model.index(0, 0)) - finally: - self.fetch.setEnabled(True) - self.unsetCursor() - self.matches.resizeColumnsToContents() - self.pi.stop() - - def terminate(self): - if hasattr(self, 'fetcher') and self.fetcher.is_alive(): - _hung_fetchers.add(self.fetcher) - if hasattr(self, '_hangcheck') and self._hangcheck.isActive(): - self._hangcheck.stop() - # Save value of auto_download_cover, since this is the only place it can - # be set. The values of the other options can be set in - # Preferences->Behavior and should not be set here as they affect bulk - # downloading as well. - if self.opt_auto_download_cover.isChecked() != config['auto_download_cover']: - config.set('auto_download_cover', self.opt_auto_download_cover.isChecked()) - - def __enter__(self, *args): - return self - - def __exit__(self, *args): - self.terminate() - - def selected_book(self): - try: - return self.matches.model().matches[self.matches.currentIndex().row()] - except: - return None - - def chosen(self, index): - self.matches.setCurrentIndex(index) - self.accept() - - def show_sort_indicator(self, *args): - self.matches.horizontalHeader().setSortIndicatorShown(True) - diff --git a/src/calibre/gui2/dialogs/fetch_metadata.ui b/src/calibre/gui2/dialogs/fetch_metadata.ui deleted file mode 100644 index b140fa158d..0000000000 --- a/src/calibre/gui2/dialogs/fetch_metadata.ui +++ /dev/null @@ -1,179 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<ui version="4.0"> - <class>FetchMetadata</class> - <widget class="QDialog" name="FetchMetadata"> - <property name="windowModality"> - <enum>Qt::WindowModal</enum> - </property> - <property name="geometry"> - <rect> - <x>0</x> - <y>0</y> - <width>890</width> - <height>642</height> - </rect> - </property> - <property name="windowTitle"> - <string>Fetch metadata</string> - </property> - <property name="windowIcon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/metadata.png</normaloff>:/images/metadata.png</iconset> - </property> - <layout class="QVBoxLayout"> - <item> - <widget class="QLabel" name="tlabel"> - <property name="text"> - <string><p>calibre can find metadata for your books from two locations: <b>Google Books</b> and <b>isbndb.com</b>. <p>To use isbndb.com you must sign up for a <a href="http://www.isbndb.com">free account</a> and enter your access key below.</string> - </property> - <property name="alignment"> - <set>Qt::AlignCenter</set> - </property> - <property name="wordWrap"> - <bool>true</bool> - </property> - <property name="openExternalLinks"> - <bool>true</bool> - </property> - </widget> - </item> - <item> - <layout class="QHBoxLayout"> - <item> - <widget class="QLabel" name="label_2"> - <property name="text"> - <string>&Access Key:</string> - </property> - <property name="buddy"> - <cstring>key</cstring> - </property> - </widget> - </item> - <item> - <widget class="QLineEdit" name="key"/> - </item> - <item> - <widget class="QPushButton" name="fetch"> - <property name="text"> - <string>Fetch</string> - </property> - </widget> - </item> - </layout> - </item> - <item> - <widget class="QLabel" name="warning"> - <property name="text"> - <string/> - </property> - <property name="wordWrap"> - <bool>true</bool> - </property> - </widget> - </item> - <item> - <widget class="QGroupBox" name="groupBox"> - <property name="title"> - <string>Matches</string> - </property> - <layout class="QVBoxLayout"> - <item> - <widget class="QLabel" name="label_3"> - <property name="text"> - <string>Select the book that most closely matches your copy from the list below</string> - </property> - </widget> - </item> - <item> - <widget class="QTableView" name="matches"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Expanding" vsizetype="Expanding"> - <horstretch>0</horstretch> - <verstretch>1</verstretch> - </sizepolicy> - </property> - <property name="alternatingRowColors"> - <bool>true</bool> - </property> - <property name="selectionMode"> - <enum>QAbstractItemView::SingleSelection</enum> - </property> - <property name="selectionBehavior"> - <enum>QAbstractItemView::SelectRows</enum> - </property> - </widget> - </item> - <item> - <widget class="QTextBrowser" name="summary"/> - </item> - </layout> - </widget> - </item> - <item> - <widget class="QCheckBox" name="opt_overwrite_author_title_metadata"> - <property name="text"> - <string>Overwrite author and title with author and title of selected book</string> - </property> - </widget> - </item> - <item> - <widget class="QCheckBox" name="opt_get_social_metadata"> - <property name="text"> - <string>Download &social metadata (tags/rating/etc.) for the selected book</string> - </property> - </widget> - </item> - <item> - <widget class="QCheckBox" name="opt_auto_download_cover"> - <property name="text"> - <string>Automatically download the cover, if available</string> - </property> - </widget> - </item> - <item> - <widget class="QDialogButtonBox" name="buttonBox"> - <property name="standardButtons"> - <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set> - </property> - </widget> - </item> - </layout> - </widget> - <resources> - <include location="../../../../resources/images.qrc"/> - </resources> - <connections> - <connection> - <sender>buttonBox</sender> - <signal>accepted()</signal> - <receiver>FetchMetadata</receiver> - <slot>accept()</slot> - <hints> - <hint type="sourcelabel"> - <x>460</x> - <y>599</y> - </hint> - <hint type="destinationlabel"> - <x>657</x> - <y>530</y> - </hint> - </hints> - </connection> - <connection> - <sender>buttonBox</sender> - <signal>rejected()</signal> - <receiver>FetchMetadata</receiver> - <slot>reject()</slot> - <hints> - <hint type="sourcelabel"> - <x>417</x> - <y>599</y> - </hint> - <hint type="destinationlabel"> - <x>0</x> - <y>491</y> - </hint> - </hints> - </connection> - </connections> -</ui> diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py deleted file mode 100644 index 4776562c29..0000000000 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ /dev/null @@ -1,1031 +0,0 @@ -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' - -''' -The dialog used to edit meta information for a book as well as -add/remove formats -''' - -import os, re, time, traceback, textwrap -from functools import partial -from threading import Thread - -from PyQt4.Qt import SIGNAL, QObject, Qt, QTimer, QDate, \ - QPixmap, QListWidgetItem, QDialog, pyqtSignal, QIcon, \ - QPushButton, QKeySequence - -from calibre.gui2 import error_dialog, file_icon_provider, dynamic, \ - choose_files, choose_images, ResizableDialog, \ - warning_dialog, question_dialog, UNDEFINED_QDATE -from calibre.gui2.dialogs.metadata_single_ui import Ui_MetadataSingleDialog -from calibre.gui2.dialogs.fetch_metadata import FetchMetadata -from calibre.gui2.dialogs.tag_editor import TagEditor -from calibre.gui2.widgets import ProgressIndicator -from calibre.ebooks import BOOK_EXTENSIONS -from calibre.ebooks.metadata import string_to_authors, \ - authors_to_string, check_isbn, title_sort -from calibre.ebooks.metadata.covers import download_cover -from calibre.ebooks.metadata import MetaInformation -from calibre.utils.config import prefs, tweaks -from calibre.utils.date import qt_to_dt, local_tz, utcfromtimestamp -from calibre.utils.icu import sort_key -from calibre.customize.ui import run_plugins_on_import, get_isbndb_key -from calibre.gui2.preferences.social import SocialMetadata -from calibre.gui2.custom_column_widgets import populate_metadata_page -from calibre import strftime -from calibre.library.comments import comments_to_html - -class CoverFetcher(Thread): # {{{ - - def __init__(self, username, password, isbn, timeout, title, author): - Thread.__init__(self) - self.daemon = True - - self.username = username.strip() if username else username - self.password = password.strip() if password else password - self.timeout = timeout - self.isbn = isbn - self.title = title - self.needs_isbn = False - self.author = author - self.exception = self.traceback = self.cover_data = self.errors = None - - def run(self): - try: - au = self.author if self.author else None - mi = MetaInformation(self.title, [au]) - if not self.isbn: - from calibre.ebooks.metadata.fetch import search - if not self.title: - self.needs_isbn = True - return - key = get_isbndb_key() - if not key: - key = None - results = search(title=self.title, author=au, - isbndb_key=key)[0] - results = sorted([x.isbn for x in results if x.isbn], - cmp=lambda x,y:cmp(len(x),len(y)), reverse=True) - if not results: - self.needs_isbn = True - return - self.isbn = results[0] - - mi.isbn = self.isbn - - self.cover_data, self.errors = download_cover(mi, - timeout=self.timeout) - except Exception as e: - self.exception = e - self.traceback = traceback.format_exc() - print self.traceback - -# }}} - -class Format(QListWidgetItem): # {{{ - - def __init__(self, parent, ext, size, path=None, timestamp=None): - self.path = path - self.ext = ext - self.size = float(size)/(1024*1024) - text = '%s (%.2f MB)'%(self.ext.upper(), self.size) - QListWidgetItem.__init__(self, file_icon_provider().icon_from_ext(ext), - text, parent, QListWidgetItem.UserType) - if timestamp is not None: - ts = timestamp.astimezone(local_tz) - t = strftime('%a, %d %b %Y [%H:%M:%S]', ts.timetuple()) - text = _('Last modified: %s')%t - self.setToolTip(text) - self.setStatusTip(text) - -# }}} - - -class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): - - COVER_FETCH_TIMEOUT = 240 # seconds - view_format = pyqtSignal(object) - - # Cover processing {{{ - - def set_cover(self): - mi, ext = self.get_selected_format_metadata() - if mi is None: - return - cdata = None - if mi.cover and os.access(mi.cover, os.R_OK): - cdata = open(mi.cover).read() - elif mi.cover_data[1] is not None: - cdata = mi.cover_data[1] - if cdata is None: - error_dialog(self, _('Could not read cover'), - _('Could not read cover from %s format')%ext).exec_() - return - pix = QPixmap() - pix.loadFromData(cdata) - if pix.isNull(): - error_dialog(self, _('Could not read cover'), - _('The cover in the %s format is invalid')%ext).exec_() - return - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - self.cover_data = cdata - - def trim_cover(self, *args): - from calibre.utils.magick import Image - cdata = self.cover_data - if not cdata: - return - im = Image() - im.load(cdata) - im.trim(10) - cdata = im.export('png') - pix = QPixmap() - pix.loadFromData(cdata) - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - self.cover_data = cdata - - - - def update_cover_tooltip(self): - p = self.cover.pixmap() - self.cover.setToolTip(_('Cover size: %dx%d pixels') % - (p.width(), p.height())) - - - def do_reset_cover(self, *args): - pix = QPixmap(I('default_cover.png')) - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cover_data = None - - def select_cover(self, checked): - files = choose_images(self, 'change cover dialog', - _('Choose cover for ') + unicode(self.title.text())) - if not files: - return - _file = files[0] - if _file: - _file = os.path.abspath(_file) - if not os.access(_file, os.R_OK): - d = error_dialog(self, _('Cannot read'), - _('You do not have permission to read the file: ') + _file) - d.exec_() - return - cf, cover = None, None - try: - cf = open(_file, "rb") - cover = cf.read() - except IOError as e: - d = error_dialog(self, _('Error reading file'), - _("<p>There was an error reading from file: <br /><b>") + _file + "</b></p><br />"+str(e)) - d.exec_() - if cover: - pix = QPixmap() - pix.loadFromData(cover) - if pix.isNull(): - d = error_dialog(self, - _("Not a valid picture"), - _file + _(" is not a valid picture")) - d.exec_() - else: - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - self.cover_data = cover - - def generate_cover(self, *args): - from calibre.ebooks import calibre_cover - from calibre.ebooks.metadata import fmt_sidx - from calibre.gui2 import config - title = unicode(self.title.text()).strip() - author = unicode(self.authors.text()).strip() - if author.endswith('&'): - author = author[:-1].strip() - if not title or not author: - return error_dialog(self, _('Specify title and author'), - _('You must specify a title and author before generating ' - 'a cover'), show=True) - series = unicode(self.series.text()).strip() - series_string = None - if series: - series_string = _('Book %s of %s')%( - fmt_sidx(self.series_index.value(), - use_roman=config['use_roman_numerals_for_series_number']), series) - self.cover_data = calibre_cover(title, author, - series_string=series_string) - pix = QPixmap() - pix.loadFromData(self.cover_data) - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - - def cover_dropped(self, cover_data): - self.cover_changed = True - self.cover_data = cover_data - self.update_cover_tooltip() - - def fetch_cover(self): - isbn = re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text())).strip() - self.fetch_cover_button.setEnabled(False) - self.setCursor(Qt.WaitCursor) - title, author = map(unicode, (self.title.text(), self.authors.text())) - self.cover_fetcher = CoverFetcher(None, None, isbn, - self.timeout, title, author) - self.cover_fetcher.start() - self.cf_start_time = time.time() - self.pi.start(_('Downloading cover...')) - QTimer.singleShot(100, self.hangcheck) - - def hangcheck(self): - cf = self.cover_fetcher - if cf is None: - # Called after dialog closed - return - - if cf.is_alive() and \ - time.time()-self.cf_start_time < self.COVER_FETCH_TIMEOUT: - QTimer.singleShot(100, self.hangcheck) - return - - try: - if cf.is_alive(): - error_dialog(self, _('Cannot fetch cover'), - _('<b>Could not fetch cover.</b><br/>')+ - _('The download timed out.')).exec_() - return - if cf.needs_isbn: - error_dialog(self, _('Cannot fetch cover'), - _('Could not find cover for this book. Try ' - 'specifying the ISBN first.')).exec_() - return - if cf.exception is not None: - err = cf.exception - error_dialog(self, _('Cannot fetch cover'), - _('<b>Could not fetch cover.</b><br/>')+unicode(err)).exec_() - return - if cf.errors and cf.cover_data is None: - details = u'\n\n'.join([e[-1] + ': ' + e[1] for e in cf.errors]) - error_dialog(self, _('Cannot fetch cover'), - _('<b>Could not fetch cover.</b><br/>') + - _('For the error message from each cover source, ' - 'click Show details below.'), det_msg=details, show=True) - return - - pix = QPixmap() - pix.loadFromData(cf.cover_data) - if pix.isNull(): - error_dialog(self, _('Bad cover'), - _('The cover is not a valid picture')).exec_() - else: - self.cover.setPixmap(pix) - self.update_cover_tooltip() - self.cover_changed = True - self.cpixmap = pix - self.cover_data = cf.cover_data - finally: - self.fetch_cover_button.setEnabled(True) - self.unsetCursor() - if self.pi is not None: - self.pi.stop() - - - # }}} - - # Formats processing {{{ - def add_format(self, x): - files = choose_files(self, 'add formats dialog', - _("Choose formats for ") + unicode((self.title.text())), - [(_('Books'), BOOK_EXTENSIONS)]) - self._add_formats(files) - - def _add_formats(self, paths): - added = False - if not paths: - return added - bad_perms = [] - for _file in paths: - _file = os.path.abspath(_file) - if not os.access(_file, os.R_OK): - bad_perms.append(_file) - continue - - nfile = run_plugins_on_import(_file) - if nfile is not None: - _file = nfile - stat = os.stat(_file) - size = stat.st_size - ext = os.path.splitext(_file)[1].lower().replace('.', '') - timestamp = utcfromtimestamp(stat.st_mtime) - for row in range(self.formats.count()): - fmt = self.formats.item(row) - if fmt.ext.lower() == ext: - self.formats.takeItem(row) - break - Format(self.formats, ext, size, path=_file, timestamp=timestamp) - self.formats_changed = True - added = True - if bad_perms: - error_dialog(self, _('No permission'), - _('You do not have ' - 'permission to read the following files:'), - det_msg='\n'.join(bad_perms), show=True) - - return added - - def formats_dropped(self, event, paths): - if self._add_formats(paths): - event.accept() - - def remove_format(self, *args): - rows = self.formats.selectionModel().selectedRows(0) - for row in rows: - self.formats.takeItem(row.row()) - self.formats_changed = True - - def get_selected_format_metadata(self): - from calibre.ebooks.metadata.meta import get_metadata - old = prefs['read_file_metadata'] - if not old: - prefs['read_file_metadata'] = True - try: - row = self.formats.currentRow() - fmt = self.formats.item(row) - if fmt is None: - if self.formats.count() == 1: - fmt = self.formats.item(0) - if fmt is None: - error_dialog(self, _('No format selected'), - _('No format selected')).exec_() - return None, None - ext = fmt.ext.lower() - if fmt.path is None: - stream = self.db.format(self.row, ext, as_file=True) - else: - stream = open(fmt.path, 'r+b') - try: - mi = get_metadata(stream, ext) - return mi, ext - except: - error_dialog(self, _('Could not read metadata'), - _('Could not read metadata from %s format')%ext).exec_() - return None, None - finally: - if old != prefs['read_file_metadata']: - prefs['read_file_metadata'] = old - - def set_metadata_from_format(self): - mi, ext = self.get_selected_format_metadata() - if mi is None: - return - if mi.title: - self.title.setText(mi.title) - if mi.authors: - self.authors.setEditText(authors_to_string(mi.authors)) - if mi.author_sort: - self.author_sort.setText(mi.author_sort) - if mi.rating is not None: - try: - self.rating.setValue(mi.rating) - except: - pass - if mi.publisher: - self.publisher.setEditText(mi.publisher) - if mi.tags: - self.tags.setText(', '.join(mi.tags)) - if mi.isbn: - self.isbn.setText(mi.isbn) - if mi.pubdate: - self.pubdate.setDate(QDate(mi.pubdate.year, mi.pubdate.month, - mi.pubdate.day)) - if mi.series and mi.series.strip(): - self.series.setEditText(mi.series) - if mi.series_index is not None: - self.series_index.setValue(float(mi.series_index)) - if mi.comments and mi.comments.strip(): - comments = comments_to_html(mi.comments) - self.comments.html = comments - - - def sync_formats(self): - old_extensions, new_extensions, paths = set(), set(), {} - for row in range(self.formats.count()): - fmt = self.formats.item(row) - ext, path = fmt.ext.lower(), fmt.path - if 'unknown' in ext.lower(): - ext = None - if path: - new_extensions.add(ext) - paths[ext] = path - else: - old_extensions.add(ext) - for ext in new_extensions: - self.db.add_format(self.row, ext, open(paths[ext], 'rb'), notify=False) - dbfmts = self.db.formats(self.row) - db_extensions = set([f.lower() for f in (dbfmts.split(',') if dbfmts - else [])]) - extensions = new_extensions.union(old_extensions) - for ext in db_extensions: - if ext not in extensions and ext in self.original_formats: - self.db.remove_format(self.row, ext, notify=False) - - def show_format(self, item, *args): - fmt = item.ext - self.view_format.emit(fmt) - - # }}} - - def __init__(self, window, row, db, prev=None, - next_=None): - ResizableDialog.__init__(self, window) - self.cover_fetcher = None - self.bc_box.layout().setAlignment(self.cover, Qt.AlignCenter|Qt.AlignHCenter) - base = unicode(self.author_sort.toolTip()) - ok_tooltip = '<p>' + textwrap.fill(base+'<br><br>'+ - _(' The green color indicates that the current ' - 'author sort matches the current author')) - bad_tooltip = '<p>'+textwrap.fill(base + '<br><br>'+ - _(' The red color indicates that the current ' - 'author sort does not match the current author. ' - 'No action is required if this is what you want.')) - self.aus_tooltips = (ok_tooltip, bad_tooltip) - - base = unicode(self.title_sort.toolTip()) - ok_tooltip = '<p>' + textwrap.fill(base+'<br><br>'+ - _(' The green color indicates that the current ' - 'title sort matches the current title')) - bad_tooltip = '<p>'+textwrap.fill(base + '<br><br>'+ - _(' The red color warns that the current ' - 'title sort does not match the current title. ' - 'No action is required if this is what you want.')) - self.ts_tooltips = (ok_tooltip, bad_tooltip) - self.row_delta = 0 - if prev: - self.prev_button = QPushButton(QIcon(I('back.png')), _('Previous'), - self) - self.button_box.addButton(self.prev_button, self.button_box.ActionRole) - tip = (_('Save changes and edit the metadata of %s')+' [Alt+Left]')%prev - self.prev_button.setToolTip(tip) - self.prev_button.clicked.connect(partial(self.next_triggered, - -1)) - self.prev_button.setShortcut(QKeySequence('Alt+Left')) - if next_: - self.next_button = QPushButton(QIcon(I('forward.png')), _('Next'), - self) - self.button_box.addButton(self.next_button, self.button_box.ActionRole) - tip = (_('Save changes and edit the metadata of %s')+' [Alt+Right]')%next_ - self.next_button.setToolTip(tip) - self.next_button.clicked.connect(partial(self.next_triggered, 1)) - self.next_button.setShortcut(QKeySequence('Alt+Right')) - - self.splitter.setStretchFactor(100, 1) - self.read_state() - self.db = db - self.pi = ProgressIndicator(self) - self.id = db.id(row) - self.row = row - self.cover_data = None - self.formats_changed = False - self.formats.setAcceptDrops(True) - self.cover_changed = False - self.cpixmap = None - self.pubdate.setMinimumDate(UNDEFINED_QDATE) - pubdate_format = tweaks['gui_pubdate_display_format'] - if pubdate_format is not None: - self.pubdate.setDisplayFormat(pubdate_format) - self.date.setMinimumDate(UNDEFINED_QDATE) - self.pubdate.setSpecialValueText(_('Undefined')) - self.date.setSpecialValueText(_('Undefined')) - self.clear_pubdate_button.clicked.connect(self.clear_pubdate) - - - self.connect(self.cover, SIGNAL('cover_changed(PyQt_PyObject)'), self.cover_dropped) - QObject.connect(self.cover_button, SIGNAL("clicked(bool)"), \ - self.select_cover) - QObject.connect(self.add_format_button, SIGNAL("clicked(bool)"), \ - self.add_format) - self.connect(self.formats, - SIGNAL('formats_dropped(PyQt_PyObject,PyQt_PyObject)'), - self.formats_dropped) - QObject.connect(self.remove_format_button, SIGNAL("clicked(bool)"), \ - self.remove_format) - QObject.connect(self.fetch_metadata_button, SIGNAL('clicked()'), - self.fetch_metadata) - - QObject.connect(self.fetch_cover_button, SIGNAL('clicked()'), - self.fetch_cover) - QObject.connect(self.tag_editor_button, SIGNAL('clicked()'), - self.edit_tags) - QObject.connect(self.remove_series_button, SIGNAL('clicked()'), - self.remove_unused_series) - QObject.connect(self.auto_author_sort, SIGNAL('clicked()'), - self.deduce_author_sort) - QObject.connect(self.auto_title_sort, SIGNAL('clicked()'), - self.deduce_title_sort) - self.trim_cover_button.clicked.connect(self.trim_cover) - self.connect(self.title_sort, SIGNAL('textChanged(const QString&)'), - self.title_sort_box_changed) - self.connect(self.title, SIGNAL('textChanged(const QString&)'), - self.title_box_changed) - self.connect(self.author_sort, SIGNAL('textChanged(const QString&)'), - self.author_sort_box_changed) - self.connect(self.authors, SIGNAL('editTextChanged(const QString&)'), - self.authors_box_changed) - self.connect(self.formats, SIGNAL('itemDoubleClicked(QListWidgetItem*)'), - self.show_format) - self.connect(self.formats, SIGNAL('delete_format()'), self.remove_format) - self.connect(self.button_set_cover, SIGNAL('clicked()'), self.set_cover) - self.connect(self.button_set_metadata, SIGNAL('clicked()'), - self.set_metadata_from_format) - self.connect(self.reset_cover, SIGNAL('clicked()'), self.do_reset_cover) - self.connect(self.swap_button, SIGNAL('clicked()'), self.swap_title_author) - self.timeout = float(prefs['network_timeout']) - - - self.title.setText(db.title(row)) - self.title_sort.setText(db.title_sort(row)) - isbn = db.isbn(self.id, index_is_id=True) - if not isbn: - isbn = '' - self.isbn.textChanged.connect(self.validate_isbn) - self.isbn.setText(isbn) - aus = self.db.author_sort(row) - self.author_sort.setText(aus if aus else '') - tags = self.db.tags(row) - self.original_tags = ', '.join(tags.split(',')) if tags else '' - self.tags.setText(self.original_tags) - self.tags.update_items_cache(self.db.all_tags()) - rating = self.db.rating(row) - if rating > 0: - self.rating.setValue(int(rating/2.)) - comments = self.db.comments(row) - if comments and comments.strip(): - comments = comments_to_html(comments) - self.comments.html = comments - cover = self.db.cover(row) - pubdate = db.pubdate(self.id, index_is_id=True) - self.pubdate.setDate(QDate(pubdate.year, pubdate.month, - pubdate.day)) - timestamp = db.timestamp(self.id, index_is_id=True) - self.date.setDate(QDate(timestamp.year, timestamp.month, - timestamp.day)) - self.orig_date = qt_to_dt(self.date.date()) - - exts = self.db.formats(row) - self.original_formats = [] - if exts: - exts = exts.split(',') - for ext in exts: - if not ext: - ext = '' - size = self.db.sizeof_format(row, ext) - timestamp = self.db.format_last_modified(self.id, ext) - if size is None: - continue - Format(self.formats, ext, size, timestamp=timestamp) - self.original_formats.append(ext.lower()) - - - self.initialize_combos() - si = self.db.series_index(row) - if si is None: - si = 1.0 - try: - self.series_index.setValue(float(si)) - except: - self.series_index.setValue(1.0) - QObject.connect(self.series, SIGNAL('currentIndexChanged(int)'), self.enable_series_index) - QObject.connect(self.series, SIGNAL('editTextChanged(QString)'), self.enable_series_index) - self.series.lineEdit().editingFinished.connect(self.increment_series_index) - - pm = QPixmap() - if cover: - pm.loadFromData(cover) - if pm.isNull(): - pm = QPixmap(I('default_cover.png')) - else: - self.cover_data = cover - self.cover.setPixmap(pm) - self.update_cover_tooltip() - self.original_series_name = unicode(self.series.text()).strip() - if len(db.custom_column_label_map) == 0: - self.central_widget.tabBar().setVisible(False) - self.central_widget.setTabEnabled(1, False) - else: - self.create_custom_column_editors() - self.generate_cover_button.clicked.connect(self.generate_cover) - - self.original_author = unicode(self.authors.text()).strip() - self.original_title = unicode(self.title.text()).strip() - self.books_to_refresh = set() - - self.show() - - def clear_pubdate(self, *args): - self.pubdate.setDate(UNDEFINED_QDATE) - - def create_custom_column_editors(self): - w = self.central_widget.widget(1) - layout = w.layout() - self.custom_column_widgets, self.__cc_spacers = \ - populate_metadata_page(layout, self.db, self.id, parent=w, bulk=False, - two_column=tweaks['metadata_single_use_2_cols_for_custom_fields']) - self.__custom_col_layouts = [layout] - ans = self.custom_column_widgets - for i in range(len(ans)-1): - if len(ans[i+1].widgets) == 2: - w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[1]) - else: - w.setTabOrder(ans[i].widgets[-1], ans[i+1].widgets[0]) - for c in range(2, len(ans[i].widgets), 2): - w.setTabOrder(ans[i].widgets[c-1], ans[i].widgets[c+1]) - - def title_box_changed(self, txt): - ts = unicode(txt) - ts = title_sort(ts) - self.mark_box_as_ok(control = self.title_sort, tt=self.ts_tooltips, - normal=(unicode(self.title_sort.text()) == ts)) - - def title_sort_box_changed(self, txt): - ts = unicode(txt) - self.mark_box_as_ok(control = self.title_sort, tt=self.ts_tooltips, - normal=(title_sort(unicode(self.title.text())) == ts)) - - def authors_box_changed(self, txt): - aus = unicode(txt) - aus = re.sub(r'\s+et al\.$', '', aus) - aus = self.db.author_sort_from_authors(string_to_authors(aus)) - self.mark_box_as_ok(control = self.author_sort, tt=self.aus_tooltips, - normal=(unicode(self.author_sort.text()) == aus)) - - def author_sort_box_changed(self, txt): - au = unicode(self.authors.text()) - au = re.sub(r'\s+et al\.$', '', au) - au = self.db.author_sort_from_authors(string_to_authors(au)) - self.mark_box_as_ok(control = self.author_sort, tt=self.aus_tooltips, - normal=(au == txt)) - - def mark_box_as_ok(self, control, tt, normal=True): - if normal: - col = 'rgb(0, 255, 0, 20%)' - else: - col = 'rgb(255, 0, 0, 20%)' - control.setStyleSheet('QLineEdit { color: black; ' - 'background-color: %s; }'%col) - tt = tt[0] if normal else tt[1] - control.setToolTip(tt) - - def validate_isbn(self, isbn): - isbn = unicode(isbn).strip() - if not isbn: - self.isbn.setStyleSheet('QLineEdit { background-color: rgba(0,255,0,0%) }') - self.isbn.setToolTip(_('This ISBN number is valid')) - return - - if check_isbn(isbn): - self.isbn.setStyleSheet('QLineEdit { background-color: rgba(0,255,0,20%) }') - self.isbn.setToolTip(_('This ISBN number is valid')) - else: - self.isbn.setStyleSheet('QLineEdit { background-color: rgba(255,0,0,20%) }') - self.isbn.setToolTip(_('This ISBN number is invalid')) - - def deduce_author_sort(self): - au = unicode(self.authors.text()) - au = re.sub(r'\s+et al\.$', '', au) - authors = string_to_authors(au) - self.author_sort.setText(self.db.author_sort_from_authors(authors)) - - def deduce_title_sort(self): - ts = unicode(self.title.text()) - self.title_sort.setText(title_sort(ts)) - - def swap_title_author(self): - title = self.title.text() - self.title.setText(self.authors.text()) - self.authors.setText(title) - self.deduce_author_sort() - self.deduce_title_sort() - - def initialize_combos(self): - self.initalize_authors() - self.initialize_series() - self.initialize_publisher() - - self.layout().activate() - - def initalize_authors(self): - all_authors = self.db.all_authors() - all_authors.sort(key=lambda x : sort_key(x[1])) - for i in all_authors: - id, name = i - name = [name.strip().replace('|', ',') for n in name.split(',')] - self.authors.addItem(authors_to_string(name)) - - au = self.db.authors(self.row) - if not au: - au = _('Unknown') - au = ' & '.join([a.strip().replace('|', ',') for a in au.split(',')]) - self.authors.setEditText(au) - - self.authors.set_separator('&') - self.authors.set_space_before_sep(True) - self.authors.set_add_separator(tweaks['authors_completer_append_separator']) - self.authors.update_items_cache(self.db.all_author_names()) - - def initialize_series(self): - self.series.setSizeAdjustPolicy(self.series.AdjustToContentsOnFirstShow) - all_series = self.db.all_series() - all_series.sort(key=lambda x : sort_key(x[1])) - self.series.set_separator(None) - self.series.update_items_cache([x[1] for x in all_series]) - series_id = self.db.series_id(self.row) - idx, c = None, 0 - for i in all_series: - id, name = i - if id == series_id: - idx = c - self.series.addItem(name) - c += 1 - - self.series.lineEdit().setText('') - if idx is not None: - self.series.setCurrentIndex(idx) - self.enable_series_index() - - def initialize_publisher(self): - all_publishers = self.db.all_publishers() - all_publishers.sort(key=lambda x : sort_key(x[1])) - self.publisher.set_separator(None) - self.publisher.update_items_cache([x[1] for x in all_publishers]) - publisher_id = self.db.publisher_id(self.row) - idx, c = None, 0 - for i in all_publishers: - id, name = i - if id == publisher_id: - idx = c - self.publisher.addItem(name) - c += 1 - - self.publisher.setEditText('') - if idx is not None: - self.publisher.setCurrentIndex(idx) - - def edit_tags(self): - if self.tags.text() != self.original_tags: - if question_dialog(self, _('Tags changed'), - _('You have changed the tags. In order to use the tags' - ' editor, you must either discard or apply these ' - 'changes. Apply changes?'), show_copy_button=False): - self.books_to_refresh |= self.apply_tags(commit=True, - notify=True) - self.original_tags = unicode(self.tags.text()) - else: - self.tags.setText(self.original_tags) - d = TagEditor(self, self.db, self.id) - d.exec_() - if d.result() == QDialog.Accepted: - tag_string = ', '.join(d.tags) - self.tags.setText(tag_string) - self.tags.update_items_cache(self.db.all_tags()) - - - def fetch_metadata(self): - isbn = re.sub(r'[^0-9a-zA-Z]', '', unicode(self.isbn.text())) - title = unicode(self.title.text()) - try: - author = string_to_authors(unicode(self.authors.text()))[0] - except: - author = '' - publisher = unicode(self.publisher.currentText()) - if isbn or title or author or publisher: - d = FetchMetadata(self, isbn, title, author, publisher, self.timeout) - self._fetch_metadata_scope = d - with d: - if d.exec_() == QDialog.Accepted: - book = d.selected_book() - if book: - if d.opt_get_social_metadata.isChecked(): - d2 = SocialMetadata(book, self) - d2.exec_() - if d2.timed_out: - warning_dialog(self, _('Timed out'), - _('The download of social' - ' metadata timed out, the servers are' - ' probably busy. Try again later.'), - show=True) - elif d2.exceptions: - det = '\n'.join([x[0]+'\n\n'+x[-1]+'\n\n\n' for - x in d2.exceptions]) - warning_dialog(self, _('There were errors'), - _('There were errors downloading social metadata'), - det_msg=det, show=True) - else: - book.tags = [] - if d.opt_overwrite_author_title_metadata.isChecked(): - self.title.setText(book.title) - self.authors.setText(authors_to_string(book.authors)) - if book.author_sort: self.author_sort.setText(book.author_sort) - if book.publisher: self.publisher.setEditText(book.publisher) - if book.isbn: self.isbn.setText(book.isbn) - if book.pubdate: - dt = book.pubdate - self.pubdate.setDate(QDate(dt.year, dt.month, dt.day)) - summ = book.comments - if summ: - prefix = self.comments.html - if prefix: - prefix += '\n' - self.comments.html = prefix + comments_to_html(summ) - if book.rating is not None: - self.rating.setValue(int(book.rating)) - if book.tags: - self.tags.setText(', '.join(book.tags)) - if book.series is not None: - if self.series.text() is None or self.series.text() == '': - self.series.setText(book.series) - if book.series_index is not None: - self.series_index.setValue(book.series_index) - if book.has_cover: - if d.opt_auto_download_cover.isChecked(): - self.fetch_cover() - else: - self.fetch_cover_button.setFocus(Qt.OtherFocusReason) - else: - error_dialog(self, _('Cannot fetch metadata'), - _('You must specify at least one of ISBN, Title, ' - 'Authors or Publisher'), show=True) - self.title.setFocus(Qt.OtherFocusReason) - - def enable_series_index(self, *args): - self.series_index.setEnabled(True) - - def increment_series_index(self): - if self.db is not None: - try: - series = unicode(self.series.text()).strip() - if series and series != self.original_series_name: - ns = 1 - if tweaks['series_index_auto_increment'] != 'const': - ns = self.db.get_next_series_num_for(series) - self.series_index.setValue(ns) - self.original_series_name = series - except: - traceback.print_exc() - - def remove_unused_series(self): - self.db.remove_unused_series() - idx = unicode(self.series.currentText()) - self.series.clear() - self.initialize_series() - if idx: - for i in range(self.series.count()): - if unicode(self.series.itemText(i)) == idx: - self.series.setCurrentIndex(i) - break - - def apply_tags(self, commit=False, notify=False): - return self.db.set_tags(self.id, [x.strip() for x in - unicode(self.tags.text()).split(',')], - notify=notify, commit=commit, allow_case_change=True) - - def next_triggered(self, row_delta, *args): - self.row_delta = row_delta - self.accept() - - def accept(self): - try: - if self.formats_changed: - self.sync_formats() - title = unicode(self.title.text()).strip() - if title != self.original_title: - self.db.set_title(self.id, title, notify=False) - # This must be after setting the title because of the DB update trigger - ts = unicode(self.title_sort.text()).strip() - if ts: - self.db.set_title_sort(self.id, ts, notify=False, commit=False) - au = unicode(self.authors.text()).strip() - if au and au != self.original_author: - self.books_to_refresh |= self.db.set_authors(self.id, - string_to_authors(au), - notify=False, - allow_case_change=True) - aus = unicode(self.author_sort.text()).strip() - if aus: - self.db.set_author_sort(self.id, aus, notify=False, commit=False) - self.db.set_isbn(self.id, - re.sub(r'[^0-9a-zA-Z]', '', - unicode(self.isbn.text()).strip()), - notify=False, commit=False) - self.db.set_rating(self.id, 2*self.rating.value(), notify=False, - commit=False) - self.books_to_refresh |= self.apply_tags() - self.books_to_refresh |= self.db.set_publisher(self.id, - unicode(self.publisher.currentText()).strip(), - notify=False, commit=False, allow_case_change=True) - self.books_to_refresh |= self.db.set_series(self.id, - unicode(self.series.currentText()).strip(), notify=False, - commit=False, allow_case_change=True) - self.db.set_series_index(self.id, self.series_index.value(), - notify=False, commit=False) - self.db.set_comment(self.id, - self.comments.html, - notify=False, commit=False) - d = self.pubdate.date() - d = qt_to_dt(d) - self.db.set_pubdate(self.id, d, notify=False, commit=False) - d = self.date.date() - d = qt_to_dt(d) - if d != self.orig_date: - self.db.set_timestamp(self.id, d, notify=False, commit=False) - self.db.commit() - - if self.cover_changed: - if self.cover_data is not None: - self.db.set_cover(self.id, self.cover_data) - else: - self.db.remove_cover(self.id) - for w in getattr(self, 'custom_column_widgets', []): - self.books_to_refresh |= w.commit(self.id) - self.db.commit() - except (IOError, OSError) as err: - if getattr(err, 'errno', -1) == 13: # Permission denied - fname = err.filename if err.filename else 'file' - return error_dialog(self, _('Permission denied'), - _('Could not open %s. Is it being used by another' - ' program?')%fname, det_msg=traceback.format_exc(), - show=True) - raise - self.save_state() - self.cover_fetcher = None - QDialog.accept(self) - - def reject(self, *args): - self.save_state() - self.cover_fetcher = None - QDialog.reject(self, *args) - - def read_state(self): - wg = dynamic.get('metasingle_window_geometry2', None) - ss = dynamic.get('metasingle_splitter_state2', None) - if wg is not None: - self.restoreGeometry(wg) - if ss is not None: - self.splitter.restoreState(ss) - - def save_state(self): - dynamic.set('metasingle_window_geometry2', bytes(self.saveGeometry())) - dynamic.set('metasingle_splitter_state2', - bytes(self.splitter.saveState())) - - def break_cycles(self): - # Break any reference cycles that could prevent python - # from garbage collecting this dialog - def disconnect(signal): - try: - signal.disconnect() - except: - pass # Fails if view format was never connected - disconnect(self.view_format) - for b in ('next_button', 'prev_button'): - x = getattr(self, b, None) - if x is not None: - disconnect(x.clicked) - -if __name__ == '__main__': - from calibre.library import db - from PyQt4.Qt import QApplication - from calibre.utils.mem import memory - import gc - - - app = QApplication([]) - db = db() - - # Initialize all Qt Objects once - d = MetadataSingleDialog(None, 4, db) - d.break_cycles() - d.reject() - del d - - for i in range(5): - gc.collect() - before = memory() - - d = MetadataSingleDialog(None, 4, db) - d.reject() - d.break_cycles() - del d - - for i in range(5): - gc.collect() - print 'Used memory:', memory(before)/1024.**2, 'MB' - - diff --git a/src/calibre/gui2/dialogs/metadata_single.ui b/src/calibre/gui2/dialogs/metadata_single.ui deleted file mode 100644 index ced5030f94..0000000000 --- a/src/calibre/gui2/dialogs/metadata_single.ui +++ /dev/null @@ -1,937 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<ui version="4.0"> - <class>MetadataSingleDialog</class> - <widget class="QDialog" name="MetadataSingleDialog"> - <property name="geometry"> - <rect> - <x>0</x> - <y>0</y> - <width>994</width> - <height>716</height> - </rect> - </property> - <property name="sizePolicy"> - <sizepolicy hsizetype="MinimumExpanding" vsizetype="MinimumExpanding"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="windowTitle"> - <string>Edit Meta Information</string> - </property> - <property name="windowIcon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset> - </property> - <property name="sizeGripEnabled"> - <bool>true</bool> - </property> - <property name="modal"> - <bool>true</bool> - </property> - <layout class="QVBoxLayout" name="verticalLayout_6"> - <item> - <widget class="QScrollArea" name="scrollArea"> - <property name="frameShape"> - <enum>QFrame::NoFrame</enum> - </property> - <property name="widgetResizable"> - <bool>true</bool> - </property> - <widget class="QWidget" name="scrollAreaWidgetContents"> - <property name="geometry"> - <rect> - <x>0</x> - <y>0</y> - <width>986</width> - <height>677</height> - </rect> - </property> - <layout class="QVBoxLayout" name="verticalLayout_5"> - <property name="margin"> - <number>0</number> - </property> - <item> - <widget class="QTabWidget" name="central_widget"> - <property name="minimumSize"> - <size> - <width>800</width> - <height>665</height> - </size> - </property> - <property name="currentIndex"> - <number>0</number> - </property> - <widget class="QWidget" name="central_tabWidgetPage1"> - <attribute name="title"> - <string>&Basic metadata</string> - </attribute> - <layout class="QGridLayout" name="gridLayout_5"> - <item row="0" column="0"> - <widget class="QSplitter" name="splitter"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <widget class="QWidget" name="layoutWidget"> - <layout class="QVBoxLayout"> - <item> - <widget class="QGroupBox" name="meta_box"> - <property name="title"> - <string>Meta information</string> - </property> - <layout class="QGridLayout" name="gridLayout_3"> - <item row="0" column="0"> - <widget class="QLabel" name="label"> - <property name="text"> - <string>&Title: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>title</cstring> - </property> - </widget> - </item> - <item row="0" column="1"> - <widget class="EnLineEdit" name="title"> - <property name="toolTip"> - <string>Change the title of this book</string> - </property> - </widget> - </item> - <item row="0" column="2" rowspan="4"> - <layout class="QVBoxLayout" name="verticalLayout_7"> - <item> - <spacer name="verticalSpacer_3"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - <item> - <widget class="QToolButton" name="auto_title_sort"> - <property name="toolTip"> - <string>Automatically create the title sort entry based on the current title entry. -Using this button to create title sort will change title sort from red to green.</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/auto_author_sort.png</normaloff>:/images/auto_author_sort.png</iconset> - </property> - </widget> - </item> - <item> - <spacer name="verticalSpacer"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - <item> - <widget class="QToolButton" name="swap_button"> - <property name="toolTip"> - <string>Swap the author and title</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/swap.png</normaloff>:/images/swap.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>16</width> - <height>16</height> - </size> - </property> - </widget> - </item> - <item> - <spacer name="verticalSpacer_2"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - <item> - <widget class="QToolButton" name="auto_author_sort"> - <property name="toolTip"> - <string>Automatically create the author sort entry based on the current author entry. -Using this button to create author sort will change author sort from red to green.</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/auto_author_sort.png</normaloff>:/images/auto_author_sort.png</iconset> - </property> - </widget> - </item> - <item> - <spacer name="verticalSpacer_4"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - </layout> - </item> - <item row="1" column="0"> - <widget class="QLabel" name="label"> - <property name="text"> - <string>Title &sort: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>title_sort</cstring> - </property> - </widget> - </item> - <item row="1" column="1"> - <widget class="EnLineEdit" name="title_sort"> - <property name="toolTip"> - <string>Specify how this book should be sorted when by title. For example, The Exorcist might be sorted as Exorcist, The.</string> - </property> - </widget> - </item> - <item row="2" column="0"> - <widget class="QLabel" name="label_2"> - <property name="text"> - <string>&Author(s): </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>authors</cstring> - </property> - </widget> - </item> - <item row="2" column="1"> - <widget class="MultiCompleteComboBox" name="authors"> - <property name="editable"> - <bool>true</bool> - </property> - </widget> - </item> - <item row="3" column="0"> - <widget class="QLabel" name="label_8"> - <property name="text"> - <string>Author S&ort: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>author_sort</cstring> - </property> - </widget> - </item> - <item row="3" column="1"> - <widget class="EnLineEdit" name="author_sort"> - <property name="toolTip"> - <string>Specify how the author(s) of this book should be sorted. For example Charles Dickens should be sorted as Dickens, Charles. -If the box is colored green, then text matches the individual author's sort strings. If it is colored red, then the authors and this text do not match.</string> - </property> - </widget> - </item> - <item row="4" column="0"> - <widget class="QLabel" name="label_6"> - <property name="text"> - <string>&Rating:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>rating</cstring> - </property> - </widget> - </item> - <item row="4" column="1" colspan="2"> - <widget class="QSpinBox" name="rating"> - <property name="toolTip"> - <string>Rating of this book. 0-5 stars</string> - </property> - <property name="whatsThis"> - <string>Rating of this book. 0-5 stars</string> - </property> - <property name="buttonSymbols"> - <enum>QAbstractSpinBox::PlusMinus</enum> - </property> - <property name="suffix"> - <string> stars</string> - </property> - <property name="maximum"> - <number>5</number> - </property> - </widget> - </item> - <item row="5" column="0"> - <widget class="QLabel" name="label_3"> - <property name="text"> - <string>&Publisher: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>publisher</cstring> - </property> - </widget> - </item> - <item row="5" column="1" colspan="2"> - <widget class="MultiCompleteComboBox" name="publisher"> - <property name="editable"> - <bool>true</bool> - </property> - </widget> - </item> - <item row="6" column="0"> - <widget class="QLabel" name="label_4"> - <property name="text"> - <string>Ta&gs: </string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>tags</cstring> - </property> - </widget> - </item> - <item row="6" column="1"> - <layout class="QHBoxLayout" name="_2"> - <item> - <widget class="MultiCompleteLineEdit" name="tags"> - <property name="toolTip"> - <string>Tags categorize the book. This is particularly useful while searching. <br><br>They can be any words or phrases, separated by commas.</string> - </property> - </widget> - </item> - </layout> - </item> - <item row="6" column="2"> - <widget class="QToolButton" name="tag_editor_button"> - <property name="toolTip"> - <string>Open Tag Editor</string> - </property> - <property name="text"> - <string>Open Tag Editor</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/chapters.png</normaloff>:/images/chapters.png</iconset> - </property> - </widget> - </item> - <item row="7" column="0"> - <widget class="QLabel" name="label_7"> - <property name="text"> - <string>&Series:</string> - </property> - <property name="textFormat"> - <enum>Qt::PlainText</enum> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>series</cstring> - </property> - </widget> - </item> - <item row="7" column="1"> - <layout class="QHBoxLayout" name="_3"> - <property name="spacing"> - <number>5</number> - </property> - <item> - <widget class="MultiCompleteComboBox" name="series"> - <property name="toolTip"> - <string>List of known series. You can add new series.</string> - </property> - <property name="whatsThis"> - <string>List of known series. You can add new series.</string> - </property> - <property name="editable"> - <bool>true</bool> - </property> - <property name="insertPolicy"> - <enum>QComboBox::InsertAlphabetically</enum> - </property> - </widget> - </item> - </layout> - </item> - <item row="7" column="2"> - <widget class="QToolButton" name="remove_series_button"> - <property name="toolTip"> - <string>Remove unused series (Series that have no books)</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> - </property> - </widget> - </item> - <item row="8" column="1" colspan="2"> - <widget class="QDoubleSpinBox" name="series_index"> - <property name="enabled"> - <bool>false</bool> - </property> - <property name="prefix"> - <string>Book </string> - </property> - <property name="maximum"> - <double>99999999.989999994635582</double> - </property> - </widget> - </item> - <item row="9" column="0"> - <widget class="QLabel" name="label_9"> - <property name="text"> - <string>IS&BN:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>isbn</cstring> - </property> - </widget> - </item> - <item row="9" column="1" colspan="2"> - <widget class="QLineEdit" name="isbn"/> - </item> - <item row="10" column="0"> - <widget class="QLabel" name="label_11"> - <property name="text"> - <string>&Date:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>date</cstring> - </property> - </widget> - </item> - <item row="10" column="1" colspan="2"> - <widget class="QDateEdit" name="date"> - <property name="displayFormat"> - <string>dd MMM yyyy</string> - </property> - <property name="calendarPopup"> - <bool>true</bool> - </property> - </widget> - </item> - <item row="11" column="0"> - <widget class="QLabel" name="label_10"> - <property name="text"> - <string>Publishe&d:</string> - </property> - <property name="alignment"> - <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> - </property> - <property name="buddy"> - <cstring>pubdate</cstring> - </property> - </widget> - </item> - <item row="11" column="1"> - <widget class="QDateEdit" name="pubdate"> - <property name="displayFormat"> - <string>MMM yyyy</string> - </property> - <property name="calendarPopup"> - <bool>true</bool> - </property> - </widget> - </item> - <item row="11" column="2"> - <widget class="QToolButton" name="clear_pubdate_button"> - <property name="toolTip"> - <string>Clear published date</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> - </property> - </widget> - </item> - </layout> - </widget> - </item> - <item> - <widget class="QPushButton" name="fetch_metadata_button"> - <property name="text"> - <string>&Fetch metadata from server</string> - </property> - </widget> - </item> - <item> - <spacer name="verticalSpacer_5"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeType"> - <enum>QSizePolicy::Fixed</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>40</height> - </size> - </property> - </spacer> - </item> - </layout> - </widget> - <widget class="QWidget" name="layoutWidget_2"> - <layout class="QVBoxLayout" name="verticalLayout_2"> - <item> - <widget class="QGroupBox" name="bc_box"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Preferred" vsizetype="Expanding"> - <horstretch>0</horstretch> - <verstretch>10</verstretch> - </sizepolicy> - </property> - <property name="title"> - <string>Book Cover</string> - </property> - <layout class="QVBoxLayout" name="verticalLayout_4"> - <item> - <widget class="ImageView" name="cover" native="true"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Expanding" vsizetype="Expanding"> - <horstretch>0</horstretch> - <verstretch>100</verstretch> - </sizepolicy> - </property> - </widget> - </item> - <item> - <layout class="QVBoxLayout" name="_4"> - <property name="spacing"> - <number>6</number> - </property> - <property name="sizeConstraint"> - <enum>QLayout::SetMaximumSize</enum> - </property> - <property name="margin"> - <number>0</number> - </property> - <item> - <widget class="QLabel" name="label_5"> - <property name="text"> - <string>Change &cover image:</string> - </property> - <property name="buddy"> - <cstring>cover_button</cstring> - </property> - </widget> - </item> - <item> - <layout class="QHBoxLayout" name="_5"> - <property name="spacing"> - <number>6</number> - </property> - <property name="margin"> - <number>0</number> - </property> - <item> - <widget class="QPushButton" name="cover_button"> - <property name="text"> - <string>&Browse</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/document_open.png</normaloff>:/images/document_open.png</iconset> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="trim_cover_button"> - <property name="toolTip"> - <string>Remove border (if any) from cover</string> - </property> - <property name="text"> - <string>T&rim</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trim.png</normaloff>:/images/trim.png</iconset> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="reset_cover"> - <property name="toolTip"> - <string>Reset cover to default</string> - </property> - <property name="text"> - <string>&Remove</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> - </property> - </widget> - </item> - </layout> - </item> - </layout> - </item> - <item> - <layout class="QHBoxLayout" name="_6"> - <item> - <widget class="QPushButton" name="fetch_cover_button"> - <property name="text"> - <string>Download co&ver</string> - </property> - </widget> - </item> - <item> - <widget class="QPushButton" name="generate_cover_button"> - <property name="toolTip"> - <string>Generate a default cover based on the title and author</string> - </property> - <property name="text"> - <string>&Generate cover</string> - </property> - </widget> - </item> - </layout> - </item> - </layout> - </widget> - </item> - </layout> - </widget> - <widget class="QWidget" name="layoutWidget"> - <layout class="QVBoxLayout" name="verticalLayout_3"> - <item> - <widget class="QGroupBox" name="af_group_box"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Preferred" vsizetype="Minimum"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="title"> - <string>Available Formats</string> - </property> - <layout class="QVBoxLayout" name="verticalLayout"> - <item> - <layout class="QGridLayout" name="gridLayout"> - <item row="0" column="1" rowspan="3"> - <widget class="FormatList" name="formats"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Minimum" vsizetype="Minimum"> - <horstretch>0</horstretch> - <verstretch>0</verstretch> - </sizepolicy> - </property> - <property name="maximumSize"> - <size> - <width>16777215</width> - <height>140</height> - </size> - </property> - <property name="baseSize"> - <size> - <width>100</width> - <height>0</height> - </size> - </property> - <property name="dragDropMode"> - <enum>QAbstractItemView::DropOnly</enum> - </property> - <property name="iconSize"> - <size> - <width>64</width> - <height>64</height> - </size> - </property> - </widget> - </item> - <item row="0" column="2"> - <widget class="QToolButton" name="add_format_button"> - <property name="toolTip"> - <string>Add a new format for this book to the database</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/add_book.png</normaloff>:/images/add_book.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>32</width> - <height>32</height> - </size> - </property> - </widget> - </item> - <item row="2" column="2"> - <widget class="QToolButton" name="remove_format_button"> - <property name="toolTip"> - <string>Remove the selected formats for this book from the database.</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>32</width> - <height>32</height> - </size> - </property> - </widget> - </item> - <item row="0" column="0"> - <widget class="QToolButton" name="button_set_cover"> - <property name="toolTip"> - <string>Set the cover for the book from the selected format</string> - </property> - <property name="text"> - <string>...</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/book.png</normaloff>:/images/book.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>32</width> - <height>32</height> - </size> - </property> - </widget> - </item> - <item row="2" column="0"> - <widget class="QToolButton" name="button_set_metadata"> - <property name="toolTip"> - <string>Update metadata from the metadata in the selected format</string> - </property> - <property name="text"> - <string/> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/edit_input.png</normaloff>:/images/edit_input.png</iconset> - </property> - <property name="iconSize"> - <size> - <width>32</width> - <height>32</height> - </size> - </property> - </widget> - </item> - </layout> - </item> - </layout> - <zorder></zorder> - </widget> - </item> - <item> - <widget class="QGroupBox" name="groupBox"> - <property name="sizePolicy"> - <sizepolicy hsizetype="Preferred" vsizetype="Preferred"> - <horstretch>0</horstretch> - <verstretch>10</verstretch> - </sizepolicy> - </property> - <property name="title"> - <string>&Comments</string> - </property> - <layout class="QVBoxLayout" name="verticalLayout_8"> - <property name="margin"> - <number>0</number> - </property> - <item> - <widget class="Editor" name="comments" native="true"/> - </item> - </layout> - </widget> - </item> - </layout> - </widget> - </widget> - </item> - </layout> - </widget> - <widget class="QWidget" name="tab"> - <attribute name="title"> - <string>&Custom metadata</string> - </attribute> - <layout class="QGridLayout" name="gridLayout_2"/> - </widget> - </widget> - </item> - </layout> - </widget> - </widget> - </item> - <item> - <widget class="QDialogButtonBox" name="button_box"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="standardButtons"> - <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set> - </property> - </widget> - </item> - </layout> - </widget> - <customwidgets> - <customwidget> - <class>EnLineEdit</class> - <extends>QLineEdit</extends> - <header>widgets.h</header> - </customwidget> - <customwidget> - <class>MultiCompleteLineEdit</class> - <extends>QLineEdit</extends> - <header>calibre/gui2/complete.h</header> - </customwidget> - <customwidget> - <class>MultiCompleteComboBox</class> - <extends>QComboBox</extends> - <header>calibre/gui2/complete.h</header> - </customwidget> - <customwidget> - <class>FormatList</class> - <extends>QListWidget</extends> - <header location="global">calibre/gui2/widgets.h</header> - </customwidget> - <customwidget> - <class>ImageView</class> - <extends>QWidget</extends> - <header>calibre/gui2/widgets.h</header> - <container>1</container> - </customwidget> - <customwidget> - <class>Editor</class> - <extends>QWidget</extends> - <header location="global">calibre/gui2/comments_editor.h</header> - <container>1</container> - </customwidget> - </customwidgets> - <tabstops> - <tabstop>title</tabstop> - <tabstop>auto_title_sort</tabstop> - <tabstop>title_sort</tabstop> - <tabstop>swap_button</tabstop> - <tabstop>authors</tabstop> - <tabstop>auto_author_sort</tabstop> - <tabstop>author_sort</tabstop> - <tabstop>rating</tabstop> - <tabstop>publisher</tabstop> - <tabstop>tags</tabstop> - <tabstop>tag_editor_button</tabstop> - <tabstop>series</tabstop> - <tabstop>remove_series_button</tabstop> - <tabstop>series_index</tabstop> - <tabstop>isbn</tabstop> - <tabstop>date</tabstop> - <tabstop>pubdate</tabstop> - <tabstop>fetch_metadata_button</tabstop> - <tabstop>button_set_cover</tabstop> - <tabstop>button_set_metadata</tabstop> - <tabstop>formats</tabstop> - <tabstop>add_format_button</tabstop> - <tabstop>remove_format_button</tabstop> - <tabstop>cover_button</tabstop> - <tabstop>trim_cover_button</tabstop> - <tabstop>reset_cover</tabstop> - <tabstop>fetch_cover_button</tabstop> - <tabstop>generate_cover_button</tabstop> - <tabstop>button_box</tabstop> - <tabstop>scrollArea</tabstop> - <tabstop>central_widget</tabstop> - </tabstops> - <resources> - <include location="../../../../resources/images.qrc"/> - </resources> - <connections> - <connection> - <sender>button_box</sender> - <signal>accepted()</signal> - <receiver>MetadataSingleDialog</receiver> - <slot>accept()</slot> - <hints> - <hint type="sourcelabel"> - <x>261</x> - <y>710</y> - </hint> - <hint type="destinationlabel"> - <x>157</x> - <y>274</y> - </hint> - </hints> - </connection> - <connection> - <sender>button_box</sender> - <signal>rejected()</signal> - <receiver>MetadataSingleDialog</receiver> - <slot>reject()</slot> - <hints> - <hint type="sourcelabel"> - <x>329</x> - <y>710</y> - </hint> - <hint type="destinationlabel"> - <x>286</x> - <y>274</y> - </hint> - </hints> - </connection> - </connections> -</ui> diff --git a/src/calibre/gui2/metadata/bulk_download.py b/src/calibre/gui2/metadata/bulk_download.py index 7a7f49dabf..2a307fc902 100644 --- a/src/calibre/gui2/metadata/bulk_download.py +++ b/src/calibre/gui2/metadata/bulk_download.py @@ -1,308 +1,195 @@ #!/usr/bin/env python # vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement +from __future__ import (unicode_literals, division, absolute_import, + print_function) __license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' +__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' -import traceback -from threading import Thread -from Queue import Queue, Empty from functools import partial +from itertools import izip +from threading import Event -from PyQt4.Qt import QObject, QTimer, QDialog, \ - QVBoxLayout, QTextBrowser, QLabel, QGroupBox, QDialogButtonBox +from PyQt4.Qt import (QIcon, QDialog, + QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt) -from calibre.ebooks.metadata.fetch import search, get_social_metadata -from calibre.gui2 import config, error_dialog -from calibre.gui2.dialogs.progress import ProgressDialog -from calibre.ebooks.metadata.covers import download_cover -from calibre.customize.ui import get_isbndb_key +from calibre.gui2.threaded_jobs import ThreadedJob +from calibre.ebooks.metadata.sources.identify import identify, msprefs +from calibre.ebooks.metadata.sources.covers import download_cover +from calibre.ebooks.metadata.book.base import Metadata +from calibre.customize.ui import metadata_plugins +from calibre.ptempfile import PersistentTemporaryFile -class Worker(Thread): - 'Cover downloader' +# Start download {{{ +def show_config(gui, parent): + from calibre.gui2.preferences import show_config_widget + show_config_widget('Sharing', 'Metadata download', parent=parent, + gui=gui, never_shutdown=True) - def __init__(self): - Thread.__init__(self) - self.daemon = True - self.jobs = Queue() - self.results = Queue() +class ConfirmDialog(QDialog): - def run(self): - while True: - id, mi = self.jobs.get() - if not getattr(mi, 'isbn', False): - break + def __init__(self, ids, parent): + QDialog.__init__(self, parent) + self.setWindowTitle(_('Schedule download?')) + self.setWindowIcon(QIcon(I('dialog_question.png'))) + + l = self.l = QGridLayout() + self.setLayout(l) + + i = QLabel(self) + i.setPixmap(QPixmap(I('dialog_question.png'))) + l.addWidget(i, 0, 0) + + t = QLabel( + '<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will' + ' run in the background. Proceed?')%len(ids) + + '<p>'+_('You can monitor the progress of the download ' + 'by clicking the rotating spinner in the bottom right ' + 'corner.') + + '<p>'+_('When the download completes you will be asked for' + ' confirmation before calibre applies the downloaded metadata.') + ) + t.setWordWrap(True) + l.addWidget(t, 0, 1) + l.setColumnStretch(0, 1) + l.setColumnStretch(1, 100) + + self.identify = self.covers = True + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) + self.bb.rejected.connect(self.reject) + b = self.bb.addButton(_('Download only &metadata'), + self.bb.AcceptRole) + b.clicked.connect(self.only_metadata) + b.setIcon(QIcon(I('edit_input.png'))) + b = self.bb.addButton(_('Download only &covers'), + self.bb.AcceptRole) + b.clicked.connect(self.only_covers) + b.setIcon(QIcon(I('default_cover.png'))) + b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole) + b.setIcon(QIcon(I('config.png'))) + b.clicked.connect(partial(show_config, parent, self)) + l.addWidget(self.bb, 1, 0, 1, 2) + b = self.bb.addButton(_('Download &both'), + self.bb.AcceptRole) + b.clicked.connect(self.accept) + b.setDefault(True) + b.setAutoDefault(True) + b.setIcon(QIcon(I('ok.png'))) + + self.resize(self.sizeHint()) + b.setFocus(Qt.OtherFocusReason) + + def only_metadata(self): + self.covers = False + self.accept() + + def only_covers(self): + self.identify = False + self.accept() + +def start_download(gui, ids, callback): + d = ConfirmDialog(ids, gui) + ret = d.exec_() + d.b.clicked.disconnect() + if ret != d.Accepted: + return + + job = ThreadedJob('metadata bulk download', + _('Download metadata for %d books')%len(ids), + download, (ids, gui.current_db, d.identify, d.covers), {}, callback) + gui.job_manager.run_threaded_job(job) + gui.status_bar.show_message(_('Metadata download started'), 3000) +# }}} + +def get_job_details(job): + id_map, failed_ids, failed_covers, title_map, all_failed = job.result + det_msg = [] + for i in failed_ids | failed_covers: + title = title_map[i] + if i in failed_ids: + title += (' ' + _('(Failed metadata)')) + if i in failed_covers: + title += (' ' + _('(Failed cover)')) + det_msg.append(title) + det_msg = '\n'.join(det_msg) + return id_map, failed_ids, failed_covers, all_failed, det_msg + +def merge_result(oldmi, newmi): + dummy = Metadata(_('Unknown')) + for f in msprefs['ignore_fields']: + if ':' not in f: + setattr(newmi, f, getattr(dummy, f)) + fields = set() + for plugin in metadata_plugins(['identify']): + fields |= plugin.touched_fields + + for f in fields: + # Optimize so that set_metadata does not have to do extra work later + if not f.startswith('identifier:'): + if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): + setattr(newmi, f, getattr(dummy, f)) + + newmi.last_modified = oldmi.last_modified + + return newmi + +def download(ids, db, do_identify, covers, + log=None, abort=None, notifications=None): + ids = list(ids) + metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) + for i in ids] + failed_ids = set() + failed_covers = set() + title_map = {} + ans = {} + count = 0 + all_failed = True + ''' + # Test apply dialog + all_failed = do_identify = covers = False + ''' + for i, mi in izip(ids, metadata): + if abort.is_set(): + log.error('Aborting...') + break + title, authors, identifiers = mi.title, mi.authors, mi.identifiers + title_map[i] = title + if do_identify: + results = [] try: - cdata, errors = download_cover(mi) - if cdata: - self.results.put((id, mi, True, cdata)) - else: - msg = [] - for e in errors: - if not e[0]: - msg.append(e[-1] + ' - ' + e[1]) - self.results.put((id, mi, False, '\n'.join(msg))) + results = identify(log, Event(), title=title, authors=authors, + identifiers=identifiers) except: - self.results.put((id, mi, False, traceback.format_exc())) - - def __enter__(self): - self.start() - return self - - def __exit__(self, *args): - self.jobs.put((False, False)) - - -class DownloadMetadata(Thread): - 'Metadata downloader' - - def __init__(self, db, ids, get_covers, set_metadata=True, - get_social_metadata=True): - Thread.__init__(self) - self.daemon = True - self.metadata = {} - self.covers = {} - self.set_metadata = set_metadata - self.get_social_metadata = get_social_metadata - self.social_metadata_exceptions = [] - self.db = db - self.updated = set([]) - self.get_covers = get_covers - self.worker = Worker() - self.results = Queue() - self.keep_going = True - for id in ids: - self.metadata[id] = db.get_metadata(id, index_is_id=True) - self.metadata[id].rating = None - self.total = len(ids) - if self.get_covers: - self.total += len(ids) - self.fetched_metadata = {} - self.fetched_covers = {} - self.failures = {} - self.cover_failures = {} - self.exception = self.tb = None - - def run(self): - try: - self._run() - except Exception as e: - self.exception = e - self.tb = traceback.format_exc() - - def _run(self): - self.key = get_isbndb_key() - if not self.key: - self.key = None - with self.worker: - for id, mi in self.metadata.items(): - if not self.keep_going: - break - args = {} - if mi.isbn: - args['isbn'] = mi.isbn - else: - if mi.is_null('title'): - self.failures[id] = \ - _('Book has neither title nor ISBN') - continue - args['title'] = mi.title - if mi.authors and mi.authors[0] != _('Unknown'): - args['author'] = mi.authors[0] - if self.key: - args['isbndb_key'] = self.key - results, exceptions = search(**args) - if results: - fmi = results[0] - self.fetched_metadata[id] = fmi - if self.get_covers: - if fmi.isbn: - self.worker.jobs.put((id, fmi)) - else: - self.results.put((id, 'cover', False, mi.title)) - if (not config['overwrite_author_title_metadata']): - fmi.authors = mi.authors - fmi.author_sort = mi.author_sort - fmi.title = mi.title - mi.smart_update(fmi) - if mi.isbn and self.get_social_metadata: - self.social_metadata_exceptions = get_social_metadata(mi) - if mi.rating: - mi.rating *= 2 - if not self.get_social_metadata: - mi.tags = [] - self.results.put((id, 'metadata', True, mi.title)) - else: - self.failures[id] = _('No matches found for this book') - self.results.put((id, 'metadata', False, mi.title)) - self.results.put((id, 'cover', False, mi.title)) - self.commit_covers() - - self.commit_covers(True) - - def commit_covers(self, all=False): - if all: - self.worker.jobs.put((False, False)) - while True: - try: - id, fmi, ok, cdata = self.worker.results.get_nowait() - if ok: - self.fetched_covers[id] = cdata - self.results.put((id, 'cover', ok, fmi.title)) - else: - self.results.put((id, 'cover', ok, fmi.title)) - try: - self.cover_failures[id] = unicode(cdata) - except: - self.cover_failures[id] = repr(cdata) - except Empty: - if not all or not self.worker.is_alive(): - return - -class DoDownload(QObject): - - def __init__(self, parent, title, db, ids, get_covers, set_metadata=True, - get_social_metadata=True): - QObject.__init__(self, parent) - self.pd = ProgressDialog(title, min=0, max=0, parent=parent) - self.pd.canceled_signal.connect(self.cancel) - self.downloader = None - self.create = partial(DownloadMetadata, db, ids, get_covers, - set_metadata=set_metadata, - get_social_metadata=get_social_metadata) - self.get_covers = get_covers - self.db = db - self.updated = set([]) - self.total = len(ids) - self.keep_going = True - - def exec_(self): - QTimer.singleShot(50, self.do_one) - ret = self.pd.exec_() - if getattr(self.downloader, 'exception', None) is not None and \ - ret == self.pd.Accepted: - error_dialog(self.parent(), _('Failed'), - _('Failed to download metadata'), show=True) - else: - self.show_report() - return ret - - def cancel(self, *args): - self.keep_going = False - self.downloader.keep_going = False - self.pd.reject() - - def do_one(self): - try: - if not self.keep_going: - return - if self.downloader is None: - self.downloader = self.create() - self.downloader.start() - self.pd.set_min(0) - self.pd.set_max(self.downloader.total) - try: - r = self.downloader.results.get_nowait() - self.handle_result(r) - except Empty: pass - if not self.downloader.is_alive(): - while True: - try: - r = self.downloader.results.get_nowait() - self.handle_result(r) - except Empty: - break - self.pd.accept() - return - except: - self.cancel() - raise - QTimer.singleShot(50, self.do_one) - - def handle_result(self, r): - id_, typ, ok, title = r - what = _('cover') if typ == 'cover' else _('metadata') - which = _('Downloaded') if ok else _('Failed to get') - if self.get_covers or typ != 'cover' or ok: - # Do not show message when cover fetch fails if user didn't ask to - # download covers - self.pd.set_msg(_('%s %s for: %s') % (which, what, title)) - self.pd.value += 1 - if ok: - self.updated.add(id_) - if typ == 'cover': - try: - self.db.set_cover(id_, - self.downloader.fetched_covers.pop(id_)) - except: - self.downloader.cover_failures[id_] = \ - traceback.format_exc() + if results: + all_failed = False + mi = merge_result(mi, results[0]) + identifiers = mi.identifiers + if not mi.is_null('rating'): + # set_metadata expects a rating out of 10 + mi.rating *= 2 else: - try: - self.set_metadata(id_) - except: - self.downloader.failures[id_] = \ - traceback.format_exc() - - def set_metadata(self, id_): - mi = self.downloader.metadata[id_] - if self.downloader.set_metadata: - self.db.set_metadata(id_, mi) - if not self.downloader.set_metadata and self.downloader.get_social_metadata: - if mi.rating: - self.db.set_rating(id_, mi.rating) - if mi.tags: - self.db.set_tags(id_, mi.tags) - if mi.comments: - self.db.set_comment(id_, mi.comments) - if mi.series: - self.db.set_series(id_, mi.series) - if mi.series_index is not None: - self.db.set_series_index(id_, mi.series_index) - - def show_report(self): - f, cf = self.downloader.failures, self.downloader.cover_failures - report = [] - if f: - report.append( - '<h3>Failed to download metadata for the following:</h3><ol>') - for id_, err in f.items(): - mi = self.downloader.metadata[id_] - report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title, - unicode(err))) - report.append('</ol>') - if cf: - report.append( - '<h3>Failed to download cover for the following:</h3><ol>') - for id_, err in cf.items(): - mi = self.downloader.metadata[id_] - report.append('<li><b>%s</b><pre>%s</pre></li>' % (mi.title, - unicode(err))) - report.append('</ol>') - - if len(self.updated) != self.total or report: - d = QDialog(self.parent()) - bb = QDialogButtonBox(QDialogButtonBox.Ok, parent=d) - v1 = QVBoxLayout() - d.setLayout(v1) - d.setWindowTitle(_('Done')) - v1.addWidget(QLabel(_('Successfully downloaded metadata for %d out of %d books') % - (len(self.updated), self.total))) - gb = QGroupBox(_('Details'), self.parent()) - v2 = QVBoxLayout() - gb.setLayout(v2) - b = QTextBrowser(self.parent()) - v2.addWidget(b) - b.setHtml('\n'.join(report)) - v1.addWidget(gb) - v1.addWidget(bb) - bb.accepted.connect(d.accept) - d.resize(800, 600) - d.exec_() - + log.error('Failed to download metadata for', title) + failed_ids.add(i) + # We don't want set_metadata operating on anything but covers + mi = merge_result(mi, mi) + if covers: + cdata = download_cover(log, title=title, authors=authors, + identifiers=identifiers) + if cdata is not None: + with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: + f.write(cdata[-1]) + mi.cover = f.name + all_failed = False + else: + failed_covers.add(i) + ans[i] = mi + count += 1 + notifications.put((count/len(ids), + _('Downloaded %d of %d')%(count, len(ids)))) + log('Download complete, with %d failures'%len(failed_ids)) + return (ans, failed_ids, failed_covers, title_map, all_failed) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py deleted file mode 100644 index 2a307fc902..0000000000 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import (unicode_literals, division, absolute_import, - print_function) - -__license__ = 'GPL v3' -__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -from functools import partial -from itertools import izip -from threading import Event - -from PyQt4.Qt import (QIcon, QDialog, - QDialogButtonBox, QLabel, QGridLayout, QPixmap, Qt) - -from calibre.gui2.threaded_jobs import ThreadedJob -from calibre.ebooks.metadata.sources.identify import identify, msprefs -from calibre.ebooks.metadata.sources.covers import download_cover -from calibre.ebooks.metadata.book.base import Metadata -from calibre.customize.ui import metadata_plugins -from calibre.ptempfile import PersistentTemporaryFile - -# Start download {{{ -def show_config(gui, parent): - from calibre.gui2.preferences import show_config_widget - show_config_widget('Sharing', 'Metadata download', parent=parent, - gui=gui, never_shutdown=True) - -class ConfirmDialog(QDialog): - - def __init__(self, ids, parent): - QDialog.__init__(self, parent) - self.setWindowTitle(_('Schedule download?')) - self.setWindowIcon(QIcon(I('dialog_question.png'))) - - l = self.l = QGridLayout() - self.setLayout(l) - - i = QLabel(self) - i.setPixmap(QPixmap(I('dialog_question.png'))) - l.addWidget(i, 0, 0) - - t = QLabel( - '<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will' - ' run in the background. Proceed?')%len(ids) + - '<p>'+_('You can monitor the progress of the download ' - 'by clicking the rotating spinner in the bottom right ' - 'corner.') + - '<p>'+_('When the download completes you will be asked for' - ' confirmation before calibre applies the downloaded metadata.') - ) - t.setWordWrap(True) - l.addWidget(t, 0, 1) - l.setColumnStretch(0, 1) - l.setColumnStretch(1, 100) - - self.identify = self.covers = True - self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) - self.bb.rejected.connect(self.reject) - b = self.bb.addButton(_('Download only &metadata'), - self.bb.AcceptRole) - b.clicked.connect(self.only_metadata) - b.setIcon(QIcon(I('edit_input.png'))) - b = self.bb.addButton(_('Download only &covers'), - self.bb.AcceptRole) - b.clicked.connect(self.only_covers) - b.setIcon(QIcon(I('default_cover.png'))) - b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole) - b.setIcon(QIcon(I('config.png'))) - b.clicked.connect(partial(show_config, parent, self)) - l.addWidget(self.bb, 1, 0, 1, 2) - b = self.bb.addButton(_('Download &both'), - self.bb.AcceptRole) - b.clicked.connect(self.accept) - b.setDefault(True) - b.setAutoDefault(True) - b.setIcon(QIcon(I('ok.png'))) - - self.resize(self.sizeHint()) - b.setFocus(Qt.OtherFocusReason) - - def only_metadata(self): - self.covers = False - self.accept() - - def only_covers(self): - self.identify = False - self.accept() - -def start_download(gui, ids, callback): - d = ConfirmDialog(ids, gui) - ret = d.exec_() - d.b.clicked.disconnect() - if ret != d.Accepted: - return - - job = ThreadedJob('metadata bulk download', - _('Download metadata for %d books')%len(ids), - download, (ids, gui.current_db, d.identify, d.covers), {}, callback) - gui.job_manager.run_threaded_job(job) - gui.status_bar.show_message(_('Metadata download started'), 3000) -# }}} - -def get_job_details(job): - id_map, failed_ids, failed_covers, title_map, all_failed = job.result - det_msg = [] - for i in failed_ids | failed_covers: - title = title_map[i] - if i in failed_ids: - title += (' ' + _('(Failed metadata)')) - if i in failed_covers: - title += (' ' + _('(Failed cover)')) - det_msg.append(title) - det_msg = '\n'.join(det_msg) - return id_map, failed_ids, failed_covers, all_failed, det_msg - -def merge_result(oldmi, newmi): - dummy = Metadata(_('Unknown')) - for f in msprefs['ignore_fields']: - if ':' not in f: - setattr(newmi, f, getattr(dummy, f)) - fields = set() - for plugin in metadata_plugins(['identify']): - fields |= plugin.touched_fields - - for f in fields: - # Optimize so that set_metadata does not have to do extra work later - if not f.startswith('identifier:'): - if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): - setattr(newmi, f, getattr(dummy, f)) - - newmi.last_modified = oldmi.last_modified - - return newmi - -def download(ids, db, do_identify, covers, - log=None, abort=None, notifications=None): - ids = list(ids) - metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) - for i in ids] - failed_ids = set() - failed_covers = set() - title_map = {} - ans = {} - count = 0 - all_failed = True - ''' - # Test apply dialog - all_failed = do_identify = covers = False - ''' - for i, mi in izip(ids, metadata): - if abort.is_set(): - log.error('Aborting...') - break - title, authors, identifiers = mi.title, mi.authors, mi.identifiers - title_map[i] = title - if do_identify: - results = [] - try: - results = identify(log, Event(), title=title, authors=authors, - identifiers=identifiers) - except: - pass - if results: - all_failed = False - mi = merge_result(mi, results[0]) - identifiers = mi.identifiers - if not mi.is_null('rating'): - # set_metadata expects a rating out of 10 - mi.rating *= 2 - else: - log.error('Failed to download metadata for', title) - failed_ids.add(i) - # We don't want set_metadata operating on anything but covers - mi = merge_result(mi, mi) - if covers: - cdata = download_cover(log, title=title, authors=authors, - identifiers=identifiers) - if cdata is not None: - with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: - f.write(cdata[-1]) - mi.cover = f.name - all_failed = False - else: - failed_covers.add(i) - ans[i] = mi - count += 1 - notifications.put((count/len(ids), - _('Downloaded %d of %d')%(count, len(ids)))) - log('Download complete, with %d failures'%len(failed_ids)) - return (ans, failed_ids, failed_covers, title_map, all_failed) - - - diff --git a/src/calibre/gui2/preferences/behavior.py b/src/calibre/gui2/preferences/behavior.py index b376d067bc..e062ae2662 100644 --- a/src/calibre/gui2/preferences/behavior.py +++ b/src/calibre/gui2/preferences/behavior.py @@ -19,7 +19,6 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.oeb.iterator import is_supported from calibre.constants import iswindows from calibre.utils.icu import sort_key -from calibre.utils.config import test_eight_code class OutputFormatSetting(Setting): @@ -40,12 +39,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('network_timeout', prefs) - - r('overwrite_author_title_metadata', config) - r('get_social_metadata', config) - if test_eight_code: - self.opt_overwrite_author_title_metadata.setVisible(False) - self.opt_get_social_metadata.setVisible(False) r('new_version_notification', config) r('upload_news_to_device', config) r('delete_news_from_library_on_upload', config) @@ -67,13 +60,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): signal.connect(self.internally_viewed_formats_changed) r('bools_are_tristate', db.prefs, restart_required=True) - if test_eight_code: - r = self.register - choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')] - r('edit_metadata_single_layout', gprefs, choices=choices) - else: - self.opt_edit_metadata_single_layout.setVisible(False) - self.edit_metadata_single_label.setVisible(False) + r = self.register + choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')] + r('edit_metadata_single_layout', gprefs, choices=choices) def initialize(self): ConfigWidgetBase.initialize(self) diff --git a/src/calibre/gui2/preferences/behavior.ui b/src/calibre/gui2/preferences/behavior.ui index 69ebce6acf..ffd59d72bb 100644 --- a/src/calibre/gui2/preferences/behavior.ui +++ b/src/calibre/gui2/preferences/behavior.ui @@ -14,41 +14,14 @@ <string>Form</string> </property> <layout class="QGridLayout" name="gridLayout"> - <item row="0" column="1"> - <spacer> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>10</width> - <height>0</height> - </size> - </property> - </spacer> - </item> - <item row="0" column="0"> - <widget class="QCheckBox" name="opt_overwrite_author_title_metadata"> - <property name="text"> - <string>&Overwrite author and title by default when fetching metadata</string> - </property> - </widget> - </item> - <item row="0" column="2"> - <widget class="QCheckBox" name="opt_get_social_metadata"> - <property name="text"> - <string>Download &social metadata (tags/ratings/etc.) by default</string> - </property> - </widget> - </item> - <item row="2" column="0"> + <item row="1" column="0"> <widget class="QCheckBox" name="opt_new_version_notification"> <property name="text"> <string>Show notification when &new version is available</string> </property> </widget> </item> - <item row="2" column="2"> + <item row="1" column="1"> <widget class="QCheckBox" name="opt_bools_are_tristate"> <property name="toolTip"> <string>If checked, Yes/No custom columns values can be Yes, No, or Unknown. @@ -59,21 +32,21 @@ If not checked, the values can be Yes or No.</string> </property> </widget> </item> - <item row="4" column="0"> + <item row="3" column="0"> <widget class="QCheckBox" name="opt_upload_news_to_device"> <property name="text"> <string>Automatically send downloaded &news to ebook reader</string> </property> </widget> </item> - <item row="4" column="2"> + <item row="3" column="1"> <widget class="QCheckBox" name="opt_delete_news_from_library_on_upload"> <property name="text"> <string>&Delete news from library when it is automatically sent to reader</string> </property> </widget> </item> - <item row="6" column="0"> + <item row="5" column="0"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="label_23"> @@ -97,7 +70,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="6" column="2"> + <item row="5" column="1"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="label_2"> @@ -130,7 +103,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="8" column="0"> + <item row="7" column="0"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="priority_label"> @@ -169,7 +142,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="8" column="2"> + <item row="7" column="1"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="label_170"> @@ -202,7 +175,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="9" column="0"> + <item row="8" column="0"> <layout class="QHBoxLayout"> <item> <widget class="QLabel" name="edit_metadata_single_label"> @@ -223,7 +196,7 @@ If not checked, the values can be Yes or No.</string> </item> </layout> </item> - <item row="20" column="0"> + <item row="19" column="0"> <widget class="QGroupBox" name="groupBox_5"> <property name="title"> <string>Preferred &input format order:</string> @@ -285,7 +258,7 @@ If not checked, the values can be Yes or No.</string> </layout> </widget> </item> - <item row="20" column="2"> + <item row="19" column="1"> <widget class="QGroupBox" name="groupBox_3"> <property name="title"> <string>Use internal &viewer for:</string> @@ -304,7 +277,7 @@ If not checked, the values can be Yes or No.</string> </layout> </widget> </item> - <item row="9" column="2"> + <item row="8" column="1"> <widget class="QPushButton" name="reset_confirmation_button"> <property name="text"> <string>Reset all disabled &confirmation dialogs</string> diff --git a/src/calibre/gui2/preferences/social.py b/src/calibre/gui2/preferences/social.py deleted file mode 100644 index a22bcce091..0000000000 --- a/src/calibre/gui2/preferences/social.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement - -__license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - -import time -from threading import Thread - -from PyQt4.Qt import QDialog, QDialogButtonBox, Qt, QLabel, QVBoxLayout, \ - QTimer - -from calibre.ebooks.metadata import MetaInformation - -class Worker(Thread): - - def __init__(self, mi): - Thread.__init__(self) - self.daemon = True - self.mi = MetaInformation(mi) - self.exceptions = [] - - def run(self): - from calibre.ebooks.metadata.fetch import get_social_metadata - self.exceptions = get_social_metadata(self.mi) - -class SocialMetadata(QDialog): - - TIMEOUT = 300 # seconds - - def __init__(self, mi, parent): - QDialog.__init__(self, parent) - - self.bbox = QDialogButtonBox(QDialogButtonBox.Cancel, Qt.Horizontal, self) - self.mi = mi - self.layout = QVBoxLayout(self) - self.label = QLabel(_('Downloading social metadata, please wait...'), self) - self.label.setWordWrap(True) - self.layout.addWidget(self.label) - self.layout.addWidget(self.bbox) - - self.worker = Worker(mi) - self.bbox.rejected.connect(self.reject) - self.worker.start() - self.start_time = time.time() - self.timed_out = False - self.rejected = False - QTimer.singleShot(50, self.update) - - def reject(self): - self.rejected = True - QDialog.reject(self) - - def update(self): - if self.rejected: - return - if time.time() - self.start_time > self.TIMEOUT: - self.timed_out = True - self.reject() - return - if not self.worker.is_alive(): - self.accept() - return - QTimer.singleShot(50, self.update) - - def accept(self): - self.mi.tags = self.worker.mi.tags - self.mi.rating = self.worker.mi.rating - self.mi.comments = self.worker.mi.comments - if self.worker.mi.series: - self.mi.series = self.worker.mi.series - self.mi.series_index = self.worker.mi.series_index - QDialog.accept(self) - - @property - def exceptions(self): - return self.worker.exceptions diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 8b23cf3071..0fcd047619 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -24,8 +24,6 @@ if False: OptionSet, ConfigInterface, read_tweaks, write_tweaks read_raw_tweaks, tweaks, plugin_dir -test_eight_code = tweaks.get('test_eight_code', False) - def check_config_write_access(): return os.access(config_dir, os.W_OK) and os.access(config_dir, os.X_OK) From 75a43d9c1a5a6ac84fae363734e17952b327f364 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 30 Apr 2011 13:20:38 -0600 Subject: [PATCH 02/39] ISBN checking now correctly flags ISBNs with all same digits as invalid --- src/calibre/ebooks/metadata/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 2ae5f3ade5..9c7838cb2c 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -274,6 +274,9 @@ def check_isbn(isbn): if not isbn: return None isbn = re.sub(r'[^0-9X]', '', isbn.upper()) + all_same = re.match(r'(\d)\1{9,12}$', isbn) + if all_same is not None: + return None if len(isbn) == 10: return check_isbn10(isbn) if len(isbn) == 13: From 8862bc694e2424c36dd2e39060c1cc8485318b87 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 30 Apr 2011 16:32:48 -0600 Subject: [PATCH 03/39] Fix #774457 (Downloading metadata gives error) --- src/calibre/ebooks/metadata/sources/identify.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index 9a9e5aa164..3d4807ac02 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -400,6 +400,9 @@ def identify(log, abort, # {{{ and plugin.get_cached_cover_url(result.identifiers) is not None) result.identify_plugin = plugin + if msprefs['txt_comments']: + if plugin.has_html_comments and result.comments: + result.comments = html2text(r.comments) log('The identify phase took %.2f seconds'%(time.time() - start_time)) log('The longest time (%f) was taken by:'%longest, lp) @@ -410,10 +413,6 @@ def identify(log, abort, # {{{ log('We have %d merged results, merging took: %.2f seconds' % (len(results), time.time() - start_time)) - if msprefs['txt_comments']: - for r in results: - if r.identify_plugin.has_html_comments and r.comments: - r.comments = html2text(r.comments) max_tags = msprefs['max_tags'] for r in results: From 3cfbf1cccd705dc49dbeeaf26e9d35821fe195cd Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sat, 30 Apr 2011 16:42:11 -0600 Subject: [PATCH 04/39] ... --- src/calibre/devices/misc.py | 2 +- src/calibre/manual/faq.rst | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index b9710d1958..a0cb819cb9 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -187,7 +187,7 @@ class LUMIREAD(USBMS): cfilepath = cfilepath.replace(os.sep+'books'+os.sep, os.sep+'covers'+os.sep, 1) pdir = os.path.dirname(cfilepath) - if not os.exists(pdir): + if not os.path.exists(pdir): os.makedirs(pdir) with open(cfilepath+'.jpg', 'wb') as f: f.write(metadata.thumbnail[-1]) diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 56d1832440..816ce7c496 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -468,6 +468,18 @@ If it still wont launch, start a command prompt (press the windows key and R; th Post any output you see in a help message on the `Forum <http://www.mobileread.com/forums/forumdisplay.php?f=166>`_. +|app| freeze when I click on anything? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +There are three possible things I know of, that can cause this: + + * You recently connected an external monitor or TV to your computer. In this case, whenever |app| opens a new window like the edit metadata window or the conversion dialog, it appears on the second monitor where you dont notice it and so you think |app| has frozen. Disconnect your second monitor and restart calibre. + + * You are using a Wacom branded mouse. There is an incompatibility between Wacom mice and the graphics toolkit |app| uses. Try using a non-Wacom mouse. + + * You have invalid files in your fonts folder. If this is the case, start |app| in debug mode as desribed in the previous answer and you will get messages about invalid files in :file:`C:\\Windows\\fonts`. Delete these files and you will be fine. + + |app| is not starting on OS X? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 91350fc1284bb3207a62be4552cdf5e0bdde8408 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Sun, 1 May 2011 16:21:13 +0100 Subject: [PATCH 05/39] Add another Samsung card ID (SGH-T849_CARD) to WINDOWS_CARD_A_MEM. --- src/calibre/devices/android/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 359dae89fe..f500560f97 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -112,7 +112,7 @@ class ANDROID(USBMS): 'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE', 'A7EB'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', - 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB'] + 'A70S', 'A101IT', '7', 'INCREDIBLE', 'A7EB', 'SGH-T849_CARD'] OSX_MAIN_MEM = 'Android Device Main Memory' From bab502e6543b2997158b23a72d6f41e119e15136 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 1 May 2011 09:58:55 -0600 Subject: [PATCH 06/39] Fix Brand Eins --- recipes/brand_eins.recipe | 13 ++++++++----- src/calibre/manual/faq.rst | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/recipes/brand_eins.recipe b/recipes/brand_eins.recipe index 15e1d3ccca..e6fe57b334 100644 --- a/recipes/brand_eins.recipe +++ b/recipes/brand_eins.recipe @@ -3,7 +3,8 @@ __license__ = 'GPL v3' __copyright__ = '2010, Constantin Hofstetter <consti at consti.de>, Steffen Siebert <calibre at steffensiebert.de>' -__version__ = '0.98' # 2011-04-10 +__version__ = '0.98' + ''' http://brandeins.de - Wirtschaftsmagazin ''' import re import string @@ -13,8 +14,8 @@ from calibre.web.feeds.recipes import BasicNewsRecipe class BrandEins(BasicNewsRecipe): title = u'brand eins' - __author__ = 'Constantin Hofstetter; Steffen Siebert' - description = u'Wirtschaftsmagazin: Gets the last full issue on default. Set a integer value for the username-field to get older issues: 1 -> the newest (but not complete) issue, 2 -> the last complete issue (default), 3 -> the issue before 2 etc.' + __author__ = 'Constantin Hofstetter' + description = u'Wirtschaftsmagazin' publisher ='brandeins.de' category = 'politics, business, wirtschaft, Germany' use_embedded_content = False @@ -105,10 +106,11 @@ class BrandEins(BasicNewsRecipe): keys = issue_map.keys() keys.sort() keys.reverse() - selected_issue = issue_map[keys[issue-1]] + selected_issue_key = keys[issue - 1] + selected_issue = issue_map[selected_issue_key] url = selected_issue.get('href', False) # Get the title for the magazin - build it out of the title of the cover - take the issue and year; - self.title = "brand eins "+ re.search(r"(?P<date>\d\d\/\d\d\d\d)", selected_issue.find('img').get('title', False)).group('date') + self.title = "brand eins " + selected_issue_key[4:] + "/" + selected_issue_key[0:4] url = 'http://brandeins.de/'+url # url = "http://www.brandeins.de/archiv/magazin/tierisch.html" @@ -161,3 +163,4 @@ class BrandEins(BasicNewsRecipe): current_articles.append({'title': title, 'url': url, 'description': description, 'date':''}) titles_and_articles.append([chapter_title, current_articles]) return titles_and_articles + diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 816ce7c496..08ebb6506b 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -468,7 +468,7 @@ If it still wont launch, start a command prompt (press the windows key and R; th Post any output you see in a help message on the `Forum <http://www.mobileread.com/forums/forumdisplay.php?f=166>`_. -|app| freeze when I click on anything? +|app| freezes when I click on anything? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ There are three possible things I know of, that can cause this: From 186a47da8ce95287db54d0706ff556a4363d0860 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 1 May 2011 10:33:10 -0600 Subject: [PATCH 07/39] Fix #774743 (Calibre Crash on read [open by (v) from edit-screen]) --- src/calibre/gui2/actions/view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/view.py b/src/calibre/gui2/actions/view.py index c93e69f0fc..6cf5c5d5af 100644 --- a/src/calibre/gui2/actions/view.py +++ b/src/calibre/gui2/actions/view.py @@ -60,7 +60,7 @@ class ViewAction(InterfaceAction): def build_menus(self, db): self.view_menu.clear() - self.view_menu.addAction(self.qaction) + self.view_menu.addAction(self.view_action) self.view_menu.addAction(self.view_specific_action) self.view_menu.addSeparator() self.view_menu.addAction(self.action_pick_random) From 9a32f09a71ba49f727f1c298bf79915bd18f6e98 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 1 May 2011 10:46:42 -0600 Subject: [PATCH 08/39] MOBI Input: Handle MOBI files with empty <u> tags correctly. Fixes #774785 (Private bug) --- src/calibre/ebooks/mobi/reader.py | 5 +++++ src/calibre/ebooks/oeb/base.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index d9c6853795..3d858864a8 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -253,6 +253,8 @@ class MobiReader(object): .italic { font-style: italic } + .underline { text-decoration: underline } + .mbp_pagebreak { page-break-after: always; margin: 0; display: block } @@ -601,6 +603,9 @@ class MobiReader(object): elif tag.tag == 'i': tag.tag = 'span' tag.attrib['class'] = 'italic' + elif tag.tag == 'u': + tag.tag = 'span' + tag.attrib['class'] = 'underline' elif tag.tag == 'b': tag.tag = 'span' tag.attrib['class'] = 'bold' diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 1f71e32548..db83fca496 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -1049,8 +1049,8 @@ class Manifest(object): # Remove hyperlinks with no content as they cause rendering # artifacts in browser based renderers - # Also remove empty <b> and <i> tags - for a in xpath(data, '//h:a[@href]|//h:i|//h:b'): + # Also remove empty <b>, <u> and <i> tags + for a in xpath(data, '//h:a[@href]|//h:i|//h:b|//h:u'): if a.get('id', None) is None and a.get('name', None) is None \ and len(a) == 0 and not a.text: remove_elem(a) From 7bfa82b98308f18e7d53a590417d0dd378893416 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Sun, 1 May 2011 13:50:24 -0600 Subject: [PATCH 09/39] Fix #775048 (spelling mistake on website) --- src/calibre/manual/faq.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 08ebb6506b..2e2a8e5ae6 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -557,7 +557,7 @@ You have two choices: How is |app| licensed? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -|app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode <http://code.google.com/p/calibre-ebook/downloads/list>`_. You are free to use the results of conversions from |app| however you want. You cannot use code, libraries from |app| in your software without maing your software open source. For details, see `The GNU GPL v3 <http://www.gnu.org/licenses/gpl.html>`_. +|app| is licensed under the GNU General Public License v3 (an open source license). This means that you are free to redistribute |app| as long as you make the source code available. So if you want to put |app| on a CD with your product, you must also put the |app| source code on the CD. The source code is available for download `from googlecode <http://code.google.com/p/calibre-ebook/downloads/list>`_. You are free to use the results of conversions from |app| however you want. You cannot use code, libraries from |app| in your software without making your software open source. For details, see `The GNU GPL v3 <http://www.gnu.org/licenses/gpl.html>`_. How do I run calibre from my USB stick? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 255fe03b289000c46f83eb4a2c93362e8179be53 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 2 May 2011 12:25:45 +0100 Subject: [PATCH 10/39] First attempt at a user_defined device --- src/calibre/customize/builtins.py | 2 + src/calibre/devices/__init__.py | 52 ++++++++++ src/calibre/devices/user_defined/__init__.py | 0 src/calibre/devices/user_defined/driver.py | 86 +++++++++++++++++ .../gui2/preferences/device_user_defined.py | 94 +++++++++++++++++++ src/calibre/gui2/preferences/misc.py | 6 ++ src/calibre/gui2/preferences/misc.ui | 9 +- 7 files changed, 248 insertions(+), 1 deletion(-) create mode 100644 src/calibre/devices/user_defined/__init__.py create mode 100644 src/calibre/devices/user_defined/driver.py create mode 100644 src/calibre/gui2/preferences/device_user_defined.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 36bcbdbfe2..776b04d5f6 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -595,6 +595,7 @@ from calibre.devices.jetbook.driver import JETBOOK, MIBUK, JETBOOK_MINI from calibre.devices.kindle.driver import KINDLE, KINDLE2, KINDLE_DX from calibre.devices.nook.driver import NOOK, NOOK_COLOR from calibre.devices.prs505.driver import PRS505 +from calibre.devices.user_defined.driver import USER_DEFINED from calibre.devices.android.driver import ANDROID, S60 from calibre.devices.nokia.driver import N770, N810, E71X, E52 from calibre.devices.eslick.driver import ESLICK, EBK52 @@ -742,6 +743,7 @@ plugins += [ EEEREADER, NEXTBOOK, ITUNES, + USER_DEFINED, ] plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ x.__name__.endswith('MetadataReader')] diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 63b0b89a17..d151ae1844 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -156,3 +156,55 @@ def debug(ioreg_to_tmp=False, buf=None): sys.stdout = oldo sys.stderr = olde +def device_info(ioreg_to_tmp=False, buf=None): + from calibre.devices.scanner import DeviceScanner, win_pnp_drives + from calibre.constants import iswindows + from calibre import prints + import re + + res = {} + if not iswindows: + return None + try: + s = DeviceScanner() + s.scan() + devices = (s.devices) + device_details = {} + device_set = set() + for dev in devices: + vid = re.search('vid_([0-9a-f]*)&', dev) + if vid: + vid = vid.group(1) + pid = re.search('pid_([0-9a-f]*)&', dev) + if pid: + pid = pid.group(1) + rev = re.search('rev_([0-9a-f]*)$', dev) + if rev: + rev = rev.group(1) + d = vid+pid+rev + prints(d) + device_set.add(d) + device_details[d] = (vid, pid, rev) + res['device_set'] = device_set + res['device_details'] = device_details + drives = win_pnp_drives(debug=False) + drive_details = {} + print drives + drive_set = set() + for drive,details in drives.iteritems(): + order = 'ORD_' + str(drive.order) + ven = re.search('VEN_([^&]*)&', details) + if ven: + ven = ven.group(1) + prod = re.search('PROD_([^&]*)&', details) + if prod: + prod = prod.group(1) + d = (order, ven, prod) + print d + drive_details[drive] = d + drive_set.add(drive) + res['drive_details'] = drive_details + res['drive_set'] = drive_set + finally: + pass + return res \ No newline at end of file diff --git a/src/calibre/devices/user_defined/__init__.py b/src/calibre/devices/user_defined/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/calibre/devices/user_defined/driver.py b/src/calibre/devices/user_defined/driver.py new file mode 100644 index 0000000000..682ed1712e --- /dev/null +++ b/src/calibre/devices/user_defined/driver.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + +from calibre.devices.usbms.driver import USBMS +from calibre.ebooks import BOOK_EXTENSIONS + +class USER_DEFINED(USBMS): + + name = 'User Defined USB driver' + gui_name = 'User Defined phone' + author = 'Kovid Goyal' + supported_platforms = ['windows', 'osx', 'linux'] + + # Ordered list of supported formats + FORMATS = BOOK_EXTENSIONS + + VENDOR_ID = 0xFFFF + PRODUCT_ID = 0xFFFF + BCD = None + + EBOOK_DIR_MAIN = '' + EBOOK_DIR_CARD_A = '' + + VENDOR_NAME = [] + WINDOWS_MAIN_MEM = '' + WINDOWS_CARD_A_MEM = '' + + OSX_MAIN_MEM = 'Device Main Memory' + + MAIN_MEMORY_VOLUME_LABEL = 'Device Main Memory' + + SUPPORTS_SUB_DIRS = True + + EXTRA_CUSTOMIZATION_MESSAGE = [ + _('USB Vendor ID (in hex)'), + _('USB Product ID (in hex)'), + _('USB Revision ID (in hex)'), + _('Windows main memory vendor string'), + _('Windows main memory ID string'), + _('Windows card A vendor string'), + _('Windows card A ID string'), + _('Main memory folder'), + _('Card A folder'), + ] + EXTRA_CUSTOMIZATION_DEFAULT = [ + '0x0000', + '0x0000', + '0x0000', + '', + '', + '', + '', + '', + '', + ] + OPT_USB_VENDOR_ID = 0 + OPT_USB_PRODUCT_ID = 1 + OPT_USB_REVISION_ID = 2 + OPT_USB_WINDOWS_MM_VEN_ID = 3 + OPT_USB_WINDOWS_MM_ID = 4 + OPT_USB_WINDOWS_CA_VEN_ID = 5 + OPT_USB_WINDOWS_CA_ID = 6 + OPT_MAIN_MEM_FOLDER = 7 + OPT_CARD_A_FOLDER = 8 + + def __init__(self, *args): + USBMS.__init__(self, args) + try: + e = self.settings().extra_customization + self.VENDOR_ID = int(e[self.OPT_USB_VENDOR_ID], 16) + self.PRODUCT_ID = int(e[self.OPT_USB_PRODUCT_ID], 16) + self.BCD = [int(e[self.OPT_USB_REVISION_ID], 16)] + print '%x, %x, %s' %(self.VENDOR_ID, self.PRODUCT_ID, str(self.BCD)) + if e[self.OPT_USB_WINDOWS_MM_VEN_ID]: + self.VENDOR_NAME.append(e[self.OPT_USB_WINDOWS_MM_VEN_ID]) + if e[self.OPT_USB_WINDOWS_CA_VEN_ID]: + self.VENDOR_NAME.append(e[self.OPT_USB_WINDOWS_CA_VEN_ID]) + self.WINDOWS_MAIN_MEM = e[self.OPT_USB_WINDOWS_MM_ID] + self.WINDOWS_CARD_A_MEM = e[self.OPT_USB_WINDOWS_CA_ID] + self.EBOOK_DIR_MAIN = e[self.OPT_MAIN_MEM_FOLDER] + self.EBOOK_DIR_CARD_A = e[self.OPT_CARD_A_FOLDER] + except: + pass \ No newline at end of file diff --git a/src/calibre/gui2/preferences/device_user_defined.py b/src/calibre/gui2/preferences/device_user_defined.py new file mode 100644 index 0000000000..0b820bd742 --- /dev/null +++ b/src/calibre/gui2/preferences/device_user_defined.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + + +from PyQt4.Qt import QDialog, QVBoxLayout, QPlainTextEdit, QTimer, \ + QDialogButtonBox, QPushButton, QApplication, QIcon, QMessageBox + +def step_dialog(parent, title, msg, det_msg=''): + d = QMessageBox(parent) + d.setWindowTitle(title) + d.setText(msg) + d.setStandardButtons(QMessageBox.Ok | QMessageBox.Cancel) + return d.exec_() & QMessageBox.Cancel + + +class UserDefinedDevice(QDialog): + + def __init__(self, parent=None): + QDialog.__init__(self, parent) + self._layout = QVBoxLayout(self) + self.setLayout(self._layout) + self.log = QPlainTextEdit(self) + self._layout.addWidget(self.log) + self.log.setPlainText(_('Getting debug information')+'...') + self.copy = QPushButton(_('Copy to &clipboard')) + self.copy.setDefault(True) + self.setWindowTitle(_('Debug device detection')) + self.setWindowIcon(QIcon(I('debug.png'))) + self.copy.clicked.connect(self.copy_to_clipboard) + self.ok = QPushButton('&OK') + self.ok.setAutoDefault(False) + self.ok.clicked.connect(self.accept) + self.bbox = QDialogButtonBox(self) + self.bbox.addButton(self.copy, QDialogButtonBox.ActionRole) + self.bbox.addButton(self.ok, QDialogButtonBox.AcceptRole) + self._layout.addWidget(self.bbox) + self.resize(750, 500) + self.bbox.setEnabled(False) + QTimer.singleShot(1000, self.device_info) + + def device_info(self): + try: + from calibre.devices import device_info + r = step_dialog(self.parent(), _('Device Detection'), + _('Ensure your device is disconnected, then press OK')) + if r: + return + before = device_info() + r = step_dialog(self.parent(), _('Device Detection'), + _('Ensure your device is connected, then press OK')) + if r: + return + after = device_info() + new_drives = after['drive_set'] - before['drive_set'] + new_devices = after['device_set'] - before['device_set'] + res = '' + if len(new_drives) and len(new_devices) == 1: + for d in new_devices: + res = _('USB Vendor ID (in hex)') + ': 0x' + \ + after['device_details'][d][0] + '\n' + res += _('USB Product ID (in hex)') + ': 0x' + \ + after['device_details'][d][1] + '\n' + res += _('USB Revision ID (in hex)') + ': 0x' + \ + after['device_details'][d][2] + '\n' + # sort the drives by the order number + for i,d in enumerate(sorted(new_drives, + key=lambda x: after['drive_details'][x][0])): + if i == 0: + res += _('Windows main memory ID string') + ': ' + \ + after['drive_details'][d][1] + '\n' + res += _('Windows main memory ID string') + ': ' + \ + after['drive_details'][d][2] + '\n' + else: + res += _('Windows card A vendor string') + ': ' + \ + after['drive_details'][d][1] + '\n' + res += _('Windows card A ID string') + ': ' + \ + after['drive_details'][d][2] + '\n' + + self.log.setPlainText(res) + finally: + self.bbox.setEnabled(True) + + def copy_to_clipboard(self): + QApplication.clipboard().setText(self.log.toPlainText()) + +if __name__ == '__main__': + app = QApplication([]) + d = UserDefinedDevice() + d.exec_() diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py index ead5da4ce4..80bfdffcd8 100644 --- a/src/calibre/gui2/preferences/misc.py +++ b/src/calibre/gui2/preferences/misc.py @@ -30,6 +30,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('enforce_cpu_limit', config, restart_required=True) self.device_detection_button.clicked.connect(self.debug_device_detection) self.button_open_config_dir.clicked.connect(self.open_config_dir) + self.user_defined_device_button.clicked.connect(self.user_defined_device) self.button_osx_symlinks.clicked.connect(self.create_symlinks) self.button_osx_symlinks.setVisible(isosx) @@ -38,6 +39,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): d = DebugDevice(self) d.exec_() + def user_defined_device(self, *args): + from calibre.gui2.preferences.device_user_defined import UserDefinedDevice + d = UserDefinedDevice(self) + d.exec_() + def open_config_dir(self, *args): from calibre.utils.config import config_dir open_local_file(config_dir) diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui index 8b0189b0a1..cce14f5ade 100644 --- a/src/calibre/gui2/preferences/misc.ui +++ b/src/calibre/gui2/preferences/misc.ui @@ -58,7 +58,14 @@ </property> </widget> </item> - <item row="4" column="0"> + <item row="4" column="0" colspan="2"> + <widget class="QPushButton" name="user_defined_device_button"> + <property name="text"> + <string>Setup the &user defined device</string> + </property> + </widget> + </item> + <item row="5" column="0"> <spacer name="verticalSpacer_6"> <property name="orientation"> <enum>Qt::Vertical</enum> From f74bed638111653a7d302c17da8938e34d962f16 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 2 May 2011 13:29:08 +0100 Subject: [PATCH 11/39] ... --- src/calibre/devices/__init__.py | 4 ---- src/calibre/devices/user_defined/driver.py | 17 +++++++++-------- .../gui2/preferences/device_user_defined.py | 7 ++++++- src/calibre/gui2/preferences/misc.py | 3 ++- src/calibre/gui2/preferences/misc.ui | 2 +- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index d151ae1844..02c42a1d6e 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -159,7 +159,6 @@ def debug(ioreg_to_tmp=False, buf=None): def device_info(ioreg_to_tmp=False, buf=None): from calibre.devices.scanner import DeviceScanner, win_pnp_drives from calibre.constants import iswindows - from calibre import prints import re res = {} @@ -182,14 +181,12 @@ def device_info(ioreg_to_tmp=False, buf=None): if rev: rev = rev.group(1) d = vid+pid+rev - prints(d) device_set.add(d) device_details[d] = (vid, pid, rev) res['device_set'] = device_set res['device_details'] = device_details drives = win_pnp_drives(debug=False) drive_details = {} - print drives drive_set = set() for drive,details in drives.iteritems(): order = 'ORD_' + str(drive.order) @@ -200,7 +197,6 @@ def device_info(ioreg_to_tmp=False, buf=None): if prod: prod = prod.group(1) d = (order, ven, prod) - print d drive_details[drive] = d drive_set.add(drive) res['drive_details'] = drive_details diff --git a/src/calibre/devices/user_defined/driver.py b/src/calibre/devices/user_defined/driver.py index 682ed1712e..03ed7dee94 100644 --- a/src/calibre/devices/user_defined/driver.py +++ b/src/calibre/devices/user_defined/driver.py @@ -10,7 +10,7 @@ from calibre.ebooks import BOOK_EXTENSIONS class USER_DEFINED(USBMS): name = 'User Defined USB driver' - gui_name = 'User Defined phone' + gui_name = 'User Defined USB Device' author = 'Kovid Goyal' supported_platforms = ['windows', 'osx', 'linux'] @@ -66,21 +66,22 @@ class USER_DEFINED(USBMS): OPT_MAIN_MEM_FOLDER = 7 OPT_CARD_A_FOLDER = 8 - def __init__(self, *args): - USBMS.__init__(self, args) + def initialize(self): try: e = self.settings().extra_customization self.VENDOR_ID = int(e[self.OPT_USB_VENDOR_ID], 16) self.PRODUCT_ID = int(e[self.OPT_USB_PRODUCT_ID], 16) self.BCD = [int(e[self.OPT_USB_REVISION_ID], 16)] - print '%x, %x, %s' %(self.VENDOR_ID, self.PRODUCT_ID, str(self.BCD)) if e[self.OPT_USB_WINDOWS_MM_VEN_ID]: self.VENDOR_NAME.append(e[self.OPT_USB_WINDOWS_MM_VEN_ID]) - if e[self.OPT_USB_WINDOWS_CA_VEN_ID]: + if e[self.OPT_USB_WINDOWS_CA_VEN_ID] and \ + e[self.OPT_USB_WINDOWS_CA_VEN_ID] not in self.VENDOR_NAME: self.VENDOR_NAME.append(e[self.OPT_USB_WINDOWS_CA_VEN_ID]) - self.WINDOWS_MAIN_MEM = e[self.OPT_USB_WINDOWS_MM_ID] - self.WINDOWS_CARD_A_MEM = e[self.OPT_USB_WINDOWS_CA_ID] + self.WINDOWS_MAIN_MEM = e[self.OPT_USB_WINDOWS_MM_ID] + '&' + self.WINDOWS_CARD_A_MEM = e[self.OPT_USB_WINDOWS_CA_ID] + '&' self.EBOOK_DIR_MAIN = e[self.OPT_MAIN_MEM_FOLDER] self.EBOOK_DIR_CARD_A = e[self.OPT_CARD_A_FOLDER] except: - pass \ No newline at end of file + import traceback + traceback.print_exc() + USBMS.initialize(self) \ No newline at end of file diff --git a/src/calibre/gui2/preferences/device_user_defined.py b/src/calibre/gui2/preferences/device_user_defined.py index 0b820bd742..914e2b5666 100644 --- a/src/calibre/gui2/preferences/device_user_defined.py +++ b/src/calibre/gui2/preferences/device_user_defined.py @@ -81,7 +81,12 @@ class UserDefinedDevice(QDialog): res += _('Windows card A ID string') + ': ' + \ after['drive_details'][d][2] + '\n' - self.log.setPlainText(res) + trailer = _('Enter the above values into the USER_DEVICE by ' + 'customizing the device plugin. Be sure to also ' + 'enter the folders where you want the books to ' + 'be put. You must restart calibre for your changes ' + 'to take effect.\n') + self.log.setPlainText(res + '\n\n' + trailer) finally: self.bbox.setEnabled(True) diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py index 80bfdffcd8..179e8a995d 100644 --- a/src/calibre/gui2/preferences/misc.py +++ b/src/calibre/gui2/preferences/misc.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting from calibre.gui2.preferences.misc_ui import Ui_Form from calibre.gui2 import error_dialog, config, open_local_file, info_dialog -from calibre.constants import isosx +from calibre.constants import isosx, iswindows class WorkersSetting(Setting): @@ -33,6 +33,7 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.user_defined_device_button.clicked.connect(self.user_defined_device) self.button_osx_symlinks.clicked.connect(self.create_symlinks) self.button_osx_symlinks.setVisible(isosx) + self.user_defined_device_button.setVisible(iswindows) def debug_device_detection(self, *args): from calibre.gui2.preferences.device_debug import DebugDevice diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui index cce14f5ade..df530bbe9a 100644 --- a/src/calibre/gui2/preferences/misc.ui +++ b/src/calibre/gui2/preferences/misc.ui @@ -61,7 +61,7 @@ <item row="4" column="0" colspan="2"> <widget class="QPushButton" name="user_defined_device_button"> <property name="text"> - <string>Setup the &user defined device</string> + <string>Get information to setup the &user defined device (Windows only)</string> </property> </widget> </item> From 0962ebb3ba0016b223d603a819e0b59b5bfe6984 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 2 May 2011 14:58:26 +0100 Subject: [PATCH 12/39] Make user_defined_device information work on linux --- src/calibre/devices/__init__.py | 81 ++++++++++--------- .../gui2/preferences/device_user_defined.py | 36 +++++---- src/calibre/gui2/preferences/misc.py | 3 +- src/calibre/gui2/preferences/misc.ui | 2 +- 4 files changed, 66 insertions(+), 56 deletions(-) diff --git a/src/calibre/devices/__init__.py b/src/calibre/devices/__init__.py index 02c42a1d6e..e47cd82b50 100644 --- a/src/calibre/devices/__init__.py +++ b/src/calibre/devices/__init__.py @@ -162,45 +162,54 @@ def device_info(ioreg_to_tmp=False, buf=None): import re res = {} - if not iswindows: - return None + device_details = {} + device_set = set() + drive_details = {} + drive_set = set() + res['device_set'] = device_set + res['device_details'] = device_details + res['drive_details'] = drive_details + res['drive_set'] = drive_set + try: s = DeviceScanner() s.scan() devices = (s.devices) - device_details = {} - device_set = set() - for dev in devices: - vid = re.search('vid_([0-9a-f]*)&', dev) - if vid: - vid = vid.group(1) - pid = re.search('pid_([0-9a-f]*)&', dev) - if pid: - pid = pid.group(1) - rev = re.search('rev_([0-9a-f]*)$', dev) - if rev: - rev = rev.group(1) - d = vid+pid+rev - device_set.add(d) - device_details[d] = (vid, pid, rev) - res['device_set'] = device_set - res['device_details'] = device_details - drives = win_pnp_drives(debug=False) - drive_details = {} - drive_set = set() - for drive,details in drives.iteritems(): - order = 'ORD_' + str(drive.order) - ven = re.search('VEN_([^&]*)&', details) - if ven: - ven = ven.group(1) - prod = re.search('PROD_([^&]*)&', details) - if prod: - prod = prod.group(1) - d = (order, ven, prod) - drive_details[drive] = d - drive_set.add(drive) - res['drive_details'] = drive_details - res['drive_set'] = drive_set + if not iswindows: + devices = [list(x) for x in devices] + for dev in devices: + for i in range(3): + dev[i] = hex(dev[i]) + d = dev[0] + dev[1] + dev[2] + device_set.add(d) + device_details[d] = dev[0:3] + else: + for dev in devices: + vid = re.search('vid_([0-9a-f]*)&', dev) + if vid: + vid = vid.group(1) + pid = re.search('pid_([0-9a-f]*)&', dev) + if pid: + pid = pid.group(1) + rev = re.search('rev_([0-9a-f]*)$', dev) + if rev: + rev = rev.group(1) + d = vid+pid+rev + device_set.add(d) + device_details[d] = (vid, pid, rev) + + drives = win_pnp_drives(debug=False) + for drive,details in drives.iteritems(): + order = 'ORD_' + str(drive.order) + ven = re.search('VEN_([^&]*)&', details) + if ven: + ven = ven.group(1) + prod = re.search('PROD_([^&]*)&', details) + if prod: + prod = prod.group(1) + d = (order, ven, prod) + drive_details[drive] = d + drive_set.add(drive) finally: pass - return res \ No newline at end of file + return res diff --git a/src/calibre/gui2/preferences/device_user_defined.py b/src/calibre/gui2/preferences/device_user_defined.py index 914e2b5666..c2a27d3937 100644 --- a/src/calibre/gui2/preferences/device_user_defined.py +++ b/src/calibre/gui2/preferences/device_user_defined.py @@ -10,6 +10,8 @@ __docformat__ = 'restructuredtext en' from PyQt4.Qt import QDialog, QVBoxLayout, QPlainTextEdit, QTimer, \ QDialogButtonBox, QPushButton, QApplication, QIcon, QMessageBox +from calibre.constants import iswindows + def step_dialog(parent, title, msg, det_msg=''): d = QMessageBox(parent) d.setWindowTitle(title) @@ -26,10 +28,10 @@ class UserDefinedDevice(QDialog): self.setLayout(self._layout) self.log = QPlainTextEdit(self) self._layout.addWidget(self.log) - self.log.setPlainText(_('Getting debug information')+'...') + self.log.setPlainText(_('Getting device information')+'...') self.copy = QPushButton(_('Copy to &clipboard')) self.copy.setDefault(True) - self.setWindowTitle(_('Debug device detection')) + self.setWindowTitle(_('User-defined device information')) self.setWindowIcon(QIcon(I('debug.png'))) self.copy.clicked.connect(self.copy_to_clipboard) self.ok = QPushButton('&OK') @@ -59,7 +61,7 @@ class UserDefinedDevice(QDialog): new_drives = after['drive_set'] - before['drive_set'] new_devices = after['device_set'] - before['device_set'] res = '' - if len(new_drives) and len(new_devices) == 1: + if (not iswindows or len(new_drives)) and len(new_devices) == 1: for d in new_devices: res = _('USB Vendor ID (in hex)') + ': 0x' + \ after['device_details'][d][0] + '\n' @@ -67,20 +69,20 @@ class UserDefinedDevice(QDialog): after['device_details'][d][1] + '\n' res += _('USB Revision ID (in hex)') + ': 0x' + \ after['device_details'][d][2] + '\n' - # sort the drives by the order number - for i,d in enumerate(sorted(new_drives, - key=lambda x: after['drive_details'][x][0])): - if i == 0: - res += _('Windows main memory ID string') + ': ' + \ - after['drive_details'][d][1] + '\n' - res += _('Windows main memory ID string') + ': ' + \ - after['drive_details'][d][2] + '\n' - else: - res += _('Windows card A vendor string') + ': ' + \ - after['drive_details'][d][1] + '\n' - res += _('Windows card A ID string') + ': ' + \ - after['drive_details'][d][2] + '\n' - + if iswindows: + # sort the drives by the order number + for i,d in enumerate(sorted(new_drives, + key=lambda x: after['drive_details'][x][0])): + if i == 0: + res += _('Windows main memory ID string') + ': ' + \ + after['drive_details'][d][1] + '\n' + res += _('Windows main memory ID string') + ': ' + \ + after['drive_details'][d][2] + '\n' + else: + res += _('Windows card A vendor string') + ': ' + \ + after['drive_details'][d][1] + '\n' + res += _('Windows card A ID string') + ': ' + \ + after['drive_details'][d][2] + '\n' trailer = _('Enter the above values into the USER_DEVICE by ' 'customizing the device plugin. Be sure to also ' 'enter the folders where you want the books to ' diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py index 179e8a995d..80bfdffcd8 100644 --- a/src/calibre/gui2/preferences/misc.py +++ b/src/calibre/gui2/preferences/misc.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting from calibre.gui2.preferences.misc_ui import Ui_Form from calibre.gui2 import error_dialog, config, open_local_file, info_dialog -from calibre.constants import isosx, iswindows +from calibre.constants import isosx class WorkersSetting(Setting): @@ -33,7 +33,6 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.user_defined_device_button.clicked.connect(self.user_defined_device) self.button_osx_symlinks.clicked.connect(self.create_symlinks) self.button_osx_symlinks.setVisible(isosx) - self.user_defined_device_button.setVisible(iswindows) def debug_device_detection(self, *args): from calibre.gui2.preferences.device_debug import DebugDevice diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui index df530bbe9a..843f0f01b7 100644 --- a/src/calibre/gui2/preferences/misc.ui +++ b/src/calibre/gui2/preferences/misc.ui @@ -61,7 +61,7 @@ <item row="4" column="0" colspan="2"> <widget class="QPushButton" name="user_defined_device_button"> <property name="text"> - <string>Get information to setup the &user defined device (Windows only)</string> + <string>Get information to setup the &user defined device</string> </property> </widget> </item> From ffdfb6ebe552f26a8b9e3fceb6494edab328dfb6 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 2 May 2011 15:21:51 +0100 Subject: [PATCH 13/39] ... --- src/calibre/gui2/preferences/device_user_defined.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/device_user_defined.py b/src/calibre/gui2/preferences/device_user_defined.py index c2a27d3937..198ae0d7a9 100644 --- a/src/calibre/gui2/preferences/device_user_defined.py +++ b/src/calibre/gui2/preferences/device_user_defined.py @@ -51,11 +51,13 @@ class UserDefinedDevice(QDialog): r = step_dialog(self.parent(), _('Device Detection'), _('Ensure your device is disconnected, then press OK')) if r: + self.close() return before = device_info() r = step_dialog(self.parent(), _('Device Detection'), _('Ensure your device is connected, then press OK')) if r: + self.close() return after = device_info() new_drives = after['drive_set'] - before['drive_set'] @@ -83,7 +85,8 @@ class UserDefinedDevice(QDialog): after['drive_details'][d][1] + '\n' res += _('Windows card A ID string') + ': ' + \ after['drive_details'][d][2] + '\n' - trailer = _('Enter the above values into the USER_DEVICE by ' + trailer = _('Copy these values to the clipboard, paste them into an ' + 'editor, then enter them into the USER_DEVICE by ' 'customizing the device plugin. Be sure to also ' 'enter the folders where you want the books to ' 'be put. You must restart calibre for your changes ' From 106396f76806aa2647b85268c9ab18e7688301ed Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 2 May 2011 15:36:25 +0100 Subject: [PATCH 14/39] ... --- src/calibre/devices/user_defined/driver.py | 40 +++++++++++++++++----- 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/calibre/devices/user_defined/driver.py b/src/calibre/devices/user_defined/driver.py index 03ed7dee94..3211bd19ef 100644 --- a/src/calibre/devices/user_defined/driver.py +++ b/src/calibre/devices/user_defined/driver.py @@ -35,15 +35,37 @@ class USER_DEFINED(USBMS): SUPPORTS_SUB_DIRS = True EXTRA_CUSTOMIZATION_MESSAGE = [ - _('USB Vendor ID (in hex)'), - _('USB Product ID (in hex)'), - _('USB Revision ID (in hex)'), - _('Windows main memory vendor string'), - _('Windows main memory ID string'), - _('Windows card A vendor string'), - _('Windows card A ID string'), - _('Main memory folder'), - _('Card A folder'), + _('USB Vendor ID (in hex)') + ':::' + + _('Get this ID using Preferences -> Misc -> Get information to ' + 'set up the user-defined device'), + _('USB Product ID (in hex)')+ ':::' + + _('Get this ID using Preferences -> Misc -> Get information to ' + 'set up the user-defined device'), + _('USB Revision ID (in hex)')+ ':::' + + _('Get this ID using Preferences -> Misc -> Get information to ' + 'set up the user-defined device'), + _('Windows main memory vendor string') + ':::' + + _('This field is used only on windows. ' + 'Get this ID using Preferences -> Misc -> Get information to ' + 'set up the user-defined device'), + _('Windows main memory ID string') + ':::' + + _('This field is used only on windows. ' + 'Get this ID using Preferences -> Misc -> Get information to ' + 'set up the user-defined device'), + _('Windows card A vendor string') + ':::' + + _('This field is used only on windows. ' + 'Get this ID using Preferences -> Misc -> Get information to ' + 'set up the user-defined device'), + _('Windows card A ID string') + ':::' + + _('This field is used only on windows. ' + 'Get this ID using Preferences -> Misc -> Get information to ' + 'set up the user-defined device'), + _('Main memory folder') + ':::' + + _('Enter the folder where the books are to be stored. This folder ' + 'is prepended to any send_to_device template'), + _('Card A folder') + ':::' + + _('Enter the folder where the books are to be stored. This folder ' + 'is prepended to any send_to_device template'), ] EXTRA_CUSTOMIZATION_DEFAULT = [ '0x0000', From c64d180d1d470823486b871a07617c0886fbdc3e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 2 May 2011 18:07:13 +0100 Subject: [PATCH 15/39] Switch device plugin preferences to double-column if more than 6 preferences --- src/calibre/devices/usbms/deviceconfig.py | 3 ++ src/calibre/devices/user_defined/driver.py | 52 ++++++++++--------- .../gui2/device_drivers/configwidget.py | 19 +++++-- .../gui2/device_drivers/configwidget.ui | 2 +- 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/src/calibre/devices/usbms/deviceconfig.py b/src/calibre/devices/usbms/deviceconfig.py index 3c79652463..3f669f1e24 100644 --- a/src/calibre/devices/usbms/deviceconfig.py +++ b/src/calibre/devices/usbms/deviceconfig.py @@ -94,6 +94,9 @@ class DeviceConfig(object): if isinstance(cls.EXTRA_CUSTOMIZATION_MESSAGE, list): ec = [] for i in range(0, len(cls.EXTRA_CUSTOMIZATION_MESSAGE)): + if config_widget.opt_extra_customization[i] is None: + ec.append(None) + continue if hasattr(config_widget.opt_extra_customization[i], 'isChecked'): ec.append(config_widget.opt_extra_customization[i].isChecked()) else: diff --git a/src/calibre/devices/user_defined/driver.py b/src/calibre/devices/user_defined/driver.py index 3211bd19ef..f57f61fe7c 100644 --- a/src/calibre/devices/user_defined/driver.py +++ b/src/calibre/devices/user_defined/driver.py @@ -15,7 +15,7 @@ class USER_DEFINED(USBMS): supported_platforms = ['windows', 'osx', 'linux'] # Ordered list of supported formats - FORMATS = BOOK_EXTENSIONS + FORMATS = ['epub', 'mobi', 'pdf'] VENDOR_ID = 0xFFFF PRODUCT_ID = 0xFFFF @@ -35,42 +35,44 @@ class USER_DEFINED(USBMS): SUPPORTS_SUB_DIRS = True EXTRA_CUSTOMIZATION_MESSAGE = [ - _('USB Vendor ID (in hex)') + ':::' + + _('USB Vendor ID (in hex)') + ':::<p>' + _('Get this ID using Preferences -> Misc -> Get information to ' - 'set up the user-defined device'), - _('USB Product ID (in hex)')+ ':::' + + 'set up the user-defined device') + '</p>', + _('USB Product ID (in hex)')+ ':::<p>' + _('Get this ID using Preferences -> Misc -> Get information to ' - 'set up the user-defined device'), - _('USB Revision ID (in hex)')+ ':::' + + 'set up the user-defined device') + '</p>', + _('USB Revision ID (in hex)')+ ':::<p>' + _('Get this ID using Preferences -> Misc -> Get information to ' - 'set up the user-defined device'), - _('Windows main memory vendor string') + ':::' + + 'set up the user-defined device') + '</p>', + '', + _('Windows main memory vendor string') + ':::<p>' + _('This field is used only on windows. ' 'Get this ID using Preferences -> Misc -> Get information to ' - 'set up the user-defined device'), - _('Windows main memory ID string') + ':::' + + 'set up the user-defined device') + '</p>', + _('Windows main memory ID string') + ':::<p>' + _('This field is used only on windows. ' 'Get this ID using Preferences -> Misc -> Get information to ' - 'set up the user-defined device'), - _('Windows card A vendor string') + ':::' + + 'set up the user-defined device') + '</p>', + _('Windows card A vendor string') + ':::<p>' + _('This field is used only on windows. ' 'Get this ID using Preferences -> Misc -> Get information to ' - 'set up the user-defined device'), - _('Windows card A ID string') + ':::' + + 'set up the user-defined device') + '</p>', + _('Windows card A ID string') + ':::<p>' + _('This field is used only on windows. ' 'Get this ID using Preferences -> Misc -> Get information to ' - 'set up the user-defined device'), - _('Main memory folder') + ':::' + + 'set up the user-defined device') + '</p>', + _('Main memory folder') + ':::<p>' + _('Enter the folder where the books are to be stored. This folder ' - 'is prepended to any send_to_device template'), - _('Card A folder') + ':::' + + 'is prepended to any send_to_device template') + '</p>', + _('Card A folder') + ':::<p>' + _('Enter the folder where the books are to be stored. This folder ' - 'is prepended to any send_to_device template'), + 'is prepended to any send_to_device template') + '</p>', ] EXTRA_CUSTOMIZATION_DEFAULT = [ '0x0000', '0x0000', '0x0000', + None, '', '', '', @@ -81,12 +83,12 @@ class USER_DEFINED(USBMS): OPT_USB_VENDOR_ID = 0 OPT_USB_PRODUCT_ID = 1 OPT_USB_REVISION_ID = 2 - OPT_USB_WINDOWS_MM_VEN_ID = 3 - OPT_USB_WINDOWS_MM_ID = 4 - OPT_USB_WINDOWS_CA_VEN_ID = 5 - OPT_USB_WINDOWS_CA_ID = 6 - OPT_MAIN_MEM_FOLDER = 7 - OPT_CARD_A_FOLDER = 8 + OPT_USB_WINDOWS_MM_VEN_ID = 4 + OPT_USB_WINDOWS_MM_ID = 5 + OPT_USB_WINDOWS_CA_VEN_ID = 6 + OPT_USB_WINDOWS_CA_ID = 7 + OPT_MAIN_MEM_FOLDER = 8 + OPT_CARD_A_FOLDER = 9 def initialize(self): try: diff --git a/src/calibre/gui2/device_drivers/configwidget.py b/src/calibre/gui2/device_drivers/configwidget.py index fc7e16e639..e55a0c6dda 100644 --- a/src/calibre/gui2/device_drivers/configwidget.py +++ b/src/calibre/gui2/device_drivers/configwidget.py @@ -62,8 +62,18 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): if isinstance(extra_customization_message, list): self.opt_extra_customization = [] + if len(extra_customization_message) > 6: + row_func = lambda x, y: ((x/2) * 2) + y + col_func = lambda x: x%2 + else: + row_func = lambda x, y: x*2 + y + col_func = lambda x: 0 + for i, m in enumerate(extra_customization_message): label_text, tt = parse_msg(m) + if not label_text: + self.opt_extra_customization.append(None) + continue if isinstance(settings.extra_customization[i], bool): self.opt_extra_customization.append(QCheckBox(label_text)) self.opt_extra_customization[-1].setToolTip(tt) @@ -75,8 +85,9 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): l.setBuddy(self.opt_extra_customization[i]) l.setWordWrap(True) self.opt_extra_customization[i].setText(settings.extra_customization[i]) - self.extra_layout.addWidget(l) - self.extra_layout.addWidget(self.opt_extra_customization[i]) + self.extra_layout.addWidget(l, row_func(i, 0), col_func(i)) + self.extra_layout.addWidget(self.opt_extra_customization[i], + row_func(i, 1), col_func(i)) else: self.opt_extra_customization = QLineEdit() label_text, tt = parse_msg(extra_customization_message) @@ -86,8 +97,8 @@ class ConfigWidget(QWidget, Ui_ConfigWidget): l.setWordWrap(True) if settings.extra_customization: self.opt_extra_customization.setText(settings.extra_customization) - self.extra_layout.addWidget(l) - self.extra_layout.addWidget(self.opt_extra_customization) + self.extra_layout.addWidget(l, 0, 0) + self.extra_layout.addWidget(self.opt_extra_customization, 1, 0) self.opt_save_template.setText(settings.save_template) diff --git a/src/calibre/gui2/device_drivers/configwidget.ui b/src/calibre/gui2/device_drivers/configwidget.ui index 619d7052e8..92324fd1a7 100644 --- a/src/calibre/gui2/device_drivers/configwidget.ui +++ b/src/calibre/gui2/device_drivers/configwidget.ui @@ -101,7 +101,7 @@ </widget> </item> <item row="6" column="0"> - <layout class="QVBoxLayout" name="extra_layout"/> + <layout class="QGridLayout" name="extra_layout"/> </item> <item row="4" column="0"> <widget class="QLabel" name="label"> From 5d3455b184a6563fd3a43d93fea6de87f638d649 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 2 May 2011 11:52:07 -0600 Subject: [PATCH 16/39] Print out plugin load failure traceback only in DEBUG mode --- src/calibre/customize/ui.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index 151235cef9..3a2d638aab 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -19,6 +19,7 @@ from calibre.utils.config import (make_config_dir, Config, ConfigProxy, plugin_dir, OptionParser) from calibre.ebooks.epub.fix import ePubFixer from calibre.ebooks.metadata.sources.base import Source +from calibre.constants import DEBUG builtin_names = frozenset([p.name for p in builtin_plugins]) @@ -487,8 +488,9 @@ def initialize_plugins(): plugin = initialize_plugin(plugin, None if isinstance(zfp, type) else zfp) _initialized_plugins.append(plugin) except: - print 'Failed to initialize plugin...' - traceback.print_exc() + print 'Failed to initialize plugin:', repr(zfp) + if DEBUG: + traceback.print_exc() _initialized_plugins.sort(cmp=lambda x,y:cmp(x.priority, y.priority), reverse=True) reread_filetype_plugins() reread_metadata_plugins() From 58c6733d3f9af650c5b7d7dc004e551d21d0d79e Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 2 May 2011 20:27:41 +0100 Subject: [PATCH 17/39] Add some documentation for the new User Defined device. --- src/calibre/manual/faq.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 2e2a8e5ae6..0e964516c4 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -100,7 +100,9 @@ Device Integration What devices does |app| support? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -At the moment |app| has full support for the SONY PRS line, Barnes & Noble Nook line, Cybook Gen 3/Opus, Amazon Kindle line, Entourage Edge, Longshine ShineBook, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook line, Italica, eClicto, Iriver Story, Airis dBook, Hanvon N515, Binatone Readme, Teclast K3 and clones, SpringDesign Alex, Kobo Reader, various Android phones and the iPhone/iPad. In addition, using the :guilabel:`Connect to folder` function you can use it with any ebook reader that exports itself as a USB disk. +At the moment |app| has full support for the SONY PRS line, Barnes & Noble Nook line, Cybook Gen 3/Opus, Amazon Kindle line, Entourage Edge, Longshine ShineBook, Ectaco Jetbook, BeBook/BeBook Mini, Irex Illiad/DR1000, Foxit eSlick, PocketBook line, Italica, eClicto, Iriver Story, Airis dBook, Hanvon N515, Binatone Readme, Teclast K3 and clones, SpringDesign Alex, Kobo Reader, various Android phones and the iPhone/iPad. In addition, using the :guilabel:`Connect to folder` function you can use it with any ebook reader that exports itself as a USB disk. + +There is also a special ``User Defined`` device plugin that can be used to connect to arbitrary devices that present their memory as disk drives. See the device plugin ``Preferences -> Plugins -> Device Plugins -> User Defined`` and ``Preferences -> Miscelleaneous -> Get information to setup the user defined device`` for more information. How can I help get my device supported in |app|? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -133,6 +135,11 @@ Follow these steps to find the problem: * In calibre, go to Preferences->Plugins->Device Interface plugin and make sure the plugin for your device is enabled, the plugin icon next to it should be green when it is enabled. * If all the above steps fail, go to Preferences->Miscellaneous and click debug device detection with your device attached and post the output as a ticket on `the calibre bug tracker <http://bugs.calibre-ebook.com>`_. +My device is non-standard or unusual. What can I do to connect to it? +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +In addition to the ``Connect to Folder`` function found under the Connect/Share menu (see :guilabel:`Connect to folder`), |app| provides a ``User Defined`` device plugin that can be used to connect to any USB device that presents its memory as disk drives. See the device plugin ``Preferences -> Plugins -> Device Plugins -> User Defined`` and ``Preferences -> Miscelleaneous -> Get information to setup the user defined device`` for more information. + How does |app| manage collections on my SONY reader? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From fb87d6b9ad88ef54fe2bbf008a27430b6b954387 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 2 May 2011 13:46:19 -0600 Subject: [PATCH 18/39] Fix #775825 (titlecase error on the word (part) macht) --- src/calibre/utils/titlecase.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/utils/titlecase.py b/src/calibre/utils/titlecase.py index bf2f9a78d4..1f153dd5fe 100755 --- a/src/calibre/utils/titlecase.py +++ b/src/calibre/utils/titlecase.py @@ -68,7 +68,7 @@ def titlecase(text): continue match = MAC_MC.match(word) - if match and not match.group(2).startswith('hin'): + if match and not match.group(2)[:3] in ('hin', 'ht'): line.append("%s%s" % (capitalize(match.group(1)), capitalize(match.group(2)))) continue From 375529f7cba71779bc74b264ba645bcf0daa9b80 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 2 May 2011 16:41:53 -0600 Subject: [PATCH 19/39] Fix #775952 (Calibre not seeing dell streak 5 due to new USB identifier) --- src/calibre/devices/android/driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index f500560f97..ca84271778 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -62,7 +62,7 @@ class ANDROID(USBMS): 0x502 : { 0x3203 : [0x0100]}, # Dell - 0x413c : { 0xb007 : [0x0100, 0x0224]}, + 0x413c : { 0xb007 : [0x0100, 0x0224, 0x0226]}, # LG 0x1004 : { 0x61cc : [0x100], 0x61ce : [0x100], 0x618e : [0x226] }, From 5bf153338b11da59b4f24cdd1e68611513ac7daf Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 2 May 2011 18:28:38 -0600 Subject: [PATCH 20/39] BibTeX catalog: Convert all HTML comments to plain text. Fixes #775051 (catalog BIB) --- src/calibre/devices/user_defined/driver.py | 3 +-- src/calibre/library/catalog.py | 17 ++++++++++------- src/calibre/utils/bibtex.py | 2 +- 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/calibre/devices/user_defined/driver.py b/src/calibre/devices/user_defined/driver.py index f57f61fe7c..c496422255 100644 --- a/src/calibre/devices/user_defined/driver.py +++ b/src/calibre/devices/user_defined/driver.py @@ -5,7 +5,6 @@ __copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' from calibre.devices.usbms.driver import USBMS -from calibre.ebooks import BOOK_EXTENSIONS class USER_DEFINED(USBMS): @@ -108,4 +107,4 @@ class USER_DEFINED(USBMS): except: import traceback traceback.print_exc() - USBMS.initialize(self) \ No newline at end of file + USBMS.initialize(self) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 717e8e2c6b..aeecc3cfca 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -8,6 +8,7 @@ from collections import namedtuple from copy import deepcopy from xml.sax.saxutils import escape from lxml import etree +from types import StringType, UnicodeType from calibre import prints, prepare_string_for_xml, strftime from calibre.constants import preferred_encoding, DEBUG @@ -15,13 +16,16 @@ from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString from calibre.ebooks.chardet import substitute_entites +from calibre.library.save_to_disk import preprocess_template from calibre.ptempfile import PersistentTemporaryDirectory +from calibre.utils.bibtex import BibTeX from calibre.utils.config import config_dir from calibre.utils.date import format_date, isoformat, is_date_undefined, now as nowf +from calibre.utils.html2text import html2text from calibre.utils.icu import capitalize from calibre.utils.logging import default_log as log -from calibre.utils.zipfile import ZipFile, ZipInfo from calibre.utils.magick.draw import thumbnail +from calibre.utils.zipfile import ZipFile, ZipInfo FIELDS = ['all', 'title', 'author_sort', 'authors', 'comments', 'cover', 'formats','id', 'isbn', 'ondevice', 'pubdate', 'publisher', @@ -303,12 +307,6 @@ class BIBTEX(CatalogPlugin): # {{{ def run(self, path_to_output, opts, db, notification=DummyReporter()): - from types import StringType, UnicodeType - - from calibre.library.save_to_disk import preprocess_template - #Bibtex functions - from calibre.utils.bibtex import BibTeX - def create_bibtex_entry(entry, fields, mode, template_citation, bibtexdict, citation_bibtex=True, calibre_files=True): @@ -365,6 +363,11 @@ class BIBTEX(CatalogPlugin): # {{{ #\n removal item = item.replace(u'\r\n',u' ') item = item.replace(u'\n',u' ') + #html to text + try: + item = html2text(item) + except: + log.warn("Failed to convert comments to text") bibtex_entry.append(u'note = "%s"' % bibtexdict.utf8ToBibtex(item)) elif field == 'isbn' : diff --git a/src/calibre/utils/bibtex.py b/src/calibre/utils/bibtex.py index d19a6b05fe..518ec96611 100644 --- a/src/calibre/utils/bibtex.py +++ b/src/calibre/utils/bibtex.py @@ -2905,4 +2905,4 @@ class BibTeX: def bibtex_author_format(self, item): #Format authors for Bibtex compliance (get a list as input) - return self.utf8ToBibtex(u' and'.join([author for author in item])) + return self.utf8ToBibtex(u' and '.join([author for author in item])) From 25a4310fb90ccd8639f89e99cdda2387a302e2f0 Mon Sep 17 00:00:00 2001 From: "ken@szboeye.com" <> Date: Tue, 3 May 2011 14:24:21 +0800 Subject: [PATCH 21/39] Add support for the Boeye Digital Reader . --- src/calibre/customize/builtins.py | 3 + src/calibre/customize/profiles.py | 89 ++++++++++++++++++++++++++- src/calibre/devices/boeye/__init__.py | 0 src/calibre/devices/boeye/driver.py | 57 +++++++++++++++++ 4 files changed, 147 insertions(+), 2 deletions(-) create mode 100644 src/calibre/devices/boeye/__init__.py create mode 100644 src/calibre/devices/boeye/driver.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 776b04d5f6..c1da8391e0 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -613,6 +613,7 @@ from calibre.devices.misc import PALMPRE, AVANT, SWEEX, PDNOVEL, \ from calibre.devices.folder_device.driver import FOLDER_DEVICE_FOR_CONFIG from calibre.devices.kobo.driver import KOBO from calibre.devices.bambook.driver import BAMBOOK +from calibre.devices.boeye.driver import BOEYE_BEX, BOEYE_BDX from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX from calibre.ebooks.epub.fix.unmanifested import Unmanifested @@ -743,6 +744,8 @@ plugins += [ EEEREADER, NEXTBOOK, ITUNES, + BOEYE_BEX, + BOEYE_BDX, USER_DEFINED, ] plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 5c29f1e79b..e771a36c2e 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -214,11 +214,51 @@ class NookInput(InputProfile): dpi = 167 fbase = 16 fsizes = [12, 12, 14, 16, 18, 20, 22, 24] + +class BoeyeG5Input(InputProfile): + + author = 'Ken' + name = 'Boeye Digital Reader G5' + short_name = 'boeyeg5' + description = _('This profile is intended for the Boeye G5.') + + # Screen size is a best guess + screen_size = (600, 800) + dpi = 200 + fbase = 16 + fsizes = [12, 14, 16, 18, 20, 22, 24] + +class BoeyeG6Input(InputProfile): + + author = 'Ken' + name = 'Boeye Digital Reader G6' + short_name = 'boeyeg6' + description = _('This profile is intended for the Boeye G6.') + + # Screen size is a best guess + screen_size = (600, 800) + dpi = 166.66 + fbase = 16 + fsizes = [12, 14, 16, 18, 20, 22, 24] + +class BoeyeG10Input(InputProfile): + + author = 'Ken' + name = 'Boeye Digital Reader G10' + short_name = 'boeyeg10' + description = _('This profile is intended for the Boeye G10.') + + # Screen size is a best guess + screen_size = (825, 1200) + dpi = 150 + fbase = 16 + fsizes = [12, 14, 16, 18, 20, 22, 24] + input_profiles = [InputProfile, SonyReaderInput, SonyReader300Input, SonyReader900Input, MSReaderInput, MobipocketInput, HanlinV3Input, HanlinV5Input, CybookG3Input, CybookOpusInput, KindleInput, IlliadInput, - IRexDR1000Input, IRexDR800Input, NookInput] + IRexDR1000Input, IRexDR800Input, NookInput, BoeyeG5Input, BoeyeG6Input, BoeyeG10Input] input_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) @@ -730,6 +770,50 @@ class BambookOutput(OutputProfile): dpi = 168.451 fbase = 12 fsizes = [10, 12, 14, 16] + +class BoeyeG5Output(OutputProfile): + + author = 'Ken' + name = 'Boeye Digital Reader G5' + short_name = 'boeyeg5' + description = _('This profile is intended for the Boeye Digital Reader G5.') + + # Screen size is a best guess + screen_size = (600, 800) + comic_screen_size = (600, 740) + dpi = 200 + fbase = 16 + fsizes = [12, 14, 16, 18, 20, 22, 24] + + +class BoeyeG6Output(OutputProfile): + + author = 'Ken' + name = 'Boeye Digital Reader G6' + short_name = 'boeyeg6' + description = _('This profile is intended for the Boeye Digital Reader G6.') + + # Screen size is a best guess + screen_size = (600, 800) + comic_screen_size = (600, 740) + dpi = 160 + fbase = 16 + fsizes = [12, 14, 16, 18, 20, 22, 24] + +class BoeyeG10Output(OutputProfile): + + author = 'Ken' + name = 'Boeye Digital Reader G10' + short_name = 'boeyeg10' + description = _('This profile is intended for the Boeye Digital Reader G10.') + + # Screen size is a best guess + screen_size = (825, 1200) + comic_screen_size = (824, 1140) + dpi = 150 + fbase = 16 + fsizes = [12, 14, 16, 18, 20, 22, 24] + output_profiles = [OutputProfile, SonyReaderOutput, SonyReader300Output, SonyReader900Output, MSReaderOutput, MobipocketOutput, HanlinV3Output, @@ -737,6 +821,7 @@ output_profiles = [OutputProfile, SonyReaderOutput, SonyReader300Output, iPadOutput, KoboReaderOutput, TabletOutput, SamsungGalaxy, SonyReaderLandscapeOutput, KindleDXOutput, IlliadOutput, IRexDR1000Output, IRexDR800Output, JetBook5Output, NookOutput, - BambookOutput, NookColorOutput, GenericEink, GenericEinkLarge] + BambookOutput, NookColorOutput, BoeyeG5Output, BoeyeG6Output, BoeyeG10Output, + GenericEink, GenericEinkLarge] output_profiles.sort(cmp=lambda x,y:cmp(x.name.lower(), y.name.lower())) diff --git a/src/calibre/devices/boeye/__init__.py b/src/calibre/devices/boeye/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/calibre/devices/boeye/driver.py b/src/calibre/devices/boeye/driver.py new file mode 100644 index 0000000000..fcde1653d1 --- /dev/null +++ b/src/calibre/devices/boeye/driver.py @@ -0,0 +1,57 @@ +__license__ = 'GPL v3' +__copyright__ = '2011, Ken <ken at szboeye.com>' +__docformat__ = 'restructuredtext en' + +''' +Device driver for BOEYE serial readers +''' + +import re +from calibre.devices.usbms.driver import USBMS + +class BOEYE_BEX(USBMS): + name = 'BOEYE BEX reader driver' + gui_name = 'BOEYE BEX' + description = _('Communicate with BOEYE BEX Serial eBook readers.') + author = 'szboeye' + supported_platforms = ['windows', 'osx', 'linux'] + + FORMATS = ['epub', 'mobi', 'fb2', 'lit', 'prc', 'pdf', 'rtf', 'txt', 'djvu', 'doc', 'chm', 'html', 'zip', 'pdb'] + + VENDOR_ID = [0x0085] + PRODUCT_ID = [0x600] + + VENDOR_NAME = 'LINUX' + WINDOWS_MAIN_MEM = 'FILE-STOR_GADGET' + OSX_MAIN_MEM = 'Linux File-Stor Gadget Media' + + MAIN_MEMORY_VOLUME_LABEL = 'BOEYE BEX Storage Card' + + EBOOK_DIR_MAIN = 'Documents' + SUPPORTS_SUB_DIRS = True + +class BOEYE_BDX(USBMS): + name = 'BOEYE BDX reader driver' + gui_name = 'BOEYE BDX' + description = _('Communicate with BOEYE BDX serial eBook readers.') + author = 'szboeye' + supported_platforms = ['windows', 'osx', 'linux'] + + FORMATS = ['epub', 'mobi', 'fb2', 'lit', 'prc', 'pdf', 'rtf', 'txt', 'djvu', 'doc', 'chm', 'html', 'zip', 'pdb'] + + VENDOR_ID = [0x0085] + PRODUCT_ID = [0x800] + + VENDOR_NAME = 'LINUX' + WINDOWS_MAIN_MEM = 'FILE-STOR_GADGET' + WINDOWS_CARD_A_MEM = 'FILE-STOR_GADGET' + + OSX_MAIN_MEM = 'Linux File-Stor Gadget Media' + OSX_CARD_A_MEM = 'Linux File-Stor Gadget Media' + + MAIN_MEMORY_VOLUME_LABEL = 'BOEYE BDX Internal Memory' + STORAGE_CARD_VOLUME_LABEL = 'BOEYE BDX Storage Card' + + EBOOK_DIR_MAIN = 'Documents' + EBOOK_DIR_CARD_A = 'Documents' + SUPPORTS_SUB_DIRS = True From 731ff8eac4aef3bf27472f27f7d9d97c8d7d34cf Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 3 May 2011 09:39:41 -0600 Subject: [PATCH 22/39] ... --- recipes/f_secure.recipe | 1 - 1 file changed, 1 deletion(-) diff --git a/recipes/f_secure.recipe b/recipes/f_secure.recipe index f276a4961a..5a03f01ac8 100644 --- a/recipes/f_secure.recipe +++ b/recipes/f_secure.recipe @@ -12,7 +12,6 @@ class AdvancedUserRecipe1301860159(BasicNewsRecipe): max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False - language = 'en_EN' remove_javascript = True keep_only_tags = [dict(name='div', attrs={'class':'modSectionTd2'})] remove_tags = [dict(name='a'),dict(name='hr')] From 72120ed403490d4a9938b67f476639d92f49fd13 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 3 May 2011 10:15:46 -0600 Subject: [PATCH 23/39] Ensure the Preferences menu in OSX has an action to launch the preferences --- src/calibre/gui2/actions/preferences.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/actions/preferences.py b/src/calibre/gui2/actions/preferences.py index 24d20b23f9..b65967e994 100644 --- a/src/calibre/gui2/actions/preferences.py +++ b/src/calibre/gui2/actions/preferences.py @@ -10,7 +10,7 @@ from PyQt4.Qt import QIcon, QMenu, Qt from calibre.gui2.actions import InterfaceAction from calibre.gui2.preferences.main import Preferences from calibre.gui2 import error_dialog -from calibre.constants import DEBUG +from calibre.constants import DEBUG, isosx class PreferencesAction(InterfaceAction): @@ -19,7 +19,8 @@ class PreferencesAction(InterfaceAction): def genesis(self): pm = QMenu() - pm.addAction(QIcon(I('config.png')), _('Preferences'), self.do_config) + acname = _('Change calibre behavior') if isosx else _('Preferences') + pm.addAction(QIcon(I('config.png')), acname, self.do_config) pm.addAction(QIcon(I('wizard.png')), _('Run welcome wizard'), self.gui.run_wizard) if not DEBUG: From 62892d7161d414f18b62bab766f15ad5651ce47f Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 3 May 2011 14:10:35 -0600 Subject: [PATCH 24/39] Add Get Books action to main toolbar by default --- src/calibre/gui2/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 60d2a0a7dd..1dfe1d8d14 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -34,7 +34,7 @@ if isosx: ) gprefs.defaults['action-layout-toolbar'] = ( 'Add Books', 'Edit Metadata', None, 'Convert Books', 'View', None, - 'Choose Library', 'Donate', None, 'Fetch News', 'Save To Disk', + 'Choose Library', 'Donate', None, 'Fetch News', 'Store', 'Save To Disk', 'Connect Share', None, 'Remove Books', ) gprefs.defaults['action-layout-toolbar-device'] = ( @@ -48,7 +48,7 @@ else: gprefs.defaults['action-layout-menubar-device'] = () gprefs.defaults['action-layout-toolbar'] = ( 'Add Books', 'Edit Metadata', None, 'Convert Books', 'View', None, - 'Choose Library', 'Donate', None, 'Fetch News', 'Save To Disk', + 'Choose Library', 'Donate', None, 'Fetch News', 'Store', 'Save To Disk', 'Connect Share', None, 'Remove Books', None, 'Help', 'Preferences', ) gprefs.defaults['action-layout-toolbar-device'] = ( From b9973c45404f0457684bbbfa1daf0f143bd492a8 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 3 May 2011 21:19:47 +0100 Subject: [PATCH 25/39] Add an 'all metadata' edit metadata variant --- src/calibre/gui2/metadata/single.py | 136 ++++++++++++++++++++++- src/calibre/gui2/preferences/behavior.py | 3 +- 2 files changed, 134 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 63d4499966..e85a0adc13 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -13,7 +13,7 @@ from functools import partial from PyQt4.Qt import (Qt, QVBoxLayout, QHBoxLayout, QWidget, QPushButton, QGridLayout, pyqtSignal, QDialogButtonBox, QScrollArea, QFont, QTabWidget, QIcon, QToolButton, QSplitter, QGroupBox, QSpacerItem, - QSizePolicy, QPalette, QFrame, QSize, QKeySequence, QMenu) + QSizePolicy, QPalette, QFrame, QSize, QKeySequence, QMenu, QLabel) from calibre.ebooks.metadata import authors_to_string, string_to_authors from calibre.gui2 import ResizableDialog, error_dialog, gprefs, pixmap_to_data @@ -198,7 +198,7 @@ class MetadataSingleDialogBase(ResizableDialog): ans = self.custom_metadata_widgets for i in range(len(ans)-1): if before is not None and i == 0: - pass# Do something + pass if len(ans[i+1].widgets) == 2: sto(ans[i].widgets[-1], ans[i+1].widgets[1]) else: @@ -206,7 +206,7 @@ class MetadataSingleDialogBase(ResizableDialog): for c in range(2, len(ans[i].widgets), 2): sto(ans[i].widgets[c-1], ans[i].widgets[c+1]) if after is not None: - pass # Do something + pass # }}} def do_view_format(self, path, fmt): @@ -728,7 +728,135 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{ # }}} -editors = {'default': MetadataSingleDialog, 'alt1': MetadataSingleDialogAlt1} +class MetadataSingleDialogAlt2(MetadataSingleDialogBase): # {{{ + + cc_two_column = False + one_line_comments_toolbar = True + + def do_layout(self): + self.central_widget.clear() + self.labels = [] + sto = QWidget.setTabOrder + + self.central_widget.tabBar().setVisible(False) + tab0 = QWidget(self) + self.central_widget.addTab(tab0, _("&Metadata")) + l = QGridLayout() + tab0.setLayout(l) + + # Basic metadata in col 0 + tl = QGridLayout() + gb = QGroupBox(_('Basic metadata'), tab0) + l.addWidget(gb, 0, 0, 1, 1) + gb.setLayout(tl) + + self.button_box.addButton(self.fetch_metadata_button, + QDialogButtonBox.ActionRole) + self.config_metadata_button.setToolButtonStyle(Qt.ToolButtonTextOnly) + self.config_metadata_button.setText(_('Configure metadata downloading')) + self.button_box.addButton(self.config_metadata_button, + QDialogButtonBox.ActionRole) + sto(self.button_box, self.title) + + def create_row(row, widget, tab_to, button=None, icon=None, span=1): + ql = BuddyLabel(widget) + tl.addWidget(ql, row, 1, 1, 1) + tl.addWidget(widget, row, 2, 1, 1) + if button is not None: + tl.addWidget(button, row, 3, span, 1) + if icon is not None: + button.setIcon(QIcon(I(icon))) + if tab_to is not None: + if button is not None: + sto(widget, button) + sto(button, tab_to) + else: + sto(widget, tab_to) + + tl.addWidget(self.swap_title_author_button, 0, 0, 2, 1) + + create_row(0, self.title, self.title_sort, + button=self.deduce_title_sort_button, span=2, + icon='auto_author_sort.png') + create_row(1, self.title_sort, self.authors) + create_row(2, self.authors, self.author_sort, + button=self.deduce_author_sort_button, + span=2, icon='auto_author_sort.png') + create_row(3, self.author_sort, self.series) + create_row(4, self.series, self.series_index, + button=self.remove_unused_series_button, icon='trash.png') + create_row(5, self.series_index, self.tags) + create_row(6, self.tags, self.rating, button=self.tags_editor_button) + create_row(7, self.rating, self.pubdate) + create_row(8, self.pubdate, self.publisher, + button=self.pubdate.clear_button, icon='trash.png') + create_row(9, self.publisher, self.timestamp) + create_row(10, self.timestamp, self.identifiers, + button=self.timestamp.clear_button, icon='trash.png') + create_row(11, self.identifiers, self.comments, + button=self.clear_identifiers_button, icon='trash.png') + tl.addItem(QSpacerItem(1, 1, QSizePolicy.Fixed, QSizePolicy.Expanding), + 12, 1, 1 ,1) + + # Custom metadata in col 1 + w = getattr(self, 'custom_metadata_widgets_parent', None) + if w is not None: + gb = QGroupBox(_('Custom metadata'), tab0) + gbl = QVBoxLayout() + gb.setLayout(gbl) + sr = QScrollArea(gb) + sr.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) + sr.setWidgetResizable(True) + sr.setBackgroundRole(QPalette.Base) + sr.setFrameStyle(QFrame.NoFrame) + sr.setWidget(w) + gbl.addWidget(sr) + l.addWidget(gb, 0, 1, 1, 1) + sp = QSizePolicy() + sp.setVerticalStretch(10) + sp.setHorizontalPolicy(QSizePolicy.Fixed) + sp.setVerticalPolicy(QSizePolicy.Expanding) + gb.setSizePolicy(sp) + self.set_custom_metadata_tab_order() + + # comments span col 0 & 1 + w = QGroupBox(_('Comments'), tab0) + sp = QSizePolicy() + sp.setVerticalStretch(10) + sp.setHorizontalPolicy(QSizePolicy.Expanding) + sp.setVerticalPolicy(QSizePolicy.Expanding) + w.setSizePolicy(sp) + lb = QHBoxLayout() + w.setLayout(lb) + lb.addWidget(self.comments) + l.addWidget(w, 1, 0, 1, 2) + + # Cover & formats in col 3 + gb = QGroupBox(_('Cover'), tab0) + lb = QGridLayout() + gb.setLayout(lb) + lb.addWidget(self.cover, 0, 0, 1, 3, alignment=Qt.AlignCenter) + sto(self.clear_identifiers_button, self.cover.buttons[0]) + for i, b in enumerate(self.cover.buttons[:3]): + lb.addWidget(b, 1, i, 1, 1) + sto(b, self.cover.buttons[i+1]) + hl = QHBoxLayout() + for b in self.cover.buttons[3:]: + hl.addWidget(b) + sto(self.cover.buttons[-2], self.cover.buttons[-1]) + lb.addLayout(hl, 2, 0, 1, 3) + l.addWidget(gb, 0, 2, 1, 1) + l.addWidget(self.formats_manager, 1, 2, 1, 1) + sto(self.cover.buttons[-1], self.formats_manager) + + self.formats_manager.formats.setMaximumWidth(10000) + self.formats_manager.formats.setIconSize(QSize(32, 32)) + +# }}} + + +editors = {'default': MetadataSingleDialog, 'alt1': MetadataSingleDialogAlt1, + 'alt2': MetadataSingleDialogAlt2} def edit_metadata(db, row_list, current_row, parent=None, view_slot=None, set_current_callback=None): diff --git a/src/calibre/gui2/preferences/behavior.py b/src/calibre/gui2/preferences/behavior.py index e062ae2662..1247c54ec9 100644 --- a/src/calibre/gui2/preferences/behavior.py +++ b/src/calibre/gui2/preferences/behavior.py @@ -61,7 +61,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): r('bools_are_tristate', db.prefs, restart_required=True) r = self.register - choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1')] + choices = [(_('Default'), 'default'), (_('Compact Metadata'), 'alt1'), + (_('All on 1 tab'), 'alt2')] r('edit_metadata_single_layout', gprefs, choices=choices) def initialize(self): From 3ac69128e7899d8c24bb5c1cfbac1a91c9c26757 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 3 May 2011 16:04:59 -0600 Subject: [PATCH 26/39] Update FrazPC --- recipes/frazpc.recipe | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/recipes/frazpc.recipe b/recipes/frazpc.recipe index 56e45076ac..2d0d54d10c 100644 --- a/recipes/frazpc.recipe +++ b/recipes/frazpc.recipe @@ -1,7 +1,7 @@ #!/usr/bin/env python __license__ = 'GPL v3' -__copyright__ = u'2010, Tomasz Dlugosz <tomek3d@gmail.com>' +__copyright__ = u'2010-2011, Tomasz Dlugosz <tomek3d@gmail.com>' ''' frazpc.pl ''' @@ -19,17 +19,20 @@ class FrazPC(BasicNewsRecipe): use_embedded_content = False no_stylesheets = True - feeds = [(u'Aktualno\u015bci', u'http://www.frazpc.pl/feed'), (u'Recenzje', u'http://www.frazpc.pl/kat/recenzje-2/feed') ] - - keep_only_tags = [dict(name='div', attrs={'id':'FRAZ_CONTENT'})] - - remove_tags = [dict(name='p', attrs={'class':'gray tagsP fs11'})] - - preprocess_regexps = [ - (re.compile(i[0], re.IGNORECASE | re.DOTALL), i[1]) for i in - [(r'<div id="post-[0-9]*"', lambda match: '<div id="FRAZ_CONTENT"'), - (r'href="/f/news/', lambda match: 'href="http://www.frazpc.pl/f/news/'), - (r'   <a href="http://www.frazpc.pl/[^>]*?">(Skomentuj|Komentarz(e)?\([0-9]*\))</a>  \|', lambda match: '')] + feeds = [ + (u'Aktualno\u015bci', u'http://www.frazpc.pl/feed/aktualnosci'), + (u'Artyku\u0142y', u'http://www.frazpc.pl/feed/artykuly') ] + keep_only_tags = [dict(name='div', attrs={'class':'article'})] + + remove_tags = [ + dict(name='div', attrs={'class':'title-wrapper'}), + dict(name='p', attrs={'class':'tags'}), + dict(name='p', attrs={'class':'article-links'}), + dict(name='div', attrs={'class':'comments_box'}) + ] + + preprocess_regexps = [(re.compile(r'\| <a href="#comments">Komentarze \([0-9]*\)</a>'), lambda match: '')] + remove_attributes = [ 'width', 'height' ] From 0168ce58667ce362d62bbc4a1f524e5882f39184 Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Tue, 3 May 2011 19:13:57 -0400 Subject: [PATCH 27/39] GUI: OS X fix, show books in library when device connected as menu item so user can go to their library. --- src/calibre/gui2/layout.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index b5cc0163ed..b3c9bd3a02 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -44,18 +44,19 @@ class LocationManager(QObject): # {{{ receiver = partial(self._location_selected, name) ac.triggered.connect(receiver) self.tooltips[name] = tooltip + + m = QMenu(parent) + self._mem.append(m) + a = m.addAction(icon, tooltip) + a.triggered.connect(receiver) if name != 'library': - m = QMenu(parent) - self._mem.append(m) - a = m.addAction(icon, tooltip) - a.triggered.connect(receiver) self._mem.append(a) a = m.addAction(QIcon(I('eject.png')), _('Eject this device')) a.triggered.connect(self._eject_requested) - ac.setMenu(m) self._mem.append(a) else: ac.setToolTip(tooltip) + ac.setMenu(m) ac.calibre_name = name return ac @@ -71,7 +72,12 @@ class LocationManager(QObject): # {{{ def set_switch_actions(self, quick_actions, rename_actions, delete_actions, switch_actions, choose_action): - self.switch_menu = QMenu() + self.switch_menu = self.library_action.menu() + if self.switch_menu: + self.switch_menu.addSeparator() + else: + self.switch_menu = QMenu() + self.switch_menu.addAction(choose_action) self.cs_menus = [] for t, acs in [(_('Quick switch'), quick_actions), @@ -85,7 +91,9 @@ class LocationManager(QObject): # {{{ self.switch_menu.addSeparator() for ac in switch_actions: self.switch_menu.addAction(ac) - self.library_action.setMenu(self.switch_menu) + + if self.switch_menu != self.library_action.menu(): + self.library_action.setMenu(self.switch_menu) def _location_selected(self, location, *args): if location != self.current_location and hasattr(self, From ce28d66991ebd825ce41ecedb68503be2c243267 Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Tue, 3 May 2011 19:37:27 -0400 Subject: [PATCH 28/39] Store: Open external quick check in search dialog. --- src/calibre/gui2/store/search/search.py | 5 ++++- src/calibre/gui2/store/search/search.ui | 12 +++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/search/search.py b/src/calibre/gui2/store/search/search.py index 5654df8ffc..07d4afca54 100644 --- a/src/calibre/gui2/store/search/search.py +++ b/src/calibre/gui2/store/search/search.py @@ -155,6 +155,7 @@ class SearchDialog(QDialog, Ui_Dialog): self.config['results_view_column_width'] = [self.results_view.columnWidth(i) for i in range(self.results_view.model().columnCount())] self.config['sort_col'] = self.results_view.model().sort_col self.config['sort_order'] = self.results_view.model().sort_order + self.config['open_external'] = self.open_external.isChecked() store_check = {} for n in self.store_plugins: @@ -179,6 +180,8 @@ class SearchDialog(QDialog, Ui_Dialog): else: self.resize_columns() + self.open_external.setChecked(self.config.get('open_external', False)) + store_check = self.config.get('store_checked', None) if store_check: for n in store_check: @@ -212,7 +215,7 @@ class SearchDialog(QDialog, Ui_Dialog): def open_store(self, index): result = self.results_view.model().get_result(index) - self.store_plugins[result.store_name].open(self, result.detail_item) + self.store_plugins[result.store_name].open(self, result.detail_item, self.open_external.isChecked()) def check_progress(self): if not self.search_pool.threads_running() and not self.results_view.model().cover_pool.threads_running() and not self.results_view.model().details_pool.threads_running(): diff --git a/src/calibre/gui2/store/search/search.ui b/src/calibre/gui2/store/search/search.ui index 0d39a70a29..7e8dd36284 100644 --- a/src/calibre/gui2/store/search/search.ui +++ b/src/calibre/gui2/store/search/search.ui @@ -70,7 +70,7 @@ <x>0</x> <y>0</y> <width>215</width> - <height>116</height> + <height>93</height> </rect> </property> </widget> @@ -101,6 +101,16 @@ </item> </layout> </item> + <item> + <widget class="QCheckBox" name="open_external"> + <property name="toolTip"> + <string>Open a selected book in the system's web browser</string> + </property> + <property name="text"> + <string>Open external</string> + </property> + </widget> + </item> </layout> </widget> <widget class="QSplitter" name="splitter_2"> From bb0e6a60e749ad5fe9768fcee11a45b3d4ded7de Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 4 May 2011 10:24:23 -0600 Subject: [PATCH 29/39] ... --- recipes/telepolis.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes/telepolis.recipe b/recipes/telepolis.recipe index 4ca57f8275..8109e3e39a 100644 --- a/recipes/telepolis.recipe +++ b/recipes/telepolis.recipe @@ -18,7 +18,7 @@ class TelepolisNews(BasicNewsRecipe): recursion = 0 no_stylesheets = True encoding = "utf-8" - language = 'de_AT' + language = 'de' use_embedded_content =False remove_empty_feeds = True From bfbd42dd6d0bc5494162cea64c981e46ab8ab8be Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 4 May 2011 10:39:20 -0600 Subject: [PATCH 30/39] Fix USA Today --- recipes/usatoday.recipe | 397 ++-------------------------------------- 1 file changed, 20 insertions(+), 377 deletions(-) diff --git a/recipes/usatoday.recipe b/recipes/usatoday.recipe index bd47262563..a4899b7187 100644 --- a/recipes/usatoday.recipe +++ b/recipes/usatoday.recipe @@ -7,13 +7,11 @@ usatoday.com ''' from calibre.web.feeds.news import BasicNewsRecipe -from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, NavigableString, Tag -import re class USAToday(BasicNewsRecipe): title = 'USA Today' - __author__ = 'GRiker' + __author__ = 'Kovid Goyal' oldest_article = 1 timefmt = '' max_articles_per_feed = 20 @@ -31,7 +29,6 @@ class USAToday(BasicNewsRecipe): margin-bottom: 0em; \ font-size: smaller;}\n \ .articleBody {text-align: left;}\n ' - conversion_options = { 'linearize_tables' : True } #simultaneous_downloads = 1 feeds = [ ('Top Headlines', 'http://rssfeeds.usatoday.com/usatoday-NewsTopStories'), @@ -47,63 +44,26 @@ class USAToday(BasicNewsRecipe): ('Most Popular', 'http://rssfeeds.usatoday.com/Usatoday-MostViewedArticles'), ('Offbeat News', 'http://rssfeeds.usatoday.com/UsatodaycomOffbeat-TopStories'), ] - keep_only_tags = [dict(attrs={'class':[ - 'byLine', - 'inside-copy', - 'inside-head', - 'inside-head2', - 'item', - 'item-block', - 'photo-container', - ]}), - dict(id=[ - 'applyMainStoryPhoto', - 'permalink', - ])] + keep_only_tags = [dict(attrs={'class':'story'})] + remove_tags = [ + dict(attrs={'class':[ + 'share', + 'reprints', + 'inline-h3', + 'info-extras', + 'ppy-outer', + 'ppy-caption', + 'comments', + 'jump', + 'pagetools', + 'post-attributes', + 'tags', + 'bottom-tools', + 'sponsoredlinks', + ]}), + dict(id=['pluck']), + ] - remove_tags = [dict(attrs={'class':[ - 'comments', - 'jump', - 'pagetools', - 'post-attributes', - 'tags', - ]}), - dict(id=[])] - - #feeds = [('Most Popular', 'http://rssfeeds.usatoday.com/Usatoday-MostViewedArticles')] - - def dump_hex(self, src, length=16): - ''' Diagnostic ''' - FILTER=''.join([(len(repr(chr(x)))==3) and chr(x) or '.' for x in range(256)]) - N=0; result='' - while src: - s,src = src[:length],src[length:] - hexa = ' '.join(["%02X"%ord(x) for x in s]) - s = s.translate(FILTER) - result += "%04X %-*s %s\n" % (N, length*3, hexa, s) - N+=length - print result - - def fixChars(self,string): - # Replace lsquo (\x91) - fixed = re.sub("\x91","‘",string) - - # Replace rsquo (\x92) - fixed = re.sub("\x92","’",fixed) - - # Replace ldquo (\x93) - fixed = re.sub("\x93","“",fixed) - - # Replace rdquo (\x94) - fixed = re.sub("\x94","”",fixed) - - # Replace ndash (\x96) - fixed = re.sub("\x96","–",fixed) - - # Replace mdash (\x97) - fixed = re.sub("\x97","—",fixed) - - return fixed def get_masthead_url(self): masthead = 'http://i.usatoday.net/mobile/_common/_images/565x73_usat_mobile.gif' @@ -115,321 +75,4 @@ class USAToday(BasicNewsRecipe): masthead = None return masthead - def massageNCXText(self, description): - # Kindle TOC descriptions won't render certain characters - if description: - massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - # Replace '&' with '&' - massaged = re.sub("&","&", massaged) - return self.fixChars(massaged) - else: - return description - def parse_feeds(self, *args, **kwargs): - parsed_feeds = BasicNewsRecipe.parse_feeds(self, *args, **kwargs) - # Count articles for progress dialog - article_count = 0 - for feed in parsed_feeds: - article_count += len(feed) - self.log( "Queued %d articles" % article_count) - return parsed_feeds - - def preprocess_html(self, soup): - soup = self.strip_anchors(soup) - return soup - - def postprocess_html(self, soup, first_fetch): - - # Remove navLinks <div class="inside-copy" style="padding-bottom:3px"> - navLinks = soup.find(True,{'style':'padding-bottom:3px'}) - if navLinks: - navLinks.extract() - - # Remove <div class="inside-copy" style="margin-bottom:10px"> - gibberish = soup.find(True,{'style':'margin-bottom:10px'}) - if gibberish: - gibberish.extract() - - # Change <inside-head> to <h2> - headline = soup.find(True, {'class':['inside-head','inside-head2']}) - if not headline: - headline = soup.find('h3') - if headline: - tag = Tag(soup, "h2") - tag['class'] = "headline" - tag.insert(0, headline.contents[0]) - headline.replaceWith(tag) - else: - print "unable to find headline:\n%s\n" % soup - - # Change byLine to byline, change commas to middot - # Kindle renders commas in byline as '&' - byline = soup.find(True, {'class':'byLine'}) - if byline: - byline['class'] = 'byline' - # Replace comma with middot - byline.contents[0].replaceWith(re.sub(","," ·", byline.renderContents())) - - jumpout_punc_list = [':','?'] - # Remove the inline jumpouts in <div class="inside-copy"> - paras = soup.findAll(True, {'class':'inside-copy'}) - for para in paras: - if re.match("<b>[\w\W]+ ",para.renderContents()): - p = para.find('b') - for punc in jumpout_punc_list: - punc_offset = p.contents[0].find(punc) - if punc_offset == -1: - continue - if punc_offset > 1: - if p.contents[0][:punc_offset] == p.contents[0][:punc_offset].upper(): - #print "extracting \n%s\n" % para.prettify() - para.extract() - - # Reset class for remaining - paras = soup.findAll(True, {'class':'inside-copy'}) - for para in paras: - para['class'] = 'articleBody' - - # Remove inline jumpouts in <p> - paras = soup.findAll(['p']) - for p in paras: - if hasattr(p,'contents') and len(p.contents): - for punc in jumpout_punc_list: - punc_offset = p.contents[0].find(punc) - if punc_offset == -1: - continue - if punc_offset > 2 and hasattr(p,'a') and len(p.contents): - #print "evaluating %s\n" % p.contents[0][:punc_offset+1] - if p.contents[0][:punc_offset] == p.contents[0][:punc_offset].upper(): - #print "extracting \n%s\n" % p.prettify() - p.extract() - - # Capture the first img, insert after headline - imgs = soup.findAll('img') - print "postprocess_html(): %d images" % len(imgs) - if imgs: - divTag = Tag(soup, 'div') - divTag['class'] = 'image' - body = soup.find('body') - img = imgs[0] - #print "img: \n%s\n" % img.prettify() - - # Table for photo and credit - tableTag = Tag(soup,'table') - - # Photo - trimgTag = Tag(soup, 'tr') - tdimgTag = Tag(soup, 'td') - tdimgTag.insert(0,img) - trimgTag.insert(0,tdimgTag) - tableTag.insert(0,trimgTag) - - # Credit - trcreditTag = Tag(soup, 'tr') - - tdcreditTag = Tag(soup, 'td') - tdcreditTag['class'] = 'credit' - credit = soup.find('td',{'class':'photoCredit'}) - if credit: - tdcreditTag.insert(0,NavigableString(credit.renderContents())) - else: - credit = img['credit'] - if credit: - tdcreditTag.insert(0,NavigableString(credit)) - else: - tdcreditTag.insert(0,NavigableString('')) - - trcreditTag.insert(0,tdcreditTag) - tableTag.insert(1,trcreditTag) - dtc = 0 - divTag.insert(dtc,tableTag) - dtc += 1 - - if False: - # Add the caption in the table - tableCaptionTag = Tag(soup,'caption') - tableCaptionTag.insert(0,soup.find('td',{'class':'photoCredit'}).renderContents()) - tableTag.insert(1,tableCaptionTag) - divTag.insert(dtc,tableTag) - dtc += 1 - body.insert(1,divTag) - else: - # Add the caption below the table - #print "Looking for caption in this soup:\n%s" % img.prettify() - captionTag = Tag(soup,'p') - captionTag['class'] = 'caption' - if hasattr(img,'alt') and img['alt']: - captionTag.insert(0,NavigableString('<blockquote>%s</blockquote>' % img['alt'])) - divTag.insert(dtc, captionTag) - dtc += 1 - else: - try: - captionTag.insert(0,NavigableString('<blockquote>%s</blockquote>' % img['cutline'])) - divTag.insert(dtc, captionTag) - dtc += 1 - except: - pass - - hrTag = Tag(soup, 'hr') - divTag.insert(dtc, hrTag) - dtc += 1 - - # Delete <div id="applyMainStoryPhoto" - photoJunk = soup.find('div',{'id':'applyMainStoryPhoto'}) - if photoJunk: - photoJunk.extract() - - # Insert img after headline - tag = body.find(True) - insertLoc = 0 - headline_found = False - while True: - # Scan the top-level tags - insertLoc += 1 - if hasattr(tag,'class') and tag['class'] == 'headline': - headline_found = True - body.insert(insertLoc,divTag) - break - tag = tag.nextSibling - if not tag: - break - - if not headline_found: - # Monolithic <div> - restructure - tag = body.find(True) - while True: - insertLoc += 1 - try: - if hasattr(tag,'class') and tag['class'] == 'headline': - headline_found = True - tag.insert(insertLoc,divTag) - break - except: - pass - tag = tag.next - if not tag: - break - - # Yank out headline, img and caption - headline = body.find('h2','headline') - img = body.find('div','image') - caption = body.find('p''class') - - # body(0) is calibre_navbar - # body(1) is <div class="item"> - - btc = 1 - headline.extract() - body.insert(1, headline) - btc += 1 - if img: - img.extract() - body.insert(btc, img) - btc += 1 - if caption: - caption.extract() - body.insert(btc, caption) - btc += 1 - - if len(imgs) > 1: - if True: - [img.extract() for img in imgs[1:]] - else: - # Format the remaining images - # This doesn't work yet - for img in imgs[1:]: - print "img:\n%s\n" % img.prettify() - divTag = Tag(soup, 'div') - divTag['class'] = 'image' - - # Table for photo and credit - tableTag = Tag(soup,'table') - - # Photo - trimgTag = Tag(soup, 'tr') - tdimgTag = Tag(soup, 'td') - tdimgTag.insert(0,img) - trimgTag.insert(0,tdimgTag) - tableTag.insert(0,trimgTag) - - # Credit - trcreditTag = Tag(soup, 'tr') - - tdcreditTag = Tag(soup, 'td') - tdcreditTag['class'] = 'credit' - try: - tdcreditTag.insert(0,NavigableString(img['credit'])) - except: - tdcreditTag.insert(0,NavigableString('')) - trcreditTag.insert(0,tdcreditTag) - tableTag.insert(1,trcreditTag) - divTag.insert(0,tableTag) - soup.img.replaceWith(divTag) - - return soup - - def postprocess_book(self, oeb, opts, log) : - - def extract_byline(href) : - # <meta name="byline" content= - soup = BeautifulSoup(str(oeb.manifest.hrefs[href])) - byline = soup.find('div',attrs={'class':'byline'}) - if byline: - byline['class'] = 'byline' - # Replace comma with middot - byline.contents[0].replaceWith(re.sub(u",", u" ·", - byline.renderContents(encoding=None))) - return byline.renderContents(encoding=None) - else : - paras = soup.findAll(text=True) - for para in paras: - if para.startswith("Copyright"): - return para[len('Copyright xxxx '):para.find('.')] - return None - - def extract_description(href) : - soup = BeautifulSoup(str(oeb.manifest.hrefs[href])) - description = soup.find('meta',attrs={'name':'description'}) - if description : - return self.massageNCXText(description['content']) - else: - # Take first paragraph of article - articleBody = soup.find('div',attrs={'id':['articleBody','item']}) - if articleBody: - paras = articleBody.findAll('p') - for p in paras: - if p.renderContents() > '' : - return self.massageNCXText(self.tag_to_string(p,use_alt=False)) - else: - print "Didn't find <div id='articleBody'> in this soup:\n%s" % soup.prettify() - return None - - # Method entry point here - # Single section toc looks different than multi-section tocs - if oeb.toc.depth() == 2 : - for article in oeb.toc : - if article.author is None : - article.author = extract_byline(article.href) - if article.description is None : - article.description = extract_description(article.href) - elif oeb.toc.depth() == 3 : - for section in oeb.toc : - for article in section : - article.author = extract_byline(article.href) - ''' - if article.author is None : - article.author = self.massageNCXText(extract_byline(article.href)) - else: - article.author = self.massageNCXText(article.author) - ''' - if article.description is None : - article.description = extract_description(article.href) - - def strip_anchors(self,soup): - paras = soup.findAll(True) - for para in paras: - aTags = para.findAll('a') - for a in aTags: - if a.img is None: - a.replaceWith(a.renderContents().decode('cp1252','replace')) - return soup From 1400f54107185d80ea880bc786de78910a382e48 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 4 May 2011 11:02:56 -0600 Subject: [PATCH 31/39] EPUB metadata: When extracting covers from epub files handle invalid epubs that specify their content as a raster cover. Apparently PG produces these. --- src/calibre/ebooks/metadata/opf2.py | 4 +++- src/calibre/gui2/actions/choose_library.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index 58c887bfdb..1d91236757 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -966,7 +966,9 @@ class OPF(object): # {{{ cover_id = covers[0].get('content') for item in self.itermanifest(): if item.get('id', None) == cover_id: - return item.get('href', None) + mt = item.get('media-type', '') + if 'xml' not in mt: + return item.get('href', None) @dynamic_property def cover(self): diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index 4b262ad9dd..a663f288af 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -246,7 +246,7 @@ class ChooseLibraryAction(InterfaceAction): def delete_requested(self, name, location): loc = location.replace('/', os.sep) if not question_dialog(self.gui, _('Are you sure?'), '<p>'+ - _('All files from %s will be ' + _('<b>All files</b> from %s will be ' '<b>permanently deleted</b>. Are you sure?') % loc, show_copy_button=False): return From fe90a1b04fa8fecd5abfacbc795ca8b526897b11 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 4 May 2011 13:13:38 -0600 Subject: [PATCH 32/39] Implement #777001 (Add Preview ePub button to Tweak ePub dialog) --- src/calibre/gui2/dialogs/tweak_epub.py | 38 ++++++++++++++--- src/calibre/gui2/dialogs/tweak_epub.ui | 57 +++++++++++++++----------- 2 files changed, 66 insertions(+), 29 deletions(-) diff --git a/src/calibre/gui2/dialogs/tweak_epub.py b/src/calibre/gui2/dialogs/tweak_epub.py index edc274c9b2..732d74b77d 100755 --- a/src/calibre/gui2/dialogs/tweak_epub.py +++ b/src/calibre/gui2/dialogs/tweak_epub.py @@ -7,16 +7,16 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' import os, shutil -from contextlib import closing from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED from PyQt4.Qt import QDialog from calibre.constants import isosx -from calibre.gui2 import open_local_file +from calibre.gui2 import open_local_file, error_dialog from calibre.gui2.dialogs.tweak_epub_ui import Ui_Dialog from calibre.libunzip import extract as zipextract -from calibre.ptempfile import PersistentTemporaryDirectory +from calibre.ptempfile import (PersistentTemporaryDirectory, + PersistentTemporaryFile) class TweakEpub(QDialog, Ui_Dialog): ''' @@ -37,11 +37,15 @@ class TweakEpub(QDialog, Ui_Dialog): self.cancel_button.clicked.connect(self.reject) self.explode_button.clicked.connect(self.explode) self.rebuild_button.clicked.connect(self.rebuild) + self.preview_button.clicked.connect(self.preview) # Position update dialog overlaying top left of app window parent_loc = parent.pos() self.move(parent_loc.x(),parent_loc.y()) + self.gui = parent + self._preview_files = [] + def cleanup(self): if isosx: try: @@ -55,6 +59,11 @@ class TweakEpub(QDialog, Ui_Dialog): # Delete directory containing exploded ePub if self._exploded is not None: shutil.rmtree(self._exploded, ignore_errors=True) + for x in self._preview_files: + try: + os.remove(x) + except: + pass def display_exploded(self): ''' @@ -71,9 +80,8 @@ class TweakEpub(QDialog, Ui_Dialog): self.rebuild_button.setEnabled(True) self.explode_button.setEnabled(False) - def rebuild(self, *args): - self._output = os.path.join(self._exploded, 'rebuilt.epub') - with closing(ZipFile(self._output, 'w', compression=ZIP_DEFLATED)) as zf: + def do_rebuild(self, src): + with ZipFile(src, 'w', compression=ZIP_DEFLATED) as zf: # Write mimetype zf.write(os.path.join(self._exploded,'mimetype'), 'mimetype', compress_type=ZIP_STORED) # Write everything else @@ -86,5 +94,23 @@ class TweakEpub(QDialog, Ui_Dialog): zfn = os.path.relpath(absfn, self._exploded).replace(os.sep, '/') zf.write(absfn, zfn) + + def preview(self): + if not self._exploded: + return error_dialog(self, _('Cannot preview'), + _('You must first explode the epub before previewing.'), + show=True) + + tf = PersistentTemporaryFile('.epub') + tf.close() + self._preview_files.append(tf.name) + + self.do_rebuild(tf.name) + + self.gui.iactions['View']._view_file(tf.name) + + def rebuild(self, *args): + self._output = os.path.join(self._exploded, 'rebuilt.epub') + self.do_rebuild(self._output) return QDialog.accept(self) diff --git a/src/calibre/gui2/dialogs/tweak_epub.ui b/src/calibre/gui2/dialogs/tweak_epub.ui index fc6f24675f..a59af4fde1 100644 --- a/src/calibre/gui2/dialogs/tweak_epub.ui +++ b/src/calibre/gui2/dialogs/tweak_epub.ui @@ -23,6 +23,16 @@ <bool>false</bool> </property> <layout class="QGridLayout" name="gridLayout"> + <item row="0" column="0" colspan="2"> + <widget class="QLabel" name="label"> + <property name="text"> + <string><p>Explode the ePub to display contents in a file browser window. To tweak individual files, right-click, then 'Open with...' your editor of choice. When tweaks are complete, close the file browser window <b>and the editor windows you used to edit files in the epub</b>.</p><p>Rebuild the ePub, updating your calibre library.</p></string> + </property> + <property name="wordWrap"> + <bool>true</bool> + </property> + </widget> + </item> <item row="1" column="0"> <widget class="QPushButton" name="explode_button"> <property name="statusTip"> @@ -37,23 +47,6 @@ </property> </widget> </item> - <item row="2" column="0"> - <widget class="QPushButton" name="rebuild_button"> - <property name="enabled"> - <bool>false</bool> - </property> - <property name="statusTip"> - <string>Rebuild ePub from exploded contents</string> - </property> - <property name="text"> - <string>&Rebuild ePub</string> - </property> - <property name="icon"> - <iconset resource="../../../../resources/images.qrc"> - <normaloff>:/images/exec.png</normaloff>:/images/exec.png</iconset> - </property> - </widget> - </item> <item row="3" column="0"> <widget class="QPushButton" name="cancel_button"> <property name="statusTip"> @@ -68,13 +61,31 @@ </property> </widget> </item> - <item row="0" column="0"> - <widget class="QLabel" name="label"> - <property name="text"> - <string><p>Explode the ePub to display contents in a file browser window. To tweak individual files, right-click, then 'Open with...' your editor of choice. When tweaks are complete, close the file browser window <b>and the editor windows you used to edit files in the epub</b>.</p><p>Rebuild the ePub, updating your calibre library.</p></string> + <item row="3" column="1"> + <widget class="QPushButton" name="rebuild_button"> + <property name="enabled"> + <bool>false</bool> </property> - <property name="wordWrap"> - <bool>true</bool> + <property name="statusTip"> + <string>Rebuild ePub from exploded contents</string> + </property> + <property name="text"> + <string>&Rebuild ePub</string> + </property> + <property name="icon"> + <iconset resource="../../../../resources/images.qrc"> + <normaloff>:/images/exec.png</normaloff>:/images/exec.png</iconset> + </property> + </widget> + </item> + <item row="1" column="1"> + <widget class="QPushButton" name="preview_button"> + <property name="text"> + <string>&Preview ePub</string> + </property> + <property name="icon"> + <iconset resource="../../../../resources/images.qrc"> + <normaloff>:/images/view.png</normaloff>:/images/view.png</iconset> </property> </widget> </item> From c84c5f297bcfcd30f99f503768acd86235f1da43 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 4 May 2011 15:18:37 -0600 Subject: [PATCH 33/39] Add select all/none buttons to the metadata download prefs --- .../gui2/preferences/metadata_sources.py | 13 ++++++++++++- .../gui2/preferences/metadata_sources.ui | 18 ++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/preferences/metadata_sources.py b/src/calibre/gui2/preferences/metadata_sources.py index f487051d07..05ff23987d 100644 --- a/src/calibre/gui2/preferences/metadata_sources.py +++ b/src/calibre/gui2/preferences/metadata_sources.py @@ -190,7 +190,15 @@ class FieldsModel(QAbstractListModel): # {{{ return ans | Qt.ItemIsUserCheckable def restore_defaults(self): - self.overrides = dict([(f, self.state(f, True)) for f in self.fields]) + self.overrides = dict([(f, self.state(f, Qt.Checked)) for f in self.fields]) + self.reset() + + def select_all(self): + self.overrides = dict([(f, Qt.Checked) for f in self.fields]) + self.reset() + + def clear_all(self): + self.overrides = dict([(f, Qt.Unchecked) for f in self.fields]) self.reset() def setData(self, index, val, role): @@ -273,6 +281,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.fields_view.setModel(self.fields_model) self.fields_model.dataChanged.connect(self.changed_signal) + self.select_all_button.clicked.connect(self.fields_model.select_all) + self.clear_all_button.clicked.connect(self.fields_model.clear_all) + def configure_plugin(self): for index in self.sources_view.selectionModel().selectedRows(): plugin = self.sources_model.data(index, Qt.UserRole) diff --git a/src/calibre/gui2/preferences/metadata_sources.ui b/src/calibre/gui2/preferences/metadata_sources.ui index e46069b036..ff161654dd 100644 --- a/src/calibre/gui2/preferences/metadata_sources.ui +++ b/src/calibre/gui2/preferences/metadata_sources.ui @@ -77,8 +77,8 @@ <property name="title"> <string>Downloaded metadata fields</string> </property> - <layout class="QVBoxLayout" name="verticalLayout_2"> - <item> + <layout class="QGridLayout" name="gridLayout_2"> + <item row="0" column="0" colspan="2"> <widget class="QListView" name="fields_view"> <property name="toolTip"> <string>If you uncheck any fields, metadata for those fields will not be downloaded</string> @@ -88,6 +88,20 @@ </property> </widget> </item> + <item row="1" column="0"> + <widget class="QPushButton" name="select_all_button"> + <property name="text"> + <string>&Select all</string> + </property> + </widget> + </item> + <item row="1" column="1"> + <widget class="QPushButton" name="clear_all_button"> + <property name="text"> + <string>&Clear all</string> + </property> + </widget> + </item> </layout> </widget> </item> From 45ec80f4bb580b88d6c957906f5201648c2b9edf Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 4 May 2011 15:49:34 -0600 Subject: [PATCH 34/39] For the lazy among us --- src/calibre/gui2/actions/choose_library.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/choose_library.py b/src/calibre/gui2/actions/choose_library.py index a663f288af..f6b19fc4aa 100644 --- a/src/calibre/gui2/actions/choose_library.py +++ b/src/calibre/gui2/actions/choose_library.py @@ -246,7 +246,7 @@ class ChooseLibraryAction(InterfaceAction): def delete_requested(self, name, location): loc = location.replace('/', os.sep) if not question_dialog(self.gui, _('Are you sure?'), '<p>'+ - _('<b>All files</b> from %s will be ' + _('<b style="color: red">All files</b> from <br><br><b>%s</b><br><br> will be ' '<b>permanently deleted</b>. Are you sure?') % loc, show_copy_button=False): return From a34ea87a940feaf743ade8eddb893ce353aef6be Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Wed, 4 May 2011 18:56:30 -0400 Subject: [PATCH 35/39] Fix Bug #763105: RTF output displays some unicode characters incorrectly. --- src/calibre/ebooks/rtf/rtfml.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/calibre/ebooks/rtf/rtfml.py b/src/calibre/ebooks/rtf/rtfml.py index 97fa175d1a..cd877c63c2 100644 --- a/src/calibre/ebooks/rtf/rtfml.py +++ b/src/calibre/ebooks/rtf/rtfml.py @@ -79,8 +79,7 @@ def txt2rtf(text): elif val <= 127: buf.write(x) else: - repl = ascii_text(x) - c = r'\uc{2}\u{0:d}{1}'.format(val, repl, len(repl)) + c = r'\u{0:d}?'.format(val) buf.write(c) return buf.getvalue() From e15ee70a1ded03bf4b84f5a30f8fce89aeefa56e Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 4 May 2011 18:56:07 -0600 Subject: [PATCH 36/39] ODT Input: Speed up conversion of ODT files that define huge amounts of redundant style information. Fixes #777468 (Conversion from ODT to EPUB extremely slow) --- src/calibre/ebooks/odt/input.py | 55 ++++++++++++++++++++++++++++++--- src/odf/odf2xhtml.py | 14 +++++++-- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/calibre/ebooks/odt/input.py b/src/calibre/ebooks/odt/input.py index 1184148e80..10553dac2b 100644 --- a/src/calibre/ebooks/odt/input.py +++ b/src/calibre/ebooks/odt/input.py @@ -7,6 +7,8 @@ __docformat__ = 'restructuredtext en' Convert an ODT file into a Open Ebook ''' import os + +from lxml import etree from odf.odf2xhtml import ODF2XHTML from calibre import CurrentDir, walk @@ -23,7 +25,48 @@ class Extract(ODF2XHTML): with open(name, 'wb') as f: f.write(data) - def __call__(self, stream, odir): + def filter_css(self, html, log): + root = etree.fromstring(html) + style = root.xpath('//*[local-name() = "style" and @type="text/css"]') + if style: + style = style[0] + css = style.text + if css: + style.text, sel_map = self.do_filter_css(css) + for x in root.xpath('//*[@class]'): + extra = [] + orig = x.get('class') + for cls in orig.split(): + extra.extend(sel_map.get(cls, [])) + if extra: + x.set('class', orig + ' ' + ' '.join(extra)) + html = etree.tostring(root, encoding='utf-8', + xml_declaration=True) + return html + + def do_filter_css(self, css): + from cssutils import parseString + from cssutils.css import CSSRule + sheet = parseString(css) + rules = list(sheet.cssRules.rulesOfType(CSSRule.STYLE_RULE)) + sel_map = {} + count = 0 + for r in rules: + # Check if we have only class selectors for this rule + nc = [x for x in r.selectorList if not + x.selectorText.startswith('.')] + if len(r.selectorList) > 1 and not nc: + replace_name = 'c_odt%d'%count + count += 1 + for sel in r.selectorList: + s = sel.selectorText[1:] + if s not in sel_map: + sel_map[s] = [] + sel_map[s].append(replace_name) + r.selectorText = '.'+replace_name + return sheet.cssText, sel_map + + def __call__(self, stream, odir, log): from calibre.utils.zipfile import ZipFile from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata.opf2 import OPFCreator @@ -32,13 +75,17 @@ class Extract(ODF2XHTML): if not os.path.exists(odir): os.makedirs(odir) with CurrentDir(odir): - print 'Extracting ODT file...' + log('Extracting ODT file...') html = self.odf2xhtml(stream) # A blanket img specification like this causes problems - # with EPUB output as the contaiing element often has + # with EPUB output as the containing element often has # an absolute height and width set that is larger than # the available screen real estate html = html.replace('img { width: 100%; height: 100%; }', '') + try: + html = self.filter_css(html, log) + except: + log.exception('Failed to filter CSS, conversion may be slow') with open('index.xhtml', 'wb') as f: f.write(html.encode('utf-8')) zf = ZipFile(stream, 'r') @@ -67,7 +114,7 @@ class ODTInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): - return Extract()(stream, '.') + return Extract()(stream, '.', log) def postprocess_book(self, oeb, opts, log): # Fix <p><div> constructs as the asinine epubchecker complains diff --git a/src/odf/odf2xhtml.py b/src/odf/odf2xhtml.py index 26da9d9905..a04aa48bf7 100644 --- a/src/odf/odf2xhtml.py +++ b/src/odf/odf2xhtml.py @@ -841,11 +841,19 @@ ol, ul { padding-left: 2em; } self.styledict[name] = styles # Write the styles to HTML self.writeout(self.default_styles) + # Changed by Kovid to not write out endless copies of the same style + css_styles = {} for name in self.stylestack: styles = self.styledict.get(name) - css2 = self.cs.convert_styles(styles) - self.writeout("%s {\n" % name) - for style, val in css2.items(): + css2 = tuple(self.cs.convert_styles(styles).iteritems()) + if css2 in css_styles: + css_styles[css2].append(name) + else: + css_styles[css2] = [name] + + for css2, names in css_styles.iteritems(): + self.writeout("%s {\n" % ', '.join(names)) + for style, val in css2: self.writeout("\t%s: %s;\n" % (style, val) ) self.writeout("}\n") From 07da7b239c377a856fbc43f16aa2f01042964e3d Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Wed, 4 May 2011 21:48:44 -0400 Subject: [PATCH 37/39] Fix Bug #775669: HTMLZ OPF sets cover, adds cover properly when updating metadata, reads cover form guide section of opf in archive. --- src/calibre/ebooks/htmlz/input.py | 25 +++++++++++++++++-- src/calibre/ebooks/htmlz/output.py | 25 ++++++++++++++++++- src/calibre/ebooks/metadata/extz.py | 24 ++++++++++-------- src/calibre/ebooks/oeb/transforms/metadata.py | 2 +- 4 files changed, 61 insertions(+), 15 deletions(-) diff --git a/src/calibre/ebooks/htmlz/input.py b/src/calibre/ebooks/htmlz/input.py index dcf2ed0ed3..d083fcc4ab 100644 --- a/src/calibre/ebooks/htmlz/input.py +++ b/src/calibre/ebooks/htmlz/input.py @@ -7,10 +7,12 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>' __docformat__ = 'restructuredtext en' import os +import posixpath -from calibre import walk +from calibre import guess_type, walk from calibre.customize.conversion import InputFormatPlugin from calibre.ebooks.chardet import xml_to_unicode +from calibre.ebooks.metadata.opf2 import OPF from calibre.utils.zipfile import ZipFile class HTMLZInput(InputFormatPlugin): @@ -27,7 +29,7 @@ class HTMLZInput(InputFormatPlugin): # Extract content from zip archive. zf = ZipFile(stream) - zf.extractall('.') + zf.extractall() for x in walk('.'): if os.path.splitext(x)[1].lower() in ('.html', '.xhtml', '.htm'): @@ -70,5 +72,24 @@ class HTMLZInput(InputFormatPlugin): from calibre.ebooks.oeb.transforms.metadata import meta_info_to_oeb_metadata mi = get_file_type_metadata(stream, file_ext) meta_info_to_oeb_metadata(mi, oeb.metadata, log) + + # Get the cover path from the OPF. + cover_href = None + opf = None + for x in walk('.'): + if os.path.splitext(x)[1].lower() in ('.opf'): + opf = x + break + if opf: + opf = OPF(opf) + cover_href = posixpath.relpath(opf.cover, os.path.dirname(stream.name)) + # Set the cover. + if cover_href: + cdata = None + with open(cover_href, 'rb') as cf: + cdata = cf.read() + id, href = oeb.manifest.generate('cover', cover_href) + oeb.manifest.add(id, href, guess_type(cover_href)[0], data=cdata) + oeb.guide.add('cover', 'Cover', href) return oeb diff --git a/src/calibre/ebooks/htmlz/output.py b/src/calibre/ebooks/htmlz/output.py index 6d2ad54a12..a1ef57af2c 100644 --- a/src/calibre/ebooks/htmlz/output.py +++ b/src/calibre/ebooks/htmlz/output.py @@ -7,11 +7,13 @@ __copyright__ = '2011, John Schember <john@nachtimwald.com>' __docformat__ = 'restructuredtext en' import os +from cStringIO import StringIO from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation +from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile @@ -79,10 +81,31 @@ class HTMLZOutput(OutputFormatPlugin): fname = os.path.join(tdir, 'images', images[item.href]) with open(fname, 'wb') as img: img.write(data) + + # Cover + cover_path = None + try: + cover_data = None + if oeb_book.metadata.cover: + term = oeb_book.metadata.cover[0].term + cover_data = oeb_book.guide[term].item.data + if cover_data: + from calibre.utils.magick.draw import save_cover_data_to + cover_path = os.path.join(tdir, 'cover.jpg') + with open(cover_path, 'w') as cf: + cf.write('') + save_cover_data_to(cover_data, cover_path) + except: + import traceback + traceback.print_exc() # Metadata with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: - mdataf.write(etree.tostring(oeb_book.metadata.to_opf1())) + opf = OPF(StringIO(etree.tostring(oeb_book.metadata.to_opf1()))) + mi = opf.to_book_metadata() + if cover_path: + mi.cover = 'cover.jpg' + mdataf.write(metadata_to_opf(mi)) htmlz = ZipFile(output_path, 'w') htmlz.add_dir(tdir) diff --git a/src/calibre/ebooks/metadata/extz.py b/src/calibre/ebooks/metadata/extz.py index 18069b2a6a..21c10278e1 100644 --- a/src/calibre/ebooks/metadata/extz.py +++ b/src/calibre/ebooks/metadata/extz.py @@ -13,7 +13,7 @@ import posixpath from cStringIO import StringIO from calibre.ebooks.metadata import MetaInformation -from calibre.ebooks.metadata.opf2 import OPF +from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.zipfile import ZipFile, safe_replace @@ -31,9 +31,9 @@ def get_metadata(stream, extract_cover=True): opf = OPF(opf_stream) mi = opf.to_book_metadata() if extract_cover: - cover_name = opf.raster_cover - if cover_name: - mi.cover_data = ('jpg', zf.read(cover_name)) + cover_href = posixpath.relpath(opf.cover, os.path.dirname(stream.name)) + if cover_href: + mi.cover_data = ('jpg', zf.read(cover_href)) except: return mi return mi @@ -59,17 +59,19 @@ def set_metadata(stream, mi): except: pass if new_cdata: - raster_cover = opf.raster_cover - if not raster_cover: - raster_cover = 'cover.jpg' - cpath = posixpath.join(posixpath.dirname(opf_path), raster_cover) + cover = opf.cover + if not cover: + cover = 'cover.jpg' + cpath = posixpath.join(posixpath.dirname(opf_path), cover) new_cover = _write_new_cover(new_cdata, cpath) replacements[cpath] = open(new_cover.name, 'rb') + mi.cover = cover # Update the metadata. - opf.smart_update(mi, replace_metadata=True) - newopf = StringIO(opf.render()) - safe_replace(stream, opf_path, newopf, extra_replacements=replacements) + old_mi = opf.to_book_metadata() + old_mi.smart_update(mi) + newopf = StringIO(metadata_to_opf(old_mi)) + safe_replace(stream, opf_path, newopf, extra_replacements=replacements, add_missing=True) # Cleanup temporary files. try: diff --git a/src/calibre/ebooks/oeb/transforms/metadata.py b/src/calibre/ebooks/oeb/transforms/metadata.py index 19c209b74d..f719ee3eb5 100644 --- a/src/calibre/ebooks/oeb/transforms/metadata.py +++ b/src/calibre/ebooks/oeb/transforms/metadata.py @@ -36,7 +36,7 @@ def meta_info_to_oeb_metadata(mi, m, log, override_input_metadata=False): m.clear('description') m.add('description', mi.comments) elif override_input_metadata: - m.clear('description') + m.clear('description') if not mi.is_null('publisher'): m.clear('publisher') m.add('publisher', mi.publisher) From 6fbb996d2a30afd36da0e3566cc8e22e23a912ab Mon Sep 17 00:00:00 2001 From: John Schember <john@nachtimwald.com> Date: Wed, 4 May 2011 21:50:08 -0400 Subject: [PATCH 38/39] HTMLZ: Keep existing non-metadata sections when updating. --- src/calibre/ebooks/metadata/extz.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/extz.py b/src/calibre/ebooks/metadata/extz.py index 21c10278e1..1bda263015 100644 --- a/src/calibre/ebooks/metadata/extz.py +++ b/src/calibre/ebooks/metadata/extz.py @@ -70,7 +70,8 @@ def set_metadata(stream, mi): # Update the metadata. old_mi = opf.to_book_metadata() old_mi.smart_update(mi) - newopf = StringIO(metadata_to_opf(old_mi)) + opf.smart_update(metadata_to_opf(old_mi)) + newopf = StringIO(opf.render()) safe_replace(stream, opf_path, newopf, extra_replacements=replacements, add_missing=True) # Cleanup temporary files. From b461f5bc26030f3f8ce43d02facc3914ade45c5e Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 4 May 2011 20:30:34 -0600 Subject: [PATCH 39/39] ... --- src/calibre/ebooks/odt/input.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/ebooks/odt/input.py b/src/calibre/ebooks/odt/input.py index 10553dac2b..e724acb981 100644 --- a/src/calibre/ebooks/odt/input.py +++ b/src/calibre/ebooks/odt/input.py @@ -56,6 +56,9 @@ class Extract(ODF2XHTML): nc = [x for x in r.selectorList if not x.selectorText.startswith('.')] if len(r.selectorList) > 1 and not nc: + # Replace all the class selectors with a single class selector + # This will be added to the class attribute of all elements + # that have one of these selectors. replace_name = 'c_odt%d'%count count += 1 for sel in r.selectorList: