From a81e601428b80f31d1d992d6d78ba88936a8c10d Mon Sep 17 00:00:00 2001 From: Lee Date: Sun, 6 Mar 2011 17:18:54 +0800 Subject: [PATCH 01/90] added initial Overdrive support --- src/calibre/customize/builtins.py | 8 +- src/calibre/ebooks/metadata/covers.py | 34 ++ src/calibre/ebooks/metadata/fetch.py | 21 ++ src/calibre/ebooks/metadata/overdrive.py | 386 +++++++++++++++++++++++ 4 files changed, 445 insertions(+), 4 deletions(-) create mode 100644 src/calibre/ebooks/metadata/overdrive.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index cd4c866562..0c71317f8f 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -581,19 +581,19 @@ from calibre.devices.kobo.driver import KOBO from calibre.devices.bambook.driver import BAMBOOK from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \ - KentDistrictLibrary + KentDistrictLibrary, Overdrive from calibre.ebooks.metadata.douban import DoubanBooks from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers from calibre.ebooks.metadata.covers import OpenLibraryCovers, \ - AmazonCovers, DoubanCovers + AmazonCovers, DoubanCovers, OverdriveCovers from calibre.library.catalog import CSV_XML, EPUB_MOBI, BIBTEX from calibre.ebooks.epub.fix.unmanifested import Unmanifested from calibre.ebooks.epub.fix.epubcheck import Epubcheck -plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, GoogleBooks, ISBNDB, Amazon, +plugins = [HTML2ZIP, PML2PMLZ, TXT2TXTZ, ArchiveExtract, GoogleBooks, ISBNDB, Amazon, Overdrive, KentDistrictLibrary, DoubanBooks, NiceBooks, CSV_XML, EPUB_MOBI, BIBTEX, Unmanifested, Epubcheck, OpenLibraryCovers, AmazonCovers, DoubanCovers, - NiceBooksCovers] + NiceBooksCovers, OverdriveCovers] plugins += [ ComicInput, EPUBInput, diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index 15e0a05c1e..280ca077ef 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -145,6 +145,40 @@ class AmazonCovers(CoverDownload): # {{{ # }}} +class OverdriveCovers(CoverDownload): # {{{ + + name = 'overdrive.com covers' + description = _('Download covers from Overdrive') + author = 'Kovid Goyal' + + + def has_cover(self, mi, ans, timeout=5.): + if not mi.authors or not mi.title: + return False + from calibre.ebooks.metadata.overdrive import get_cover_url + br = browser() + try: + get_cover_url(mi.isbn, mi.title, mi.authors, br) + self.debug('cover for', mi.isbn, 'found') + ans.set() + except Exception, e: + self.debug(e) + + def get_covers(self, mi, result_queue, abort, timeout=5.): + if not mi.isbn: + return + from calibre.ebooks.metadata.overdrive import get_cover_url + br = browser() + try: + url = get_cover_url(mi.isbn, mi.title, mi.authors, br) + cover_data = br.open_novisit(url).read() + result_queue.put((True, cover_data, 'jpg', self.name)) + except Exception, e: + result_queue.put((False, self.exception_to_string(e), + traceback.format_exc(), self.name)) + +# }}} + def check_for_cover(mi, timeout=5.): # {{{ from calibre.customize.ui import cover_sources ans = Event() diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py index 667b4f4d7c..1f584bc107 100644 --- a/src/calibre/ebooks/metadata/fetch.py +++ b/src/calibre/ebooks/metadata/fetch.py @@ -250,6 +250,27 @@ class Amazon(MetadataSource): # {{{ # }}} +class Overdrive(MetadataSource): # {{{ + + name = 'Overdrive' + metadata_type = 'social' + description = _('Downloads metadata from the Overdrive library network') + + has_html_comments = True + + def fetch(self): + if not self.isbn: + return + from calibre.ebooks.metadata.overdrive import get_metadata + try: + self.results = get_metadata(self.title, self.book_author, + self.publisher, self.isbn) + except Exception, e: + self.exception = e + self.tb = traceback.format_exc() + + # }}} + class KentDistrictLibrary(MetadataSource): # {{{ name = 'Kent District Library' diff --git a/src/calibre/ebooks/metadata/overdrive.py b/src/calibre/ebooks/metadata/overdrive.py new file mode 100644 index 0000000000..ad512579d7 --- /dev/null +++ b/src/calibre/ebooks/metadata/overdrive.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Fetch metadata using Adobe Overdrive +''' +import sys, re, random, urllib, mechanize, copy +from threading import RLock + +from lxml import html, etree +from lxml.html import soupparser + +from calibre import browser +from calibre.ebooks.metadata import check_isbn +from calibre.ebooks.metadata.sources.base import Source +from calibre.ebooks.metadata.book.base import Metadata +from calibre.ebooks.chardet import xml_to_unicode +from calibre.library.comments import sanitize_comments_html + +ovrdrv_data_cache = {} +cover_url_cache = {} +cache_lock = RLock() +base_url = 'http://search.overdrive.com/' + +def get_base_referer(): + choices = [ + 'http://overdrive.chipublib.org/82DC601D-7DDE-4212-B43A-09D821935B01/10/375/en/', + 'http://emedia.clevnet.org/9D321DAD-EC0D-490D-BFD8-64AE2C96ECA8/10/241/en/', + 'http://singapore.lib.overdrive.com/F11D55BE-A917-4D63-8111-318E88B29740/10/382/en/', + 'http://ebooks.nypl.org/20E48048-A377-4520-BC43-F8729A42A424/10/257/en/', + 'http://spl.lib.overdrive.com/5875E082-4CB2-4689-9426-8509F354AFEF/10/335/en/' + ] + return choices[random.randint(0, len(choices)-1)] + +def format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid): + fix_slashes = re.compile(r'\\/') + thumbimage = fix_slashes.sub('/', thumbimage) + worldcatlink = fix_slashes.sub('/', worldcatlink) + cover_url = re.sub('(?P(Ima?g(eType-)?))200', '\g100', thumbimage) + social_metadata_url = base_url+'TitleInfo.aspx?ReserveID='+reserveid+'&FormatID='+formatid + series_num = '' + if not series: + if subtitle: + title = od_title+': '+subtitle + else: + title = od_title + m = re.search("([0-9]+$)", subtitle) + if m: + series_num = float(m.group(1)) + return [cover_url, social_metadata_url, worldcatlink, series, series_num, publisher, creators, reserveid, title] + +def overdrive_search(br, q, title, author): + q_query = q+'default.aspx/SearchByKeyword' + q_init_search = q+'SearchResults.aspx' + + # query terms + author_q = re.sub('\s', '+', author) + q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=10&sSearch='+author_q + query = '{"szKeyword":"'+title+'"}' + + # main query, requires specific Content Type header + req = mechanize.Request(q_query) + req.add_header('Content-Type', 'application/json; charset=utf-8') + br.open_novisit(req, query) + + print "q_init_search is "+q_init_search + + # the query must be initialized by loading an empty search results page + # this page attempts to set a cookie that Mechanize doesn't like + # copy the cookiejar to a separate instance and make a one-off request with the temp cookiejar + goodcookies = br._ua_handlers['_cookies'].cookiejar + clean_cj = mechanize.CookieJar() + cookies_to_copy = [] + for cookie in goodcookies: + copied_cookie = copy.deepcopy(cookie) + cookies_to_copy.append(copied_cookie) + for copied_cookie in cookies_to_copy: + clean_cj.set_cookie(copied_cookie) + + br.open_novisit(q_init_search) + + br.set_cookiejar(clean_cj) + + # get the search results object + xreq = mechanize.Request(q_xref) + xreq.add_header('X-Requested-With', 'XMLHttpRequest') + xreq.add_header('Referer', q_init_search) + xreq.add_header('Accept', 'application/json, text/javascript, */*') + raw = br.open_novisit(xreq).read() + print "overdrive search result is:\n"+raw + raw = re.sub('.*?\[\[(?P.*?)\]\].*', '[[\g]]', raw) + results = eval(raw) + print "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + print results + # The search results are from a keyword search (overdrive's advanced search is broken), + # sort through the results for closest match/format + for result in results: + print "\n\n\nthis result is "+str(result) + for reserveid, od_title, subtitle, edition, series, publisher, format, formatid, creators, \ + thumbimage, shortdescription, worldcatlink, excerptlink, creatorfile, sorttitle, \ + availabletolibrary, availabletoretailer, relevancyrank, unknown1, unknown2, unknown3 in results: + creators = creators.split(', ') + print "fixed creators are: "+str(creators) + # if an exact match occurs + if creators[0] == author and od_title == title and int(formatid) in [1, 50, 410, 900]: + print "Got Exact Match!!!" + return format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) + + +def library_search(br, q, title, author): + q_search = q+'AdvancedSearch.htm' + q_query = q+'BANGSearch.dll' + br.open(q_search) + # Search for cover with audiobooks lowest priority + for format in ['410','50','900','25','425']: + query = 'Title='+title+'&Creator='+author+'&Keyword=&ISBN=&Format='+format+'&Language=&Publisher=&Subject=&Award=&CollDate=&PerPage=10&Sort=SortBy%3Dtitle' + query = re.sub('\s', '+', query) + #print "search url is "+str(q_search) + print "query is "+str(query) + raw = br.open(q_query, query).read() + #print "raw html is:\n"+str(raw) + raw = xml_to_unicode(raw, strip_encoding_pats=True, + resolve_entities=True)[0] + root = html.fromstring(raw) + revs = root.xpath("//img[@class='blackborder']") + if revs: + #print "revs are "+str(revs) + # get the first match, as it's the most likely candidate + x = revs[0] + id = urllib.unquote(re.sub('.*?/(?P%7B.*?%7D).*', '\g', x.get('src'))) + curl = re.sub('(?P(Ima?g(eType-)?))200', '\g100', x.get('src')) + murl = root.xpath("//img[@class='blackborder']/parent::*") + if murl: + murl = [y.get('href') for y in murl] + print "murl is"+str(murl) + murl = q+murl[0] + else: + print "didn't get metadata URL" + print "curl is "+str(curl)+", id is "+str(id)+", murl is "+str(murl) + ovrdrv_data = [id, curl, murl] + print "revs final are "+str(revs) + return ovrdrv_data + + +def find_ovrdrv_data(br, title, author, isbn): + print "in fnd_ovrdrv_data, title is "+str(title)+", author is "+str(author) + q = base_url + if re.match('http://search\.overdrive\.', q): + return overdrive_search(br, q, title, author) + else: + return library_search(br, q, title, author) + + + +def to_ovrdrv_data(br, title, author, isbn): + print "starting to_ovrdrv_data" + with cache_lock: + ans = ovrdrv_data_cache.get(isbn, None) + if ans: + print "inside to_ovrdrv_data, ans returned positive, ans is"+str(ans) + return ans + if ans is False: + print "inside to_ovrdrv_data, ans returned False" + return None + try: + ovrdrv_data = find_ovrdrv_data(br, title, author, isbn) + print "ovrdrv_data = "+str(ovrdrv_data) + except: + import traceback + traceback.print_exc() + ovrdrv_data = None + + with cache_lock: + ovrdrv_data_cache[isbn] = ovrdrv_data if ovrdrv_data else False + return ovrdrv_data + + +def get_social_metadata(title, authors, publisher, isbn): + author = authors[0] + mi = Metadata(title, authors) + if not isbn: + return mi + isbn = check_isbn(isbn) + if not isbn: + return mi + br = browser() + ovrdrv_data = to_ovrdrv_data(br, title, authors, isbn) + if ovrdrv_data and get_metadata_detail_ovrdrv(br, ovrdrv_data, mi): + return mi + #from calibre.ebooks.metadata.xisbn import xisbn + #for i in xisbn.get_associated_isbns(isbn): + # print "xisbn isbn is "+str(i) + # ovrdrv_data = to_ovrdrv_data(br, title, author, i) + # if ovrdrv_data and get_metadata_detail(br, ovrdrv_data, mi): + # return mi + return mi + +def get_cover_url(isbn, title, author, br): + print "starting get_cover_url" + isbn = check_isbn(isbn) + print "isbn is "+str(isbn) + print "title is "+str(title) + print "author is "+str(author[0]) + cleanup = Source() + author = cleanup.get_author_tokens(author) + print "cleansed author is "+str(author) + + with cache_lock: + ans = cover_url_cache.get(isbn, None) + if ans: + print "ans returned positive" + return ans + if ans is False: + "ans returned false" + return None + print "in get_cover_url, running through ovrdrv_data function" + ovrdrv_data = to_ovrdrv_data(br, title, author, isbn) + print "ovrdrv_id is "+str(ovrdrv_data) + if ovrdrv_data: + ans = ovrdrv_data[0] + print "inside get_cover_url, ans is "+str(ans) + if ans: + with cache_lock: + cover_url_cache[isbn] = ans + return ans + #from calibre.ebooks.metadata.xisbn import xisbn + #for i in xisbn.get_associated_isbns(isbn): + # print "in get_cover_url, using xisbn list to associate other books" + # ovrdrv_data = to_ovrdrv_data(br, title, author, i) + # if ovrdrv_data: + # ans = _get_cover_url(br, ovrdrv_data) + # if ans: + # with cache_lock: + # cover_url_cache[isbn] = ans + # cover_url_cache[i] = ans + # return ans + with cache_lock: + cover_url_cache[isbn] = False + return None + +def _get_cover_url(br, ovrdrv_data): + q = ovrdrv_data[1] + try: + raw = br.open_novisit(q).read() + except Exception, e: + if callable(getattr(e, 'getcode', None)) and \ + e.getcode() == 404: + return None + raise + if '404 - ' in raw: + return None + raw = xml_to_unicode(raw, strip_encoding_pats=True, + resolve_entities=True)[0] + try: + root = soupparser.fromstring(raw) + except: + return False + + imgs = root.xpath('//img[@id="prodImage" and @src]') + if imgs: + src = imgs[0].get('src') + parts = src.split('/') + if len(parts) > 3: + bn = parts[-1] + sparts = bn.split('_') + if len(sparts) > 2: + bn = sparts[0] + sparts[-1] + return ('/'.join(parts[:-1]))+'/'+bn + return None + + +def get_metadata_detail(br, ovrdrv_data, mi): + q = ovrdrv_data[2] + try: + raw = br.open_novisit(q).read() + except Exception, e: + if callable(getattr(e, 'getcode', None)) and \ + e.getcode() == 404: + return False + raise + if '<title>404 - ' in raw: + return False + raw = xml_to_unicode(raw, strip_encoding_pats=True, + resolve_entities=True)[0] + try: + root = soupparser.fromstring(raw) + except: + return False + + # Check for series name and retrieve it + series_name = root.xpath("//td/script[re:test(text(), 'szSeries', 'i')]", + namespaces={"re": "http://exslt.org/regular-expressions"}) + if series_name: + series = html.tostring(series_name[0], method='html', encoding=unicode).strip() + series = re.sub('(?s).*?szSeries\s*=\s*\"(?P<series>.*?)\";.*', '\g<series>', series) + if len(series) > 1: + mi.series = series + # If series was successful attempt to get the series number + series_num = root.xpath("//div/strong[re:test(text(), ',\s(Book|Part|Volume)')]", + namespaces={"re": "http://exslt.org/regular-expressions"}) + if series_num: + series_num = float(re.sub('(?s).*?,\s*(Book|Part|Volume)\s*(?P<num>\d+).*', '\g<num>', + etree.tostring(series_num[0]))) + if series_num >= 1: + mi.series_index = series_num + print "series_num is "+str(series_num) + + desc = root.xpath("//td[@class='collection' and re:test(., 'Description', 'i')]/following::div[1]", + namespaces={"re": "http://exslt.org/regular-expressions"}) + if desc: + desc = desc[0] + desc = html.tostring(desc, method='html', encoding=unicode).strip() + # remove all attributes from tags + desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc) + # Remove comments + desc = re.sub(r'(?s)<!--.*?-->', '', desc) + mi.comments = sanitize_comments_html(desc) + + publisher = root.xpath("//td/strong[re:test(text(), 'Publisher\:', 'i')]/ancestor::td[1]/following-sibling::td/text()", + namespaces={"re": "http://exslt.org/regular-expressions"}) + if publisher: + mi.publisher = re.sub('^\s*(?P<pub>.*?)\s*$', '\g<pub>', publisher[0]) + print "publisher is "+str(mi.publisher) + + lang = root.xpath("//td/strong[re:test(text(), 'Language\(s\):', 'i')]/ancestor::td[1]/following-sibling::td/text()", + namespaces={"re": "http://exslt.org/regular-expressions"}) + if lang: + mi.language = re.sub('^\s*(?P<lang>.*?)\s*$', '\g<lang>', lang[0]) + print "languages is "+str(mi.language) + + isbn = root.xpath("//tr/td[re:test(text(), 'ISBN:', 'i')]/following::td/text()", + namespaces={"re": "http://exslt.org/regular-expressions"}) + if isbn: + mi.isbn = re.sub('^\s*(?P<isbn>.*?)\s*$', '\g<isbn>', isbn[0]) + print "ISBN is "+str(mi.isbn) + + subjects = root.xpath("//td/strong[re:test(text(), 'Subject', 'i')]/ancestor::td[1]/following-sibling::td/a/text()", + namespaces={"re": "http://exslt.org/regular-expressions"}) + if subjects: + mi.tags = subjects + print "tags are "+str(mi.tags) + + creators = root.xpath("//table/tr/td[re:test(text(), '\s*by', 'i')]/ancestor::tr[1]/td[2]/table/tr/td/a/text()", + namespaces={"re": "http://exslt.org/regular-expressions"}) + if creators: + print "authors are "+str(creators) + mi.authors = creators + + return True + +def main(args=sys.argv): + print "running through main tests" + import tempfile, os, time + tdir = tempfile.gettempdir() + br = browser() + for isbn, title, author in [ + #('0899661343', 'On the Road', ['Jack Kerouac']), # basic test, no series, single author + #('9780061952838', 'The Fellowship of the Ring', ['J. R. R. Tolkien']), # Series test, multi-author + #('9780061952838', 'The Two Towers', ['J. R. R. Tolkien']), # Series test, book 2 + ('9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors + #('', 'Deluge', ['Anne McCaffrey']) # Empty ISBN + #(None, 'On the Road', ['Jack Kerouac']) # Nonetype ISBN + ]: + cpath = os.path.join(tdir, title+'.jpg') + print "cpath is "+cpath + st = time.time() + curl = get_cover_url(isbn, title, author, br) + print '\n\n Took ', time.time() - st, ' to get metadata\n\n' + if curl is None: + print 'No cover found for', title + else: + print "curl is "+curl + #open(cpath, 'wb').write(br.open_novisit(curl).read()) + #print 'Cover for', title, 'saved to', cpath + + #import time + + #print get_social_metadata(title, author, None, isbn) + #print '\n\n', time.time() - st, '\n\n' + + return 0 + +if __name__ == '__main__': + sys.exit(main()) From 4e428219c94c05df0597c54aab9849f227928094 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Mon, 7 Mar 2011 04:37:47 +0800 Subject: [PATCH 02/90] ... --- src/calibre/ebooks/metadata/overdrive.py | 67 +++++++++++++----------- 1 file changed, 36 insertions(+), 31 deletions(-) diff --git a/src/calibre/ebooks/metadata/overdrive.py b/src/calibre/ebooks/metadata/overdrive.py index cb9ab2c9c9..5afb875fad 100644 --- a/src/calibre/ebooks/metadata/overdrive.py +++ b/src/calibre/ebooks/metadata/overdrive.py @@ -24,32 +24,29 @@ cover_url_cache = {} cache_lock = RLock() base_url = 'http://search.overdrive.com/' -class ContentReserve(Source): - def create_query(self, title=None, authors=None, identifiers={}): - q = '' - if title or authors: - def build_term(prefix, parts): - return ' '.join('in'+prefix + ':' + x for x in parts) - title_tokens = list(self.get_title_tokens(title)) - if title_tokens: - q += build_term('title', title_tokens) - author_tokens = self.get_author_tokens(authors, - only_first_author=True) - if author_tokens: - q += ('+' if q else '') + build_term('author', - author_tokens) - if isinstance(q, unicode): - q = q.encode('utf-8') - if not q: - return None - return BASE_URL+urlencode({ - 'q':q, - 'max-results':20, - 'start-index':1, - 'min-viewability':'none', - }) +def create_query(self, title=None, authors=None, identifiers={}): + q = '' + if title or authors: + def build_term(prefix, parts): + return ' '.join('in'+prefix + ':' + x for x in parts) + title_tokens = list(self.get_title_tokens(title)) + if title_tokens: + q += build_term('title', title_tokens) + author_tokens = self.get_author_tokens(authors, + only_first_author=True) + if author_tokens: + q += ('+' if q else '') + build_term('author', + author_tokens) + + if isinstance(q, unicode): + q = q.encode('utf-8') + if not q: + return None + return BASE_URL+urlencode({ + 'q':q, + }) def get_base_referer(): @@ -82,9 +79,20 @@ def format_results(reserveid, od_title, subtitle, series, publisher, creators, t def overdrive_search(br, q, title, author): q_query = q+'default.aspx/SearchByKeyword' q_init_search = q+'SearchResults.aspx' - + # get first author as string - convert this to a proper cleanup function later + s = Source(None) + print "printing list with string:" + print list(s.get_author_tokens(['J. R. R. Tolkien'])) + print "printing list with author "+str(author)+":" + print list(s.get_author_tokens(author)) + author = list(s.get_author_tokens(author)) + for token in author: + print "cleaned up author is: "+str(token) + author_q = '+'.join(author) + #author_q = separator.join(for x in author) # query terms - author_q = re.sub('\s', '+', author) + #author_q = re.sub('\s', '+', author_q) + print "final author query is "+str(author_q) q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=10&sSearch='+author_q query = '{"szKeyword":"'+title+'"}' @@ -231,9 +239,6 @@ def get_cover_url(isbn, title, author, br): print "isbn is "+str(isbn) print "title is "+str(title) print "author is "+str(author[0]) - cleanup = ContentReserve() - query = cleanup.create_query(author, title) - print "cleansed query is "+str(author) with cache_lock: ans = cover_url_cache.get(isbn, None) @@ -386,8 +391,8 @@ def main(args=sys.argv): for isbn, title, author in [ #('0899661343', 'On the Road', ['Jack Kerouac']), # basic test, no series, single author #('9780061952838', 'The Fellowship of the Ring', ['J. R. R. Tolkien']), # Series test, multi-author - #('9780061952838', 'The Two Towers', ['J. R. R. Tolkien']), # Series test, book 2 - ('9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors + ('9780061952838', 'The Two Towers', ['J. R. R. Tolkien']), # Series test, book 2 + #('9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors #('', 'Deluge', ['Anne McCaffrey']) # Empty ISBN #(None, 'On the Road', ['Jack Kerouac']) # Nonetype ISBN ]: From c6a2c8e82e5dcd64f0bfb605b10f3a590eb41a08 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Tue, 22 Mar 2011 13:53:09 +0800 Subject: [PATCH 03/90] further work on the overdrive plugin --- src/calibre/ebooks/metadata/covers.py | 9 +- src/calibre/ebooks/metadata/fetch.py | 6 +- src/calibre/ebooks/metadata/overdrive.py | 386 ++++++++++---------- src/calibre/ebooks/metadata/sources/base.py | 18 +- 4 files changed, 214 insertions(+), 205 deletions(-) diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index 9f5958f1ad..6ea292aa93 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -161,14 +161,7 @@ class OverdriveCovers(CoverDownload): # {{{ def has_cover(self, mi, ans, timeout=5.): if not mi.authors or not mi.title: return False - from calibre.ebooks.metadata.overdrive import get_cover_url - br = browser() - try: - get_cover_url(mi.isbn, mi.title, mi.authors, br) - self.debug('cover for', mi.isbn, 'found') - ans.set() - except Exception, e: - self.debug(e) + return True def get_covers(self, mi, result_queue, abort, timeout=5.): if not mi.isbn: diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py index 1f584bc107..0401ee78c5 100644 --- a/src/calibre/ebooks/metadata/fetch.py +++ b/src/calibre/ebooks/metadata/fetch.py @@ -261,10 +261,10 @@ class Overdrive(MetadataSource): # {{{ def fetch(self): if not self.isbn: return - from calibre.ebooks.metadata.overdrive import get_metadata + from calibre.ebooks.metadata.overdrive import get_social_metadata try: - self.results = get_metadata(self.title, self.book_author, - self.publisher, self.isbn) + self.results = get_social_metadata(self.title, self.book_author, self.isbn) + except Exception, e: self.exception = e self.tb = traceback.format_exc() diff --git a/src/calibre/ebooks/metadata/overdrive.py b/src/calibre/ebooks/metadata/overdrive.py index 5afb875fad..e72d168146 100644 --- a/src/calibre/ebooks/metadata/overdrive.py +++ b/src/calibre/ebooks/metadata/overdrive.py @@ -25,13 +25,12 @@ cache_lock = RLock() base_url = 'http://search.overdrive.com/' - def create_query(self, title=None, authors=None, identifiers={}): q = '' if title or authors: def build_term(prefix, parts): return ' '.join('in'+prefix + ':' + x for x in parts) - title_tokens = list(self.get_title_tokens(title)) + title_tokens = list(self.get_title_tokens(title, False)) if title_tokens: q += build_term('title', title_tokens) author_tokens = self.get_author_tokens(authors, @@ -58,7 +57,7 @@ def get_base_referer(): 'http://spl.lib.overdrive.com/5875E082-4CB2-4689-9426-8509F354AFEF/10/335/en/' ] return choices[random.randint(0, len(choices)-1)] - + def format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid): fix_slashes = re.compile(r'\\/') thumbimage = fix_slashes.sub('/', thumbimage) @@ -67,8 +66,10 @@ def format_results(reserveid, od_title, subtitle, series, publisher, creators, t social_metadata_url = base_url+'TitleInfo.aspx?ReserveID='+reserveid+'&FormatID='+formatid series_num = '' if not series: - if subtitle: - title = od_title+': '+subtitle + if subtitle: + title = od_title+': '+subtitle + else: + title = od_title else: title = od_title m = re.search("([0-9]+$)", subtitle) @@ -76,36 +77,12 @@ def format_results(reserveid, od_title, subtitle, series, publisher, creators, t series_num = float(m.group(1)) return [cover_url, social_metadata_url, worldcatlink, series, series_num, publisher, creators, reserveid, title] -def overdrive_search(br, q, title, author): - q_query = q+'default.aspx/SearchByKeyword' - q_init_search = q+'SearchResults.aspx' - # get first author as string - convert this to a proper cleanup function later - s = Source(None) - print "printing list with string:" - print list(s.get_author_tokens(['J. R. R. Tolkien'])) - print "printing list with author "+str(author)+":" - print list(s.get_author_tokens(author)) - author = list(s.get_author_tokens(author)) - for token in author: - print "cleaned up author is: "+str(token) - author_q = '+'.join(author) - #author_q = separator.join(for x in author) - # query terms - #author_q = re.sub('\s', '+', author_q) - print "final author query is "+str(author_q) - q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=10&sSearch='+author_q - query = '{"szKeyword":"'+title+'"}' - - # main query, requires specific Content Type header - req = mechanize.Request(q_query) - req.add_header('Content-Type', 'application/json; charset=utf-8') - br.open_novisit(req, query) - - print "q_init_search is "+q_init_search - - # the query must be initialized by loading an empty search results page - # this page attempts to set a cookie that Mechanize doesn't like - # copy the cookiejar to a separate instance and make a one-off request with the temp cookiejar +def safe_query(br, query_url): + ''' + The query must be initialized by loading an empty search results page + this page attempts to set a cookie that Mechanize doesn't like + copy the cookiejar to a separate instance and make a one-off request with the temp cookiejar + ''' goodcookies = br._ua_handlers['_cookies'].cookiejar clean_cj = mechanize.CookieJar() cookies_to_copy = [] @@ -115,10 +92,46 @@ def overdrive_search(br, q, title, author): for copied_cookie in cookies_to_copy: clean_cj.set_cookie(copied_cookie) - br.open_novisit(q_init_search) + br.open_novisit(query_url) br.set_cookiejar(clean_cj) + +def overdrive_search(br, q, title, author): + q_query = q+'default.aspx/SearchByKeyword' + q_init_search = q+'SearchResults.aspx' + # get first author as string - convert this to a proper cleanup function later + s = Source(None) + print "printing list with string:" + #print list(s.get_author_tokens(['J. R. R. Tolkien'])) + print "printing list with author "+str(author)+":" + print list(s.get_author_tokens(author)) + author_tokens = list(s.get_author_tokens(author)) + for token in author_tokens: + print "cleaned up author token is: "+str(token) + author_q = ' '.join(author_tokens) + + title_tokens = list(s.get_title_tokens(title)) + for token in title_tokens: + print "cleaned up title token is: "+str(token) + title_q = '+'.join(title_tokens) + #author_q = separator.join(for x in author) + # query terms + #author_q = re.sub('\s', '+', author_q) + print "final author query is "+str(author_q) + print "final title query is "+str(title_q) + q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=20&sSearch='+title_q + query = '{"szKeyword":"'+author_q+'"}' + + # main query, requires specific Content Type header + req = mechanize.Request(q_query) + req.add_header('Content-Type', 'application/json; charset=utf-8') + br.open_novisit(req, query) + + print "q_init_search is "+q_init_search + # initiate the search without messing up the cookiejar + safe_query(br, q_init_search) + # get the search results object xreq = mechanize.Request(q_xref) xreq.add_header('X-Requested-With', 'XMLHttpRequest') @@ -126,83 +139,102 @@ def overdrive_search(br, q, title, author): xreq.add_header('Accept', 'application/json, text/javascript, */*') raw = br.open_novisit(xreq).read() print "overdrive search result is:\n"+raw + print "\n\nsorting results" + return sort_ovrdrv_results(raw, title, title_tokens, author, author_tokens) + + +def sort_ovrdrv_results(raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None): + print "\ntitle to search for is "+str(title)+"\nauthor to search for is "+str(author) + close_matches = [] raw = re.sub('.*?\[\[(?P<content>.*?)\]\].*', '[[\g<content>]]', raw) results = eval(raw) print "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" - print results - # The search results are from a keyword search (overdrive's advanced search is broken), + #print results + # The search results are either from a keyword search or a multi-format list from a single ID, # sort through the results for closest match/format for result in results: print "\n\n\nthis result is "+str(result) for reserveid, od_title, subtitle, edition, series, publisher, format, formatid, creators, \ thumbimage, shortdescription, worldcatlink, excerptlink, creatorfile, sorttitle, \ availabletolibrary, availabletoretailer, relevancyrank, unknown1, unknown2, unknown3 in results: - creators = creators.split(', ') - print "fixed creators are: "+str(creators) - # if an exact match occurs - if creators[0] == author and od_title == title and int(formatid) in [1, 50, 410, 900]: - print "Got Exact Match!!!" - return format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) - - -def library_search(br, q, title, author): - q_search = q+'AdvancedSearch.htm' - q_query = q+'BANGSearch.dll' - br.open(q_search) - # Search for cover with audiobooks lowest priority - for format in ['410','50','900','25','425']: - query = 'Title='+title+'&Creator='+author+'&Keyword=&ISBN=&Format='+format+'&Language=&Publisher=&Subject=&Award=&CollDate=&PerPage=10&Sort=SortBy%3Dtitle' - query = re.sub('\s', '+', query) - #print "search url is "+str(q_search) - print "query is "+str(query) - raw = br.open(q_query, query).read() - #print "raw html is:\n"+str(raw) - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - root = html.fromstring(raw) - revs = root.xpath("//img[@class='blackborder']") - if revs: - #print "revs are "+str(revs) - # get the first match, as it's the most likely candidate - x = revs[0] - id = urllib.unquote(re.sub('.*?/(?P<i>%7B.*?%7D).*', '\g<i>', x.get('src'))) - curl = re.sub('(?P<img>(Ima?g(eType-)?))200', '\g<img>100', x.get('src')) - murl = root.xpath("//img[@class='blackborder']/parent::*") - if murl: - murl = [y.get('href') for y in murl] - print "murl is"+str(murl) - murl = q+murl[0] + if ovrdrv_id is not None and int(formatid) in [1, 50, 410, 900]: + print "overdrive id is not None, searching based on format type priority" + return format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) else: - print "didn't get metadata URL" - print "curl is "+str(curl)+", id is "+str(id)+", murl is "+str(murl) - ovrdrv_data = [id, curl, murl] - print "revs final are "+str(revs) - return ovrdrv_data + creators = creators.split(', ') + print "fixed creators are: "+str(creators) + # if an exact match in a preferred format occurs + if creators[0] == author[0] and od_title == title and int(formatid) in [1, 50, 410, 900]: + print "Got Exact Match!!!" + return format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) + else: + close_title_match = False + close_author_match = False + for token in title_tokens: + if od_title.lower().find(token.lower()) != -1: + close_title_match = True + else: + close_title_match = False + break + for token in author_tokens: + if creators[0].lower().find(token.lower()) != -1: + close_author_match = True + else: + close_author_match = False + break + if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900]: + close_matches.append(format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) + if close_matches: + return close_matches[0] + else: + return None -def find_ovrdrv_data(br, title, author, isbn): - print "in fnd_ovrdrv_data, title is "+str(title)+", author is "+str(author) + +def overdrive_get_record(br, q, ovrdrv_id): + search_url = q+'SearchResults.aspx?ReserveID={'+ovrdrv_id+'}' + results_url = q+'SearchResults.svc/GetResults?sEcho=1&iColumns=18&sColumns=ReserveID%2CTitle%2CSubtitle%2CEdition%2CSeries%2CPublisher%2CFormat%2CFormatID%2CCreators%2CThumbImage%2CShortDescription%2CWorldCatLink%2CExcerptLink%2CCreatorFile%2CSortTitle%2CAvailableToLibrary%2CAvailableToRetailer%2CRelevancyRank&iDisplayStart=0&iDisplayLength=10&sSearch=&bEscapeRegex=true&iSortingCols=1&iSortCol_0=17&sSortDir_0=asc' + + # get the base url to set the proper session cookie + br.open_novisit(q) + + # initialize the search + safe_query(br, search_url) + + # get the results + req = mechanize.Request(results_url) + req.add_header('X-Requested-With', 'XMLHttpRequest') + req.add_header('Referer', search_url) + req.add_header('Accept', 'application/json, text/javascript, */*') + raw = br.open_novisit(req) + raw = str(list(raw)) + return sort_ovrdrv_results(raw, None, None, None, ovrdrv_id) + + +def find_ovrdrv_data(br, title, author, isbn, ovrdrv_id=None): + print "in find_ovrdrv_data, title is "+str(title)+", author is "+str(author)+", overdrive id is "+str(ovrdrv_id) q = base_url - if re.match('http://search\.overdrive\.', q): + if ovrdrv_id is None: return overdrive_search(br, q, title, author) else: - return library_search(br, q, title, author) - + return overdrive_get_record(br, q, ovrdrv_id) -def to_ovrdrv_data(br, title, author, isbn): + +def to_ovrdrv_data(br, title, author, isbn, ovrdrv_id=None): print "starting to_ovrdrv_data" with cache_lock: ans = ovrdrv_data_cache.get(isbn, None) if ans: - print "inside to_ovrdrv_data, ans returned positive, ans is"+str(ans) + print "inside to_ovrdrv_data, cache lookup successful, ans is "+str(ans) return ans if ans is False: print "inside to_ovrdrv_data, ans returned False" return None try: - ovrdrv_data = find_ovrdrv_data(br, title, author, isbn) - print "ovrdrv_data = "+str(ovrdrv_data) + print "trying to retrieve data, running find_ovrdrv_data" + ovrdrv_data = find_ovrdrv_data(br, title, author, isbn, ovrdrv_id) + print "ovrdrv_data is "+str(ovrdrv_data) except: import traceback traceback.print_exc() @@ -210,66 +242,69 @@ def to_ovrdrv_data(br, title, author, isbn): with cache_lock: ovrdrv_data_cache[isbn] = ovrdrv_data if ovrdrv_data else False + if ovrdrv_data: + from calibre.ebooks.metadata.xisbn import xisbn + for i in xisbn.get_associated_isbns(isbn): + with cache_lock: + ovrdrv_data_cache[i] = ovrdrv_data + return ovrdrv_data -def get_social_metadata(title, authors, publisher, isbn): +def get_social_metadata(title, authors, isbn, ovrdrv_id=None): author = authors[0] mi = Metadata(title, authors) - if not isbn: - return mi - isbn = check_isbn(isbn) - if not isbn: - return mi br = browser() - ovrdrv_data = to_ovrdrv_data(br, title, authors, isbn) - if ovrdrv_data and get_metadata_detail_ovrdrv(br, ovrdrv_data, mi): + print "calling to_ovrdrv_data from inside get_social_metadata" + ovrdrv_data = to_ovrdrv_data(br, title, authors, isbn, ovrdrv_id) + + #[cover_url, social_metadata_url, worldcatlink, series, series_num, publisher, creators, reserveid, title] + + if len(ovrdrv_data[3]) > 1: + mi.series = ovrdrv_data[3] + if ovrdrv_data[4]: + mi.series_index = ovrdrv_data[4] + mi.publisher = ovrdrv_data[5] + mi.authors = ovrdrv_data[6] + if ovrdrv_id is None: + ovrdrv_id = ovrdrv_data[7] + mi.set_identifier('overdrive', ovrdrv_id) + mi.title = ovrdrv_data[8] + + if ovrdrv_data and get_metadata_detail(br, ovrdrv_data[1], mi, isbn): return mi - #from calibre.ebooks.metadata.xisbn import xisbn - #for i in xisbn.get_associated_isbns(isbn): - # print "xisbn isbn is "+str(i) - # ovrdrv_data = to_ovrdrv_data(br, title, author, i) - # if ovrdrv_data and get_metadata_detail(br, ovrdrv_data, mi): - # return mi return mi -def get_cover_url(isbn, title, author, br): +def get_cover_url(isbn, title, author, br, ovrdrv_id=None): print "starting get_cover_url" - isbn = check_isbn(isbn) - print "isbn is "+str(isbn) print "title is "+str(title) print "author is "+str(author[0]) + print "isbn is "+str(isbn) + print "ovrdrv_id is "+str(ovrdrv_id) with cache_lock: ans = cover_url_cache.get(isbn, None) + #ans = cover_url_cache.get(ovrdrv_id, None) if ans: - print "ans returned positive" + print "cover url cache lookup returned positive, ans is "+str(ans) return ans if ans is False: - "ans returned false" + "cover url cache lookup returned false" return None - print "in get_cover_url, running through ovrdrv_data function" - ovrdrv_data = to_ovrdrv_data(br, title, author, isbn) - print "ovrdrv_id is "+str(ovrdrv_data) + print "in get_cover_url, calling to_ovrdrv_data function" + ovrdrv_data = to_ovrdrv_data(br, title, author, isbn, ovrdrv_id) if ovrdrv_data: ans = ovrdrv_data[0] - print "inside get_cover_url, ans is "+str(ans) + print "inside get_cover_url, got url from to_ovrdrv_data, ans is "+str(ans) if ans: + print "writing cover url to url cache" with cache_lock: cover_url_cache[isbn] = ans + #cover_url_cache[ovrdrv_id] = ans return ans - #from calibre.ebooks.metadata.xisbn import xisbn - #for i in xisbn.get_associated_isbns(isbn): - # print "in get_cover_url, using xisbn list to associate other books" - # ovrdrv_data = to_ovrdrv_data(br, title, author, i) - # if ovrdrv_data: - # ans = _get_cover_url(br, ovrdrv_data) - # if ans: - # with cache_lock: - # cover_url_cache[isbn] = ans - # cover_url_cache[i] = ans - # return ans + with cache_lock: + print "marking cover url cache for this isbn false" cover_url_cache[isbn] = False return None @@ -303,18 +338,14 @@ def _get_cover_url(br, ovrdrv_data): return ('/'.join(parts[:-1]))+'/'+bn return None - -def get_metadata_detail(br, ovrdrv_data, mi): - q = ovrdrv_data[2] +def get_metadata_detail(br, metadata_url, mi, isbn=None): try: - raw = br.open_novisit(q).read() + raw = br.open_novisit(metadata_url).read() except Exception, e: if callable(getattr(e, 'getcode', None)) and \ e.getcode() == 404: return False - raise - if '<title>404 - ' in raw: - return False + raise raw = xml_to_unicode(raw, strip_encoding_pats=True, resolve_entities=True)[0] try: @@ -322,26 +353,28 @@ def get_metadata_detail(br, ovrdrv_data, mi): except: return False - # Check for series name and retrieve it - series_name = root.xpath("//td/script[re:test(text(), 'szSeries', 'i')]", - namespaces={"re": "http://exslt.org/regular-expressions"}) - if series_name: - series = html.tostring(series_name[0], method='html', encoding=unicode).strip() - series = re.sub('(?s).*?szSeries\s*=\s*\"(?P<series>.*?)\";.*', '\g<series>', series) - if len(series) > 1: - mi.series = series - # If series was successful attempt to get the series number - series_num = root.xpath("//div/strong[re:test(text(), ',\s(Book|Part|Volume)')]", - namespaces={"re": "http://exslt.org/regular-expressions"}) - if series_num: - series_num = float(re.sub('(?s).*?,\s*(Book|Part|Volume)\s*(?P<num>\d+).*', '\g<num>', - etree.tostring(series_num[0]))) - if series_num >= 1: - mi.series_index = series_num - print "series_num is "+str(series_num) + isbn = check_isbn(isbn) - desc = root.xpath("//td[@class='collection' and re:test(., 'Description', 'i')]/following::div[1]", - namespaces={"re": "http://exslt.org/regular-expressions"}) + pub_date = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblPubDate']/text()") + lang = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblLanguage']/text()") + subjects = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblSubjects']/text()") + ebook_isbn = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblIdentifier']/text()") + desc = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblDescription']/ancestor::div[1]") + + if pub_date: + from calibre.utils.date import parse_date + mi.pubdate = parse_date(pub_date[0].strip()) + if lang: + mi.language = lang[0].strip() + print "languages is "+str(mi.language) + if ebook_isbn and isbn is None: + print "ebook isbn is "+str(ebook_isbn[0]) + mi.set_identifier('isbn', ebook_isbn) + #elif isbn is not None: + # mi.set_identifier('isbn', isbn) + if subjects: + mi.tags = subjects + print "tags are "+str(mi.tags) if desc: desc = desc[0] desc = html.tostring(desc, method='html', encoding=unicode).strip() @@ -351,36 +384,6 @@ def get_metadata_detail(br, ovrdrv_data, mi): desc = re.sub(r'(?s)<!--.*?-->', '', desc) mi.comments = sanitize_comments_html(desc) - publisher = root.xpath("//td/strong[re:test(text(), 'Publisher\:', 'i')]/ancestor::td[1]/following-sibling::td/text()", - namespaces={"re": "http://exslt.org/regular-expressions"}) - if publisher: - mi.publisher = re.sub('^\s*(?P<pub>.*?)\s*$', '\g<pub>', publisher[0]) - print "publisher is "+str(mi.publisher) - - lang = root.xpath("//td/strong[re:test(text(), 'Language\(s\):', 'i')]/ancestor::td[1]/following-sibling::td/text()", - namespaces={"re": "http://exslt.org/regular-expressions"}) - if lang: - mi.language = re.sub('^\s*(?P<lang>.*?)\s*$', '\g<lang>', lang[0]) - print "languages is "+str(mi.language) - - isbn = root.xpath("//tr/td[re:test(text(), 'ISBN:', 'i')]/following::td/text()", - namespaces={"re": "http://exslt.org/regular-expressions"}) - if isbn: - mi.isbn = re.sub('^\s*(?P<isbn>.*?)\s*$', '\g<isbn>', isbn[0]) - print "ISBN is "+str(mi.isbn) - - subjects = root.xpath("//td/strong[re:test(text(), 'Subject', 'i')]/ancestor::td[1]/following-sibling::td/a/text()", - namespaces={"re": "http://exslt.org/regular-expressions"}) - if subjects: - mi.tags = subjects - print "tags are "+str(mi.tags) - - creators = root.xpath("//table/tr/td[re:test(text(), '\s*by', 'i')]/ancestor::tr[1]/td[2]/table/tr/td/a/text()", - namespaces={"re": "http://exslt.org/regular-expressions"}) - if creators: - print "authors are "+str(creators) - mi.authors = creators - return True def main(args=sys.argv): @@ -388,19 +391,26 @@ def main(args=sys.argv): import tempfile, os, time tdir = tempfile.gettempdir() br = browser() - for isbn, title, author in [ - #('0899661343', 'On the Road', ['Jack Kerouac']), # basic test, no series, single author - #('9780061952838', 'The Fellowship of the Ring', ['J. R. R. Tolkien']), # Series test, multi-author - ('9780061952838', 'The Two Towers', ['J. R. R. Tolkien']), # Series test, book 2 - #('9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors - #('', 'Deluge', ['Anne McCaffrey']) # Empty ISBN - #(None, 'On the Road', ['Jack Kerouac']) # Nonetype ISBN + for ovrdrv_id, isbn, title, author in [ + #(None, '0899661343', 'On the Road', ['Jack Kerouac']), # basic test, no series, single author + #(None, '9780061952838', 'The Fellowship of the Ring', ['J. R. R. Tolkien']), # Series test, multi-author + #(None, '9780061952838', 'The Two Towers', ['J. R. R. Tolkien']), # Series test, book 2 + #('57844706-20fa-4ace-b5ee-3470b1b52173', None, 'The Two Towers', ['J. R. R. Tolkien']), # Series test, w/ ovrdrv id + #(None, '9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors + #(None, None, 'Deluge', ['Anne McCaffrey']) # Empty ISBN + #(None, None, 'On the Road', ['Jack Kerouac']), # Nonetype ISBN + #(None, '9780345435279', 'A Caress of Twilight', ['Laurell K. Hamilton']), + #(None, '9780606087230', 'The Omnivore\'s Dilemma : A Natural History of Four Meals', ['Michael Pollan']), # Subtitle colon + #(None, '9780061747649', 'Mental_Floss Presents: Condensed Knowledge', ['Will Pearson', 'Mangesh Hattikudur']), + #(None, '9781400050802', 'The Zombie Survival Guide', ['Max Brooks']), # Two books with this title by this author + #(None, '9781775414315', 'The Worst Journey in the World / Antarctic 1910-1913', ['Apsley Cherry-Garrard']), # Garbage sub-title + (None, '9780440335160', 'Outlander', ['Diana Gabaldon']), # Returns lots of results to sort through to get the best match ]: cpath = os.path.join(tdir, title+'.jpg') print "cpath is "+cpath st = time.time() - curl = get_cover_url(isbn, title, author, br) - print '\n\n Took ', time.time() - st, ' to get metadata\n\n' + curl = get_cover_url(isbn, title, author, br, ovrdrv_id) + print '\n\n Took ', time.time() - st, ' to get basic metadata\n\n' if curl is None: print 'No cover found for', title else: @@ -408,9 +418,7 @@ def main(args=sys.argv): #open(cpath, 'wb').write(br.open_novisit(curl).read()) #print 'Cover for', title, 'saved to', cpath - #import time - - #print get_social_metadata(title, author, None, isbn) + print get_social_metadata(title, author, isbn, ovrdrv_id) #print '\n\n', time.time() - st, '\n\n' return 0 diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 55cc996cf7..b600eafaf2 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -87,32 +87,40 @@ class Source(Plugin): if authors: # Leave ' in there for Irish names - pat = re.compile(r'[-,:;+!@#$%^&*(){}.`~"\s\[\]/]') + remove_pat = re.compile(r'[,:;!@#$%^&*(){}.`~"\s\[\]/]') + replace_pat = re.compile(r'-+') if only_first_author: authors = authors[:1] for au in authors: + au = replace_pat.sub(' ', au) parts = au.split() if ',' in au: # au probably in ln, fn form parts = parts[1:] + parts[:1] for tok in parts: - tok = pat.sub('', tok).strip() + tok = remove_pat.sub('', tok).strip() if len(tok) > 2 and tok.lower() not in ('von', ): yield tok - def get_title_tokens(self, title): + def get_title_tokens(self, title, strip_joiners=True): ''' Take a title and return a list of tokens useful for an AND search query. Excludes connectives and punctuation. ''' if title: - pat = re.compile(r'''[-,:;+!@#$%^&*(){}.`~"'\s\[\]/]''') + # strip sub-titles + subtitle = re.compile(r'([\(\[\{].*?[\)\]\}]|[/:\\].*$)') + if len(subtitle.sub('', title)) > 1: + title = subtitle.sub('', title) + pat = re.compile(r'''([-,:;+!@#$%^&*(){}.`~"\s\[\]/]|'(?!s))''') title = pat.sub(' ', title) tokens = title.split() for token in tokens: token = token.strip() - if token and token.lower() not in ('a', 'and', 'the'): + if token and token.lower() not in ('a', 'and', 'the') and strip_joiners: + yield token + elif token: yield token def split_jobs(self, jobs, num): From 6f9fff63e03f2392c6c0e646530b5a16e804ffb2 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Wed, 23 Mar 2011 22:38:29 +0800 Subject: [PATCH 04/90] ... --- src/calibre/ebooks/metadata/overdrive.py | 19 +++++++++++++------ src/calibre/ebooks/metadata/sources/base.py | 8 ++++---- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/calibre/ebooks/metadata/overdrive.py b/src/calibre/ebooks/metadata/overdrive.py index e72d168146..61ff2ee7ae 100644 --- a/src/calibre/ebooks/metadata/overdrive.py +++ b/src/calibre/ebooks/metadata/overdrive.py @@ -120,7 +120,7 @@ def overdrive_search(br, q, title, author): #author_q = re.sub('\s', '+', author_q) print "final author query is "+str(author_q) print "final title query is "+str(title_q) - q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=20&sSearch='+title_q + q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+title_q query = '{"szKeyword":"'+author_q+'"}' # main query, requires specific Content Type header @@ -152,11 +152,11 @@ def sort_ovrdrv_results(raw, title=None, title_tokens=None, author=None, author_ #print results # The search results are either from a keyword search or a multi-format list from a single ID, # sort through the results for closest match/format - for result in results: - print "\n\n\nthis result is "+str(result) + if results: for reserveid, od_title, subtitle, edition, series, publisher, format, formatid, creators, \ thumbimage, shortdescription, worldcatlink, excerptlink, creatorfile, sorttitle, \ availabletolibrary, availabletoretailer, relevancyrank, unknown1, unknown2, unknown3 in results: + print "this record's title is "+od_title+", subtitle is "+subtitle+", author[s] are "+creators+", series is "+series if ovrdrv_id is not None and int(formatid) in [1, 50, 410, 900]: print "overdrive id is not None, searching based on format type priority" return format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) @@ -183,11 +183,16 @@ def sort_ovrdrv_results(raw, title=None, title_tokens=None, author=None, author_ close_author_match = False break if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900]: - close_matches.append(format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) + if subtitle and series: + close_matches.insert(0, format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) + else: + close_matches.append(format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) if close_matches: return close_matches[0] else: - return None + return '' + else: + return '' @@ -394,7 +399,8 @@ def main(args=sys.argv): for ovrdrv_id, isbn, title, author in [ #(None, '0899661343', 'On the Road', ['Jack Kerouac']), # basic test, no series, single author #(None, '9780061952838', 'The Fellowship of the Ring', ['J. R. R. Tolkien']), # Series test, multi-author - #(None, '9780061952838', 'The Two Towers', ['J. R. R. Tolkien']), # Series test, book 2 + #(None, '9780061952838', 'The Two Towers (The Lord of the Rings, Book II)', ['J. R. R. Tolkien']), # Series test, book 2 + #(None, '9780618153985', 'The Fellowship of the Ring (The Lord of the Rings, Part 1)', ['J.R.R. Tolkien']), #('57844706-20fa-4ace-b5ee-3470b1b52173', None, 'The Two Towers', ['J. R. R. Tolkien']), # Series test, w/ ovrdrv id #(None, '9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors #(None, None, 'Deluge', ['Anne McCaffrey']) # Empty ISBN @@ -405,6 +411,7 @@ def main(args=sys.argv): #(None, '9781400050802', 'The Zombie Survival Guide', ['Max Brooks']), # Two books with this title by this author #(None, '9781775414315', 'The Worst Journey in the World / Antarctic 1910-1913', ['Apsley Cherry-Garrard']), # Garbage sub-title (None, '9780440335160', 'Outlander', ['Diana Gabaldon']), # Returns lots of results to sort through to get the best match + (None, '9780345509741', 'The Horror Stories of Robert E. Howard', ['Robert E. Howard']), # Complex title with initials/dots stripped, some results don't have a cover ]: cpath = os.path.join(tdir, title+'.jpg') print "cpath is "+cpath diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 464d08032b..6fc52eb88b 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -102,8 +102,8 @@ class Source(Plugin): if authors: # Leave ' in there for Irish names - remove_pat = re.compile(r'[,:;!@#$%^&*(){}.`~"\s\[\]/]') - replace_pat = re.compile(r'-+') + remove_pat = re.compile(r'[,!@#$%^&*(){}`~"\s\[\]/]') + replace_pat = re.compile(r'[-+.:;]') if only_first_author: authors = authors[:1] for au in authors: @@ -128,12 +128,12 @@ class Source(Plugin): subtitle = re.compile(r'([\(\[\{].*?[\)\]\}]|[/:\\].*$)') if len(subtitle.sub('', title)) > 1: title = subtitle.sub('', title) - pat = re.compile(r'''([-,:;+!@#$%^&*(){}.`~"\s\[\]/]|'(?!s))''') + pat = re.compile(r'''([-,:;+!@#$%^*(){}.`~"\s\[\]/]|'(?!s))''') title = pat.sub(' ', title) tokens = title.split() for token in tokens: token = token.strip() - if token and token.lower() not in ('a', 'and', 'the') and strip_joiners: + if token and token.lower() not in ('a', 'and', 'the', '&') and strip_joiners: yield token elif token: yield token From 433270f20ead59bc013855d5b1403e43e1f50a02 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Mon, 28 Mar 2011 17:24:45 +0800 Subject: [PATCH 05/90] add another type of scene break to the scene break formatting logic --- src/calibre/ebooks/conversion/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/ebooks/conversion/utils.py b/src/calibre/ebooks/conversion/utils.py index f1f2f87293..1546644f95 100644 --- a/src/calibre/ebooks/conversion/utils.py +++ b/src/calibre/ebooks/conversion/utils.py @@ -764,6 +764,7 @@ class HeuristicProcessor(object): # Multiple sequential blank paragraphs are merged with appropriate margins # If non-blank scene breaks exist they are center aligned and styled with appropriate margins. if getattr(self.extra_opts, 'format_scene_breaks', False): + html = re.sub('(?i)<div[^>]*>\s*<br(\s?/)?>\s*</div>', '<p></p>', html) html = self.detect_whitespace(html) html = self.detect_soft_breaks(html) blanks_count = len(self.any_multi_blank.findall(html)) From 07733b2fc800fb135bbebbeae33153434b82daf3 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Fri, 1 Apr 2011 12:59:07 +0800 Subject: [PATCH 06/90] overdrive tweaks --- src/calibre/ebooks/metadata/overdrive.py | 58 +++++++++++++++--------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/src/calibre/ebooks/metadata/overdrive.py b/src/calibre/ebooks/metadata/overdrive.py index 61ff2ee7ae..289d6bea0e 100644 --- a/src/calibre/ebooks/metadata/overdrive.py +++ b/src/calibre/ebooks/metadata/overdrive.py @@ -107,21 +107,27 @@ def overdrive_search(br, q, title, author): print "printing list with author "+str(author)+":" print list(s.get_author_tokens(author)) author_tokens = list(s.get_author_tokens(author)) + print "there are "+str(len(author_tokens))+" author tokens" for token in author_tokens: print "cleaned up author token is: "+str(token) - author_q = ' '.join(author_tokens) + title_tokens = list(s.get_title_tokens(title)) + print "there are "+str(len(title_tokens))+" title tokens" for token in title_tokens: print "cleaned up title token is: "+str(token) - title_q = '+'.join(title_tokens) - #author_q = separator.join(for x in author) - # query terms - #author_q = re.sub('\s', '+', author_q) - print "final author query is "+str(author_q) - print "final title query is "+str(title_q) - q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+title_q - query = '{"szKeyword":"'+author_q+'"}' + + if len(title_tokens) >= len(author_tokens): + initial_q = ' '.join(title_tokens) + xref_q = '+'.join(author_tokens) + else: + initial_q = ' '.join(author_tokens) + xref_q = '+'.join(title_tokens) + + print "initial query is "+str(initial_q) + print "cross reference query is "+str(xref_q) + q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q + query = '{"szKeyword":"'+initial_q+'"}' # main query, requires specific Content Type header req = mechanize.Request(q_query) @@ -133,12 +139,21 @@ def overdrive_search(br, q, title, author): safe_query(br, q_init_search) # get the search results object - xreq = mechanize.Request(q_xref) - xreq.add_header('X-Requested-With', 'XMLHttpRequest') - xreq.add_header('Referer', q_init_search) - xreq.add_header('Accept', 'application/json, text/javascript, */*') - raw = br.open_novisit(xreq).read() - print "overdrive search result is:\n"+raw + results = False + while results == False: + xreq = mechanize.Request(q_xref) + xreq.add_header('X-Requested-With', 'XMLHttpRequest') + xreq.add_header('Referer', q_init_search) + xreq.add_header('Accept', 'application/json, text/javascript, */*') + raw = br.open_novisit(xreq).read() + print "overdrive search result is:\n"+raw + for m in re.finditer(ur'"iTotalDisplayRecords":(?P<displayrecords>\d+).*?"iTotalRecords":(?P<totalrecords>\d+)', raw): + if int(m.group('displayrecords')) >= 1: + results = True + elif int(m.group('totalrecords')) >= 1: + xref_q = '' + q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q + print "\n\nsorting results" return sort_ovrdrv_results(raw, title, title_tokens, author, author_tokens) @@ -162,7 +177,7 @@ def sort_ovrdrv_results(raw, title=None, title_tokens=None, author=None, author_ return format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) else: creators = creators.split(', ') - print "fixed creators are: "+str(creators) + print "split creators from results are: "+str(creators) # if an exact match in a preferred format occurs if creators[0] == author[0] and od_title == title and int(formatid) in [1, 50, 410, 900]: print "Got Exact Match!!!" @@ -275,9 +290,10 @@ def get_social_metadata(title, authors, isbn, ovrdrv_id=None): ovrdrv_id = ovrdrv_data[7] mi.set_identifier('overdrive', ovrdrv_id) mi.title = ovrdrv_data[8] - + print "populated basic social metadata, getting detailed metadata" if ovrdrv_data and get_metadata_detail(br, ovrdrv_data[1], mi, isbn): return mi + print "failed to get detailed metadata, returning basic info" return mi def get_cover_url(isbn, title, author, br, ovrdrv_id=None): @@ -378,7 +394,7 @@ def get_metadata_detail(br, metadata_url, mi, isbn=None): #elif isbn is not None: # mi.set_identifier('isbn', isbn) if subjects: - mi.tags = subjects + mi.tags = [tag.strip() for tag in subjects[0].split(',')] print "tags are "+str(mi.tags) if desc: desc = desc[0] @@ -410,7 +426,7 @@ def main(args=sys.argv): #(None, '9780061747649', 'Mental_Floss Presents: Condensed Knowledge', ['Will Pearson', 'Mangesh Hattikudur']), #(None, '9781400050802', 'The Zombie Survival Guide', ['Max Brooks']), # Two books with this title by this author #(None, '9781775414315', 'The Worst Journey in the World / Antarctic 1910-1913', ['Apsley Cherry-Garrard']), # Garbage sub-title - (None, '9780440335160', 'Outlander', ['Diana Gabaldon']), # Returns lots of results to sort through to get the best match + #(None, '9780440335160', 'Outlander', ['Diana Gabaldon']), # Returns lots of results to sort through to get the best match (None, '9780345509741', 'The Horror Stories of Robert E. Howard', ['Robert E. Howard']), # Complex title with initials/dots stripped, some results don't have a cover ]: cpath = os.path.join(tdir, title+'.jpg') @@ -424,9 +440,9 @@ def main(args=sys.argv): print "curl is "+curl #open(cpath, 'wb').write(br.open_novisit(curl).read()) #print 'Cover for', title, 'saved to', cpath - + st = time.time() print get_social_metadata(title, author, isbn, ovrdrv_id) - #print '\n\n', time.time() - st, '\n\n' + print '\n\n Took ', time.time() - st, ' to get detailed metadata\n\n' return 0 From c4b5c8c91665d108cceadcd648c36d1e2888c4ef Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Thu, 7 Apr 2011 13:31:41 +0800 Subject: [PATCH 07/90] ... --- src/calibre/ebooks/metadata/overdrive.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/calibre/ebooks/metadata/overdrive.py b/src/calibre/ebooks/metadata/overdrive.py index 289d6bea0e..38d6d730ff 100644 --- a/src/calibre/ebooks/metadata/overdrive.py +++ b/src/calibre/ebooks/metadata/overdrive.py @@ -153,6 +153,8 @@ def overdrive_search(br, q, title, author): elif int(m.group('totalrecords')) >= 1: xref_q = '' q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q + elif int(m.group('totalrecords')) == 0: + return '' print "\n\nsorting results" return sort_ovrdrv_results(raw, title, title_tokens, author, author_tokens) @@ -185,16 +187,23 @@ def sort_ovrdrv_results(raw, title=None, title_tokens=None, author=None, author_ else: close_title_match = False close_author_match = False + print "format id is "+str(formatid) for token in title_tokens: + print "attempting to find "+str(token)+" title token" if od_title.lower().find(token.lower()) != -1: + print "matched token" close_title_match = True else: + print "token didn't match" close_title_match = False break for token in author_tokens: + print "attempting to find "+str(token)+" author token" if creators[0].lower().find(token.lower()) != -1: + print "matched token" close_author_match = True else: + print "token didn't match" close_author_match = False break if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900]: From 361e86c6ff04cf0d6a3cb07226309e99df373128 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Thu, 7 Apr 2011 23:04:32 +0800 Subject: [PATCH 08/90] ... --- src/calibre/customize/builtins.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 475cb36687..5e50f81173 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -10,6 +10,7 @@ from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.ebooks.oeb.base import OEB_IMAGES +from calibre.utils.config import test_eight_code # To archive plugins {{{ class HTML2ZIP(FileTypePlugin): @@ -166,6 +167,14 @@ class ComicMetadataReader(MetadataReaderPlugin): description = _('Extract cover from comic files') def get_metadata(self, stream, ftype): + if hasattr(stream, 'seek') and hasattr(stream, 'tell'): + pos = stream.tell() + id_ = stream.read(3) + stream.seek(pos) + if id_ == b'Rar': + ftype = 'cbr' + elif id.startswith(b'PK'): + ftype = 'cbz' if ftype == 'cbr': from calibre.libunrar import extract_first_alphabetically as extract_first extract_first From ddb3d935d4c311382615dd646eae1f97e512c973 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Fri, 8 Apr 2011 08:37:38 +0800 Subject: [PATCH 09/90] ... --- src/calibre/customize/builtins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 5e50f81173..8dbc72f8ac 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -637,7 +637,7 @@ else: from calibre.ebooks.metadata.covers import OpenLibraryCovers, \ AmazonCovers, DoubanCovers, OverdriveCovers - plugins += [GoogleBooks, ISBNDB, Amazon, + plugins += [GoogleBooks, ISBNDB, Amazon, Overdrive, OpenLibraryCovers, AmazonCovers, DoubanCovers, OverdriveCovers, NiceBooksCovers, KentDistrictLibrary, DoubanBooks, NiceBooks] From 330d12c5eb8f41295990945d7a74ff1524825ba1 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Wed, 13 Apr 2011 23:24:34 +0800 Subject: [PATCH 10/90] ... --- src/calibre/ebooks/mobi/mobiml.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 40ad5e9e78..3feef7b6f5 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -464,9 +464,10 @@ class MobiMLizer(object): valign = style['vertical-align'] not_baseline = valign in ('super', 'sub', 'text-top', 'text-bottom') or ( - isinstance(valign, (float, int)) and abs(valign) != 0) + isinstance(valign, (float, int)) and abs(valign) != 0) or ( + tag in ('sup', 'sub')) issup = valign in ('super', 'text-top') or ( - isinstance(valign, (float, int)) and valign > 0) + isinstance(valign, (float, int)) and valign > 0) or tag == 'sup' vtag = 'sup' if issup else 'sub' if not_baseline and not ignore_valign and tag not in NOT_VTAGS and not isblock: nroot = etree.Element(XHTML('html'), nsmap=MOBI_NSMAP) From ecf21962d5fac590f9a8103fda8608e6cade3843 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Thu, 14 Apr 2011 18:34:49 +0800 Subject: [PATCH 11/90] ... --- src/calibre/ebooks/mobi/mobiml.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 8c7b740cdb..1e626cf916 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -464,10 +464,9 @@ class MobiMLizer(object): valign = style['vertical-align'] not_baseline = valign in ('super', 'sub', 'text-top', 'text-bottom') or ( - isinstance(valign, (float, int)) and abs(valign) != 0) or ( - tag in ('sup', 'sub')) + isinstance(valign, (float, int)) and abs(valign) != 0) issup = valign in ('super', 'text-top') or ( - isinstance(valign, (float, int)) and valign > 0) or tag == 'sup' + isinstance(valign, (float, int)) and valign > 0) vtag = 'sup' if issup else 'sub' if not_baseline and not ignore_valign and tag not in NOT_VTAGS and not isblock: nroot = etree.Element(XHTML('html'), nsmap=MOBI_NSMAP) From 04b543e854a409b73a7da8555815a10b5669e3d7 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Sun, 17 Apr 2011 22:42:57 +0800 Subject: [PATCH 12/90] start porting overdrive to 8 --- src/calibre/customize/builtins.py | 4 +++- src/calibre/ebooks/conversion/preprocess.py | 2 +- src/calibre/ebooks/metadata/sources/base.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 8956780e2c..6131c03f9c 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -626,8 +626,9 @@ if test_eight_code: from calibre.ebooks.metadata.sources.amazon import Amazon from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary from calibre.ebooks.metadata.sources.isbndb import ISBNDB + from calibre.ebooks.metadata.sources.overdrive import OverDrive - plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB] + plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB, OverDrive] # }}} else: @@ -1097,6 +1098,7 @@ if test_eight_code: from calibre.ebooks.metadata.sources.google import GoogleBooks from calibre.ebooks.metadata.sources.amazon import Amazon from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary +from calibre.ebooks.metadata.sources.overdrive import OverDrive plugins += [GoogleBooks, Amazon, OpenLibrary] diff --git a/src/calibre/ebooks/conversion/preprocess.py b/src/calibre/ebooks/conversion/preprocess.py index a1d5fa94d8..8822a39b87 100644 --- a/src/calibre/ebooks/conversion/preprocess.py +++ b/src/calibre/ebooks/conversion/preprocess.py @@ -399,7 +399,7 @@ class HTMLPreProcessor(object): (re.compile(u'Ë™\s*(<br.*?>)*\s*Z', re.UNICODE), lambda match: u'Å»'), # If pdf printed from a browser then the header/footer has a reliable pattern - (re.compile(r'((?<=</a>)\s*file:////?[A-Z].*<br>|file:////?[A-Z].*<br>(?=\s*<hr>))', re.IGNORECASE), lambda match: ''), + (re.compile(r'((?<=</a>)\s*file:/{2,4}[A-Z].*<br>|file:////?[A-Z].*<br>(?=\s*<hr>))', re.IGNORECASE), lambda match: ''), # Center separator lines (re.compile(u'<br>\s*(?P<break>([*#•✦=]+\s*)+)\s*<br>'), lambda match: '<p>\n<p style="text-align:center">' + match.group(1) + '</p>'), diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 56f82641ab..bfc3e498eb 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -293,7 +293,7 @@ class Source(Plugin): def get_title_tokens(self, title, strip_joiners=True): ''' Take a title and return a list of tokens useful for an AND search query. - Excludes connectives and punctuation. + Excludes connectives(optionally) and punctuation. ''' if title: # strip sub-titles From 4ea961ba6298a905bc50136e8054117d77a18575 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 08:34:22 +0100 Subject: [PATCH 13/90] From Greg --- src/calibre/customize/builtins.py | 5 +- src/calibre/devices/apple/driver.py | 51 +++++++------ .../devices/content_server/__init__.py | 10 +++ src/calibre/devices/content_server/driver.py | 74 +++++++++++++++++++ src/calibre/gui2/actions/catalog.py | 2 +- src/calibre/gui2/device.py | 2 +- src/calibre/gui2/dialogs/tweak_epub.py | 11 ++- src/calibre/library/server/content.py | 27 ++++++- 8 files changed, 154 insertions(+), 28 deletions(-) create mode 100644 src/calibre/devices/content_server/__init__.py create mode 100644 src/calibre/devices/content_server/driver.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index d3b0b8409d..458bfec3fd 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -582,6 +582,7 @@ from calibre.ebooks.snb.output import SNBOutput from calibre.customize.profiles import input_profiles, output_profiles from calibre.devices.apple.driver import ITUNES +from calibre.devices.content_server.driver import CONTENT_SERVER_FOR_CONFIG from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA from calibre.devices.blackberry.driver import BLACKBERRY from calibre.devices.cybook.driver import CYBOOK, ORIZON @@ -753,7 +754,9 @@ plugins += [ EEEREADER, NEXTBOOK, ITUNES, -] + CONTENT_SERVER_FOR_CONFIG + ] + plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ x.__name__.endswith('MetadataReader')] plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 2cc478603a..d7811f0a22 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -201,8 +201,9 @@ class ITUNES(DriverBase): # 0x1294 iPhone 3GS # 0x1297 iPhone 4 # 0x129a iPad + # 0x12a2 iPad2 VENDOR_ID = [0x05ac] - PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a] + PRODUCT_ID = [0x1292,0x1293,0x1294,0x1297,0x1299,0x129a,0x12a2] BCD = [0x01] # Plugboard ID @@ -421,7 +422,7 @@ class ITUNES(DriverBase): cached_books[this_book.path] = { 'title':book.name(), - 'author':[book.artist()], + 'author':book.artist().split(' & '), 'lib_book':library_books[this_book.path] if this_book.path in library_books else None, 'dev_book':book, 'uuid': book.composer() @@ -459,7 +460,7 @@ class ITUNES(DriverBase): cached_books[this_book.path] = { 'title':book.Name, - 'author':book.Artist, + 'author':book.artist().split(' & '), 'lib_book':library_books[this_book.path] if this_book.path in library_books else None, 'uuid': book.Composer, 'format': 'pdf' if book.KindAsString.startswith('PDF') else 'epub' @@ -1021,7 +1022,9 @@ class ITUNES(DriverBase): if isosx: for (i,file) in enumerate(files): format = file.rpartition('.')[2].lower() - path = self.path_template % (metadata[i].title, metadata[i].author[0],format) + path = self.path_template % (metadata[i].title, + authors_to_string(metadata[i].authors), + format) self._remove_existing_copy(path, metadata[i]) fpath = self._get_fpath(file, metadata[i], format, update_md=True) db_added, lb_added = self._add_new_copy(fpath, metadata[i]) @@ -1034,9 +1037,11 @@ class ITUNES(DriverBase): if DEBUG: self.log.info("ITUNES.upload_books()") self.log.info(" adding '%s' by '%s' uuid:%s to self.cached_books" % - ( metadata[i].title, metadata[i].author, metadata[i].uuid)) + (metadata[i].title, + authors_to_string(metadata[i].authors), + metadata[i].uuid)) self.cached_books[this_book.path] = { - 'author': metadata[i].author, + 'author': authors_to_string(metadata[i].authors), 'dev_book': db_added, 'format': format, 'lib_book': lb_added, @@ -1055,7 +1060,9 @@ class ITUNES(DriverBase): for (i,file) in enumerate(files): format = file.rpartition('.')[2].lower() - path = self.path_template % (metadata[i].title, metadata[i].author[0],format) + path = self.path_template % (metadata[i].title, + authors_to_string(metadata[i].authors), + format) self._remove_existing_copy(path, metadata[i]) fpath = self._get_fpath(file, metadata[i],format, update_md=True) db_added, lb_added = self._add_new_copy(fpath, metadata[i]) @@ -1075,9 +1082,11 @@ class ITUNES(DriverBase): if DEBUG: self.log.info("ITUNES.upload_books()") self.log.info(" adding '%s' by '%s' uuid:%s to self.cached_books" % - ( metadata[i].title, metadata[i].author, metadata[i].uuid)) + (metadata[i].title, + authors_to_string(metadata[i].authors), + metadata[i].uuid)) self.cached_books[this_book.path] = { - 'author': metadata[i].author[0], + 'author': authors_to_string(metadata[i].authors), 'dev_book': db_added, 'format': format, 'lib_book': lb_added, @@ -1190,7 +1199,7 @@ class ITUNES(DriverBase): base_fn = base_fn.rpartition('.')[0] db_added = self._find_device_book( { 'title': base_fn if format == 'pdf' else metadata.title, - 'author': metadata.authors[0], + 'author': authors_to_string(metadata.authors), 'uuid': metadata.uuid, 'format': format}) return db_added @@ -1255,7 +1264,7 @@ class ITUNES(DriverBase): base_fn = base_fn.rpartition('.')[0] added = self._find_library_book( { 'title': base_fn if format == 'pdf' else metadata.title, - 'author': metadata.author[0], + 'author': authors_to_string(metadata.authors), 'uuid': metadata.uuid, 'format': format}) return added @@ -1314,7 +1323,7 @@ class ITUNES(DriverBase): with open(metadata.cover,'r+b') as cd: cover_data = cd.read() except: - self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0])) + self.problem_titles.append("'%s' by %s" % (metadata.title, authors_to_string(metadata.authors))) self.log.error(" error scaling '%s' for '%s'" % (metadata.cover,metadata.title)) import traceback @@ -1389,7 +1398,7 @@ class ITUNES(DriverBase): thumb_path = path.rpartition('.')[0] + '.jpg' zfw.writestr(thumb_path, thumb) except: - self.problem_titles.append("'%s' by %s" % (metadata.title, metadata.author[0])) + self.problem_titles.append("'%s' by %s" % (metadata.title, authors_to_string(metadata.authors))) self.log.error(" error converting '%s' to thumb for '%s'" % (metadata.cover,metadata.title)) finally: try: @@ -1407,7 +1416,7 @@ class ITUNES(DriverBase): if DEBUG: self.log.info(" ITUNES._create_new_book()") - this_book = Book(metadata.title, authors_to_string(metadata.author)) + this_book = Book(metadata.title, authors_to_string(metadata.authors)) this_book.datetime = time.gmtime() this_book.db_id = None this_book.device_collections = [] @@ -2451,7 +2460,7 @@ class ITUNES(DriverBase): for book in self.cached_books: if self.cached_books[book]['uuid'] == metadata.uuid or \ (self.cached_books[book]['title'] == metadata.title and \ - self.cached_books[book]['author'] == metadata.authors[0]): + self.cached_books[book]['author'] == authors_to_string(metadata.authors)): self.update_list.append(self.cached_books[book]) self._remove_from_device(self.cached_books[book]) if DEBUG: @@ -2470,7 +2479,7 @@ class ITUNES(DriverBase): for book in self.cached_books: if self.cached_books[book]['uuid'] == metadata.uuid or \ (self.cached_books[book]['title'] == metadata.title and \ - self.cached_books[book]['author'] == metadata.authors[0]): + self.cached_books[book]['author'] == authors_to_string(metadata.authors)): self.update_list.append(self.cached_books[book]) self._remove_from_iTunes(self.cached_books[book]) if DEBUG: @@ -2939,13 +2948,13 @@ class ITUNES(DriverBase): def _xform_metadata_via_plugboard(self, book, format): ''' Transform book metadata from plugboard templates ''' if DEBUG: - self.log.info(" ITUNES._xform_metadata_via_plugboard()") + self.log.info(" ITUNES._xform_metadata_via_plugboard()") if self.plugboard_func: pb = self.plugboard_func(self.DEVICE_PLUGBOARD_NAME, format, self.plugboards) newmi = book.deepcopy_metadata() newmi.template_to_attribute(book, pb) - if DEBUG: + if pb is not None and DEBUG: self.log.info(" transforming %s using %s:" % (format, pb)) self.log.info(" title: %s %s" % (book.title, ">>> %s" % newmi.title if book.title != newmi.title else '')) @@ -3062,7 +3071,7 @@ class ITUNES_ASYNC(ITUNES): cached_books[this_book.path] = { 'title':library_books[book].name(), - 'author':[library_books[book].artist()], + 'author':library_books[book].artist().split(' & '), 'lib_book':library_books[book], 'dev_book':None, 'uuid': library_books[book].composer(), @@ -3102,7 +3111,7 @@ class ITUNES_ASYNC(ITUNES): cached_books[this_book.path] = { 'title':library_books[book].Name, - 'author':library_books[book].Artist, + 'author':library_books[book].Artist.split(' & '), 'lib_book':library_books[book], 'uuid': library_books[book].Composer, 'format': format @@ -3288,7 +3297,7 @@ class Book(Metadata): See ebooks.metadata.book.base ''' def __init__(self,title,author): - Metadata.__init__(self, title, authors=[author]) + Metadata.__init__(self, title, authors=author.split(' & ')) @property def title_sorter(self): diff --git a/src/calibre/devices/content_server/__init__.py b/src/calibre/devices/content_server/__init__.py new file mode 100644 index 0000000000..3d1a86922e --- /dev/null +++ b/src/calibre/devices/content_server/__init__.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import with_statement + +__license__ = 'GPL v3' +__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + + + diff --git a/src/calibre/devices/content_server/driver.py b/src/calibre/devices/content_server/driver.py new file mode 100644 index 0000000000..84b14f8e62 --- /dev/null +++ b/src/calibre/devices/content_server/driver.py @@ -0,0 +1,74 @@ +''' +Created on 17 Apr 2011 + +@author: GRiker, modeled on charles's Folder Device + +''' + +from calibre.constants import DEBUG +from calibre.devices.interface import DevicePlugin +from calibre.devices.usbms.deviceconfig import DeviceConfig +from calibre.devices.usbms.driver import USBMS, BookList + +class DriverBase(DeviceConfig, DevicePlugin): + # Reduce to just the formats eligible for plugboard xforms + # These formats are shown in the customization dialog + FORMATS = ['epub', 'mobi'] + USER_CAN_ADD_NEW_FORMATS = False + + # Hide the standard customization widgets + SUPPORTS_SUB_DIRS = False + MUST_READ_METADATA = True + SUPPORTS_USE_AUTHOR_SORT = False + + +# This class is added to the standard device plugin chain, so that it can +# be configured. It has invalid vendor_id etc, so it will never match a +# device. The 'real' CONTENT_SERVER will use the config from it. +class CONTENT_SERVER_FOR_CONFIG(USBMS): + name = 'Content Server Interface' + gui_name = 'Content Server' + description = _('Enables metadata plugboards to be used with Content Server.') + author = 'GRiker' + supported_platforms = ['windows', 'osx', 'linux'] + + VENDOR_ID = [0xffff] + PRODUCT_ID = [0xffff] + BCD = [0xffff] + DEVICE_PLUGBOARD_NAME = 'CONTENT_SERVER' + + def config_widget(cls): + ''' + Configure a minimal QWidget + Better to simply disable the config_widget altogether + ''' + cw = DriverBase.config_widget() + # Turn off the Save template + cw.opt_save_template.setVisible(False) + cw.label.setVisible(False) + # Hide the up/down arrows + cw.column_up.setVisible(False) + cw.column_down.setVisible(False) + # Retitle + cw.groupBox.setTitle(_("Enable metadata plugboards for the following formats:")) + return cw + +class CONTENT_SERVER(USBMS): + + FORMATS = CONTENT_SERVER_FOR_CONFIG.FORMATS + DEVICE_PLUGBOARD_NAME = 'CONTENT_SERVER' + + def __init__(self): + if DEBUG: + print("CONTENT_SERVER.init()") + pass + + def set_plugboards(self, plugboards, pb_func): + # This method is called with the plugboard that matches the format + # declared in use_plugboard_ext and a device name of CONTENT_SERVER + if DEBUG: + print("CONTENT_SERVER.set_plugboards()") + print(' using plugboard %s' % plugboards) + self.plugboards = plugboards + self.plugboard_func = pb_func + diff --git a/src/calibre/gui2/actions/catalog.py b/src/calibre/gui2/actions/catalog.py index fad6e59294..093985d041 100644 --- a/src/calibre/gui2/actions/catalog.py +++ b/src/calibre/gui2/actions/catalog.py @@ -17,7 +17,7 @@ from calibre.gui2.actions import InterfaceAction class GenerateCatalogAction(InterfaceAction): name = 'Generate Catalog' - action_spec = (_('Create a catalog of the books in your calibre library'), None, None, None) + action_spec = (_('Create a catalog of the books in your calibre library'), 'catalog.png', 'Catalog builder', None) dont_add_to = frozenset(['menubar-device', 'toolbar-device', 'context-menu-device']) def generate_catalog(self): diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 4d4f66eab1..8f21c17eaf 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -892,7 +892,7 @@ class DeviceMixin(object): # {{{ sub_dest_parts.append('') to = sub_dest_parts[0] fmts = sub_dest_parts[1] - subject = ';'.join(sub_dest_parts[2:]) + subject = ';'.join(sub_dest_parts[2:]) fmts = [x.strip().lower() for x in fmts.split(',')] self.send_by_mail(to, fmts, delete, subject=subject) diff --git a/src/calibre/gui2/dialogs/tweak_epub.py b/src/calibre/gui2/dialogs/tweak_epub.py index db6e93fd7a..a42fb07e40 100755 --- a/src/calibre/gui2/dialogs/tweak_epub.py +++ b/src/calibre/gui2/dialogs/tweak_epub.py @@ -12,6 +12,7 @@ from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED from PyQt4.Qt import QDialog +from calibre.constants import isosx, iswindows from calibre.gui2 import open_local_file from calibre.gui2.dialogs.tweak_epub_ui import Ui_Dialog from calibre.libunzip import extract as zipextract @@ -42,11 +43,19 @@ class TweakEpub(QDialog, Ui_Dialog): self.move(parent_loc.x(),parent_loc.y()) def cleanup(self): + if isosx: + try: + import appscript + self.finder = appscript.app('Finder') + self.finder.Finder_windows[os.path.basename(self._exploded)].close() + except: + # appscript fails to load on 10.4 + pass + # Delete directory containing exploded ePub if self._exploded is not None: shutil.rmtree(self._exploded, ignore_errors=True) - def display_exploded(self): ''' Generic subprocess launch of native file browser diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index 0c3edd1627..faa0a61baf 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -183,16 +183,37 @@ class ContentServer(object): if fmt is None: raise cherrypy.HTTPError(404, 'book: %d does not have format: %s'%(id, format)) if format == 'EPUB': + # Get the original metadata + mi = self.db.get_metadata(id, index_is_id=True) + + # Instantiate the CONTENT_SERVER driver + from calibre.devices.content_server.driver import CONTENT_SERVER + cs = CONTENT_SERVER() + + # Get any EPUB plugboards for the content server + from calibre.gui2.device import find_plugboard, device_name_for_plugboards + plugboards = self.db.prefs.get('plugboards', {}) + + # Transform the metadata via the plugboard + if hasattr(cs, 'set_plugboards') and callable(cs.set_plugboards): + cs.set_plugboards(plugboards, find_plugboard) + cpb = find_plugboard(device_name_for_plugboards(cs), format.lower(), plugboards) + if cpb: + newmi = mi.deepcopy_metadata() + newmi.template_to_attribute(mi, cpb) + else: + newmi = mi + + # Write the updated file from tempfile import TemporaryFile from calibre.ebooks.metadata.meta import set_metadata raw = fmt.read() fmt = TemporaryFile() fmt.write(raw) fmt.seek(0) - set_metadata(fmt, self.db.get_metadata(id, index_is_id=True, - get_cover=True), - 'epub') + set_metadata(fmt, newmi, 'epub') fmt.seek(0) + mt = guess_type('dummy.'+format.lower())[0] if mt is None: mt = 'application/octet-stream' From 91c5356ac5d14ef807cce610431ec44aa6ab0ff0 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 09:26:34 +0100 Subject: [PATCH 14/90] Suggested content server plugboard implementation for Greg --- src/calibre/customize/builtins.py | 2 - .../devices/content_server/__init__.py | 10 --- src/calibre/devices/content_server/driver.py | 74 ------------------- src/calibre/gui2/device.py | 20 +---- src/calibre/gui2/preferences/plugboard.py | 4 +- src/calibre/library/save_to_disk.py | 32 ++++---- src/calibre/library/server/content.py | 27 +++---- 7 files changed, 34 insertions(+), 135 deletions(-) delete mode 100644 src/calibre/devices/content_server/__init__.py delete mode 100644 src/calibre/devices/content_server/driver.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 458bfec3fd..8f50481f84 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -582,7 +582,6 @@ from calibre.ebooks.snb.output import SNBOutput from calibre.customize.profiles import input_profiles, output_profiles from calibre.devices.apple.driver import ITUNES -from calibre.devices.content_server.driver import CONTENT_SERVER_FOR_CONFIG from calibre.devices.hanlin.driver import HANLINV3, HANLINV5, BOOX, SPECTRA from calibre.devices.blackberry.driver import BLACKBERRY from calibre.devices.cybook.driver import CYBOOK, ORIZON @@ -754,7 +753,6 @@ plugins += [ EEEREADER, NEXTBOOK, ITUNES, - CONTENT_SERVER_FOR_CONFIG ] plugins += [x for x in list(locals().values()) if isinstance(x, type) and \ diff --git a/src/calibre/devices/content_server/__init__.py b/src/calibre/devices/content_server/__init__.py deleted file mode 100644 index 3d1a86922e..0000000000 --- a/src/calibre/devices/content_server/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python -# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai -from __future__ import with_statement - -__license__ = 'GPL v3' -__copyright__ = '2009, Kovid Goyal <kovid@kovidgoyal.net>' -__docformat__ = 'restructuredtext en' - - - diff --git a/src/calibre/devices/content_server/driver.py b/src/calibre/devices/content_server/driver.py deleted file mode 100644 index 84b14f8e62..0000000000 --- a/src/calibre/devices/content_server/driver.py +++ /dev/null @@ -1,74 +0,0 @@ -''' -Created on 17 Apr 2011 - -@author: GRiker, modeled on charles's Folder Device - -''' - -from calibre.constants import DEBUG -from calibre.devices.interface import DevicePlugin -from calibre.devices.usbms.deviceconfig import DeviceConfig -from calibre.devices.usbms.driver import USBMS, BookList - -class DriverBase(DeviceConfig, DevicePlugin): - # Reduce to just the formats eligible for plugboard xforms - # These formats are shown in the customization dialog - FORMATS = ['epub', 'mobi'] - USER_CAN_ADD_NEW_FORMATS = False - - # Hide the standard customization widgets - SUPPORTS_SUB_DIRS = False - MUST_READ_METADATA = True - SUPPORTS_USE_AUTHOR_SORT = False - - -# This class is added to the standard device plugin chain, so that it can -# be configured. It has invalid vendor_id etc, so it will never match a -# device. The 'real' CONTENT_SERVER will use the config from it. -class CONTENT_SERVER_FOR_CONFIG(USBMS): - name = 'Content Server Interface' - gui_name = 'Content Server' - description = _('Enables metadata plugboards to be used with Content Server.') - author = 'GRiker' - supported_platforms = ['windows', 'osx', 'linux'] - - VENDOR_ID = [0xffff] - PRODUCT_ID = [0xffff] - BCD = [0xffff] - DEVICE_PLUGBOARD_NAME = 'CONTENT_SERVER' - - def config_widget(cls): - ''' - Configure a minimal QWidget - Better to simply disable the config_widget altogether - ''' - cw = DriverBase.config_widget() - # Turn off the Save template - cw.opt_save_template.setVisible(False) - cw.label.setVisible(False) - # Hide the up/down arrows - cw.column_up.setVisible(False) - cw.column_down.setVisible(False) - # Retitle - cw.groupBox.setTitle(_("Enable metadata plugboards for the following formats:")) - return cw - -class CONTENT_SERVER(USBMS): - - FORMATS = CONTENT_SERVER_FOR_CONFIG.FORMATS - DEVICE_PLUGBOARD_NAME = 'CONTENT_SERVER' - - def __init__(self): - if DEBUG: - print("CONTENT_SERVER.init()") - pass - - def set_plugboards(self, plugboards, pb_func): - # This method is called with the plugboard that matches the format - # declared in use_plugboard_ext and a device name of CONTENT_SERVER - if DEBUG: - print("CONTENT_SERVER.set_plugboards()") - print(' using plugboard %s' % plugboards) - self.plugboards = plugboards - self.plugboard_func = pb_func - diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 8f21c17eaf..2e252047af 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -29,8 +29,7 @@ from calibre.ebooks.metadata.meta import set_metadata from calibre.constants import DEBUG from calibre.utils.config import prefs, tweaks from calibre.utils.magick.draw import thumbnail -from calibre.library.save_to_disk import plugboard_any_device_value, \ - plugboard_any_format_value +from calibre.library.save_to_disk import find_plugboard # }}} class DeviceJob(BaseJob): # {{{ @@ -93,23 +92,6 @@ class DeviceJob(BaseJob): # {{{ # }}} -def find_plugboard(device_name, format, plugboards): - cpb = None - if format in plugboards: - cpb = plugboards[format] - elif plugboard_any_format_value in plugboards: - cpb = plugboards[plugboard_any_format_value] - if cpb is not None: - if device_name in cpb: - cpb = cpb[device_name] - elif plugboard_any_device_value in cpb: - cpb = cpb[plugboard_any_device_value] - else: - cpb = None - if DEBUG: - prints('Device using plugboard', format, device_name, cpb) - return cpb - def device_name_for_plugboards(device_class): if hasattr(device_class, 'DEVICE_PLUGBOARD_NAME'): return device_class.DEVICE_PLUGBOARD_NAME diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py index 8f2b084d76..c5db7074dc 100644 --- a/src/calibre/gui2/preferences/plugboard.py +++ b/src/calibre/gui2/preferences/plugboard.py @@ -15,6 +15,7 @@ from calibre.gui2.preferences.plugboard_ui import Ui_Form from calibre.customize.ui import metadata_writers, device_plugins from calibre.library.save_to_disk import plugboard_any_format_value, \ plugboard_any_device_value, plugboard_save_to_disk_value +from calibre.library.server.content import plugboard_content_server_value from calibre.utils.formatter import validation_formatter @@ -74,7 +75,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.devices.append(n) self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) self.devices.insert(1, plugboard_save_to_disk_value) - self.devices.insert(2, plugboard_any_device_value) + self.devices.insert(1, plugboard_content_server_value) + self.devices.insert(1, plugboard_any_device_value) self.new_device.addItems(self.devices) self.formats = [''] diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 96c42e6e0e..3c57af40a8 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -51,6 +51,23 @@ for x in FORMAT_ARG_DESCS: FORMAT_ARGS[x] = '' +def find_plugboard(device_name, format, plugboards): + cpb = None + if format in plugboards: + cpb = plugboards[format] + elif plugboard_any_format_value in plugboards: + cpb = plugboards[plugboard_any_format_value] + if cpb is not None: + if device_name in cpb: + cpb = cpb[device_name] + elif plugboard_any_device_value in cpb: + cpb = cpb[plugboard_any_device_value] + else: + cpb = None + if DEBUG: + prints('Device using plugboard', format, device_name, cpb) + return cpb + def config(defaults=None): if defaults is None: c = Config('save_to_disk', _('Options to control saving to disk')) @@ -279,20 +296,7 @@ def do_save_book_to_disk(id_, mi, cover, plugboards, written = False for fmt in formats: global plugboard_save_to_disk_value, plugboard_any_format_value - dev_name = plugboard_save_to_disk_value - cpb = None - if fmt in plugboards: - cpb = plugboards[fmt] - if dev_name in cpb: - cpb = cpb[dev_name] - else: - cpb = None - if cpb is None and plugboard_any_format_value in plugboards: - cpb = plugboards[plugboard_any_format_value] - if dev_name in cpb: - cpb = cpb[dev_name] - else: - cpb = None + cpb = find_plugboard(plugboard_save_to_disk_value, fmt, plugboards) # Leave this here for a while, in case problems arise. if cpb is not None: prints('Save-to-disk using plugboard:', fmt, cpb) diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index faa0a61baf..8d9e71c528 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -12,9 +12,13 @@ import cherrypy from calibre import fit_image, guess_type from calibre.utils.date import fromtimestamp from calibre.library.caches import SortKeyGenerator +from calibre.library.save_to_disk import find_plugboard + from calibre.utils.magick.draw import save_cover_data_to, Image, \ thumbnail as generate_thumbnail +plugboard_content_server_value = 'content_server' + class CSSortKeyGenerator(SortKeyGenerator): def __init__(self, fields, fm, db_prefs): @@ -186,23 +190,16 @@ class ContentServer(object): # Get the original metadata mi = self.db.get_metadata(id, index_is_id=True) - # Instantiate the CONTENT_SERVER driver - from calibre.devices.content_server.driver import CONTENT_SERVER - cs = CONTENT_SERVER() - # Get any EPUB plugboards for the content server - from calibre.gui2.device import find_plugboard, device_name_for_plugboards plugboards = self.db.prefs.get('plugboards', {}) - - # Transform the metadata via the plugboard - if hasattr(cs, 'set_plugboards') and callable(cs.set_plugboards): - cs.set_plugboards(plugboards, find_plugboard) - cpb = find_plugboard(device_name_for_plugboards(cs), format.lower(), plugboards) - if cpb: - newmi = mi.deepcopy_metadata() - newmi.template_to_attribute(mi, cpb) - else: - newmi = mi + cpb = find_plugboard(plugboard_content_server_value, + 'epub', plugboards) + if cpb: + # Transform the metadata via the plugboard + newmi = mi.deepcopy_metadata() + newmi.template_to_attribute(mi, cpb) + else: + newmi = mi # Write the updated file from tempfile import TemporaryFile From 3709dcbc621e152472184bacbb056e16e1aef1fe Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 10:43:20 +0100 Subject: [PATCH 15/90] Add check for valid formats --- src/calibre/gui2/preferences/plugboard.py | 16 +++++++++++++++- src/calibre/library/server/content.py | 1 + 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/preferences/plugboard.py b/src/calibre/gui2/preferences/plugboard.py index c5db7074dc..7036ddf8f3 100644 --- a/src/calibre/gui2/preferences/plugboard.py +++ b/src/calibre/gui2/preferences/plugboard.py @@ -15,7 +15,8 @@ from calibre.gui2.preferences.plugboard_ui import Ui_Form from calibre.customize.ui import metadata_writers, device_plugins from calibre.library.save_to_disk import plugboard_any_format_value, \ plugboard_any_device_value, plugboard_save_to_disk_value -from calibre.library.server.content import plugboard_content_server_value +from calibre.library.server.content import plugboard_content_server_value, \ + plugboard_content_server_formats from calibre.utils.formatter import validation_formatter @@ -69,13 +70,17 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.device_label.setText(_('Device currently connected: None')) self.devices = ['', 'APPLE', 'FOLDER_DEVICE'] + self.device_to_formats_map = {} for device in device_plugins(): n = device_name_for_plugboards(device) + self.device_to_formats_map[n] = device.FORMATS if n not in self.devices: self.devices.append(n) self.devices.sort(cmp=lambda x, y: cmp(x.lower(), y.lower())) self.devices.insert(1, plugboard_save_to_disk_value) self.devices.insert(1, plugboard_content_server_value) + self.device_to_formats_map[plugboard_content_server_value] = \ + plugboard_content_server_formats self.devices.insert(1, plugboard_any_device_value) self.new_device.addItems(self.devices) @@ -232,6 +237,15 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): show=True) self.new_device.setCurrentIndex(0) return + if self.current_device in self.device_to_formats_map: + allowable_formats = self.device_to_formats_map[self.current_device] + if self.current_format not in allowable_formats: + error_dialog(self, '', + _('The {0} device does not support the {1} format.'). + format(self.current_device, self.current_format), + show=True) + self.new_device.setCurrentIndex(0) + return self.set_fields() def new_format_changed(self, txt): diff --git a/src/calibre/library/server/content.py b/src/calibre/library/server/content.py index 8d9e71c528..08de4faecd 100644 --- a/src/calibre/library/server/content.py +++ b/src/calibre/library/server/content.py @@ -18,6 +18,7 @@ from calibre.utils.magick.draw import save_cover_data_to, Image, \ thumbnail as generate_thumbnail plugboard_content_server_value = 'content_server' +plugboard_content_server_formats = ['epub'] class CSSortKeyGenerator(SortKeyGenerator): From 09da88b6d18d0c4bf09e126f6af7195069b15863 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Mon, 18 Apr 2011 18:06:10 +0800 Subject: [PATCH 16/90] port overdrive plugin to 8.x framework, remove from 7.x framework --- src/calibre/customize/builtins.py | 8 +- src/calibre/ebooks/metadata/covers.py | 27 - src/calibre/ebooks/metadata/fetch.py | 21 - src/calibre/ebooks/metadata/overdrive.py | 459 ---------------- src/calibre/ebooks/metadata/sources/base.py | 4 +- .../ebooks/metadata/sources/overdrive.py | 510 ++++++++++++++++++ 6 files changed, 516 insertions(+), 513 deletions(-) delete mode 100644 src/calibre/ebooks/metadata/overdrive.py create mode 100755 src/calibre/ebooks/metadata/sources/overdrive.py diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index f4a8c6b6bc..75c02c7e00 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -633,14 +633,14 @@ if test_eight_code: # }}} else: from calibre.ebooks.metadata.fetch import GoogleBooks, ISBNDB, Amazon, \ - KentDistrictLibrary, Overdrive + KentDistrictLibrary from calibre.ebooks.metadata.douban import DoubanBooks from calibre.ebooks.metadata.nicebooks import NiceBooks, NiceBooksCovers from calibre.ebooks.metadata.covers import OpenLibraryCovers, \ - AmazonCovers, DoubanCovers, OverdriveCovers + AmazonCovers, DoubanCovers - plugins += [GoogleBooks, ISBNDB, Amazon, Overdrive, - OpenLibraryCovers, AmazonCovers, DoubanCovers, OverdriveCovers, + plugins += [GoogleBooks, ISBNDB, Amazon, + OpenLibraryCovers, AmazonCovers, DoubanCovers, NiceBooksCovers, KentDistrictLibrary, DoubanBooks, NiceBooks] plugins += [ diff --git a/src/calibre/ebooks/metadata/covers.py b/src/calibre/ebooks/metadata/covers.py index f705317f59..10acff4e61 100644 --- a/src/calibre/ebooks/metadata/covers.py +++ b/src/calibre/ebooks/metadata/covers.py @@ -151,33 +151,6 @@ class AmazonCovers(CoverDownload): # {{{ # }}} -class OverdriveCovers(CoverDownload): # {{{ - - name = 'overdrive.com covers' - description = _('Download covers from Overdrive') - author = 'Kovid Goyal' - - - def has_cover(self, mi, ans, timeout=5.): - if not mi.authors or not mi.title: - return False - return True - - def get_covers(self, mi, result_queue, abort, timeout=5.): - if not mi.isbn: - return - from calibre.ebooks.metadata.overdrive import get_cover_url - br = browser() - try: - url = get_cover_url(mi.isbn, mi.title, mi.authors, br) - cover_data = br.open_novisit(url).read() - result_queue.put((True, cover_data, 'jpg', self.name)) - except Exception, e: - result_queue.put((False, self.exception_to_string(e), - traceback.format_exc(), self.name)) - -# }}} - def check_for_cover(mi, timeout=5.): # {{{ from calibre.customize.ui import cover_sources ans = Event() diff --git a/src/calibre/ebooks/metadata/fetch.py b/src/calibre/ebooks/metadata/fetch.py index fb01c5dd71..e1fac50d16 100644 --- a/src/calibre/ebooks/metadata/fetch.py +++ b/src/calibre/ebooks/metadata/fetch.py @@ -250,27 +250,6 @@ class Amazon(MetadataSource): # {{{ # }}} -class Overdrive(MetadataSource): # {{{ - - name = 'Overdrive' - metadata_type = 'social' - description = _('Downloads metadata from the Overdrive library network') - - has_html_comments = True - - def fetch(self): - if not self.isbn: - return - from calibre.ebooks.metadata.overdrive import get_social_metadata - try: - self.results = get_social_metadata(self.title, self.book_author, self.isbn) - - except Exception, e: - self.exception = e - self.tb = traceback.format_exc() - - # }}} - class KentDistrictLibrary(MetadataSource): # {{{ name = 'Kent District Library' diff --git a/src/calibre/ebooks/metadata/overdrive.py b/src/calibre/ebooks/metadata/overdrive.py deleted file mode 100644 index 38d6d730ff..0000000000 --- a/src/calibre/ebooks/metadata/overdrive.py +++ /dev/null @@ -1,459 +0,0 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' -__docformat__ = 'restructuredtext en' - -''' -Fetch metadata using Overdrive Content Reserve -''' -import sys, re, random, urllib, mechanize, copy -from threading import RLock - -from lxml import html, etree -from lxml.html import soupparser - -from calibre import browser -from calibre.ebooks.metadata import check_isbn -from calibre.ebooks.metadata.sources.base import Source -from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.chardet import xml_to_unicode -from calibre.library.comments import sanitize_comments_html - -ovrdrv_data_cache = {} -cover_url_cache = {} -cache_lock = RLock() -base_url = 'http://search.overdrive.com/' - - -def create_query(self, title=None, authors=None, identifiers={}): - q = '' - if title or authors: - def build_term(prefix, parts): - return ' '.join('in'+prefix + ':' + x for x in parts) - title_tokens = list(self.get_title_tokens(title, False)) - if title_tokens: - q += build_term('title', title_tokens) - author_tokens = self.get_author_tokens(authors, - only_first_author=True) - if author_tokens: - q += ('+' if q else '') + build_term('author', - author_tokens) - - if isinstance(q, unicode): - q = q.encode('utf-8') - if not q: - return None - return BASE_URL+urlencode({ - 'q':q, - }) - - -def get_base_referer(): - choices = [ - 'http://overdrive.chipublib.org/82DC601D-7DDE-4212-B43A-09D821935B01/10/375/en/', - 'http://emedia.clevnet.org/9D321DAD-EC0D-490D-BFD8-64AE2C96ECA8/10/241/en/', - 'http://singapore.lib.overdrive.com/F11D55BE-A917-4D63-8111-318E88B29740/10/382/en/', - 'http://ebooks.nypl.org/20E48048-A377-4520-BC43-F8729A42A424/10/257/en/', - 'http://spl.lib.overdrive.com/5875E082-4CB2-4689-9426-8509F354AFEF/10/335/en/' - ] - return choices[random.randint(0, len(choices)-1)] - -def format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid): - fix_slashes = re.compile(r'\\/') - thumbimage = fix_slashes.sub('/', thumbimage) - worldcatlink = fix_slashes.sub('/', worldcatlink) - cover_url = re.sub('(?P<img>(Ima?g(eType-)?))200', '\g<img>100', thumbimage) - social_metadata_url = base_url+'TitleInfo.aspx?ReserveID='+reserveid+'&FormatID='+formatid - series_num = '' - if not series: - if subtitle: - title = od_title+': '+subtitle - else: - title = od_title - else: - title = od_title - m = re.search("([0-9]+$)", subtitle) - if m: - series_num = float(m.group(1)) - return [cover_url, social_metadata_url, worldcatlink, series, series_num, publisher, creators, reserveid, title] - -def safe_query(br, query_url): - ''' - The query must be initialized by loading an empty search results page - this page attempts to set a cookie that Mechanize doesn't like - copy the cookiejar to a separate instance and make a one-off request with the temp cookiejar - ''' - goodcookies = br._ua_handlers['_cookies'].cookiejar - clean_cj = mechanize.CookieJar() - cookies_to_copy = [] - for cookie in goodcookies: - copied_cookie = copy.deepcopy(cookie) - cookies_to_copy.append(copied_cookie) - for copied_cookie in cookies_to_copy: - clean_cj.set_cookie(copied_cookie) - - br.open_novisit(query_url) - - br.set_cookiejar(clean_cj) - - -def overdrive_search(br, q, title, author): - q_query = q+'default.aspx/SearchByKeyword' - q_init_search = q+'SearchResults.aspx' - # get first author as string - convert this to a proper cleanup function later - s = Source(None) - print "printing list with string:" - #print list(s.get_author_tokens(['J. R. R. Tolkien'])) - print "printing list with author "+str(author)+":" - print list(s.get_author_tokens(author)) - author_tokens = list(s.get_author_tokens(author)) - print "there are "+str(len(author_tokens))+" author tokens" - for token in author_tokens: - print "cleaned up author token is: "+str(token) - - - title_tokens = list(s.get_title_tokens(title)) - print "there are "+str(len(title_tokens))+" title tokens" - for token in title_tokens: - print "cleaned up title token is: "+str(token) - - if len(title_tokens) >= len(author_tokens): - initial_q = ' '.join(title_tokens) - xref_q = '+'.join(author_tokens) - else: - initial_q = ' '.join(author_tokens) - xref_q = '+'.join(title_tokens) - - print "initial query is "+str(initial_q) - print "cross reference query is "+str(xref_q) - q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q - query = '{"szKeyword":"'+initial_q+'"}' - - # main query, requires specific Content Type header - req = mechanize.Request(q_query) - req.add_header('Content-Type', 'application/json; charset=utf-8') - br.open_novisit(req, query) - - print "q_init_search is "+q_init_search - # initiate the search without messing up the cookiejar - safe_query(br, q_init_search) - - # get the search results object - results = False - while results == False: - xreq = mechanize.Request(q_xref) - xreq.add_header('X-Requested-With', 'XMLHttpRequest') - xreq.add_header('Referer', q_init_search) - xreq.add_header('Accept', 'application/json, text/javascript, */*') - raw = br.open_novisit(xreq).read() - print "overdrive search result is:\n"+raw - for m in re.finditer(ur'"iTotalDisplayRecords":(?P<displayrecords>\d+).*?"iTotalRecords":(?P<totalrecords>\d+)', raw): - if int(m.group('displayrecords')) >= 1: - results = True - elif int(m.group('totalrecords')) >= 1: - xref_q = '' - q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q - elif int(m.group('totalrecords')) == 0: - return '' - - print "\n\nsorting results" - return sort_ovrdrv_results(raw, title, title_tokens, author, author_tokens) - - -def sort_ovrdrv_results(raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None): - print "\ntitle to search for is "+str(title)+"\nauthor to search for is "+str(author) - close_matches = [] - raw = re.sub('.*?\[\[(?P<content>.*?)\]\].*', '[[\g<content>]]', raw) - results = eval(raw) - print "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" - #print results - # The search results are either from a keyword search or a multi-format list from a single ID, - # sort through the results for closest match/format - if results: - for reserveid, od_title, subtitle, edition, series, publisher, format, formatid, creators, \ - thumbimage, shortdescription, worldcatlink, excerptlink, creatorfile, sorttitle, \ - availabletolibrary, availabletoretailer, relevancyrank, unknown1, unknown2, unknown3 in results: - print "this record's title is "+od_title+", subtitle is "+subtitle+", author[s] are "+creators+", series is "+series - if ovrdrv_id is not None and int(formatid) in [1, 50, 410, 900]: - print "overdrive id is not None, searching based on format type priority" - return format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) - else: - creators = creators.split(', ') - print "split creators from results are: "+str(creators) - # if an exact match in a preferred format occurs - if creators[0] == author[0] and od_title == title and int(formatid) in [1, 50, 410, 900]: - print "Got Exact Match!!!" - return format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) - else: - close_title_match = False - close_author_match = False - print "format id is "+str(formatid) - for token in title_tokens: - print "attempting to find "+str(token)+" title token" - if od_title.lower().find(token.lower()) != -1: - print "matched token" - close_title_match = True - else: - print "token didn't match" - close_title_match = False - break - for token in author_tokens: - print "attempting to find "+str(token)+" author token" - if creators[0].lower().find(token.lower()) != -1: - print "matched token" - close_author_match = True - else: - print "token didn't match" - close_author_match = False - break - if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900]: - if subtitle and series: - close_matches.insert(0, format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) - else: - close_matches.append(format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) - if close_matches: - return close_matches[0] - else: - return '' - else: - return '' - - - -def overdrive_get_record(br, q, ovrdrv_id): - search_url = q+'SearchResults.aspx?ReserveID={'+ovrdrv_id+'}' - results_url = q+'SearchResults.svc/GetResults?sEcho=1&iColumns=18&sColumns=ReserveID%2CTitle%2CSubtitle%2CEdition%2CSeries%2CPublisher%2CFormat%2CFormatID%2CCreators%2CThumbImage%2CShortDescription%2CWorldCatLink%2CExcerptLink%2CCreatorFile%2CSortTitle%2CAvailableToLibrary%2CAvailableToRetailer%2CRelevancyRank&iDisplayStart=0&iDisplayLength=10&sSearch=&bEscapeRegex=true&iSortingCols=1&iSortCol_0=17&sSortDir_0=asc' - - # get the base url to set the proper session cookie - br.open_novisit(q) - - # initialize the search - safe_query(br, search_url) - - # get the results - req = mechanize.Request(results_url) - req.add_header('X-Requested-With', 'XMLHttpRequest') - req.add_header('Referer', search_url) - req.add_header('Accept', 'application/json, text/javascript, */*') - raw = br.open_novisit(req) - raw = str(list(raw)) - return sort_ovrdrv_results(raw, None, None, None, ovrdrv_id) - - -def find_ovrdrv_data(br, title, author, isbn, ovrdrv_id=None): - print "in find_ovrdrv_data, title is "+str(title)+", author is "+str(author)+", overdrive id is "+str(ovrdrv_id) - q = base_url - if ovrdrv_id is None: - return overdrive_search(br, q, title, author) - else: - return overdrive_get_record(br, q, ovrdrv_id) - - - -def to_ovrdrv_data(br, title, author, isbn, ovrdrv_id=None): - print "starting to_ovrdrv_data" - with cache_lock: - ans = ovrdrv_data_cache.get(isbn, None) - if ans: - print "inside to_ovrdrv_data, cache lookup successful, ans is "+str(ans) - return ans - if ans is False: - print "inside to_ovrdrv_data, ans returned False" - return None - try: - print "trying to retrieve data, running find_ovrdrv_data" - ovrdrv_data = find_ovrdrv_data(br, title, author, isbn, ovrdrv_id) - print "ovrdrv_data is "+str(ovrdrv_data) - except: - import traceback - traceback.print_exc() - ovrdrv_data = None - - with cache_lock: - ovrdrv_data_cache[isbn] = ovrdrv_data if ovrdrv_data else False - if ovrdrv_data: - from calibre.ebooks.metadata.xisbn import xisbn - for i in xisbn.get_associated_isbns(isbn): - with cache_lock: - ovrdrv_data_cache[i] = ovrdrv_data - - return ovrdrv_data - - -def get_social_metadata(title, authors, isbn, ovrdrv_id=None): - author = authors[0] - mi = Metadata(title, authors) - br = browser() - print "calling to_ovrdrv_data from inside get_social_metadata" - ovrdrv_data = to_ovrdrv_data(br, title, authors, isbn, ovrdrv_id) - - #[cover_url, social_metadata_url, worldcatlink, series, series_num, publisher, creators, reserveid, title] - - if len(ovrdrv_data[3]) > 1: - mi.series = ovrdrv_data[3] - if ovrdrv_data[4]: - mi.series_index = ovrdrv_data[4] - mi.publisher = ovrdrv_data[5] - mi.authors = ovrdrv_data[6] - if ovrdrv_id is None: - ovrdrv_id = ovrdrv_data[7] - mi.set_identifier('overdrive', ovrdrv_id) - mi.title = ovrdrv_data[8] - print "populated basic social metadata, getting detailed metadata" - if ovrdrv_data and get_metadata_detail(br, ovrdrv_data[1], mi, isbn): - return mi - print "failed to get detailed metadata, returning basic info" - return mi - -def get_cover_url(isbn, title, author, br, ovrdrv_id=None): - print "starting get_cover_url" - print "title is "+str(title) - print "author is "+str(author[0]) - print "isbn is "+str(isbn) - print "ovrdrv_id is "+str(ovrdrv_id) - - with cache_lock: - ans = cover_url_cache.get(isbn, None) - #ans = cover_url_cache.get(ovrdrv_id, None) - if ans: - print "cover url cache lookup returned positive, ans is "+str(ans) - return ans - if ans is False: - "cover url cache lookup returned false" - return None - print "in get_cover_url, calling to_ovrdrv_data function" - ovrdrv_data = to_ovrdrv_data(br, title, author, isbn, ovrdrv_id) - if ovrdrv_data: - ans = ovrdrv_data[0] - print "inside get_cover_url, got url from to_ovrdrv_data, ans is "+str(ans) - if ans: - print "writing cover url to url cache" - with cache_lock: - cover_url_cache[isbn] = ans - #cover_url_cache[ovrdrv_id] = ans - return ans - - with cache_lock: - print "marking cover url cache for this isbn false" - cover_url_cache[isbn] = False - return None - -def _get_cover_url(br, ovrdrv_data): - q = ovrdrv_data[1] - try: - raw = br.open_novisit(q).read() - except Exception, e: - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return None - raise - if '<title>404 - ' in raw: - return None - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - root = soupparser.fromstring(raw) - except: - return False - - imgs = root.xpath('//img[@id="prodImage" and @src]') - if imgs: - src = imgs[0].get('src') - parts = src.split('/') - if len(parts) > 3: - bn = parts[-1] - sparts = bn.split('_') - if len(sparts) > 2: - bn = sparts[0] + sparts[-1] - return ('/'.join(parts[:-1]))+'/'+bn - return None - -def get_metadata_detail(br, metadata_url, mi, isbn=None): - try: - raw = br.open_novisit(metadata_url).read() - except Exception, e: - if callable(getattr(e, 'getcode', None)) and \ - e.getcode() == 404: - return False - raise - raw = xml_to_unicode(raw, strip_encoding_pats=True, - resolve_entities=True)[0] - try: - root = soupparser.fromstring(raw) - except: - return False - - isbn = check_isbn(isbn) - - pub_date = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblPubDate']/text()") - lang = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblLanguage']/text()") - subjects = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblSubjects']/text()") - ebook_isbn = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblIdentifier']/text()") - desc = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblDescription']/ancestor::div[1]") - - if pub_date: - from calibre.utils.date import parse_date - mi.pubdate = parse_date(pub_date[0].strip()) - if lang: - mi.language = lang[0].strip() - print "languages is "+str(mi.language) - if ebook_isbn and isbn is None: - print "ebook isbn is "+str(ebook_isbn[0]) - mi.set_identifier('isbn', ebook_isbn) - #elif isbn is not None: - # mi.set_identifier('isbn', isbn) - if subjects: - mi.tags = [tag.strip() for tag in subjects[0].split(',')] - print "tags are "+str(mi.tags) - if desc: - desc = desc[0] - desc = html.tostring(desc, method='html', encoding=unicode).strip() - # remove all attributes from tags - desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc) - # Remove comments - desc = re.sub(r'(?s)<!--.*?-->', '', desc) - mi.comments = sanitize_comments_html(desc) - - return True - -def main(args=sys.argv): - print "running through main tests" - import tempfile, os, time - tdir = tempfile.gettempdir() - br = browser() - for ovrdrv_id, isbn, title, author in [ - #(None, '0899661343', 'On the Road', ['Jack Kerouac']), # basic test, no series, single author - #(None, '9780061952838', 'The Fellowship of the Ring', ['J. R. R. Tolkien']), # Series test, multi-author - #(None, '9780061952838', 'The Two Towers (The Lord of the Rings, Book II)', ['J. R. R. Tolkien']), # Series test, book 2 - #(None, '9780618153985', 'The Fellowship of the Ring (The Lord of the Rings, Part 1)', ['J.R.R. Tolkien']), - #('57844706-20fa-4ace-b5ee-3470b1b52173', None, 'The Two Towers', ['J. R. R. Tolkien']), # Series test, w/ ovrdrv id - #(None, '9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors - #(None, None, 'Deluge', ['Anne McCaffrey']) # Empty ISBN - #(None, None, 'On the Road', ['Jack Kerouac']), # Nonetype ISBN - #(None, '9780345435279', 'A Caress of Twilight', ['Laurell K. Hamilton']), - #(None, '9780606087230', 'The Omnivore\'s Dilemma : A Natural History of Four Meals', ['Michael Pollan']), # Subtitle colon - #(None, '9780061747649', 'Mental_Floss Presents: Condensed Knowledge', ['Will Pearson', 'Mangesh Hattikudur']), - #(None, '9781400050802', 'The Zombie Survival Guide', ['Max Brooks']), # Two books with this title by this author - #(None, '9781775414315', 'The Worst Journey in the World / Antarctic 1910-1913', ['Apsley Cherry-Garrard']), # Garbage sub-title - #(None, '9780440335160', 'Outlander', ['Diana Gabaldon']), # Returns lots of results to sort through to get the best match - (None, '9780345509741', 'The Horror Stories of Robert E. Howard', ['Robert E. Howard']), # Complex title with initials/dots stripped, some results don't have a cover - ]: - cpath = os.path.join(tdir, title+'.jpg') - print "cpath is "+cpath - st = time.time() - curl = get_cover_url(isbn, title, author, br, ovrdrv_id) - print '\n\n Took ', time.time() - st, ' to get basic metadata\n\n' - if curl is None: - print 'No cover found for', title - else: - print "curl is "+curl - #open(cpath, 'wb').write(br.open_novisit(curl).read()) - #print 'Cover for', title, 'saved to', cpath - st = time.time() - print get_social_metadata(title, author, isbn, ovrdrv_id) - print '\n\n Took ', time.time() - st, ' to get detailed metadata\n\n' - - return 0 - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 5911a357ac..53fe9a4c2d 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -313,8 +313,8 @@ class Source(Plugin): (r'(\d+),(\d+)', r'\1\2'), # Remove hyphens only if they have whitespace before them (r'(\s-)', ' '), - # Remove single quotes - (r"'", ''), + # Remove single quotes not followed by 's' + (r"'(?!s)", ''), # Replace other special chars with a space (r'''[:,;+!@#$%^&*(){}.`~"\s\[\]/]''', ' ') ]] diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py new file mode 100755 index 0000000000..6950711da4 --- /dev/null +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -0,0 +1,510 @@ +#!/usr/bin/env python +__license__ = 'GPL v3' +__copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' +__docformat__ = 'restructuredtext en' + +''' +Fetch metadata using Overdrive Content Reserve +''' +import sys, re, random, urllib, mechanize, copy +from threading import RLock +from Queue import Queue, Empty + +from lxml import html, etree +from lxml.html import soupparser + +from calibre import browser +from calibre.ebooks.metadata import check_isbn +from calibre.ebooks.metadata.sources.base import Source +from calibre.ebooks.metadata.book.base import Metadata +from calibre.ebooks.chardet import xml_to_unicode +from calibre.library.comments import sanitize_comments_html + +ovrdrv_data_cache = {} +cover_url_cache = {} +cache_lock = RLock() +base_url = 'http://search.overdrive.com/' + + +class OverDrive(Source): + + name = 'Overdrive' + description = _('Downloads metadata from Overdrive\'s Content Reserve') + + capabilities = frozenset(['identify', 'cover']) + touched_fields = frozenset(['title', 'authors', 'tags', 'pubdate', + 'comments', 'publisher', 'identifier:isbn', 'series', 'series_num', + 'language', 'identifier:overdrive']) + has_html_comments = True + supports_gzip_transfer_encoding = False + cached_cover_url_is_reliable = True + + def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ + identifiers={}, timeout=30): + ovrdrv_id = identifiers.get('overdrive', None) + isbn = identifiers.get('isbn', None) + + br = self.browser + print "in identify, calling to_ovrdrv_data" + ovrdrv_data = self.to_ovrdrv_data(br, title, authors, ovrdrv_id) + if ovrdrv_data: + title = ovrdrv_data[8] + authors = ovrdrv_data[6] + mi = Metadata(title, authors) + self.parse_search_results(ovrdrv_data, mi) + if ovrdrv_id is None: + ovrdrv_id = ovrdrv_data[7] + if isbn is not None: + self.cache_isbn_to_identifier(isbn, ovrdrv_id) + + self.get_book_detail(br, ovrdrv_data[1], mi, ovrdrv_id, log) + + result_queue.put(mi) + + return None + # }}} + + + def get_book_url(self, identifiers): # {{{ + ovrdrv_id = identifiers.get('overdrive', None) + if ovrdrv_id is not None: + ovrdrv_data = ovrdrv_data_cache.get(ovrdrv_id, None) + if ovrdrv_data: + return ovrdrv_data[1] + else: + br = browser() + ovrdrv_data = self.to_ovrdrv_data(br, None, None, ovrdrv_id) + return ovrdrv_data[1] + # }}} + + def download_cover(self, log, result_queue, abort, # {{{ + title=None, authors=None, identifiers={}, timeout=30): + cached_url = self.get_cached_cover_url(identifiers) + if cached_url is None: + log.info('No cached cover found, running identify') + rq = Queue() + print "inside download cover, calling identify" + self.identify(log, rq, abort, title=title, authors=authors, + identifiers=identifiers) + if abort.is_set(): + return + results = [] + while True: + try: + results.append(rq.get_nowait()) + except Empty: + break + results.sort(key=self.identify_results_keygen( + title=title, authors=authors, identifiers=identifiers)) + for mi in results: + cached_url = self.get_cached_cover_url(mi.identifiers) + if cached_url is not None: + break + if cached_url is None: + log.info('No cover found') + return + + if abort.is_set(): + return + + ovrdrv_id = identifiers.get('overdrive', None) + br = self.browser + referer = self.get_base_referer()+'ContentDetails-Cover.htm?ID='+ovrdrv_id + print "downloading cover, referer is "+str(referer) + req = mechanize.Request(cached_url) + req.add_header('referer', referer) + log('Downloading cover from:', cached_url) + try: + cdata = br.open_novisit(req, timeout=timeout).read() + result_queue.put((self, cdata)) + except: + log.exception('Failed to download cover from:', cached_url) + # }}} + + def get_cached_cover_url(self, identifiers): # {{{ + url = None + ovrdrv_id = identifiers.get('overdrive', None) + print "inside get_cached_cover_url, ovrdrv_id is "+str(ovrdrv_id) + if ovrdrv_id is None: + isbn = identifiers.get('isbn', None) + if isbn is not None: + ovrdrv_id = self.cached_isbn_to_identifier(isbn) + if ovrdrv_id is not None: + url = self.cached_identifier_to_cover_url(ovrdrv_id) + + return url + # }}} + + def create_query(self, title=None, authors=None, identifiers={}): + q = '' + if title or authors: + def build_term(prefix, parts): + return ' '.join('in'+prefix + ':' + x for x in parts) + title_tokens = list(self.get_title_tokens(title, False, True)) + if title_tokens: + q += build_term('title', title_tokens) + author_tokens = self.get_author_tokens(authors, + only_first_author=True) + if author_tokens: + q += ('+' if q else '') + build_term('author', + author_tokens) + + if isinstance(q, unicode): + q = q.encode('utf-8') + if not q: + return None + return BASE_URL+urlencode({ + 'q':q, + }) + + def get_base_referer(self): # to be used for passing referrer headers to cover download + choices = [ + 'http://overdrive.chipublib.org/82DC601D-7DDE-4212-B43A-09D821935B01/10/375/en/', + 'http://emedia.clevnet.org/9D321DAD-EC0D-490D-BFD8-64AE2C96ECA8/10/241/en/', + 'http://singapore.lib.overdrive.com/F11D55BE-A917-4D63-8111-318E88B29740/10/382/en/', + 'http://ebooks.nypl.org/20E48048-A377-4520-BC43-F8729A42A424/10/257/en/', + 'http://spl.lib.overdrive.com/5875E082-4CB2-4689-9426-8509F354AFEF/10/335/en/' + ] + return choices[random.randint(0, len(choices)-1)] + + def format_results(self, reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid): + fix_slashes = re.compile(r'\\/') + thumbimage = fix_slashes.sub('/', thumbimage) + worldcatlink = fix_slashes.sub('/', worldcatlink) + cover_url = re.sub('(?P<img>(Ima?g(eType-)?))200', '\g<img>100', thumbimage) + social_metadata_url = base_url+'TitleInfo.aspx?ReserveID='+reserveid+'&FormatID='+formatid + series_num = '' + if not series: + if subtitle: + title = od_title+': '+subtitle + else: + title = od_title + else: + title = od_title + m = re.search("([0-9]+$)", subtitle) + if m: + series_num = float(m.group(1)) + return [cover_url, social_metadata_url, worldcatlink, series, series_num, publisher, creators, reserveid, title] + + def safe_query(self, br, query_url, post=''): + ''' + The query must be initialized by loading an empty search results page + this page attempts to set a cookie that Mechanize doesn't like + copy the cookiejar to a separate instance and make a one-off request with the temp cookiejar + ''' + goodcookies = br._ua_handlers['_cookies'].cookiejar + clean_cj = mechanize.CookieJar() + cookies_to_copy = [] + for cookie in goodcookies: + copied_cookie = copy.deepcopy(cookie) + cookies_to_copy.append(copied_cookie) + for copied_cookie in cookies_to_copy: + clean_cj.set_cookie(copied_cookie) + + if post: + br.open_novisit(query_url, post) + else: + br.open_novisit(query_url) + + br.set_cookiejar(clean_cj) + + + def overdrive_search(self, br, q, title, author): + # re-initialize the cookiejar to so that it's clean + clean_cj = mechanize.CookieJar() + br.set_cookiejar(clean_cj) + q_query = q+'default.aspx/SearchByKeyword' + q_init_search = q+'SearchResults.aspx' + # get first author as string - convert this to a proper cleanup function later + s = Source(None) + print "printing list with author "+str(author)+":" + author_tokens = list(s.get_author_tokens(author)) + print list(author_tokens) + title_tokens = list(s.get_title_tokens(title, False, True)) + print "there are "+str(len(title_tokens))+" title tokens" + for token in title_tokens: + print "cleaned up title token is: "+str(token) + + if len(title_tokens) >= len(author_tokens): + initial_q = ' '.join(title_tokens) + xref_q = '+'.join(author_tokens) + else: + initial_q = ' '.join(author_tokens) + xref_q = '+'.join(title_tokens) + + print "initial query is "+str(initial_q) + print "cross reference query is "+str(xref_q) + q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q + query = '{"szKeyword":"'+initial_q+'"}' + + # main query, requires specific Content Type header + req = mechanize.Request(q_query) + req.add_header('Content-Type', 'application/json; charset=utf-8') + br.open_novisit(req, query) + + print "q_init_search is "+q_init_search + # initiate the search without messing up the cookiejar + self.safe_query(br, q_init_search) + + # get the search results object + results = False + while results == False: + xreq = mechanize.Request(q_xref) + xreq.add_header('X-Requested-With', 'XMLHttpRequest') + xreq.add_header('Referer', q_init_search) + xreq.add_header('Accept', 'application/json, text/javascript, */*') + raw = br.open_novisit(xreq).read() + print "overdrive search result is:\n"+raw + for m in re.finditer(ur'"iTotalDisplayRecords":(?P<displayrecords>\d+).*?"iTotalRecords":(?P<totalrecords>\d+)', raw): + if int(m.group('displayrecords')) >= 1: + results = True + elif int(m.group('totalrecords')) >= 1: + xref_q = '' + q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q + elif int(m.group('totalrecords')) == 0: + return '' + + print "\n\nsorting results" + return self.sort_ovrdrv_results(raw, title, title_tokens, author, author_tokens) + + + def sort_ovrdrv_results(self, raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None): + print "\ntitle to search for is "+str(title)+"\nauthor to search for is "+str(author) + close_matches = [] + raw = re.sub('.*?\[\[(?P<content>.*?)\]\].*', '[[\g<content>]]', raw) + results = eval(raw) + print "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + #print results + # The search results are either from a keyword search or a multi-format list from a single ID, + # sort through the results for closest match/format + if results: + for reserveid, od_title, subtitle, edition, series, publisher, format, formatid, creators, \ + thumbimage, shortdescription, worldcatlink, excerptlink, creatorfile, sorttitle, \ + availabletolibrary, availabletoretailer, relevancyrank, unknown1, unknown2, unknown3 in results: + print "this record's title is "+od_title+", subtitle is "+subtitle+", author[s] are "+creators+", series is "+series + if ovrdrv_id is not None and int(formatid) in [1, 50, 410, 900]: + print "overdrive id is not None, searching based on format type priority" + return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) + else: + creators = creators.split(', ') + print "split creators from results are: "+str(creators) + # if an exact match in a preferred format occurs + if creators[0] == author[0] and od_title == title and int(formatid) in [1, 50, 410, 900]: + print "Got Exact Match!!!" + return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) + else: + close_title_match = False + close_author_match = False + print "format id is "+str(formatid) + for token in title_tokens: + print "attempting to find "+str(token)+" title token" + if od_title.lower().find(token.lower()) != -1: + print "matched token" + close_title_match = True + else: + print "token didn't match" + close_title_match = False + break + for token in author_tokens: + print "attempting to find "+str(token)+" author token" + if creators[0].lower().find(token.lower()) != -1: + print "matched token" + close_author_match = True + else: + print "token didn't match" + close_author_match = False + break + if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900]: + if subtitle and series: + close_matches.insert(0, self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) + else: + close_matches.append(self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) + if close_matches: + return close_matches[0] + else: + return '' + else: + return '' + + + def overdrive_get_record(self, br, q, ovrdrv_id): + search_url = q+'SearchResults.aspx?ReserveID={'+ovrdrv_id+'}' + results_url = q+'SearchResults.svc/GetResults?sEcho=1&iColumns=18&sColumns=ReserveID%2CTitle%2CSubtitle%2CEdition%2CSeries%2CPublisher%2CFormat%2CFormatID%2CCreators%2CThumbImage%2CShortDescription%2CWorldCatLink%2CExcerptLink%2CCreatorFile%2CSortTitle%2CAvailableToLibrary%2CAvailableToRetailer%2CRelevancyRank&iDisplayStart=0&iDisplayLength=10&sSearch=&bEscapeRegex=true&iSortingCols=1&iSortCol_0=17&sSortDir_0=asc' + + # get the base url to set the proper session cookie + br.open_novisit(q) + + # initialize the search + self.safe_query(br, search_url) + + # get the results + req = mechanize.Request(results_url) + req.add_header('X-Requested-With', 'XMLHttpRequest') + req.add_header('Referer', search_url) + req.add_header('Accept', 'application/json, text/javascript, */*') + raw = br.open_novisit(req) + raw = str(list(raw)) + clean_cj = mechanize.CookieJar() + br.set_cookiejar(clean_cj) + return self.sort_ovrdrv_results(raw, None, None, None, ovrdrv_id) + + + def find_ovrdrv_data(self, br, title, author, isbn, ovrdrv_id=None): + print "in find_ovrdrv_data, title is "+str(title)+", author is "+str(author)+", overdrive id is "+str(ovrdrv_id) + q = base_url + if ovrdrv_id is None: + return self.overdrive_search(br, q, title, author) + else: + return self.overdrive_get_record(br, q, ovrdrv_id) + + + + def to_ovrdrv_data(self, br, title=None, author=None, ovrdrv_id=None): + ''' + Takes either a title/author combo or an Overdrive ID. One of these + two must be passed to this function. + ''' + print "starting to_ovrdrv_data" + if ovrdrv_id is not None: + with cache_lock: + ans = ovrdrv_data_cache.get(ovrdrv_id, None) + if ans: + print "inside to_ovrdrv_data, cache lookup successful, ans is "+str(ans) + return ans + elif ans is False: + print "inside to_ovrdrv_data, ans returned False" + return None + else: + ovrdrv_data = self.find_ovrdrv_data(br, title, author, ovrdrv_id) + else: + try: + print "trying to retrieve data, running find_ovrdrv_data" + ovrdrv_data = self.find_ovrdrv_data(br, title, author, ovrdrv_id) + print "ovrdrv_data is "+str(ovrdrv_data) + except: + import traceback + traceback.print_exc() + ovrdrv_data = None + print "writing results to ovrdrv_data cache" + with cache_lock: + ovrdrv_data_cache[ovrdrv_id] = ovrdrv_data if ovrdrv_data else False + + return ovrdrv_data if ovrdrv_data else False + + + def parse_search_results(self, ovrdrv_data, mi): + ''' + Parse the formatted search results from the initial Overdrive query and + add the values to the metadta. + + The list object has these values: + [cover_url[0], social_metadata_url[1], worldcatlink[2], series[3], series_num[4], + publisher[5], creators[6], reserveid[7], title[8]] + + ''' + print "inside parse_search_results, writing the metadata results" + ovrdrv_id = ovrdrv_data[7] + mi.set_identifier('overdrive', ovrdrv_id) + + if len(ovrdrv_data[3]) > 1: + mi.series = ovrdrv_data[3] + if ovrdrv_data[4]: + mi.series_index = ovrdrv_data[4] + mi.publisher = ovrdrv_data[5] + mi.authors = ovrdrv_data[6] + mi.title = ovrdrv_data[8] + cover_url = ovrdrv_data[0] + if cover_url: + self.cache_identifier_to_cover_url(ovrdrv_id, + cover_url) + + + def get_book_detail(self, br, metadata_url, mi, ovrdrv_id, log): + try: + raw = br.open_novisit(metadata_url).read() + except Exception, e: + if callable(getattr(e, 'getcode', None)) and \ + e.getcode() == 404: + return False + raise + raw = xml_to_unicode(raw, strip_encoding_pats=True, + resolve_entities=True)[0] + try: + root = soupparser.fromstring(raw) + except: + return False + + pub_date = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblPubDate']/text()") + lang = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblLanguage']/text()") + subjects = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblSubjects']/text()") + ebook_isbn = root.xpath("//td/label[@id='ctl00_ContentPlaceHolder1_lblIdentifier']/text()") + desc = root.xpath("//div/label[@id='ctl00_ContentPlaceHolder1_lblDescription']/ancestor::div[1]") + + if pub_date: + from calibre.utils.date import parse_date + mi.pubdate = parse_date(pub_date[0].strip()) + if lang: + mi.language = lang[0].strip() + print "languages is "+str(mi.language) + #if ebook_isbn: + # print "ebook isbn is "+str(ebook_isbn[0]) + # isbn = check_isbn(ebook_isbn[0].strip()) + # if isbn: + # self.cache_isbn_to_identifier(isbn, ovrdrv_id) + # mi.isbn = isbn + if subjects: + mi.tags = [tag.strip() for tag in subjects[0].split(',')] + print "tags are "+str(mi.tags) + if desc: + desc = desc[0] + desc = html.tostring(desc, method='html', encoding=unicode).strip() + # remove all attributes from tags + desc = re.sub(r'<([a-zA-Z0-9]+)\s[^>]+>', r'<\1>', desc) + # Remove comments + desc = re.sub(r'(?s)<!--.*?-->', '', desc) + mi.comments = sanitize_comments_html(desc) + + return None + + +def main(args=sys.argv): + print "running through main tests" + import tempfile, os, time + tdir = tempfile.gettempdir() + br = browser() + for ovrdrv_id, isbn, title, author in [ + #(None, '0899661343', 'On the Road', ['Jack Kerouac']), # basic test, no series, single author + #(None, '9780061952838', 'The Fellowship of the Ring', ['J. R. R. Tolkien']), # Series test, multi-author + #(None, '9780061952838', 'The Two Towers (The Lord of the Rings, Book II)', ['J. R. R. Tolkien']), # Series test, book 2 + #(None, '9780618153985', 'The Fellowship of the Ring (The Lord of the Rings, Part 1)', ['J.R.R. Tolkien']), + #('57844706-20fa-4ace-b5ee-3470b1b52173', None, 'The Two Towers', ['J. R. R. Tolkien']), # Series test, w/ ovrdrv id + #(None, '9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors + #(None, None, 'Deluge', ['Anne McCaffrey']) # Empty ISBN + #(None, None, 'On the Road', ['Jack Kerouac']), # Nonetype ISBN + #(None, '9780345435279', 'A Caress of Twilight', ['Laurell K. Hamilton']), + #(None, '9780606087230', 'The Omnivore\'s Dilemma : A Natural History of Four Meals', ['Michael Pollan']), # Subtitle colon + #(None, '9780061747649', 'Mental_Floss Presents: Condensed Knowledge', ['Will Pearson', 'Mangesh Hattikudur']), + #(None, '9781400050802', 'The Zombie Survival Guide', ['Max Brooks']), # Two books with this title by this author + #(None, '9781775414315', 'The Worst Journey in the World / Antarctic 1910-1913', ['Apsley Cherry-Garrard']), # Garbage sub-title + #(None, '9780440335160', 'Outlander', ['Diana Gabaldon']), # Returns lots of results to sort through to get the best match + (None, '9780345509741', 'The Horror Stories of Robert E. Howard', ['Robert E. Howard']), # Complex title with initials/dots stripped, some results don't have a cover + ]: + cpath = os.path.join(tdir, title+'.jpg') + print "cpath is "+cpath + st = time.time() + curl = get_cover_url(isbn, title, author, br, ovrdrv_id) + print '\n\n Took ', time.time() - st, ' to get basic metadata\n\n' + if curl is None: + print 'No cover found for', title + else: + print "curl is "+curl + #open(cpath, 'wb').write(br.open_novisit(curl).read()) + #print 'Cover for', title, 'saved to', cpath + st = time.time() + print get_social_metadata(title, author, isbn, ovrdrv_id) + print '\n\n Took ', time.time() - st, ' to get detailed metadata\n\n' + + return 0 + +if __name__ == '__main__': + sys.exit(main()) From ef7070b455afdf75c93dc3f8b9baac6e361ba264 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 11:09:53 +0100 Subject: [PATCH 17/90] Fix adding a comma to custom series values when using completion. --- src/calibre/gui2/custom_column_widgets.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/calibre/gui2/custom_column_widgets.py b/src/calibre/gui2/custom_column_widgets.py index 81016d3c6a..d1acd2ed83 100644 --- a/src/calibre/gui2/custom_column_widgets.py +++ b/src/calibre/gui2/custom_column_widgets.py @@ -289,6 +289,7 @@ class Series(Base): values = self.all_values = list(self.db.all_custom(num=self.col_id)) values.sort(key=sort_key) w = MultiCompleteComboBox(parent) + w.set_separator(None) w.setSizeAdjustPolicy(w.AdjustToMinimumContentsLengthWithIcon) w.setMinimumContentsLength(25) self.name_widget = w From aa30f306b5f8894641bc5536559b427c02c5303d Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Mon, 18 Apr 2011 18:17:54 +0800 Subject: [PATCH 18/90] ... --- .../ebooks/metadata/sources/overdrive.py | 64 ++++--------------- 1 file changed, 14 insertions(+), 50 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 6950711da4..dd2e8b2a85 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -45,7 +45,6 @@ class OverDrive(Source): isbn = identifiers.get('isbn', None) br = self.browser - print "in identify, calling to_ovrdrv_data" ovrdrv_data = self.to_ovrdrv_data(br, title, authors, ovrdrv_id) if ovrdrv_data: title = ovrdrv_data[8] @@ -83,7 +82,6 @@ class OverDrive(Source): if cached_url is None: log.info('No cached cover found, running identify') rq = Queue() - print "inside download cover, calling identify" self.identify(log, rq, abort, title=title, authors=authors, identifiers=identifiers) if abort.is_set(): @@ -110,7 +108,6 @@ class OverDrive(Source): ovrdrv_id = identifiers.get('overdrive', None) br = self.browser referer = self.get_base_referer()+'ContentDetails-Cover.htm?ID='+ovrdrv_id - print "downloading cover, referer is "+str(referer) req = mechanize.Request(cached_url) req.add_header('referer', referer) log('Downloading cover from:', cached_url) @@ -124,7 +121,6 @@ class OverDrive(Source): def get_cached_cover_url(self, identifiers): # {{{ url = None ovrdrv_id = identifiers.get('overdrive', None) - print "inside get_cached_cover_url, ovrdrv_id is "+str(ovrdrv_id) if ovrdrv_id is None: isbn = identifiers.get('isbn', None) if isbn is not None: @@ -217,14 +213,9 @@ class OverDrive(Source): q_init_search = q+'SearchResults.aspx' # get first author as string - convert this to a proper cleanup function later s = Source(None) - print "printing list with author "+str(author)+":" author_tokens = list(s.get_author_tokens(author)) - print list(author_tokens) title_tokens = list(s.get_title_tokens(title, False, True)) - print "there are "+str(len(title_tokens))+" title tokens" - for token in title_tokens: - print "cleaned up title token is: "+str(token) - + if len(title_tokens) >= len(author_tokens): initial_q = ' '.join(title_tokens) xref_q = '+'.join(author_tokens) @@ -232,8 +223,6 @@ class OverDrive(Source): initial_q = ' '.join(author_tokens) xref_q = '+'.join(title_tokens) - print "initial query is "+str(initial_q) - print "cross reference query is "+str(xref_q) q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q query = '{"szKeyword":"'+initial_q+'"}' @@ -242,7 +231,6 @@ class OverDrive(Source): req.add_header('Content-Type', 'application/json; charset=utf-8') br.open_novisit(req, query) - print "q_init_search is "+q_init_search # initiate the search without messing up the cookiejar self.safe_query(br, q_init_search) @@ -254,7 +242,6 @@ class OverDrive(Source): xreq.add_header('Referer', q_init_search) xreq.add_header('Accept', 'application/json, text/javascript, */*') raw = br.open_novisit(xreq).read() - print "overdrive search result is:\n"+raw for m in re.finditer(ur'"iTotalDisplayRecords":(?P<displayrecords>\d+).*?"iTotalRecords":(?P<totalrecords>\d+)', raw): if int(m.group('displayrecords')) >= 1: results = True @@ -264,54 +251,40 @@ class OverDrive(Source): elif int(m.group('totalrecords')) == 0: return '' - print "\n\nsorting results" return self.sort_ovrdrv_results(raw, title, title_tokens, author, author_tokens) def sort_ovrdrv_results(self, raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None): - print "\ntitle to search for is "+str(title)+"\nauthor to search for is "+str(author) close_matches = [] raw = re.sub('.*?\[\[(?P<content>.*?)\]\].*', '[[\g<content>]]', raw) results = eval(raw) - print "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" - #print results + # The search results are either from a keyword search or a multi-format list from a single ID, # sort through the results for closest match/format if results: for reserveid, od_title, subtitle, edition, series, publisher, format, formatid, creators, \ thumbimage, shortdescription, worldcatlink, excerptlink, creatorfile, sorttitle, \ availabletolibrary, availabletoretailer, relevancyrank, unknown1, unknown2, unknown3 in results: - print "this record's title is "+od_title+", subtitle is "+subtitle+", author[s] are "+creators+", series is "+series if ovrdrv_id is not None and int(formatid) in [1, 50, 410, 900]: - print "overdrive id is not None, searching based on format type priority" return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) else: creators = creators.split(', ') - print "split creators from results are: "+str(creators) # if an exact match in a preferred format occurs if creators[0] == author[0] and od_title == title and int(formatid) in [1, 50, 410, 900]: - print "Got Exact Match!!!" return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) else: close_title_match = False close_author_match = False - print "format id is "+str(formatid) for token in title_tokens: - print "attempting to find "+str(token)+" title token" if od_title.lower().find(token.lower()) != -1: - print "matched token" close_title_match = True else: - print "token didn't match" close_title_match = False break for token in author_tokens: - print "attempting to find "+str(token)+" author token" if creators[0].lower().find(token.lower()) != -1: - print "matched token" close_author_match = True else: - print "token didn't match" close_author_match = False break if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900]: @@ -350,7 +323,6 @@ class OverDrive(Source): def find_ovrdrv_data(self, br, title, author, isbn, ovrdrv_id=None): - print "in find_ovrdrv_data, title is "+str(title)+", author is "+str(author)+", overdrive id is "+str(ovrdrv_id) q = base_url if ovrdrv_id is None: return self.overdrive_search(br, q, title, author) @@ -364,28 +336,22 @@ class OverDrive(Source): Takes either a title/author combo or an Overdrive ID. One of these two must be passed to this function. ''' - print "starting to_ovrdrv_data" if ovrdrv_id is not None: with cache_lock: ans = ovrdrv_data_cache.get(ovrdrv_id, None) if ans: - print "inside to_ovrdrv_data, cache lookup successful, ans is "+str(ans) return ans elif ans is False: - print "inside to_ovrdrv_data, ans returned False" return None else: ovrdrv_data = self.find_ovrdrv_data(br, title, author, ovrdrv_id) else: try: - print "trying to retrieve data, running find_ovrdrv_data" ovrdrv_data = self.find_ovrdrv_data(br, title, author, ovrdrv_id) - print "ovrdrv_data is "+str(ovrdrv_data) except: import traceback traceback.print_exc() ovrdrv_data = None - print "writing results to ovrdrv_data cache" with cache_lock: ovrdrv_data_cache[ovrdrv_id] = ovrdrv_data if ovrdrv_data else False @@ -402,7 +368,6 @@ class OverDrive(Source): publisher[5], creators[6], reserveid[7], title[8]] ''' - print "inside parse_search_results, writing the metadata results" ovrdrv_id = ovrdrv_data[7] mi.set_identifier('overdrive', ovrdrv_id) @@ -445,7 +410,7 @@ class OverDrive(Source): mi.pubdate = parse_date(pub_date[0].strip()) if lang: mi.language = lang[0].strip() - print "languages is "+str(mi.language) + #if ebook_isbn: # print "ebook isbn is "+str(ebook_isbn[0]) # isbn = check_isbn(ebook_isbn[0].strip()) @@ -454,7 +419,7 @@ class OverDrive(Source): # mi.isbn = isbn if subjects: mi.tags = [tag.strip() for tag in subjects[0].split(',')] - print "tags are "+str(mi.tags) + if desc: desc = desc[0] desc = html.tostring(desc, method='html', encoding=unicode).strip() @@ -468,7 +433,6 @@ class OverDrive(Source): def main(args=sys.argv): - print "running through main tests" import tempfile, os, time tdir = tempfile.gettempdir() br = browser() @@ -490,19 +454,19 @@ def main(args=sys.argv): (None, '9780345509741', 'The Horror Stories of Robert E. Howard', ['Robert E. Howard']), # Complex title with initials/dots stripped, some results don't have a cover ]: cpath = os.path.join(tdir, title+'.jpg') - print "cpath is "+cpath + #print "cpath is "+cpath st = time.time() curl = get_cover_url(isbn, title, author, br, ovrdrv_id) - print '\n\n Took ', time.time() - st, ' to get basic metadata\n\n' - if curl is None: - print 'No cover found for', title - else: - print "curl is "+curl - #open(cpath, 'wb').write(br.open_novisit(curl).read()) - #print 'Cover for', title, 'saved to', cpath + #print '\n\n Took ', time.time() - st, ' to get basic metadata\n\n' + #if curl is None: + # print 'No cover found for', title + #else: + # print "curl is "+curl + # open(cpath, 'wb').write(br.open_novisit(curl).read()) + # print 'Cover for', title, 'saved to', cpath st = time.time() - print get_social_metadata(title, author, isbn, ovrdrv_id) - print '\n\n Took ', time.time() - st, ' to get detailed metadata\n\n' + #print get_social_metadata(title, author, isbn, ovrdrv_id) + #print '\n\n Took ', time.time() - st, ' to get detailed metadata\n\n' return 0 From 2b82d4944859d22e56daf21f030586f4fb8977b3 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Mon, 18 Apr 2011 20:56:11 +0800 Subject: [PATCH 19/90] fixed multiple author sorting --- .../ebooks/metadata/sources/overdrive.py | 43 ++++++++++++++----- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index dd2e8b2a85..42b320745a 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -212,10 +212,15 @@ class OverDrive(Source): q_query = q+'default.aspx/SearchByKeyword' q_init_search = q+'SearchResults.aspx' # get first author as string - convert this to a proper cleanup function later - s = Source(None) - author_tokens = list(s.get_author_tokens(author)) - title_tokens = list(s.get_title_tokens(title, False, True)) - + print "printing list with author "+str(author)+":" + author_tokens = list(self.get_author_tokens(author, + only_first_author=True)) + print list(author_tokens) + title_tokens = list(self.get_title_tokens(title, False, True)) + print "there are "+str(len(title_tokens))+" title tokens" + for token in title_tokens: + print "cleaned up title token is: "+str(token) + if len(title_tokens) >= len(author_tokens): initial_q = ' '.join(title_tokens) xref_q = '+'.join(author_tokens) @@ -251,41 +256,59 @@ class OverDrive(Source): elif int(m.group('totalrecords')) == 0: return '' + print "\n\nsorting results" return self.sort_ovrdrv_results(raw, title, title_tokens, author, author_tokens) def sort_ovrdrv_results(self, raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None): + print "\ntitle to search for is "+str(title)+"\nauthor to search for is "+str(author) close_matches = [] raw = re.sub('.*?\[\[(?P<content>.*?)\]\].*', '[[\g<content>]]', raw) results = eval(raw) - + print "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" + #print results # The search results are either from a keyword search or a multi-format list from a single ID, # sort through the results for closest match/format if results: for reserveid, od_title, subtitle, edition, series, publisher, format, formatid, creators, \ thumbimage, shortdescription, worldcatlink, excerptlink, creatorfile, sorttitle, \ availabletolibrary, availabletoretailer, relevancyrank, unknown1, unknown2, unknown3 in results: + print "this record's title is "+od_title+", subtitle is "+subtitle+", author[s] are "+creators+", series is "+series if ovrdrv_id is not None and int(formatid) in [1, 50, 410, 900]: + print "overdrive id is not None, searching based on format type priority" return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) else: creators = creators.split(', ') + print "split creators from results are: "+str(creators)+", there are "+str(len(creators))+" total" # if an exact match in a preferred format occurs if creators[0] == author[0] and od_title == title and int(formatid) in [1, 50, 410, 900]: + print "Got Exact Match!!!" return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) else: close_title_match = False close_author_match = False + print "format id is "+str(formatid) for token in title_tokens: + print "attempting to find "+str(token)+" title token" if od_title.lower().find(token.lower()) != -1: + print "matched token" close_title_match = True else: + print "token didn't match" close_title_match = False break - for token in author_tokens: - if creators[0].lower().find(token.lower()) != -1: - close_author_match = True - else: - close_author_match = False + for author in creators: + print "matching tokens for "+str(author) + for token in author_tokens: + print "attempting to find "+str(token)+" author token" + if author.lower().find(token.lower()) != -1: + print "matched token" + close_author_match = True + else: + print "token didn't match" + close_author_match = False + break + if close_author_match: break if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900]: if subtitle and series: From 7b196c762bb63b482bef111fcabe553d74ed8395 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Mon, 18 Apr 2011 21:07:26 +0800 Subject: [PATCH 20/90] prioritized results with covers, cleaned up print statements --- .../ebooks/metadata/sources/overdrive.py | 69 ++++++++----------- 1 file changed, 27 insertions(+), 42 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 42b320745a..4fc8dbab1b 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -55,9 +55,9 @@ class OverDrive(Source): ovrdrv_id = ovrdrv_data[7] if isbn is not None: self.cache_isbn_to_identifier(isbn, ovrdrv_id) - + self.get_book_detail(br, ovrdrv_data[1], mi, ovrdrv_id, log) - + result_queue.put(mi) return None @@ -144,7 +144,7 @@ class OverDrive(Source): if author_tokens: q += ('+' if q else '') + build_term('author', author_tokens) - + if isinstance(q, unicode): q = q.encode('utf-8') if not q: @@ -162,7 +162,7 @@ class OverDrive(Source): 'http://spl.lib.overdrive.com/5875E082-4CB2-4689-9426-8509F354AFEF/10/335/en/' ] return choices[random.randint(0, len(choices)-1)] - + def format_results(self, reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid): fix_slashes = re.compile(r'\\/') thumbimage = fix_slashes.sub('/', thumbimage) @@ -181,7 +181,7 @@ class OverDrive(Source): if m: series_num = float(m.group(1)) return [cover_url, social_metadata_url, worldcatlink, series, series_num, publisher, creators, reserveid, title] - + def safe_query(self, br, query_url, post=''): ''' The query must be initialized by loading an empty search results page @@ -212,33 +212,29 @@ class OverDrive(Source): q_query = q+'default.aspx/SearchByKeyword' q_init_search = q+'SearchResults.aspx' # get first author as string - convert this to a proper cleanup function later - print "printing list with author "+str(author)+":" author_tokens = list(self.get_author_tokens(author, only_first_author=True)) - print list(author_tokens) - title_tokens = list(self.get_title_tokens(title, False, True)) - print "there are "+str(len(title_tokens))+" title tokens" - for token in title_tokens: - print "cleaned up title token is: "+str(token) - + title_tokens = list(self.get_title_tokens(title, + strip_joiners=False, strip_subtitle=True)) + if len(title_tokens) >= len(author_tokens): initial_q = ' '.join(title_tokens) xref_q = '+'.join(author_tokens) else: initial_q = ' '.join(author_tokens) xref_q = '+'.join(title_tokens) - + q_xref = q+'SearchResults.svc/GetResults?iDisplayLength=50&sSearch='+xref_q query = '{"szKeyword":"'+initial_q+'"}' - + # main query, requires specific Content Type header req = mechanize.Request(q_query) req.add_header('Content-Type', 'application/json; charset=utf-8') br.open_novisit(req, query) - + # initiate the search without messing up the cookiejar self.safe_query(br, q_init_search) - + # get the search results object results = False while results == False: @@ -256,16 +252,13 @@ class OverDrive(Source): elif int(m.group('totalrecords')) == 0: return '' - print "\n\nsorting results" return self.sort_ovrdrv_results(raw, title, title_tokens, author, author_tokens) - - + + def sort_ovrdrv_results(self, raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None): - print "\ntitle to search for is "+str(title)+"\nauthor to search for is "+str(author) close_matches = [] raw = re.sub('.*?\[\[(?P<content>.*?)\]\].*', '[[\g<content>]]', raw) results = eval(raw) - print "\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n" #print results # The search results are either from a keyword search or a multi-format list from a single ID, # sort through the results for closest match/format @@ -273,44 +266,36 @@ class OverDrive(Source): for reserveid, od_title, subtitle, edition, series, publisher, format, formatid, creators, \ thumbimage, shortdescription, worldcatlink, excerptlink, creatorfile, sorttitle, \ availabletolibrary, availabletoretailer, relevancyrank, unknown1, unknown2, unknown3 in results: - print "this record's title is "+od_title+", subtitle is "+subtitle+", author[s] are "+creators+", series is "+series + #print "this record's title is "+od_title+", subtitle is "+subtitle+", author[s] are "+creators+", series is "+series if ovrdrv_id is not None and int(formatid) in [1, 50, 410, 900]: - print "overdrive id is not None, searching based on format type priority" - return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) + #print "overdrive id is not None, searching based on format type priority" + return self.format_results(reserveid, od_title, subtitle, series, publisher, + creators, thumbimage, worldcatlink, formatid) else: creators = creators.split(', ') - print "split creators from results are: "+str(creators)+", there are "+str(len(creators))+" total" # if an exact match in a preferred format occurs if creators[0] == author[0] and od_title == title and int(formatid) in [1, 50, 410, 900]: - print "Got Exact Match!!!" - return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) + return self.format_results(reserveid, od_title, subtitle, series, publisher, + creators, thumbimage, worldcatlink, formatid) else: close_title_match = False close_author_match = False - print "format id is "+str(formatid) for token in title_tokens: - print "attempting to find "+str(token)+" title token" if od_title.lower().find(token.lower()) != -1: - print "matched token" close_title_match = True else: - print "token didn't match" close_title_match = False break for author in creators: - print "matching tokens for "+str(author) for token in author_tokens: - print "attempting to find "+str(token)+" author token" if author.lower().find(token.lower()) != -1: - print "matched token" close_author_match = True else: - print "token didn't match" close_author_match = False break if close_author_match: break - if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900]: + if close_title_match and close_author_match and int(formatid) in [1, 50, 410, 900] and thumbimage: if subtitle and series: close_matches.insert(0, self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid)) else: @@ -321,18 +306,18 @@ class OverDrive(Source): return '' else: return '' - - + + def overdrive_get_record(self, br, q, ovrdrv_id): search_url = q+'SearchResults.aspx?ReserveID={'+ovrdrv_id+'}' results_url = q+'SearchResults.svc/GetResults?sEcho=1&iColumns=18&sColumns=ReserveID%2CTitle%2CSubtitle%2CEdition%2CSeries%2CPublisher%2CFormat%2CFormatID%2CCreators%2CThumbImage%2CShortDescription%2CWorldCatLink%2CExcerptLink%2CCreatorFile%2CSortTitle%2CAvailableToLibrary%2CAvailableToRetailer%2CRelevancyRank&iDisplayStart=0&iDisplayLength=10&sSearch=&bEscapeRegex=true&iSortingCols=1&iSortCol_0=17&sSortDir_0=asc' - + # get the base url to set the proper session cookie br.open_novisit(q) - + # initialize the search self.safe_query(br, search_url) - + # get the results req = mechanize.Request(results_url) req.add_header('X-Requested-With', 'XMLHttpRequest') @@ -385,7 +370,7 @@ class OverDrive(Source): ''' Parse the formatted search results from the initial Overdrive query and add the values to the metadta. - + The list object has these values: [cover_url[0], social_metadata_url[1], worldcatlink[2], series[3], series_num[4], publisher[5], creators[6], reserveid[7], title[8]] From f7535a51edf98b802218bffd2d2eccbbc5bf913f Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Mon, 18 Apr 2011 21:30:35 +0800 Subject: [PATCH 21/90] re-initialize the cookie jar for book records that already have an overdrive identifier --- src/calibre/ebooks/metadata/sources/overdrive.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 4fc8dbab1b..1b237ad683 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -312,6 +312,9 @@ class OverDrive(Source): search_url = q+'SearchResults.aspx?ReserveID={'+ovrdrv_id+'}' results_url = q+'SearchResults.svc/GetResults?sEcho=1&iColumns=18&sColumns=ReserveID%2CTitle%2CSubtitle%2CEdition%2CSeries%2CPublisher%2CFormat%2CFormatID%2CCreators%2CThumbImage%2CShortDescription%2CWorldCatLink%2CExcerptLink%2CCreatorFile%2CSortTitle%2CAvailableToLibrary%2CAvailableToRetailer%2CRelevancyRank&iDisplayStart=0&iDisplayLength=10&sSearch=&bEscapeRegex=true&iSortingCols=1&iSortCol_0=17&sSortDir_0=asc' + # re-initialize the cookiejar to so that it's clean + clean_cj = mechanize.CookieJar() + br.set_cookiejar(clean_cj) # get the base url to set the proper session cookie br.open_novisit(q) From 9b9c62b7654e3a22b20c2b9e0d4f1c6cfb38e65a Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 08:29:55 -0600 Subject: [PATCH 22/90] series_test() --- src/calibre/ebooks/metadata/sources/test.py | 17 +++++++++++++++++ src/calibre/gui2/dialogs/tweak_epub.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/sources/test.py b/src/calibre/ebooks/metadata/sources/test.py index e280b0c038..c55f963003 100644 --- a/src/calibre/ebooks/metadata/sources/test.py +++ b/src/calibre/ebooks/metadata/sources/test.py @@ -67,6 +67,23 @@ def authors_test(authors): return test +def series_test(series, series_index): + series = series.lower() + + def test(mi): + ms = mi.series.lower() if mi.series else '' + if (ms == series) and (series_index == mi.series_index): + return True + if mi.series: + prints('Series test failed. Expected: \'%s [%d]\' found \'%s[%d]\''% \ + (series, series_index, ms, mi.series_index)) + else: + prints('Series test failed. Expected: \'%s [%d]\' found no series'% \ + (series, series_index)) + return False + + return test + def init_test(tdir_name): tdir = tempfile.gettempdir() lf = os.path.join(tdir, tdir_name.replace(' ', '')+'_identify_test.txt') diff --git a/src/calibre/gui2/dialogs/tweak_epub.py b/src/calibre/gui2/dialogs/tweak_epub.py index a42fb07e40..edc274c9b2 100755 --- a/src/calibre/gui2/dialogs/tweak_epub.py +++ b/src/calibre/gui2/dialogs/tweak_epub.py @@ -12,7 +12,7 @@ from zipfile import ZipFile, ZIP_DEFLATED, ZIP_STORED from PyQt4.Qt import QDialog -from calibre.constants import isosx, iswindows +from calibre.constants import isosx from calibre.gui2 import open_local_file from calibre.gui2.dialogs.tweak_epub_ui import Ui_Dialog from calibre.libunzip import extract as zipextract From 89d2b56ce89d9a916eb5f6c105bd7fb87f5fabf6 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 08:39:42 -0600 Subject: [PATCH 23/90] Fix #764472 (Set author from author sort button ignores tweak) --- src/calibre/gui2/metadata/basic_widgets.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index 9502fcb205..f918009a56 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -278,11 +278,13 @@ class AuthorSortEdit(EnLineEdit): def copy_to_authors(self): aus = self.current_val + meth = tweaks['author_sort_copy_method'] if aus: ln, _, rest = aus.partition(',') if rest: - au = rest.strip() + ' ' + ln.strip() - self.authors_edit.current_val = [au] + if meth in ('invert', 'nocomma'): + aus = rest.strip() + ' ' + ln.strip() + self.authors_edit.current_val = [aus] def auto_generate(self, *args): au = unicode(self.authors_edit.text()) From 27cd0b68734f76417ffedda91911dfe890522bd4 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 08:43:55 -0600 Subject: [PATCH 24/90] Updated Arcamax --- recipes/arcamax.recipe | 48 +++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/recipes/arcamax.recipe b/recipes/arcamax.recipe index 39fa199cc3..bcd468307e 100644 --- a/recipes/arcamax.recipe +++ b/recipes/arcamax.recipe @@ -6,12 +6,13 @@ __copyright__ = 'Copyright 2010 Starson17' www.arcamax.com ''' from calibre.web.feeds.news import BasicNewsRecipe +from calibre.ebooks.BeautifulSoup import Tag class Arcamax(BasicNewsRecipe): title = 'Arcamax' __author__ = 'Starson17' - __version__ = '1.03' - __date__ = '25 November 2010' + __version__ = '1.04' + __date__ = '18 April 2011' description = u'Family Friendly Comics - Customize for more days/comics: Defaults to 7 days, 25 comics - 20 general, 5 editorial.' category = 'news, comics' language = 'en' @@ -30,8 +31,15 @@ class Arcamax(BasicNewsRecipe): , 'language' : language } - keep_only_tags = [dict(name='div', attrs={'class':['toon']}), - ] + keep_only_tags = [dict(name='div', attrs={'class':['comics-header']}), + dict(name='b', attrs={'class':['current']}), + dict(name='article', attrs={'class':['comic']}), + ] + + remove_tags = [dict(name='div', attrs={'id':['comicfull' ]}), + dict(name='div', attrs={'class':['calendar' ]}), + dict(name='nav', attrs={'class':['calendar-nav' ]}), + ] def parse_index(self): feeds = [] @@ -71,7 +79,6 @@ class Arcamax(BasicNewsRecipe): #(u"Rugrats", u"http://www.arcamax.com/rugrats"), (u"Speed Bump", u"http://www.arcamax.com/speedbump"), (u"Wizard of Id", u"http://www.arcamax.com/wizardofid"), - (u"Dilbert", u"http://www.arcamax.com/dilbert"), (u"Zits", u"http://www.arcamax.com/zits"), ]: articles = self.make_links(url) @@ -86,24 +93,41 @@ class Arcamax(BasicNewsRecipe): for page in pages: page_soup = self.index_to_soup(url) if page_soup: - title = page_soup.find(name='div', attrs={'class':'toon'}).p.img['alt'] + title = page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0] + print 'title is: ', title page_url = url - prev_page_url = 'http://www.arcamax.com' + page_soup.find('a', attrs={'class':'next'}, text='Previous').parent['href'] - current_articles.append({'title': title, 'url': page_url, 'description':'', 'date':''}) + print 'url is: ', url + # orig prev_page_url = 'http://www.arcamax.com' + page_soup.find('a', attrs={'class':'prev'}, text='Previous').parent['href'] + prev_page_url = 'http://www.arcamax.com' + page_soup.find('span', text='Previous').parent.parent['href'] + print 'prev_page_url is: ', prev_page_url + date = self.tag_to_string(page_soup.find(name='b', attrs={'class':['current']})) + print 'date is: ', date + current_articles.append({'title': title, 'url': page_url, 'description':'', 'date': date}) url = prev_page_url current_articles.reverse() return current_articles def preprocess_html(self, soup): - main_comic = soup.find('p',attrs={'class':'m0'}) - if main_comic.a['target'] == '_blank': - main_comic.a.img['id'] = 'main_comic' + for img_tag in soup.findAll('img'): + parent_tag = img_tag.parent + if parent_tag.name == 'a': + new_tag = Tag(soup,'p') + new_tag.insert(0,img_tag) + parent_tag.replaceWith(new_tag) + elif parent_tag.name == 'p': + if not self.tag_to_string(parent_tag) == '': + new_div = Tag(soup,'div') + new_tag = Tag(soup,'p') + new_tag.insert(0,img_tag) + parent_tag.replaceWith(new_div) + new_div.insert(0,new_tag) + new_div.insert(1,parent_tag) return soup extra_css = ''' h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} - img#main_comic {max-width:100%; min-width:100%;} + img {max-width:100%; min-width:100%;} p{font-family:Arial,Helvetica,sans-serif;font-size:small;} body{font-family:Helvetica,Arial,sans-serif;font-size:small;} ''' From 4be922103a367d8820430ca1f7d3ab7981d09d3d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 08:46:18 -0600 Subject: [PATCH 25/90] ... --- src/calibre/gui2/metadata/basic_widgets.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index f918009a56..e8fc3766ee 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -467,16 +467,22 @@ class FormatsManager(QWidget): # {{{ self.metadata_from_format_button = QToolButton(self) self.metadata_from_format_button.setIcon(QIcon(I('edit_input.png'))) self.metadata_from_format_button.setIconSize(QSize(32, 32)) + self.metadata_from_format_button.setToolTip( + _('Set metadata for the book from the selected format')) self.add_format_button = QToolButton(self) self.add_format_button.setIcon(QIcon(I('add_book.png'))) self.add_format_button.setIconSize(QSize(32, 32)) self.add_format_button.clicked.connect(self.add_format) + self.add_format_button.setToolTip( + _('Add a format to this book')) self.remove_format_button = QToolButton(self) self.remove_format_button.setIcon(QIcon(I('trash.png'))) self.remove_format_button.setIconSize(QSize(32, 32)) self.remove_format_button.clicked.connect(self.remove_format) + self.remove_format_button.setToolTip( + _('Remove the selected format from this book')) self.formats = FormatList(self) self.formats.setAcceptDrops(True) From 0991d2ca3ccccf5a9226007e9fd8d7eafdcd624d Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Mon, 18 Apr 2011 22:53:19 +0800 Subject: [PATCH 26/90] get rid of test code that applied to the old plugin, set defaults --- .../ebooks/metadata/sources/overdrive.py | 40 +++---------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 1b237ad683..56a905de03 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -39,6 +39,11 @@ class OverDrive(Source): supports_gzip_transfer_encoding = False cached_cover_url_is_reliable = True + def __init__(self, *args, **kwargs): + Source.__init__(self, *args, **kwargs) + self.prefs.defaults['ignore_fields'] =['tags', 'pubdate', 'comments', 'identifier:isbn', 'language'] + + def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): ovrdrv_id = identifiers.get('overdrive', None) @@ -444,41 +449,6 @@ class OverDrive(Source): def main(args=sys.argv): - import tempfile, os, time - tdir = tempfile.gettempdir() - br = browser() - for ovrdrv_id, isbn, title, author in [ - #(None, '0899661343', 'On the Road', ['Jack Kerouac']), # basic test, no series, single author - #(None, '9780061952838', 'The Fellowship of the Ring', ['J. R. R. Tolkien']), # Series test, multi-author - #(None, '9780061952838', 'The Two Towers (The Lord of the Rings, Book II)', ['J. R. R. Tolkien']), # Series test, book 2 - #(None, '9780618153985', 'The Fellowship of the Ring (The Lord of the Rings, Part 1)', ['J.R.R. Tolkien']), - #('57844706-20fa-4ace-b5ee-3470b1b52173', None, 'The Two Towers', ['J. R. R. Tolkien']), # Series test, w/ ovrdrv id - #(None, '9780345505057', 'Deluge', ['Anne McCaffrey']) # Multiple authors - #(None, None, 'Deluge', ['Anne McCaffrey']) # Empty ISBN - #(None, None, 'On the Road', ['Jack Kerouac']), # Nonetype ISBN - #(None, '9780345435279', 'A Caress of Twilight', ['Laurell K. Hamilton']), - #(None, '9780606087230', 'The Omnivore\'s Dilemma : A Natural History of Four Meals', ['Michael Pollan']), # Subtitle colon - #(None, '9780061747649', 'Mental_Floss Presents: Condensed Knowledge', ['Will Pearson', 'Mangesh Hattikudur']), - #(None, '9781400050802', 'The Zombie Survival Guide', ['Max Brooks']), # Two books with this title by this author - #(None, '9781775414315', 'The Worst Journey in the World / Antarctic 1910-1913', ['Apsley Cherry-Garrard']), # Garbage sub-title - #(None, '9780440335160', 'Outlander', ['Diana Gabaldon']), # Returns lots of results to sort through to get the best match - (None, '9780345509741', 'The Horror Stories of Robert E. Howard', ['Robert E. Howard']), # Complex title with initials/dots stripped, some results don't have a cover - ]: - cpath = os.path.join(tdir, title+'.jpg') - #print "cpath is "+cpath - st = time.time() - curl = get_cover_url(isbn, title, author, br, ovrdrv_id) - #print '\n\n Took ', time.time() - st, ' to get basic metadata\n\n' - #if curl is None: - # print 'No cover found for', title - #else: - # print "curl is "+curl - # open(cpath, 'wb').write(br.open_novisit(curl).read()) - # print 'Cover for', title, 'saved to', cpath - st = time.time() - #print get_social_metadata(title, author, isbn, ovrdrv_id) - #print '\n\n Took ', time.time() - st, ' to get detailed metadata\n\n' - return 0 if __name__ == '__main__': From 4fdb7163b67cd26bd9901fd040e5861bfe9fc508 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 08:54:38 -0600 Subject: [PATCH 27/90] ... --- recipes/arcamax.recipe | 4 ---- 1 file changed, 4 deletions(-) diff --git a/recipes/arcamax.recipe b/recipes/arcamax.recipe index bcd468307e..db4d753cef 100644 --- a/recipes/arcamax.recipe +++ b/recipes/arcamax.recipe @@ -94,14 +94,10 @@ class Arcamax(BasicNewsRecipe): page_soup = self.index_to_soup(url) if page_soup: title = page_soup.find(name='div', attrs={'class':'comics-header'}).h1.contents[0] - print 'title is: ', title page_url = url - print 'url is: ', url # orig prev_page_url = 'http://www.arcamax.com' + page_soup.find('a', attrs={'class':'prev'}, text='Previous').parent['href'] prev_page_url = 'http://www.arcamax.com' + page_soup.find('span', text='Previous').parent.parent['href'] - print 'prev_page_url is: ', prev_page_url date = self.tag_to_string(page_soup.find(name='b', attrs={'class':['current']})) - print 'date is: ', date current_articles.append({'title': title, 'url': page_url, 'description':'', 'date': date}) url = prev_page_url current_articles.reverse() From c48ba61a4699780431a5d1a82ca361ad554fe62e Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 09:02:20 -0600 Subject: [PATCH 28/90] When displaying the list of identifiers in the edit metadata dialog, always display isbn first --- src/calibre/gui2/metadata/basic_widgets.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index e8fc3766ee..1fc84776df 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -947,7 +947,13 @@ class IdentifiersEdit(QLineEdit): # {{{ def fset(self, val): if not val: val = {} - txt = ', '.join(['%s:%s'%(k, v) for k, v in val.iteritems()]) + def keygen(x): + x = x[0] + if x == 'isbn': + x = '00isbn' + return x + ids = sorted(val.iteritems(), key=keygen) + txt = ', '.join(['%s:%s'%(k, v) for k, v in ids]) self.setText(txt.strip()) self.setCursorPosition(0) return property(fget=fget, fset=fset) From 724c549ac8cff4196ab5089bd77b130206a7116c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 09:05:19 -0600 Subject: [PATCH 29/90] ... --- src/calibre/gui2/metadata/basic_widgets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index 1fc84776df..ac41a72def 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -282,7 +282,7 @@ class AuthorSortEdit(EnLineEdit): if aus: ln, _, rest = aus.partition(',') if rest: - if meth in ('invert', 'nocomma'): + if meth in ('invert', 'nocomma', 'comma'): aus = rest.strip() + ' ' + ln.strip() self.authors_edit.current_val = [aus] From 7281c8de791e5dec9f984f6307b80840bad346ca Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 09:48:23 -0600 Subject: [PATCH 30/90] On linux ensure the cover is always drawn on top of the selection rect in the cover download dialog --- src/calibre/gui2/metadata/single_download.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index 8f01c6df1e..7e30f02420 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -30,6 +30,7 @@ from calibre.ebooks.metadata.book.base import Metadata from calibre.gui2 import error_dialog, NONE from calibre.utils.date import utcnow, fromordinal, format_date from calibre.library.comments import comments_to_html +from calibre.constants import islinux from calibre import force_unicode # }}} @@ -116,6 +117,12 @@ class CoverDelegate(QStyledItemDelegate): # {{{ def paint(self, painter, option, index): QStyledItemDelegate.paint(self, painter, option, index) + if islinux: + # On linux for some reason the selected color is drawn on top of + # the decoration + style = QApplication.style() + style.drawItemPixmap(painter, option.rect, Qt.AlignTop|Qt.AlignHCenter, + QPixmap(index.data(Qt.DecorationRole))) if self.timer.isActive() and index.data(Qt.UserRole).toBool(): rect = QRect(0, 0, self.spinner_width, self.spinner_width) rect.moveCenter(option.rect.center()) @@ -945,7 +952,7 @@ class CoverFetch(QDialog): # {{{ # }}} if __name__ == '__main__': - #DEBUG_DIALOG = True + DEBUG_DIALOG = True app = QApplication([]) d = FullFetch() d.start(title='great gatsby', authors=['fitzgerald']) From ba71b95f789a64405c5bd365aaa107dd608c5693 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 10:07:41 -0600 Subject: [PATCH 31/90] Add button to clear identifiers edit --- src/calibre/gui2/metadata/basic_widgets.py | 2 +- src/calibre/gui2/metadata/single.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index ac41a72def..593a3839ac 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -973,7 +973,7 @@ class IdentifiersEdit(QLineEdit): # {{{ tt = self.BASE_TT extra = '' if not isbn: - col = 'rgba(0,255,0,0%)' + col = 'none' elif check_isbn(isbn) is not None: col = 'rgba(0,255,0,20%)' extra = '\n\n'+_('This ISBN number is valid') diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 52b9e99872..495c89532c 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -156,6 +156,9 @@ class MetadataSingleDialogBase(ResizableDialog): self.identifiers = IdentifiersEdit(self) self.basic_metadata_widgets.append(self.identifiers) + self.clear_identifiers_button = QToolButton(self) + self.clear_identifiers_button.setIcon(QIcon(I('trash.png'))) + self.clear_identifiers_button.clicked.connect(self.identifiers.clear) self.publisher = PublisherEdit(self) self.basic_metadata_widgets.append(self.publisher) @@ -541,8 +544,8 @@ class MetadataSingleDialog(MetadataSingleDialogBase): # {{{ sto(self.rating, self.tags) create_row2(2, self.tags, self.tags_editor_button) sto(self.tags_editor_button, self.identifiers) - create_row2(3, self.identifiers) - sto(self.identifiers, self.timestamp) + create_row2(3, self.identifiers, self.clear_identifiers_button) + sto(self.clear_identifiers_button, self.timestamp) create_row2(4, self.timestamp, self.timestamp.clear_button) sto(self.timestamp.clear_button, self.pubdate) create_row2(5, self.pubdate, self.pubdate.clear_button) From b368790f138fc2a52d61a021f77d274e907fcba5 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 17:16:37 +0100 Subject: [PATCH 32/90] Add clear identifiers button to alt layout --- src/calibre/gui2/metadata/single.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 495c89532c..d527dda022 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -660,7 +660,8 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{ create_row(9, self.publisher, self.timestamp) create_row(10, self.timestamp, self.identifiers, button=self.timestamp.clear_button, icon='trash.png') - create_row(11, self.identifiers, self.comments) + create_row(11, self.identifiers, self.comments, + button=self.clear_identifiers_button, icon='trash.png') tl.addItem(QSpacerItem(1, 1, QSizePolicy.Fixed, QSizePolicy.Expanding), 12, 1, 1 ,1) From 9278da958c420cfe4cfaeb37b5302ccef8d1c358 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 11:06:19 -0600 Subject: [PATCH 33/90] Bulk metadata download: Make the confirm dialog more useful --- src/calibre/gui2/actions/edit_metadata.py | 10 +-- src/calibre/gui2/metadata/bulk_download2.py | 81 +++++++++++++++++---- 2 files changed, 71 insertions(+), 20 deletions(-) diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 18a73fb282..9d4d3891ca 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -37,8 +37,6 @@ class EditMetadataAction(InterfaceAction): md.addSeparator() if test_eight_code: dall = self.download_metadata - dident = partial(self.download_metadata, covers=False) - dcovers = partial(self.download_metadata, identify=False) else: dall = partial(self.download_metadata_old, False, covers=True) dident = partial(self.download_metadata_old, False, covers=False) @@ -47,9 +45,9 @@ class EditMetadataAction(InterfaceAction): md.addAction(_('Download metadata and covers'), dall, Qt.ControlModifier+Qt.Key_D) - md.addAction(_('Download only metadata'), dident) - md.addAction(_('Download only covers'), dcovers) if not test_eight_code: + md.addAction(_('Download only metadata'), dident) + md.addAction(_('Download only covers'), dcovers) md.addAction(_('Download only social metadata'), partial(self.download_metadata_old, False, covers=False, set_metadata=False, set_social_metadata=True)) @@ -80,7 +78,7 @@ class EditMetadataAction(InterfaceAction): self.qaction.setEnabled(enabled) self.action_merge.setEnabled(enabled) - def download_metadata(self, identify=True, covers=True, ids=None): + def download_metadata(self, ids=None): if ids is None: rows = self.gui.library_view.selectionModel().selectedRows() if not rows or len(rows) == 0: @@ -90,7 +88,7 @@ class EditMetadataAction(InterfaceAction): ids = [db.id(row.row()) for row in rows] from calibre.gui2.metadata.bulk_download2 import start_download start_download(self.gui, ids, - Dispatcher(self.bulk_metadata_downloaded), identify, covers) + Dispatcher(self.bulk_metadata_downloaded)) def bulk_metadata_downloaded(self, job): if job.failed: diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 5f0af1b316..11cbc65680 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -12,7 +12,8 @@ from functools import partial from itertools import izip from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, QSize, - QDialogButtonBox, QApplication, QTimer, QLabel, QProgressBar) + QDialogButtonBox, QApplication, QTimer, QLabel, QProgressBar, + QGridLayout, QPixmap, Qt) from calibre.gui2.dialogs.message_box import MessageBox from calibre.gui2.threaded_jobs import ThreadedJob @@ -25,37 +26,86 @@ from calibre.ebooks.metadata.book.base import Metadata from calibre.customize.ui import metadata_plugins from calibre.ptempfile import PersistentTemporaryFile +# Start download {{{ def show_config(gui, parent): from calibre.gui2.preferences import show_config_widget show_config_widget('Sharing', 'Metadata download', parent=parent, gui=gui, never_shutdown=True) -def start_download(gui, ids, callback, identify, covers): - q = MessageBox(MessageBox.QUESTION, _('Schedule download?'), +class ConfirmDialog(QDialog): + + def __init__(self, ids, parent): + QDialog.__init__(self, parent) + self.setWindowTitle(_('Schedule download?')) + self.setWindowIcon(QIcon(I('dialog_question.png'))) + + l = self.l = QGridLayout() + self.setLayout(l) + + i = QLabel(self) + i.setPixmap(QPixmap(I('dialog_question.png'))) + l.addWidget(i, 0, 0) + + t = QLabel( '<p>'+_('The download of metadata for the <b>%d selected book(s)</b> will' ' run in the background. Proceed?')%len(ids) + '<p>'+_('You can monitor the progress of the download ' 'by clicking the rotating spinner in the bottom right ' 'corner.') + '<p>'+_('When the download completes you will be asked for' - ' confirmation before calibre applies the downloaded metadata.'), - show_copy_button=False, parent=gui) - b = q.bb.addButton(_('Configure download'), q.bb.ActionRole) - b.setIcon(QIcon(I('config.png'))) - b.clicked.connect(partial(show_config, gui, q)) - q.det_msg_toggle.setVisible(False) + ' confirmation before calibre applies the downloaded metadata.') + ) + t.setWordWrap(True) + l.addWidget(t, 0, 1) + l.setColumnStretch(0, 1) + l.setColumnStretch(1, 100) - ret = q.exec_() - b.clicked.disconnect() - if ret != q.Accepted: + self.identify = self.covers = True + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) + self.bb.rejected.connect(self.reject) + b = self.bb.addButton(_('Download only metadata'), + self.bb.AcceptRole) + b.clicked.connect(self.only_metadata) + b.setIcon(QIcon(I('edit_input.png'))) + b = self.bb.addButton(_('Download only covers'), + self.bb.AcceptRole) + b.clicked.connect(self.only_covers) + b.setIcon(QIcon(I('default_cover.png'))) + b = self.b = self.bb.addButton(_('Configure download'), self.bb.ActionRole) + b.setIcon(QIcon(I('config.png'))) + b.clicked.connect(partial(show_config, parent, self)) + l.addWidget(self.bb, 1, 0, 1, 2) + b = self.bb.addButton(_('Download both'), + self.bb.AcceptRole) + b.clicked.connect(self.accept) + b.setDefault(True) + b.setAutoDefault(True) + b.setIcon(QIcon(I('ok.png'))) + + self.resize(self.sizeHint()) + b.setFocus(Qt.OtherFocusReason) + + def only_metadata(self): + self.covers = False + self.accept() + + def only_covers(self): + self.identify = False + self.accept() + +def start_download(gui, ids, callback): + d = ConfirmDialog(ids, gui) + ret = d.exec_() + d.b.clicked.disconnect() + if ret != d.Accepted: return job = ThreadedJob('metadata bulk download', _('Download metadata for %d books')%len(ids), - download, (ids, gui.current_db, identify, covers), {}, callback) + download, (ids, gui.current_db, d.identify, d.covers), {}, callback) gui.job_manager.run_threaded_job(job) gui.status_bar.show_message(_('Metadata download started'), 3000) - +# }}} class ViewLog(QDialog): # {{{ @@ -93,6 +143,7 @@ def view_log(job, parent): # }}} +# Apply downloaded metadata {{{ class ApplyDialog(QDialog): def __init__(self, id_map, gui): @@ -248,6 +299,8 @@ def proceed(gui, job): q.show() q.finished.connect(partial(apply_metadata, job, gui, q)) +# }}} + def merge_result(oldmi, newmi): dummy = Metadata(_('Unknown')) for f in msprefs['ignore_fields']: From ab1ad20dba92e3be931a80ae2eaf50625341564c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 11:11:11 -0600 Subject: [PATCH 34/90] ... --- src/calibre/gui2/metadata/bulk_download2.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 11cbc65680..a95c8b52c7 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -63,19 +63,19 @@ class ConfirmDialog(QDialog): self.identify = self.covers = True self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) self.bb.rejected.connect(self.reject) - b = self.bb.addButton(_('Download only metadata'), + b = self.bb.addButton(_('Download only &metadata'), self.bb.AcceptRole) b.clicked.connect(self.only_metadata) b.setIcon(QIcon(I('edit_input.png'))) - b = self.bb.addButton(_('Download only covers'), + b = self.bb.addButton(_('Download only &covers'), self.bb.AcceptRole) b.clicked.connect(self.only_covers) b.setIcon(QIcon(I('default_cover.png'))) - b = self.b = self.bb.addButton(_('Configure download'), self.bb.ActionRole) + b = self.b = self.bb.addButton(_('&Configure download'), self.bb.ActionRole) b.setIcon(QIcon(I('config.png'))) b.clicked.connect(partial(show_config, parent, self)) l.addWidget(self.bb, 1, 0, 1, 2) - b = self.bb.addButton(_('Download both'), + b = self.bb.addButton(_('Download &both'), self.bb.AcceptRole) b.clicked.connect(self.accept) b.setDefault(True) From 8611632ea4a110f0b2003b994dd05bab96d29597 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 11:51:09 -0600 Subject: [PATCH 35/90] Nicer implementation of apply metadata dialog --- src/calibre/gui2/metadata/bulk_download2.py | 40 ++++++++++++--------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index a95c8b52c7..4aa4561078 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -146,7 +146,7 @@ def view_log(job, parent): # Apply downloaded metadata {{{ class ApplyDialog(QDialog): - def __init__(self, id_map, gui): + def __init__(self, gui): QDialog.__init__(self, gui) self.l = l = QVBoxLayout() @@ -155,27 +155,33 @@ class ApplyDialog(QDialog): self.pb = QProgressBar(self) l.addWidget(self.pb) - self.pb.setMinimum(0) - self.pb.setMaximum(len(id_map)) self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) self.bb.rejected.connect(self.reject) - self.bb.accepted.connect(self.accept) l.addWidget(self.bb) self.gui = gui + self.timer = QTimer(self) + self.timer.timeout.connect(self.do_one) + + def start(self, id_map): self.id_map = list(id_map.iteritems()) self.current_idx = 0 - self.failures = [] self.ids = [] self.canceled = False - - QTimer.singleShot(20, self.do_one) + self.pb.setMinimum(0) + self.pb.setMaximum(len(id_map)) + self.timer.start(50) def do_one(self): if self.canceled: return + if self.current_idx >= len(self.id_map): + self.timer.stop() + self.finalize() + return + i, mi = self.id_map[self.current_idx] db = self.gui.current_db try: @@ -195,15 +201,11 @@ class ApplyDialog(QDialog): pass self.pb.setValue(self.pb.value()+1) - - if self.current_idx >= len(self.id_map) - 1: - self.finalize() - else: - self.current_idx += 1 - QTimer.singleShot(20, self.do_one) + self.current_idx += 1 def reject(self): self.canceled = True + self.timer.stop() QDialog.reject(self) def finalize(self): @@ -220,17 +222,18 @@ class ApplyDialog(QDialog): title += ' - ' + authors_to_string(authors) msg.append(title+'\n\n'+tb+'\n'+('*'*80)) - error_dialog(self, _('Some failures'), + parent = self if self.isVisible() else self.parent() + error_dialog(parent, _('Some failures'), _('Failed to apply updated metadata for some books' ' in your library. Click "Show Details" to see ' 'details.'), det_msg='\n\n'.join(msg), show=True) - self.accept() if self.ids: cr = self.gui.library_view.currentIndex().row() self.gui.library_view.model().refresh_ids( self.ids, cr) if self.gui.cover_flow: self.gui.cover_flow.dataChanged() + self.accept() _amd = None def apply_metadata(job, gui, q, result): @@ -268,8 +271,11 @@ def apply_metadata(job, gui, q, result): 'Do you want to proceed?'), det_msg='\n'.join(modified)): return - _amd = ApplyDialog(id_map, gui) - _amd.exec_() + if _amd is None: + _amd = ApplyDialog(gui) + _amd.start(id_map) + if len(id_map) > 3: + _amd.exec_() def proceed(gui, job): gui.status_bar.show_message(_('Metadata download completed'), 3000) From 204b95289276a73ff54a3352c43a829f702602d1 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 19:23:53 +0100 Subject: [PATCH 36/90] Add signal to indicate that the metadata for books on the connected device is available --- src/calibre/gui2/device.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 4d4f66eab1..f012028c8a 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -607,6 +607,8 @@ class DeviceMenu(QMenu): # {{{ class DeviceMixin(object): # {{{ + device_metadata_available = pyqtSignal() + def __init__(self): self.device_error_dialog = error_dialog(self, _('Error'), _('Error communicating with device'), ' ') @@ -791,6 +793,7 @@ class DeviceMixin(object): # {{{ self.sync_news() self.sync_catalogs() self.refresh_ondevice() + self.device_metadata_available.emit() def refresh_ondevice(self, reset_only = False): ''' @@ -892,7 +895,7 @@ class DeviceMixin(object): # {{{ sub_dest_parts.append('') to = sub_dest_parts[0] fmts = sub_dest_parts[1] - subject = ';'.join(sub_dest_parts[2:]) + subject = ';'.join(sub_dest_parts[2:]) fmts = [x.strip().lower() for x in fmts.split(',')] self.send_by_mail(to, fmts, delete, subject=subject) From b0ec35f0d310d861aa72a423a6337acd0bb25da5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 14:06:07 -0600 Subject: [PATCH 37/90] ... --- src/calibre/gui2/metadata/single_download.py | 11 ++++------- src/calibre/gui2/preferences/__init__.py | 8 +++++++- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/calibre/gui2/metadata/single_download.py b/src/calibre/gui2/metadata/single_download.py index 7e30f02420..c4e13a90f8 100644 --- a/src/calibre/gui2/metadata/single_download.py +++ b/src/calibre/gui2/metadata/single_download.py @@ -30,7 +30,6 @@ from calibre.ebooks.metadata.book.base import Metadata from calibre.gui2 import error_dialog, NONE from calibre.utils.date import utcnow, fromordinal, format_date from calibre.library.comments import comments_to_html -from calibre.constants import islinux from calibre import force_unicode # }}} @@ -117,12 +116,10 @@ class CoverDelegate(QStyledItemDelegate): # {{{ def paint(self, painter, option, index): QStyledItemDelegate.paint(self, painter, option, index) - if islinux: - # On linux for some reason the selected color is drawn on top of - # the decoration - style = QApplication.style() - style.drawItemPixmap(painter, option.rect, Qt.AlignTop|Qt.AlignHCenter, - QPixmap(index.data(Qt.DecorationRole))) + # Ensure the cover is rendered over any selection rect + style = QApplication.style() + style.drawItemPixmap(painter, option.rect, Qt.AlignTop|Qt.AlignHCenter, + QPixmap(index.data(Qt.DecorationRole))) if self.timer.isActive() and index.data(Qt.UserRole).toBool(): rect = QRect(0, 0, self.spinner_width, self.spinner_width) rect.moveCenter(option.rect.center()) diff --git a/src/calibre/gui2/preferences/__init__.py b/src/calibre/gui2/preferences/__init__.py index 649a58448d..5b0a05ba40 100644 --- a/src/calibre/gui2/preferences/__init__.py +++ b/src/calibre/gui2/preferences/__init__.py @@ -337,7 +337,13 @@ def show_config_widget(category, name, gui=None, show_restart_msg=False, bb.button(bb.RestoreDefaults).setEnabled(w.supports_restoring_to_defaults) bb.button(bb.Apply).setEnabled(False) bb.button(bb.Apply).clicked.connect(d.accept) - w.changed_signal.connect(lambda : bb.button(bb.Apply).setEnabled(True)) + def onchange(): + b = bb.button(bb.Apply) + b.setEnabled(True) + b.setDefault(True) + b.setAutoDefault(True) + w.changed_signal.connect(onchange) + bb.button(bb.Cancel).setFocus(True) l = QVBoxLayout() d.setLayout(l) l.addWidget(w) From 97c5bf39c13ec466712869526bc82d9f4566ef62 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 14:08:03 -0600 Subject: [PATCH 38/90] ... --- src/calibre/devices/android/driver.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index 44d9bc1e49..7fe246f450 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -108,10 +108,10 @@ class ANDROID(USBMS): 'SGH-T849', '_MB300', 'A70S', 'S_ANDROID', 'A101IT', 'A70H', 'IDEOS_TABLET', 'MYTOUCH_4G', 'UMS_COMPOSITE', 'SCH-I800_CARD', '7', 'A956', 'A955', 'A43', 'ANDROID_PLATFORM', 'TEGRA_2', - 'MB860', 'MULTI-CARD', 'MID7015A'] + 'MB860', 'MULTI-CARD', 'MID7015A', 'INCREDIBLE'] WINDOWS_CARD_A_MEM = ['ANDROID_PHONE', 'GT-I9000_CARD', 'SGH-I897', 'FILE-STOR_GADGET', 'SGH-T959', 'SAMSUNG_ANDROID', 'GT-P1000_CARD', - 'A70S', 'A101IT', '7'] + 'A70S', 'A101IT', '7', 'INCREDIBLE'] OSX_MAIN_MEM = 'Android Device Main Memory' From 1267df3a69db2c048be36065f89d1d2d1132caa9 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 21:16:01 +0100 Subject: [PATCH 39/90] Add the device_connection_changed signal --- src/calibre/gui2/device.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index f012028c8a..8b0e6eefef 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -608,6 +608,7 @@ class DeviceMenu(QMenu): # {{{ class DeviceMixin(object): # {{{ device_metadata_available = pyqtSignal() + device_connection_changed = pyqtSignal(object) def __init__(self): self.device_error_dialog = error_dialog(self, _('Error'), @@ -755,6 +756,7 @@ class DeviceMixin(object): # {{{ self.location_manager.update_devices() self.library_view.set_device_connected(self.device_connected) self.refresh_ondevice() + self.device_connection_changed.emit(connected) def info_read(self, job): ''' From b79faeff5691fb11e110c24bad69cb60fc05ce82 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 21:38:27 +0100 Subject: [PATCH 40/90] Change author_sort_copy_method default from invert to comma. --- resources/default_tweaks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index c4c951f980..091aa9a34d 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -48,7 +48,7 @@ authors_completer_append_separator = False # When this tweak is changed, the author_sort values stored with each author # must be recomputed by right-clicking on an author in the left-hand tags pane, # selecting 'manage authors', and pressing 'Recalculate all author sort values'. -author_sort_copy_method = 'invert' +author_sort_copy_method = 'comma' #: Use author sort in Tag Browser # Set which author field to display in the tags pane (the list of authors, From 3e0c43283d54a36c2574801070171cee33d974af Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Mon, 18 Apr 2011 21:43:10 +0100 Subject: [PATCH 41/90] Change author_sort_copy_method default from invert to comma. --- resources/default_tweaks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/default_tweaks.py b/resources/default_tweaks.py index c4c951f980..091aa9a34d 100644 --- a/resources/default_tweaks.py +++ b/resources/default_tweaks.py @@ -48,7 +48,7 @@ authors_completer_append_separator = False # When this tweak is changed, the author_sort values stored with each author # must be recomputed by right-clicking on an author in the left-hand tags pane, # selecting 'manage authors', and pressing 'Recalculate all author sort values'. -author_sort_copy_method = 'invert' +author_sort_copy_method = 'comma' #: Use author sort in Tag Browser # Set which author field to display in the tags pane (the list of authors, From 8d174eaffdfcda971885b80e0705bd221ea11f79 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 18 Apr 2011 20:56:56 -0600 Subject: [PATCH 42/90] ... --- src/calibre/ebooks/metadata/sources/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 37407a0656..86a9fe1133 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -377,8 +377,9 @@ class Source(Plugin): This URL must be browseable to by a human using a browser. It is meant to provide a clickable link for the user to easily visit the books page at this source. - If no URL is found, return None. This method must be quick, either it - should construct the URL using a known URL scheme or use a cached URL. + If no URL is found, return None. This method must be quick, and + consistent, so only implement it if it is possible to construct the URL + from a known scheme given identifiers. ''' return None From acb26328c8d36921dfe6aff0ef0361f3e42b0924 Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Tue, 19 Apr 2011 12:29:08 +0800 Subject: [PATCH 43/90] handle referer header when overdrive ID is unknown --- src/calibre/ebooks/metadata/sources/overdrive.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 26c90f08fe..2443836b37 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -23,7 +23,6 @@ from calibre.ebooks.chardet import xml_to_unicode from calibre.library.comments import sanitize_comments_html ovrdrv_data_cache = {} -cover_url_cache = {} cache_lock = RLock() base_url = 'http://search.overdrive.com/' @@ -100,8 +99,10 @@ class OverDrive(Source): ovrdrv_id = identifiers.get('overdrive', None) br = self.browser - referer = self.get_base_referer()+'ContentDetails-Cover.htm?ID='+ovrdrv_id req = mechanize.Request(cached_url) + if ovrdrv_id is not None: + referer = self.get_base_referer()+'ContentDetails-Cover.htm?ID='+ovrdrv_id + req.add_header('referer', referer) req.add_header('referer', referer) log('Downloading cover from:', cached_url) try: From 112168f3f69b07251b8979a8a0b722245d0e7e5a Mon Sep 17 00:00:00 2001 From: Lee <ldolse@yahoo.com> Date: Tue, 19 Apr 2011 12:53:03 +0800 Subject: [PATCH 44/90] check for image in exact match scenario --- src/calibre/ebooks/metadata/sources/overdrive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 2443836b37..39fa2bc6ea 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -245,7 +245,7 @@ class OverDrive(Source): else: creators = creators.split(', ') # if an exact match in a preferred format occurs - if (author and creators[0] == author[0]) and od_title == title and int(formatid) in [1, 50, 410, 900]: + if (author and creators[0] == author[0]) and od_title == title and int(formatid) in [1, 50, 410, 900] and thumbimage: return self.format_results(reserveid, od_title, subtitle, series, publisher, creators, thumbimage, worldcatlink, formatid) else: From 5312b49d85132787bc4f18d91fa7a8230561e298 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 19 Apr 2011 08:59:53 +0100 Subject: [PATCH 45/90] Divide ratings by 2 when used in a template. --- src/calibre/ebooks/metadata/book/base.py | 2 ++ src/calibre/library/save_to_disk.py | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index 167ae52fa3..57c267319f 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -625,6 +625,8 @@ class Metadata(object): res = res + ' [%s]'%self.format_series_index() elif datatype == 'datetime': res = format_date(res, fmeta['display'].get('date_format','dd MMM yyyy')) + elif datatype == 'rating': + res = res/2 return (name, unicode(res), orig_res, fmeta) return (None, None, None, None) diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 3c57af40a8..42e6c8b156 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -198,7 +198,6 @@ def get_components(template, mi, id, timefmt='%b %Y', length=250, for key in custom_metadata: if key in format_args: cm = custom_metadata[key] - ## TODO: NEWMETA: should ratings be divided by 2? The standard rating isn't... if cm['datatype'] == 'series': format_args[key] = title_sort(format_args[key], order=tsorder) if key+'_index' in format_args: From 1e1b530113636d344c59e2c93fdabc3ca974a5cf Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 19 Apr 2011 09:46:43 +0100 Subject: [PATCH 46/90] Add the ability to get the calibre field 'size' using a formatter function booksize() --- src/calibre/library/database2.py | 1 + src/calibre/manual/template_lang.rst | 1 + src/calibre/utils/formatter_functions.py | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index bdcefd13a2..d585c60aef 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -853,6 +853,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.pubdate = row[fm['pubdate']] mi.uuid = row[fm['uuid']] mi.title_sort = row[fm['sort']] + mi.book_size = row[fm['size']] mi.last_modified = row[fm['last_modified']] formats = row[fm['formats']] if not formats: diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index cdb8df2e2b..a77f0d1697 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -230,6 +230,7 @@ The following functions are available in addition to those described in single-f * ``add(x, y)`` -- returns x + y. Throws an exception if either x or y are not numbers. * ``assign(id, val)`` -- assigns val to id, then returns val. id must be an identifier, not an expression + * ``booksize()`` -- returns the value of the |app| 'size' field. Returns '' if there are no formats. * ``cmp(x, y, lt, eq, gt)`` -- compares x and y after converting both to numbers. Returns ``lt`` if x < y. Returns ``eq`` if x == y. Otherwise returns ``gt``. * ``divide(x, y)`` -- returns x / y. Throws an exception if either x or y are not numbers. * ``field(name)`` -- returns the metadata field named by ``name``. diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 7957bd0749..aa8e4fb3a3 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -549,8 +549,22 @@ class BuiltinCapitalize(BuiltinFormatterFunction): def evaluate(self, formatter, kwargs, mi, locals, val): return capitalize(val) +class BuiltinBooksize(BuiltinFormatterFunction): + name = 'booksize' + arg_count = 0 + doc = _('booksize() -- return value of the field capitalized') + + def evaluate(self, formatter, kwargs, mi, locals): + if mi.book_size is not None: + try: + return str(mi.book_size) + except: + pass + return '' + builtin_add = BuiltinAdd() builtin_assign = BuiltinAssign() +builtin_booksize = BuiltinBooksize() builtin_capitalize = BuiltinCapitalize() builtin_cmp = BuiltinCmp() builtin_contains = BuiltinContains() From d95d6eca67f7fc6f6b06cf2ab8d70477ffec9048 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 19 Apr 2011 10:07:10 +0100 Subject: [PATCH 47/90] Improve performance of get_categories. --- src/calibre/library/database2.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index d585c60aef..bc0a8235e4 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -1379,13 +1379,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): for (cat, dex, mult, is_comp) in md: if not book[dex]: continue + tid_cat = tids[cat] + tcats_cat = tcategories[cat] if not mult: val = book[dex] if is_comp: - item = tcategories[cat].get(val, None) + item = tcats_cat.get(val, None) if not item: item = tag_class(val, val) - tcategories[cat][val] = item + tcats_cat[val] = item item.c += 1 item.id = val if rating > 0: @@ -1393,11 +1395,11 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): item.rc += 1 continue try: - (item_id, sort_val) = tids[cat][val] # let exceptions fly - item = tcategories[cat].get(val, None) + (item_id, sort_val) = tid_cat[val] # let exceptions fly + item = tcats_cat.get(val, None) if not item: item = tag_class(val, sort_val) - tcategories[cat][val] = item + tcats_cat[val] = item item.c += 1 item.id_set.add(book[0]) item.id = item_id @@ -1411,21 +1413,15 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if is_comp: vals = [v.strip() for v in vals if v.strip()] for val in vals: - if val not in tids: - tids[cat][val] = (val, val) - item = tcategories[cat].get(val, None) - if not item: - item = tag_class(val, val) - tcategories[cat][val] = item - item.c += 1 - item.id = val + if val not in tid_cat: + tid_cat[val] = (val, val) for val in vals: try: - (item_id, sort_val) = tids[cat][val] # let exceptions fly - item = tcategories[cat].get(val, None) + (item_id, sort_val) = tid_cat[val] # let exceptions fly + item = tcats_cat.get(val, None) if not item: item = tag_class(val, sort_val) - tcategories[cat][val] = item + tcats_cat[val] = item item.c += 1 item.id_set.add(book[0]) item.id = item_id From b4dda2792357736ec155412f46facce563afb274 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 19 Apr 2011 13:42:09 +0100 Subject: [PATCH 48/90] Very strange change to prevent calibre from dieing when device_metadata_available or device_connection_changed is emitted. --- src/calibre/gui2/device.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index 99cb5848ba..d00aa031f5 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -7,7 +7,7 @@ import os, traceback, Queue, time, cStringIO, re, sys from threading import Thread from PyQt4.Qt import QMenu, QAction, QActionGroup, QIcon, SIGNAL, \ - Qt, pyqtSignal, QDialog + Qt, pyqtSignal, QDialog, QObject from calibre.customize.ui import available_input_formats, available_output_formats, \ device_plugins @@ -587,8 +587,7 @@ class DeviceMenu(QMenu): # {{{ # }}} -class DeviceMixin(object): # {{{ - +class DeviceSignals(QObject): #: This signal is emitted once, after metadata is downloaded from the #: connected device. #: The sequence: gui.device_manager.is_device_connected will become True, @@ -599,6 +598,10 @@ class DeviceMixin(object): # {{{ device_metadata_available = pyqtSignal() device_connection_changed = pyqtSignal(object) +device_signals = DeviceSignals() + +class DeviceMixin(object): # {{{ + def __init__(self): self.device_error_dialog = error_dialog(self, _('Error'), _('Error communicating with device'), ' ') @@ -745,7 +748,7 @@ class DeviceMixin(object): # {{{ self.location_manager.update_devices() self.library_view.set_device_connected(self.device_connected) self.refresh_ondevice() - self.device_connection_changed.emit(connected) + device_signals.device_connection_changed.emit(connected) def info_read(self, job): ''' @@ -784,7 +787,7 @@ class DeviceMixin(object): # {{{ self.sync_news() self.sync_catalogs() self.refresh_ondevice() - self.device_metadata_available.emit() + device_signals.device_metadata_available.emit() def refresh_ondevice(self, reset_only = False): ''' From d7f17e9a1975af262e2c894bfa0815cd04fdb9c1 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 19 Apr 2011 14:28:59 +0100 Subject: [PATCH 49/90] Improvements to using the current search as a restriction --- src/calibre/gui2/layout.py | 2 -- src/calibre/gui2/search_restriction_mixin.py | 33 ++++++++++++++------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index c72b074463..7250103615 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -156,8 +156,6 @@ class SearchBar(QWidget): # {{{ x = ComboBoxWithHelp(self) x.setMaximumSize(QSize(150, 16777215)) x.setObjectName("search_restriction") - x.setToolTip(_('Books display will be restricted to those matching the ' - 'selected saved search')) l.addWidget(x) parent.search_restriction = x diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 8ef02b34b0..ffebc9e131 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -17,6 +17,10 @@ class SearchRestrictionMixin(object): self.search_restriction.setMinimumContentsLength(10) self.search_restriction.setStatusTip(self.search_restriction.toolTip()) self.search_count.setText(_("(all books)")) + self.search_restriction_tooltip = \ + _('Books display will be restricted to those matching a ' + 'selected saved search') + self.search_restriction.setToolTip(self.search_restriction_tooltip) def apply_named_search_restriction(self, name): if not name: @@ -30,29 +34,38 @@ class SearchRestrictionMixin(object): self.apply_search_restriction(r) def apply_text_search_restriction(self, search): + search = unicode(search) if not search: - self.search_restriction.setItemText(1, _('*Current search')) self.search_restriction.setCurrentIndex(0) else: - self.search_restriction.setCurrentIndex(1) - self.search_restriction.setItemText(1, search) + s = '*' + search + if self.search_restriction.count() > 1: + txt = unicode(self.search_restriction.itemText(2)) + if txt.startswith('*'): + self.search_restriction.setItemText(2, s) + else: + self.search_restriction.insertItem(2, s) + else: + self.search_restriction.insertItem(2, s) + self.search_restriction.setCurrentIndex(2) + self.search_restriction.setToolTip('<p>' + + self.search_restriction_tooltip + + _(' or the search ') + "'" + search + "'</p>") self._apply_search_restriction(search) def apply_search_restriction(self, i): - self.search_restriction.setItemText(1, _('*Current search')) if i == 1: - restriction = unicode(self.search.currentText()) - if not restriction: - self.search_restriction.setCurrentIndex(0) - else: - self.search_restriction.setItemText(1, restriction) + self.apply_text_search_restriction(unicode(self.search.currentText())) + elif i == 2 and unicode(self.search_restriction.currentText()).startswith('*'): + self.apply_text_search_restriction( + unicode(self.search_restriction.currentText())[1:]) else: r = unicode(self.search_restriction.currentText()) if r is not None and r != '': restriction = 'search:"%s"'%(r) else: restriction = '' - self._apply_search_restriction(restriction) + self._apply_search_restriction(restriction) def _apply_search_restriction(self, restriction): self.saved_search.clear() From 640c4ff7848ffc7944b970f480dc493a4f4e1060 Mon Sep 17 00:00:00 2001 From: Charles Haley <> Date: Tue, 19 Apr 2011 15:02:37 +0100 Subject: [PATCH 50/90] Fix non-escaped '|' when searching for commas in authors using REGEXP_MATCH --- src/calibre/library/caches.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 9523795f28..92c5ca9b3c 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -707,7 +707,10 @@ class ResultCache(SearchQueryParser): # {{{ for loc in location: # location is now an array of field indices if loc == db_col['authors']: ### DB stores authors with commas changed to bars, so change query - q = query.replace(',', '|'); + if matchkind == REGEXP_MATCH: + q = query.replace(',', r'\|'); + else: + q = query.replace(',', '|'); else: q = query From ccaa7143b5e23785c683f568518d3b2942c270bc Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 19 Apr 2011 09:45:42 -0600 Subject: [PATCH 51/90] timing worker launches --- src/calibre/utils/ipc/server.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index e3b7bfd449..5bbc6bafc9 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -113,6 +113,7 @@ class Server(Thread): self.start() def launch_worker(self, gui=False, redirect_output=None): + #start = time.time() with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count @@ -136,6 +137,7 @@ class Server(Thread): break if isinstance(cw, basestring): raise CriticalError('Failed to launch worker process:\n'+cw) + #print 'Launch took:', time.time() - start return cw def do_launch(self, env, gui, redirect_output, rfile): From 6b52f4ad89a14c625d084050abda4ec222195aea Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 19 Apr 2011 11:24:51 -0600 Subject: [PATCH 52/90] Launch worker processes on demand --- src/calibre/gui2/__init__.py | 4 +++- src/calibre/gui2/preferences/misc.py | 14 +++++++++++--- src/calibre/gui2/preferences/misc.ui | 10 ++-------- src/calibre/manual/faq.rst | 11 ----------- src/calibre/utils/ipc/server.py | 25 +++++++------------------ 5 files changed, 23 insertions(+), 41 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 773aea3002..3ad8a81ffa 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -156,7 +156,9 @@ def _config(): c.add_opt('plugin_search_history', default=[], help='Search history for the recipe scheduler') c.add_opt('worker_limit', default=6, - help=_('Maximum number of waiting worker processes')) + help=_( + 'Maximum number of simultaneous conversion/news download jobs. ' + 'This number is twice the actual value for historical reasons.')) c.add_opt('get_social_metadata', default=True, help=_('Download social metadata (tags/rating/etc.)')) c.add_opt('overwrite_author_title_metadata', default=True, diff --git a/src/calibre/gui2/preferences/misc.py b/src/calibre/gui2/preferences/misc.py index 330332a716..ead5da4ce4 100644 --- a/src/calibre/gui2/preferences/misc.py +++ b/src/calibre/gui2/preferences/misc.py @@ -6,19 +6,27 @@ __copyright__ = '2010, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' -from calibre.gui2.preferences import ConfigWidgetBase, test_widget +from calibre.gui2.preferences import ConfigWidgetBase, test_widget, Setting from calibre.gui2.preferences.misc_ui import Ui_Form from calibre.gui2 import error_dialog, config, open_local_file, info_dialog from calibre.constants import isosx -# Check Integrity {{{ +class WorkersSetting(Setting): + + def set_gui_val(self, val): + val = val//2 + Setting.set_gui_val(self, val) + + def get_gui_val(self): + val = Setting.get_gui_val(self) + return val * 2 class ConfigWidget(ConfigWidgetBase, Ui_Form): def genesis(self, gui): self.gui = gui r = self.register - r('worker_limit', config, restart_required=True) + r('worker_limit', config, restart_required=True, setting=WorkersSetting) r('enforce_cpu_limit', config, restart_required=True) self.device_detection_button.clicked.connect(self.debug_device_detection) self.button_open_config_dir.clicked.connect(self.open_config_dir) diff --git a/src/calibre/gui2/preferences/misc.ui b/src/calibre/gui2/preferences/misc.ui index c036cb971b..8b0189b0a1 100644 --- a/src/calibre/gui2/preferences/misc.ui +++ b/src/calibre/gui2/preferences/misc.ui @@ -17,7 +17,7 @@ <item row="0" column="0"> <widget class="QLabel" name="label_5"> <property name="text"> - <string>&Maximum number of waiting worker processes (needs restart):</string> + <string>Max. simultaneous conversion/news download jobs:</string> </property> <property name="buddy"> <cstring>opt_worker_limit</cstring> @@ -27,13 +27,7 @@ <item row="0" column="1"> <widget class="QSpinBox" name="opt_worker_limit"> <property name="minimum"> - <number>2</number> - </property> - <property name="maximum"> - <number>10000</number> - </property> - <property name="singleStep"> - <number>2</number> + <number>1</number> </property> </widget> </item> diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index ef4da23826..3dce13f144 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -549,17 +549,6 @@ How do I run calibre from my USB stick? A portable version of calibre is available at: `portableapps.com <http://portableapps.com/node/20518>`_. However, this is usually out of date. You can also setup your own portable calibre install by following :ref:`these instructions <portablecalibre>`. -Why are there so many calibre-parallel processes on my system? -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -|app| maintains two separate worker process pools. One is used for adding books/saving to disk and the other for conversions. You can control the number of worker processes via :guilabel:`Preferences->Advanced->Miscellaneous`. So if you set it to 6 that means a maximum of 3 conversions will run simultaneously. And that is why you will see the number of worker processes changes by two when you use the up and down arrows. On windows, you can set the priority that these processes run with. This can be useful on older, single CPU machines, if you find them slowing down to a crawl when conversions are running. - -In addition to this some conversion plugins run tasks in their own pool of processes, so for example if you bulk convert comics, each comic conversion will use three separate processes to render the images. The job manager knows this so it will run only a single comic conversion simultaneously. - -And since I'm sure someone will ask: The reason adding/saving books are in separate processes is because of PDF. PDF processing libraries can crash on reading PDFs and I dont want the crash to take down all of calibre. Also when adding EPUB books, in order to extract the cover you have to sometimes render the HTML of the first page, which means that it either has to run in the GUI thread of the main process or in a separate process. - -Finally, the reason calibre keep workers alive and idle instead of launching on demand is to workaround the slow startup time of python processes. - How do I run parts of |app| like news download and the content server on my own linux server? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/calibre/utils/ipc/server.py b/src/calibre/utils/ipc/server.py index 5bbc6bafc9..ea6ce88ad6 100644 --- a/src/calibre/utils/ipc/server.py +++ b/src/calibre/utils/ipc/server.py @@ -17,7 +17,7 @@ from binascii import hexlify from calibre.utils.ipc.launch import Worker from calibre.utils.ipc.worker import PARALLEL_FUNCS from calibre import detect_ncpus as cpu_count -from calibre.constants import iswindows +from calibre.constants import iswindows, DEBUG from calibre.ptempfile import base_dir _counter = 0 @@ -106,14 +106,14 @@ class Server(Thread): self.add_jobs_queue, self.changed_jobs_queue = Queue(), Queue() self.kill_queue = Queue() self.waiting_jobs = [] - self.pool, self.workers = deque(), deque() + self.workers = deque() self.launched_worker_count = 0 self._worker_launch_lock = RLock() self.start() def launch_worker(self, gui=False, redirect_output=None): - #start = time.time() + start = time.time() with self._worker_launch_lock: self.launched_worker_count += 1 id = self.launched_worker_count @@ -137,7 +137,8 @@ class Server(Thread): break if isinstance(cw, basestring): raise CriticalError('Failed to launch worker process:\n'+cw) - #print 'Launch took:', time.time() - start + if DEBUG: + print 'Worker Launch took:', time.time() - start return cw def do_launch(self, env, gui, redirect_output, rfile): @@ -206,13 +207,6 @@ class Server(Thread): job.duration = time.time() - job.start_time self.changed_jobs_queue.put(job) - # Start new workers - if len(self.pool) + len(self.workers) < self.pool_size: - try: - self.pool.append(self.launch_worker()) - except Exception: - pass - # Start waiting jobs sj = self.suitable_waiting_job() if sj is not None: @@ -224,7 +218,7 @@ class Server(Thread): job.killed = job.failed = True job.result = None else: - worker = self.pool.pop() + worker = self.launch_worker() worker.start_job(job) self.workers.append(worker) job.log_path = worker.log_path @@ -238,7 +232,7 @@ class Server(Thread): break def suitable_waiting_job(self): - available_workers = len(self.pool) + available_workers = self.pool_size - len(self.workers) for worker in self.workers: job = worker.job if job.core_usage == -1: @@ -304,11 +298,6 @@ class Server(Thread): worker.kill() except: pass - for worker in list(self.pool): - try: - worker.kill() - except: - pass def __enter__(self): return self From e835131c82d01b08a09e2713dfa39a6bff997538 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 19 Apr 2011 12:24:41 -0600 Subject: [PATCH 53/90] Halve the startup time of worker processes by delay loading cssutils, oeb.stylizer and oeb.base --- src/calibre/__init__.py | 4 ---- src/calibre/customize/builtins.py | 3 ++- src/calibre/ebooks/fb2/fb2ml.py | 14 ++++++++++---- src/calibre/ebooks/html/input.py | 3 ++- src/calibre/ebooks/html/meta.py | 2 +- src/calibre/ebooks/html/output.py | 2 +- src/calibre/ebooks/htmlz/output.py | 5 +++-- src/calibre/ebooks/oeb/base.py | 15 ++++++++------- src/calibre/ebooks/oeb/reader.py | 2 +- src/calibre/ebooks/oeb/stylizer.py | 7 ++++--- src/calibre/ebooks/oeb/transforms/filenames.py | 2 +- src/calibre/ebooks/oeb/transforms/trimmanifest.py | 3 +-- src/calibre/ebooks/pdb/ereader/writer.py | 2 +- src/calibre/ebooks/pml/output.py | 2 +- src/calibre/ebooks/pml/pmlml.py | 12 +++++++++--- src/calibre/ebooks/rb/rbml.py | 9 +++++++-- src/calibre/ebooks/rb/writer.py | 2 +- src/calibre/ebooks/rtf/rtfml.py | 9 ++++++--- src/calibre/ebooks/snb/input.py | 2 +- src/calibre/ebooks/snb/snbml.py | 5 +++-- src/calibre/ebooks/txt/output.py | 10 +++++----- src/calibre/ebooks/txt/txtml.py | 9 +++++---- src/calibre/library/catalog.py | 3 ++- 23 files changed, 75 insertions(+), 52 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 0fddb9de9d..7c9638ade1 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -33,9 +33,6 @@ if False: fcntl, win32event, isfrozen, __author__, terminal_controller winerror, win32api, isfreebsd, guess_type -import cssutils -cssutils.log.setLevel(logging.WARN) - def to_unicode(raw, encoding='utf-8', errors='strict'): if isinstance(raw, unicode): return raw @@ -679,4 +676,3 @@ main() ipshell() sys.argv = old_argv - diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 75c02c7e00..00af4e5117 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -9,7 +9,6 @@ from calibre.customize import FileTypePlugin, MetadataReaderPlugin, \ from calibre.constants import numeric_version from calibre.ebooks.metadata.archive import ArchiveExtract, get_cbz_metadata from calibre.ebooks.metadata.opf2 import metadata_to_opf -from calibre.ebooks.oeb.base import OEB_IMAGES from calibre.utils.config import test_eight_code # To archive plugins {{{ @@ -98,6 +97,8 @@ class TXT2TXTZ(FileTypePlugin): on_import = True def _get_image_references(self, txt, base_dir): + from calibre.ebooks.oeb.base import OEB_IMAGES + images = [] # Textile diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 8d1164e026..385c4a5310 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -18,9 +18,6 @@ from lxml import etree from calibre import prepare_string_for_xml from calibre.constants import __appname__, __version__ -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer -from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES, OPF from calibre.utils.magick import Image class FB2MLizer(object): @@ -71,7 +68,7 @@ class FB2MLizer(object): return u'<?xml version="1.0" encoding="UTF-8"?>' + output def clean_text(self, text): - # Condense empty paragraphs into a line break. + # Condense empty paragraphs into a line break. text = re.sub(r'(?miu)(<p>\s*</p>\s*){3,}', '<empty-line />', text) # Remove empty paragraphs. text = re.sub(r'(?miu)<p>\s*</p>', '', text) @@ -100,6 +97,7 @@ class FB2MLizer(object): return text def fb2_header(self): + from calibre.ebooks.oeb.base import OPF metadata = {} metadata['title'] = self.oeb_book.metadata.title[0].value metadata['appname'] = __appname__ @@ -180,6 +178,8 @@ class FB2MLizer(object): return u'</FictionBook>' def get_cover(self): + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES + cover_href = None # Get the raster cover if it's available. @@ -213,6 +213,8 @@ class FB2MLizer(object): return u'' def get_text(self): + from calibre.ebooks.oeb.base import XHTML + from calibre.ebooks.oeb.stylizer import Stylizer text = ['<body>'] # Create main section if there are no others to create @@ -248,6 +250,8 @@ class FB2MLizer(object): ''' This function uses the self.image_hrefs dictionary mapping. It is populated by the dump_text function. ''' + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES + images = [] for item in self.oeb_book.manifest: # Don't write the image if it's not referenced in the document's text. @@ -344,6 +348,8 @@ class FB2MLizer(object): @return: List of string representing the XHTML converted to FB2 markup. ''' + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace + # Ensure what we are converting is not a string and that the fist tag is part of the XHTML namespace. if not isinstance(elem_tree.tag, basestring) or namespace(elem_tree.tag) != XHTML_NS: return [] diff --git a/src/calibre/ebooks/html/input.py b/src/calibre/ebooks/html/input.py index dd0a247a67..079e990de3 100644 --- a/src/calibre/ebooks/html/input.py +++ b/src/calibre/ebooks/html/input.py @@ -315,7 +315,8 @@ class HTMLInput(InputFormatPlugin): from calibre import guess_type from calibre.ebooks.oeb.transforms.metadata import \ meta_info_to_oeb_metadata - import cssutils + import cssutils, logging + cssutils.log.setLevel(logging.WARN) self.OEB_STYLES = OEB_STYLES oeb = create_oebbook(log, None, opts, self, encoding=opts.input_encoding, populate=False) diff --git a/src/calibre/ebooks/html/meta.py b/src/calibre/ebooks/html/meta.py index 9a088efb16..07cf9236fc 100644 --- a/src/calibre/ebooks/html/meta.py +++ b/src/calibre/ebooks/html/meta.py @@ -4,7 +4,6 @@ __copyright__ = '2010, Fabian Grassl <fg@jusmeum.de>' __docformat__ = 'restructuredtext en' -from calibre.ebooks.oeb.base import namespace, barename, DC11_NS class EasyMeta(object): @@ -12,6 +11,7 @@ class EasyMeta(object): self.meta = meta def __iter__(self): + from calibre.ebooks.oeb.base import namespace, barename, DC11_NS meta = self.meta for item_name in meta.items: for item in meta[item_name]: diff --git a/src/calibre/ebooks/html/output.py b/src/calibre/ebooks/html/output.py index 5c984162ac..fe7b4cf274 100644 --- a/src/calibre/ebooks/html/output.py +++ b/src/calibre/ebooks/html/output.py @@ -12,7 +12,6 @@ from os.path import dirname, abspath, relpath, exists, basename from lxml import etree from templite import Templite -from calibre.ebooks.oeb.base import element from calibre.customize.conversion import OutputFormatPlugin, OptionRecommendation from calibre import CurrentDir from calibre.ptempfile import PersistentTemporaryDirectory @@ -51,6 +50,7 @@ class HTMLOutput(OutputFormatPlugin): ''' Generate table of contents ''' + from calibre.ebooks.oeb.base import element with CurrentDir(output_dir): def build_node(current_node, parent=None): if parent is None: diff --git a/src/calibre/ebooks/htmlz/output.py b/src/calibre/ebooks/htmlz/output.py index 03fe12c89e..6d2ad54a12 100644 --- a/src/calibre/ebooks/htmlz/output.py +++ b/src/calibre/ebooks/htmlz/output.py @@ -12,7 +12,6 @@ from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile @@ -42,6 +41,8 @@ class HTMLZOutput(OutputFormatPlugin): ]) def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME + # HTML if opts.htmlz_css_type == 'inline': from calibre.ebooks.htmlz.oeb2html import OEB2HTMLInlineCSSizer @@ -72,7 +73,7 @@ class HTMLZOutput(OutputFormatPlugin): for item in oeb_book.manifest: if item.media_type in OEB_IMAGES and item.href in images: if item.media_type == SVG_MIME: - data = unicode(etree.tostring(item.data, encoding=unicode)) + data = unicode(etree.tostring(item.data, encoding=unicode)) else: data = item.data fname = os.path.join(tdir, 'images', images[item.href]) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index 58083f807f..ce75c97d78 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -15,11 +15,7 @@ from urlparse import urldefrag, urlparse, urlunparse, urljoin from urllib import unquote as urlunquote from lxml import etree, html -from cssutils import CSSParser, parseString, parseStyle, replaceUrls -from cssutils.css import CSSRule - -import calibre -from calibre.constants import filesystem_encoding +from calibre.constants import filesystem_encoding, __version__ from calibre.translations.dynamic import translate from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.oeb.entitydefs import ENTITYDEFS @@ -179,6 +175,9 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False): If the ``link_repl_func`` returns None, the attribute or tag text will be removed completely. ''' + from cssutils import parseString, parseStyle, replaceUrls, log + log.setLevel(logging.WARN) + if resolve_base_href: resolve_base_href(root) for el, attrib, link, pos in iterlinks(root, find_links_in_css=False): @@ -1075,7 +1074,9 @@ class Manifest(object): def _parse_css(self, data): - + from cssutils.css import CSSRule + from cssutils import CSSParser, log + log.setLevel(logging.WARN) def get_style_rules_from_import(import_rule): ans = [] if not import_rule.styleSheet: @@ -2011,7 +2012,7 @@ class OEBBook(object): name='dtb:uid', content=unicode(self.uid)) etree.SubElement(head, NCX('meta'), name='dtb:depth', content=str(self.toc.depth())) - generator = ''.join(['calibre (', calibre.__version__, ')']) + generator = ''.join(['calibre (', __version__, ')']) etree.SubElement(head, NCX('meta'), name='dtb:generator', content=generator) etree.SubElement(head, NCX('meta'), diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index ebc2f30d00..1c42a5a242 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -14,7 +14,6 @@ from mimetypes import guess_type from collections import defaultdict from lxml import etree -import cssutils from calibre.ebooks.oeb.base import OPF1_NS, OPF2_NS, OPF2_NSMAP, DC11_NS, \ DC_NSES, OPF, xml2text @@ -172,6 +171,7 @@ class OEBReader(object): return bad def _manifest_add_missing(self, invalid): + import cssutils manifest = self.oeb.manifest known = set(manifest.hrefs) unchecked = set(manifest.values()) diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index 634f7f5fce..39ab41eede 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -12,17 +12,18 @@ import os, itertools, re, logging, copy, unicodedata from weakref import WeakKeyDictionary from xml.dom import SyntaxErr as CSSSyntaxError import cssutils -from cssutils.css import CSSStyleRule, CSSPageRule, CSSStyleDeclaration, \ - CSSValueList, CSSFontFaceRule, cssproperties +from cssutils.css import (CSSStyleRule, CSSPageRule, CSSStyleDeclaration, + CSSValueList, CSSFontFaceRule, cssproperties) from cssutils import profile as cssprofiles from lxml import etree from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError - from calibre import force_unicode from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES from calibre.ebooks.oeb.base import XPNSMAP, xpath, urlnormalize from calibre.ebooks.oeb.profile import PROFILES +cssutils.log.setLevel(logging.WARN) + _html_css_stylesheet = None def html_css_stylesheet(): diff --git a/src/calibre/ebooks/oeb/transforms/filenames.py b/src/calibre/ebooks/oeb/transforms/filenames.py index bad75b9a6f..c3c7f091c3 100644 --- a/src/calibre/ebooks/oeb/transforms/filenames.py +++ b/src/calibre/ebooks/oeb/transforms/filenames.py @@ -9,7 +9,6 @@ import posixpath from urlparse import urldefrag, urlparse from lxml import etree -import cssutils from calibre.ebooks.oeb.base import rewrite_links, urlnormalize @@ -25,6 +24,7 @@ class RenameFiles(object): # {{{ self.renamed_items_map = renamed_items_map def __call__(self, oeb, opts): + import cssutils self.log = oeb.logger self.opts = opts self.oeb = oeb diff --git a/src/calibre/ebooks/oeb/transforms/trimmanifest.py b/src/calibre/ebooks/oeb/transforms/trimmanifest.py index 0baacfd1f9..95501dbb9b 100644 --- a/src/calibre/ebooks/oeb/transforms/trimmanifest.py +++ b/src/calibre/ebooks/oeb/transforms/trimmanifest.py @@ -8,8 +8,6 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>' from urlparse import urldefrag -import cssutils - from calibre.ebooks.oeb.base import CSS_MIME, OEB_DOCS from calibre.ebooks.oeb.base import urlnormalize, iterlinks @@ -23,6 +21,7 @@ class ManifestTrimmer(object): return cls() def __call__(self, oeb, context): + import cssutils oeb.logger.info('Trimming unused files from manifest...') self.opts = context used = set() diff --git a/src/calibre/ebooks/pdb/ereader/writer.py b/src/calibre/ebooks/pdb/ereader/writer.py index 4fbd343a6b..eb023c594b 100644 --- a/src/calibre/ebooks/pdb/ereader/writer.py +++ b/src/calibre/ebooks/pdb/ereader/writer.py @@ -21,7 +21,6 @@ except ImportError: import cStringIO from calibre.ebooks.pdb.formatwriter import FormatWriter -from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES from calibre.ebooks.pdb.header import PdbHeaderBuilder from calibre.ebooks.pml.pmlml import PMLMLizer @@ -135,6 +134,7 @@ class Writer(FormatWriter): 62-...: Raw image data in 8 bit PNG format. ''' images = [] + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES for item in manifest: if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys(): diff --git a/src/calibre/ebooks/pml/output.py b/src/calibre/ebooks/pml/output.py index 9d2ddc6ca6..63d8a8b220 100644 --- a/src/calibre/ebooks/pml/output.py +++ b/src/calibre/ebooks/pml/output.py @@ -18,7 +18,6 @@ from calibre.customize.conversion import OutputFormatPlugin from calibre.customize.conversion import OptionRecommendation from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile -from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES from calibre.ebooks.pml.pmlml import PMLMLizer class PMLOutput(OutputFormatPlugin): @@ -60,6 +59,7 @@ class PMLOutput(OutputFormatPlugin): pmlz.add_dir(tdir) def write_images(self, manifest, image_hrefs, out_dir, opts): + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES for item in manifest: if item.media_type in OEB_RASTER_IMAGES and item.href in image_hrefs.keys(): if opts.full_image_depth: diff --git a/src/calibre/ebooks/pml/pmlml.py b/src/calibre/ebooks/pml/pmlml.py index 779e75d713..b04aaacaec 100644 --- a/src/calibre/ebooks/pml/pmlml.py +++ b/src/calibre/ebooks/pml/pmlml.py @@ -12,8 +12,6 @@ import re from lxml import etree -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer from calibre.ebooks.pdb.ereader import image_name from calibre.ebooks.pml import unipmlcode @@ -110,6 +108,9 @@ class PMLMLizer(object): return output def get_cover_page(self): + from calibre.ebooks.oeb.stylizer import Stylizer + from calibre.ebooks.oeb.base import XHTML + output = u'' if 'cover' in self.oeb_book.guide: output += '\\m="cover.png"\n' @@ -125,6 +126,9 @@ class PMLMLizer(object): return output def get_text(self): + from calibre.ebooks.oeb.stylizer import Stylizer + from calibre.ebooks.oeb.base import XHTML + text = [u''] for item in self.oeb_book.spine: self.log.debug('Converting %s to PML markup...' % item.href) @@ -180,7 +184,7 @@ class PMLMLizer(object): links = set(re.findall(r'(?<=\\q="#).+?(?=")', text)) for unused in anchors.difference(links): text = text.replace('\\Q="%s"' % unused, '') - + # Remove \Cn tags that are within \x and \Xn tags text = re.sub(ur'(?msu)(?P<t>\\(x|X[0-4]))(?P<a>.*?)(?P<c>\\C[0-4]\s*=\s*"[^"]*")(?P<b>.*?)(?P=t)', '\g<t>\g<a>\g<b>\g<t>', text) @@ -214,6 +218,8 @@ class PMLMLizer(object): return text def dump_text(self, elem, stylizer, page, tag_stack=[]): + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace + if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: return [] diff --git a/src/calibre/ebooks/rb/rbml.py b/src/calibre/ebooks/rb/rbml.py index 50153d7d4d..8cf63e334c 100644 --- a/src/calibre/ebooks/rb/rbml.py +++ b/src/calibre/ebooks/rb/rbml.py @@ -11,8 +11,6 @@ Transform OEB content into RB compatible markup. import re from calibre import prepare_string_for_xml -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer from calibre.ebooks.rb import unique_name TAGS = [ @@ -81,6 +79,8 @@ class RBMLizer(object): return output def get_cover_page(self): + from calibre.ebooks.oeb.stylizer import Stylizer + from calibre.ebooks.oeb.base import XHTML output = u'' if 'cover' in self.oeb_book.guide: if self.name_map.get(self.oeb_book.guide['cover'].href, None): @@ -109,6 +109,9 @@ class RBMLizer(object): return ''.join(toc) def get_text(self): + from calibre.ebooks.oeb.stylizer import Stylizer + from calibre.ebooks.oeb.base import XHTML + output = [u''] for item in self.oeb_book.spine: self.log.debug('Converting %s to RocketBook HTML...' % item.href) @@ -137,6 +140,8 @@ class RBMLizer(object): return text def dump_text(self, elem, stylizer, page, tag_stack=[]): + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace + if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: return [u''] diff --git a/src/calibre/ebooks/rb/writer.py b/src/calibre/ebooks/rb/writer.py index c8908ee95f..f71b103fbd 100644 --- a/src/calibre/ebooks/rb/writer.py +++ b/src/calibre/ebooks/rb/writer.py @@ -18,7 +18,6 @@ import cStringIO from calibre.ebooks.rb.rbml import RBMLizer from calibre.ebooks.rb import HEADER from calibre.ebooks.rb import unique_name -from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES from calibre.constants import __appname__, __version__ TEXT_RECORD_SIZE = 4096 @@ -111,6 +110,7 @@ class RBWriter(object): return (size, pages) def _images(self, manifest): + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES images = [] used_names = [] diff --git a/src/calibre/ebooks/rtf/rtfml.py b/src/calibre/ebooks/rtf/rtfml.py index f739207018..97fa175d1a 100644 --- a/src/calibre/ebooks/rtf/rtfml.py +++ b/src/calibre/ebooks/rtf/rtfml.py @@ -14,9 +14,6 @@ import cStringIO from lxml import etree -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace, \ - OEB_RASTER_IMAGES -from calibre.ebooks.oeb.stylizer import Stylizer from calibre.ebooks.metadata import authors_to_string from calibre.utils.filenames import ascii_text from calibre.utils.magick.draw import save_cover_data_to, identify_data @@ -100,6 +97,8 @@ class RTFMLizer(object): return self.mlize_spine() def mlize_spine(self): + from calibre.ebooks.oeb.base import XHTML + from calibre.ebooks.oeb.stylizer import Stylizer output = self.header() if 'titlepage' in self.oeb_book.guide: href = self.oeb_book.guide['titlepage'].href @@ -154,6 +153,8 @@ class RTFMLizer(object): return ' }' def insert_images(self, text): + from calibre.ebooks.oeb.base import OEB_RASTER_IMAGES + for item in self.oeb_book.manifest: if item.media_type in OEB_RASTER_IMAGES: src = os.path.basename(item.href) @@ -201,6 +202,8 @@ class RTFMLizer(object): return text def dump_text(self, elem, stylizer, tag_stack=[]): + from calibre.ebooks.oeb.base import XHTML_NS, namespace, barename + if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: p = elem.getparent() diff --git a/src/calibre/ebooks/snb/input.py b/src/calibre/ebooks/snb/input.py index 100ac1447f..13b1ca45f9 100755 --- a/src/calibre/ebooks/snb/input.py +++ b/src/calibre/ebooks/snb/input.py @@ -7,7 +7,6 @@ __docformat__ = 'restructuredtext en' import os, uuid from calibre.customize.conversion import InputFormatPlugin -from calibre.ebooks.oeb.base import DirContainer from calibre.ebooks.snb.snbfile import SNBFile from calibre.ptempfile import TemporaryDirectory from calibre.utils.filenames import ascii_filename @@ -30,6 +29,7 @@ class SNBInput(InputFormatPlugin): def convert(self, stream, options, file_ext, log, accelerators): + from calibre.ebooks.oeb.base import DirContainer log.debug("Parsing SNB file...") snbFile = SNBFile() try: diff --git a/src/calibre/ebooks/snb/snbml.py b/src/calibre/ebooks/snb/snbml.py index 078e7ebe76..a501de1ff0 100644 --- a/src/calibre/ebooks/snb/snbml.py +++ b/src/calibre/ebooks/snb/snbml.py @@ -13,8 +13,6 @@ import re from lxml import etree -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer def ProcessFileName(fileName): # Flat the path @@ -81,6 +79,8 @@ class SNBMLizer(object): body.append(entity) def mlize(self): + from calibre.ebooks.oeb.base import XHTML + from calibre.ebooks.oeb.stylizer import Stylizer output = [ u'' ] stylizer = Stylizer(self.item.data, self.item.href, self.oeb_book, self.opts, self.opts.output_profile) content = unicode(etree.tostring(self.item.data.find(XHTML('body')), encoding=unicode)) @@ -208,6 +208,7 @@ class SNBMLizer(object): return text def dump_text(self, subitems, elem, stylizer, end='', pre=False, li = ''): + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: diff --git a/src/calibre/ebooks/txt/output.py b/src/calibre/ebooks/txt/output.py index 4e54a97b45..ac63690996 100644 --- a/src/calibre/ebooks/txt/output.py +++ b/src/calibre/ebooks/txt/output.py @@ -11,7 +11,6 @@ from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.oeb.base import OEB_IMAGES from calibre.ebooks.txt.txtml import TXTMLizer from calibre.ebooks.txt.newlines import TxtNewlines, specified_newlines from calibre.ptempfile import TemporaryDirectory, TemporaryFile @@ -103,12 +102,13 @@ class TXTOutput(OutputFormatPlugin): class TXTZOutput(TXTOutput): - + name = 'TXTZ Output' author = 'John Schember' file_type = 'txtz' def convert(self, oeb_book, output_path, input_plugin, opts, log): + from calibre.ebooks.oeb.base import OEB_IMAGES with TemporaryDirectory('_txtz_output') as tdir: # TXT with TemporaryFile('index.txt') as tf: @@ -123,10 +123,10 @@ class TXTZOutput(TXTOutput): os.makedirs(path) with open(os.path.join(tdir, item.href), 'wb') as imgf: imgf.write(item.data) - + # Metadata - with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: + with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: mdataf.write(etree.tostring(oeb_book.metadata.to_opf1())) - + txtz = ZipFile(output_path, 'w') txtz.add_dir(tdir) diff --git a/src/calibre/ebooks/txt/txtml.py b/src/calibre/ebooks/txt/txtml.py index fa7bfbb380..2320fbbbc7 100644 --- a/src/calibre/ebooks/txt/txtml.py +++ b/src/calibre/ebooks/txt/txtml.py @@ -12,8 +12,6 @@ import re from lxml import etree -from calibre.ebooks.oeb.base import XHTML, XHTML_NS, barename, namespace -from calibre.ebooks.oeb.stylizer import Stylizer BLOCK_TAGS = [ 'div', @@ -58,12 +56,14 @@ class TXTMLizer(object): self.toc_titles = [] self.toc_ids = [] self.last_was_heading = False - + self.create_flat_toc(self.oeb_book.toc) return self.mlize_spine() def mlize_spine(self): + from calibre.ebooks.oeb.base import XHTML + from calibre.ebooks.oeb.stylizer import Stylizer output = [u''] output.append(self.get_toc()) for item in self.oeb_book.spine: @@ -139,7 +139,7 @@ class TXTMLizer(object): # when remove paragraph spacing is enabled. text = re.sub('(?imu)^[ ]+', '', text) text = re.sub('(?imu)[ ]+$', '', text) - + # Remove empty space and newlines at the beginning of the document. text = re.sub(r'(?u)^[ \n]+', '', text) @@ -185,6 +185,7 @@ class TXTMLizer(object): @stylizer: The style information attached to the element. @page: OEB page used to determine absolute urls. ''' + from calibre.ebooks.oeb.base import XHTML_NS, barename, namespace if not isinstance(elem.tag, basestring) \ or namespace(elem.tag) != XHTML_NS: diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index ffa08eaed2..717e8e2c6b 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -15,7 +15,6 @@ from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString from calibre.ebooks.chardet import substitute_entites -from calibre.ebooks.oeb.base import XHTML_NS from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.config import config_dir from calibre.utils.date import format_date, isoformat, is_date_undefined, now as nowf @@ -4322,6 +4321,8 @@ Author '{0}': ''' Generate description header from template ''' + from calibre.ebooks.oeb.base import XHTML_NS + def generate_html(): args = dict( author=author, From f76307f9d54e6253ca4fc06be6777f9ac2bc3881 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 19 Apr 2011 12:33:38 -0600 Subject: [PATCH 54/90] Speedup import of gui2.__init__ --- src/calibre/gui2/__init__.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 3ad8a81ffa..de066359ed 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -4,19 +4,17 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>' import os, sys, Queue, threading from threading import RLock from urllib import unquote - -from PyQt4.Qt import QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, \ - QByteArray, QTranslator, QCoreApplication, QThread, \ - QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, \ - QFileDialog, QFileIconProvider, \ - QIcon, QApplication, QDialog, QUrl, QFont +from PyQt4.Qt import (QVariant, QFileInfo, QObject, SIGNAL, QBuffer, Qt, + QByteArray, QTranslator, QCoreApplication, QThread, + QEvent, QTimer, pyqtSignal, QDate, QDesktopServices, + QFileDialog, QFileIconProvider, + QIcon, QApplication, QDialog, QUrl, QFont) ORG_NAME = 'KovidsBrain' APP_UID = 'libprs500' from calibre.constants import islinux, iswindows, isfreebsd, isfrozen, isosx from calibre.utils.config import Config, ConfigProxy, dynamic, JSONConfig from calibre.utils.localization import set_qt_translator -from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats from calibre.ebooks.metadata import MetaInformation from calibre.utils.date import UNDEFINED_DATE @@ -332,6 +330,7 @@ class GetMetadata(QObject): id, args, kwargs) def _from_formats(self, id, args, kwargs): + from calibre.ebooks.metadata.meta import metadata_from_formats try: mi = metadata_from_formats(*args, **kwargs) except: @@ -339,6 +338,7 @@ class GetMetadata(QObject): self.emit(SIGNAL('metadataf(PyQt_PyObject, PyQt_PyObject)'), id, mi) def _get_metadata(self, id, args, kwargs): + from calibre.ebooks.metadata.meta import get_metadata try: mi = get_metadata(*args, **kwargs) except: @@ -740,3 +740,4 @@ def build_forms(srcdir, info=None): _df = os.environ.get('CALIBRE_DEVELOP_FROM', None) if _df and os.path.exists(_df): build_forms(_df) + From 1206cb3304f8482e7809cec0132f96ab45f4ef5a Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 19 Apr 2011 12:43:09 -0600 Subject: [PATCH 55/90] Delay load the metadata read/write subsystem --- src/calibre/gui2/device.py | 2 +- src/calibre/gui2/dialogs/metadata_bulk.py | 2 +- src/calibre/gui2/dialogs/metadata_single.py | 2 +- src/calibre/gui2/library/models.py | 3 ++- src/calibre/gui2/widgets.py | 2 +- src/calibre/library/database2.py | 12 +++++++++--- src/calibre/library/save_to_disk.py | 2 +- 7 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/calibre/gui2/device.py b/src/calibre/gui2/device.py index db3a43e47d..49542abdc1 100644 --- a/src/calibre/gui2/device.py +++ b/src/calibre/gui2/device.py @@ -25,7 +25,6 @@ from calibre.devices.errors import FreeSpaceError from calibre.devices.apple.driver import ITUNES_ASYNC from calibre.devices.folder_device.driver import FOLDER_DEVICE from calibre.devices.bambook.driver import BAMBOOK, BAMBOOKWifi -from calibre.ebooks.metadata.meta import set_metadata from calibre.constants import DEBUG from calibre.utils.config import prefs, tweaks from calibre.utils.magick.draw import thumbnail @@ -334,6 +333,7 @@ class DeviceManager(Thread): # {{{ def _upload_books(self, files, names, on_card=None, metadata=None, plugboards=None): '''Upload books to device: ''' + from calibre.ebooks.metadata.meta import set_metadata if hasattr(self.connected_device, 'set_plugboards') and \ callable(self.connected_device.set_plugboards): self.connected_device.set_plugboards(plugboards, find_plugboard) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 8a97183ffe..66cf55a9b2 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -13,7 +13,6 @@ from calibre.gui2.dialogs.metadata_bulk_ui import Ui_MetadataBulkDialog from calibre.gui2.dialogs.tag_editor import TagEditor from calibre.ebooks.metadata import string_to_authors, authors_to_string, title_sort from calibre.ebooks.metadata.book.base import composite_formatter -from calibre.ebooks.metadata.meta import get_metadata from calibre.gui2.custom_column_widgets import populate_metadata_page from calibre.gui2 import error_dialog, ResizableDialog, UNDEFINED_QDATE, \ gprefs, question_dialog @@ -26,6 +25,7 @@ from calibre.utils.magick.draw import identify_data from calibre.utils.date import qt_to_dt def get_cover_data(path): # {{{ + from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: prefs['read_file_metadata'] = True diff --git a/src/calibre/gui2/dialogs/metadata_single.py b/src/calibre/gui2/dialogs/metadata_single.py index f6b7b94453..4776562c29 100644 --- a/src/calibre/gui2/dialogs/metadata_single.py +++ b/src/calibre/gui2/dialogs/metadata_single.py @@ -25,7 +25,6 @@ from calibre.ebooks import BOOK_EXTENSIONS from calibre.ebooks.metadata import string_to_authors, \ authors_to_string, check_isbn, title_sort from calibre.ebooks.metadata.covers import download_cover -from calibre.ebooks.metadata.meta import get_metadata from calibre.ebooks.metadata import MetaInformation from calibre.utils.config import prefs, tweaks from calibre.utils.date import qt_to_dt, local_tz, utcfromtimestamp @@ -353,6 +352,7 @@ class MetadataSingleDialog(ResizableDialog, Ui_MetadataSingleDialog): self.formats_changed = True def get_selected_format_metadata(self): + from calibre.ebooks.metadata.meta import get_metadata old = prefs['read_file_metadata'] if not old: prefs['read_file_metadata'] = True diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index 8d89ec76ed..0bd3f2133a 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -18,7 +18,6 @@ from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.config import tweaks, prefs from calibre.utils.date import dt_factory, qt_to_dt, isoformat from calibre.utils.icu import sort_key -from calibre.ebooks.metadata.meta import set_metadata as _set_metadata from calibre.utils.search_query_parser import SearchQueryParser from calibre.library.caches import _match, CONTAINS_MATCH, EQUALS_MATCH, \ REGEXP_MATCH, MetadataBackup, force_to_bool @@ -478,6 +477,7 @@ class BooksModel(QAbstractTableModel): # {{{ def get_preferred_formats_from_ids(self, ids, formats, set_metadata=False, specific_format=None, exclude_auto=False, mode='r+b'): + from calibre.ebooks.metadata.meta import set_metadata as _set_metadata ans = [] need_auto = [] if specific_format is not None: @@ -526,6 +526,7 @@ class BooksModel(QAbstractTableModel): # {{{ def get_preferred_formats(self, rows, formats, paths=False, set_metadata=False, specific_format=None, exclude_auto=False): + from calibre.ebooks.metadata.meta import set_metadata as _set_metadata ans = [] need_auto = [] if specific_format is not None: diff --git a/src/calibre/gui2/widgets.py b/src/calibre/gui2/widgets.py index ea0d2570e5..a7ecdf7b88 100644 --- a/src/calibre/gui2/widgets.py +++ b/src/calibre/gui2/widgets.py @@ -19,7 +19,6 @@ from calibre.gui2 import NONE, error_dialog, pixmap_to_data, gprefs from calibre.gui2.filename_pattern_ui import Ui_Form from calibre import fit_image from calibre.ebooks import BOOK_EXTENSIONS -from calibre.ebooks.metadata.meta import metadata_from_filename from calibre.utils.config import prefs, XMLConfig, tweaks from calibre.gui2.progress_indicator import ProgressIndicator as _ProgressIndicator from calibre.gui2.dnd import dnd_has_image, dnd_get_image, dnd_get_files, \ @@ -95,6 +94,7 @@ class FilenamePattern(QWidget, Ui_Form): self.re.setCurrentIndex(0) def do_test(self): + from calibre.ebooks.metadata.meta import metadata_from_filename try: pat = self.pattern() except Exception as err: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index bc0a8235e4..d7f6c22925 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -15,7 +15,8 @@ from math import ceil from PyQt4.QtGui import QImage from calibre import prints -from calibre.ebooks.metadata import title_sort, author_to_author_sort +from calibre.ebooks.metadata import (title_sort, author_to_author_sort, + string_to_authors, authors_to_string) from calibre.ebooks.metadata.opf2 import metadata_to_opf from calibre.library.database import LibraryDatabase from calibre.library.field_metadata import FieldMetadata, TagsIcons @@ -24,9 +25,7 @@ from calibre.library.caches import ResultCache from calibre.library.custom_columns import CustomColumns from calibre.library.sqlite import connect, IntegrityError from calibre.library.prefs import DBPrefs -from calibre.ebooks.metadata import string_to_authors, authors_to_string from calibre.ebooks.metadata.book.base import Metadata -from calibre.ebooks.metadata.meta import get_metadata, metadata_from_formats from calibre.constants import preferred_encoding, iswindows, isosx, filesystem_encoding from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import run_plugins_on_import @@ -2729,6 +2728,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): self.set_identifier(id_, 'isbn', isbn, notify=notify, commit=commit) def add_catalog(self, path, title): + from calibre.ebooks.metadata.meta import get_metadata + format = os.path.splitext(path)[1][1:].lower() with lopen(path, 'rb') as stream: matches = self.data.get_matches('title', '='+title) @@ -2764,6 +2765,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): def add_news(self, path, arg): + from calibre.ebooks.metadata.meta import get_metadata + format = os.path.splitext(path)[1][1:].lower() stream = path if hasattr(path, 'read') else lopen(path, 'rb') stream.seek(0) @@ -3157,6 +3160,8 @@ books_series_link feeds yield formats def import_book_directory_multiple(self, dirpath, callback=None): + from calibre.ebooks.metadata.meta import metadata_from_formats + duplicates = [] for formats in self.find_books_in_directory(dirpath, False): mi = metadata_from_formats(formats) @@ -3172,6 +3177,7 @@ books_series_link feeds return duplicates def import_book_directory(self, dirpath, callback=None): + from calibre.ebooks.metadata.meta import metadata_from_formats dirpath = os.path.abspath(dirpath) formats = self.find_books_in_directory(dirpath, True) formats = list(formats)[0] diff --git a/src/calibre/library/save_to_disk.py b/src/calibre/library/save_to_disk.py index 42e6c8b156..f7f5559412 100644 --- a/src/calibre/library/save_to_disk.py +++ b/src/calibre/library/save_to_disk.py @@ -14,7 +14,6 @@ from calibre.utils.formatter import TemplateFormatter from calibre.utils.filenames import shorten_components_to, supports_long_names, \ ascii_filename from calibre.ebooks.metadata.opf2 import metadata_to_opf -from calibre.ebooks.metadata.meta import set_metadata from calibre.constants import preferred_encoding from calibre.ebooks.metadata import fmt_sidx from calibre.ebooks.metadata import title_sort @@ -251,6 +250,7 @@ def save_book_to_disk(id_, db, root, opts, length): def do_save_book_to_disk(id_, mi, cover, plugboards, format_map, root, opts, length): + from calibre.ebooks.metadata.meta import set_metadata available_formats = [x.lower().strip() for x in format_map.keys()] if opts.formats == 'all': asked_formats = available_formats From e776b5b1f6ddd107dd94272c3067dcc426407313 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 19 Apr 2011 14:18:28 -0600 Subject: [PATCH 56/90] ... --- src/calibre/__init__.py | 5 +++-- src/calibre/ebooks/conversion/plumber.py | 2 ++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 7c9638ade1..0bf48a8a97 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -4,8 +4,7 @@ __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' import uuid, sys, os, re, logging, time, random, \ - __builtin__, warnings, multiprocessing -from contextlib import closing + __builtin__, warnings from urllib import getproxies from urllib2 import unquote as urllib2_unquote __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) @@ -383,6 +382,7 @@ class StreamReadWrapper(object): def detect_ncpus(): """Detects the number of effective CPUs in the system""" + import multiprocessing ans = -1 try: ans = multiprocessing.cpu_count() @@ -547,6 +547,7 @@ def get_download_filename(url, cookie_file=None): ''' Get a local filename for a URL using the content disposition header ''' + from contextlib import closing filename = '' br = browser() diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index b26befe075..8706d6eb2b 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -875,6 +875,8 @@ OptionRecommendation(name='sr3_replace', if self.opts.verbose: self.log.filter_level = self.log.DEBUG self.flush() + import cssutils, logging + cssutils.log.setLevel(logging.WARN) if self.opts.debug_pipeline is not None: self.opts.verbose = max(self.opts.verbose, 4) From 3f71ad9420ab3bf359bc667743b6935affdaf00a Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 19 Apr 2011 14:54:41 -0600 Subject: [PATCH 57/90] Fix Newsweek --- recipes/newsweek.recipe | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/recipes/newsweek.recipe b/recipes/newsweek.recipe index 73837c1872..97abd69aac 100644 --- a/recipes/newsweek.recipe +++ b/recipes/newsweek.recipe @@ -1,4 +1,3 @@ -import string from calibre.web.feeds.news import BasicNewsRecipe class Newsweek(BasicNewsRecipe): @@ -11,7 +10,6 @@ class Newsweek(BasicNewsRecipe): no_stylesheets = True BASE_URL = 'http://www.newsweek.com' - INDEX = BASE_URL+'/topics.html' keep_only_tags = dict(name='article', attrs={'class':'article-text'}) remove_tags = [dict(attrs={'data-dartad':True})] @@ -23,11 +21,14 @@ class Newsweek(BasicNewsRecipe): return soup def newsweek_sections(self): - soup = self.index_to_soup(self.INDEX) - for a in soup.findAll('a', title='Primary tag', href=True): - yield (string.capitalize(self.tag_to_string(a)), - self.BASE_URL+a['href']) - + return [ + ('Nation', 'http://www.newsweek.com/tag/nation.html'), + ('Society', 'http://www.newsweek.com/tag/society.html'), + ('Culture', 'http://www.newsweek.com/tag/culture.html'), + ('World', 'http://www.newsweek.com/tag/world.html'), + ('Politics', 'http://www.newsweek.com/tag/politics.html'), + ('Business', 'http://www.newsweek.com/tag/business.html'), + ] def newsweek_parse_section_page(self, soup): for article in soup.findAll('article', about=True, From 2b21ea3d9b8e571885607351cf7148df6f4eea41 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Tue, 19 Apr 2011 18:06:56 -0600 Subject: [PATCH 58/90] Der Spiegel by Nikolas Mangold --- recipes/der_spiegel.recipe | 83 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 recipes/der_spiegel.recipe diff --git a/recipes/der_spiegel.recipe b/recipes/der_spiegel.recipe new file mode 100644 index 0000000000..1e94785233 --- /dev/null +++ b/recipes/der_spiegel.recipe @@ -0,0 +1,83 @@ +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = '2011, Nikolas Mangold <nmangold at gmail.com>' +''' +spiegel.de +''' +from calibre.web.feeds.news import BasicNewsRecipe +from calibre import strftime +from calibre import re + +class DerSpiegel(BasicNewsRecipe): + title = 'Der Spiegel' + __author__ = 'Nikolas Mangold' + description = 'Der Spiegel, Printed Edition. Access to paid content.' + publisher = 'SPIEGEL-VERLAG RUDOLF AUGSTEIN GMBH & CO. KG' + category = 'news, politics, Germany' + no_stylesheets = True + encoding = 'cp1252' + needs_subscription = True + remove_empty_feeds = True + delay = 1 + PREFIX = 'http://m.spiegel.de' + INDEX = PREFIX + '/spiegel/print/epaper/index-heftaktuell.html' + use_embedded_content = False + masthead_url = 'http://upload.wikimedia.org/wikipedia/en/thumb/1/17/Der_Spiegel_logo.svg/200px-Der_Spiegel_logo.svg.png' + language = 'de' + publication_type = 'magazine' + extra_css = ' body{font-family: Arial,Helvetica,sans-serif} ' + timefmt = '[%W/%Y]' + empty_articles = ['Titelbild'] + preprocess_regexps = [ + (re.compile(r'<p>◆</p>', re.DOTALL|re.IGNORECASE), lambda match: '<hr>'), + ] + + def get_browser(self): + def has_login_name(form): + try: + form.find_control(name="f.loginName") + except: + return False + else: + return True + + br = BasicNewsRecipe.get_browser() + if self.username is not None and self.password is not None: + br.open(self.PREFIX + '/meinspiegel/login.html') + br.select_form(predicate=has_login_name) + br['f.loginName' ] = self.username + br['f.password'] = self.password + br.submit() + return br + + remove_tags_before = dict(attrs={'class':'spArticleContent'}) + remove_tags_after = dict(attrs={'class':'spArticleCredit'}) + + def parse_index(self): + soup = self.index_to_soup(self.INDEX) + + cover = soup.find('img', width=248) + if cover is not None: + self.cover_url = cover['src'] + + index = soup.find('dl') + + feeds = [] + for section in index.findAll('dt'): + section_title = self.tag_to_string(section).strip() + self.log('Found section ', section_title) + + articles = [] + for article in section.findNextSiblings(['dd','dt']): + if article.name == 'dt': + break + link = article.find('a') + title = self.tag_to_string(link).strip() + if title in self.empty_articles: + continue + self.log('Found article ', title) + url = self.PREFIX + link['href'] + articles.append({'title' : title, 'date' : strftime(self.timefmt), 'url' : url}) + feeds.append((section_title,articles)) + return feeds; From cbe800e423fa05461556821114621b8720976080 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 09:43:14 -0600 Subject: [PATCH 59/90] Enable Qt SSL in the calibre windows build and a couple of fixes for the overdrive plugin --- setup/installer/windows/freeze.py | 5 +- setup/installer/windows/notes.rst | 15 +++++- .../ebooks/metadata/sources/overdrive.py | 10 ++-- src/calibre/gui2/store/mobileread_plugin.py | 50 +++++++++---------- 4 files changed, 49 insertions(+), 31 deletions(-) diff --git a/setup/installer/windows/freeze.py b/setup/installer/windows/freeze.py index cf4dcd5f9d..f666427598 100644 --- a/setup/installer/windows/freeze.py +++ b/setup/installer/windows/freeze.py @@ -13,7 +13,8 @@ from setup import Command, modules, functions, basenames, __version__, \ from setup.build_environment import msvc, MT, RC from setup.installer.windows.wix import WixMixIn -QT_DIR = 'Q:\\Qt\\4.7.1' +OPENSSL_DIR = r'Q:\openssl' +QT_DIR = 'Q:\\Qt\\4.7.2' QT_DLLS = ['Core', 'Gui', 'Network', 'Svg', 'WebKit', 'Xml', 'XmlPatterns'] LIBUSB_DIR = 'C:\\libusb' LIBUNRAR = 'C:\\Program Files\\UnrarDLL\\unrar.dll' @@ -108,6 +109,8 @@ class Win32Freeze(Command, WixMixIn): self.dll_dir = self.j(self.base, 'DLLs') shutil.copytree(r'C:\Python%s\DLLs'%self.py_ver, self.dll_dir, ignore=shutil.ignore_patterns('msvc*.dll', 'Microsoft.*')) + for x in glob.glob(self.j(OPENSSL_DIR, 'bin', '*.dll')): + shutil.copy2(x, self.dll_dir) for x in QT_DLLS: x += '4.dll' if not x.startswith('phonon'): x = 'Qt'+x diff --git a/setup/installer/windows/notes.rst b/setup/installer/windows/notes.rst index 5dfd956ce2..ce6ca650a4 100644 --- a/setup/installer/windows/notes.rst +++ b/setup/installer/windows/notes.rst @@ -53,12 +53,25 @@ SQLite Put sqlite3*.h from the sqlite windows amlgamation in ~/sw/include +OpenSSL +-------- + +First install ActiveState Perl if you dont already have perl in windows +Download and untar the openssl tarball, follow the instructions in INSTALL.W32 (use no-asm) +to install use prefix q:\openssl + +perl Configure VC-WIN32 no-asm enable-static-engine --prefix=Q:/openssl +ms\do_ms.bat +nmake -f ms\ntdll.mak +nmake -f ms\ntdll.mak test +nmake -f ms\ntdll.mak install + Qt -------- Extract Qt sourcecode to C:\Qt\4.x.x. Run configure and make:: - configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs && nmake + configure -opensource -release -qt-zlib -qt-gif -qt-libmng -qt-libpng -qt-libtiff -qt-libjpeg -release -platform win32-msvc2008 -no-qt3support -webkit -xmlpatterns -no-phonon -no-style-plastique -no-style-cleanlooks -no-style-motif -no-style-cde -no-declarative -no-scripttools -no-audio-backend -no-multimedia -no-dbus -no-openvg -no-opengl -no-qt3support -confirm-license -nomake examples -nomake demos -nomake docs -openssl -I Q:\openssl\include -L Q:\openssl\lib && nmake SIP ----- diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 26c90f08fe..5f70802314 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -9,7 +9,7 @@ __docformat__ = 'restructuredtext en' ''' Fetch metadata using Overdrive Content Reserve ''' -import re, random, mechanize, copy +import re, random, mechanize, copy, json from threading import RLock from Queue import Queue, Empty @@ -43,7 +43,7 @@ class OverDrive(Source): def __init__(self, *args, **kwargs): Source.__init__(self, *args, **kwargs) - self.prefs.defaults['ignore_fields'] =['tags', 'pubdate', 'comments', 'identifier:isbn', 'language'] + self.prefs.defaults['ignore_fields'] =['tags', 'pubdate', 'comments', 'identifier:isbn'] def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): @@ -228,7 +228,7 @@ class OverDrive(Source): def sort_ovrdrv_results(self, raw, title=None, title_tokens=None, author=None, author_tokens=None, ovrdrv_id=None): close_matches = [] raw = re.sub('.*?\[\[(?P<content>.*?)\]\].*', '[[\g<content>]]', raw) - results = eval(raw) + results = json.loads(raw) #print results # The search results are either from a keyword search or a multi-format list from a single ID, # sort through the results for closest match/format @@ -392,7 +392,9 @@ class OverDrive(Source): from calibre.utils.date import parse_date mi.pubdate = parse_date(pub_date[0].strip()) if lang: - mi.language = lang[0].strip() + lang = lang[0].strip().lower() + mi.language = {'english':'en', 'french':'fr', 'german':'de', + 'spanish':'es'}.get(lang, None) if ebook_isbn: #print "ebook isbn is "+str(ebook_isbn[0]) diff --git a/src/calibre/gui2/store/mobileread_plugin.py b/src/calibre/gui2/store/mobileread_plugin.py index 49c265d7fe..b65748ac57 100644 --- a/src/calibre/gui2/store/mobileread_plugin.py +++ b/src/calibre/gui2/store/mobileread_plugin.py @@ -28,15 +28,15 @@ from calibre.utils.config import DynamicConfig from calibre.utils.icu import sort_key class MobileReadStore(BasicStoreConfig, StorePlugin): - + def genesis(self): self.config = DynamicConfig('store_' + self.name) self.rlock = RLock() - + def open(self, parent=None, detail_item=None, external=False): settings = self.get_settings() url = 'http://www.mobileread.com/' - + if external or settings.get(self.name + '_open_external', False): open_url(QUrl(detail_item if detail_item else url)) else: @@ -71,7 +71,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): ratio += s.ratio() if ratio > 0: matches.append((ratio, x)) - + # Move the best scorers to head of list. matches = heapq.nlargest(max_results, matches) for score, book in matches: @@ -81,21 +81,21 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): def update_book_list(self, timeout=10): with self.rlock: url = 'http://www.mobileread.com/forums/ebooks.php?do=getlist&type=html' - + last_download = self.config.get(self.name + '_last_download', None) # Don't update the book list if our cache is less than one week old. if last_download and (time.time() - last_download) < 604800: return - + # Download the book list HTML file from MobileRead. br = browser() raw_data = None with closing(br.open(url, timeout=timeout)) as f: raw_data = f.read() - + if not raw_data: return - + # Turn books listed in the HTML file into BookRef's. books = [] try: @@ -105,7 +105,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): book.detail_item = ''.join(book_data.xpath('.//a/@href')) book.format = ''.join(book_data.xpath('.//i/text()')) book.format = book.format.strip() - + text = ''.join(book_data.xpath('.//a/text()')) if ':' in text: book.author, q, text = text.partition(':') @@ -114,7 +114,7 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): books.append(book) except: pass - + # Save the book list and it's create time. if books: self.config[self.name + '_last_download'] = time.time() @@ -126,21 +126,21 @@ class MobileReadStore(BasicStoreConfig, StorePlugin): class BookRef(SearchResult): - + def __init__(self): SearchResult.__init__(self) - + self.format = '' class MobeReadStoreDialog(QDialog, Ui_Dialog): - + def __init__(self, plugin, *args): QDialog.__init__(self, *args) self.setupUi(self) self.plugin = plugin - + self.model = BooksModel() self.results_view.setModel(self.model) self.results_view.model().set_books(self.plugin.get_book_list()) @@ -150,14 +150,14 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog): self.search_query.textChanged.connect(self.model.set_filter) self.results_view.model().total_changed.connect(self.total.setText) self.finished.connect(self.dialog_closed) - + self.restore_state() - + def open_store(self, index): result = self.results_view.model().get_book(index) if result: self.plugin.open(self, result.detail_item) - + def restore_state(self): geometry = self.plugin.config['store_mobileread_dialog_geometry'] if geometry: @@ -172,7 +172,7 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog): else: for i in xrange(self.results_view.model().columnCount()): self.results_view.resizeColumnToContents(i) - + self.results_view.model().sort_col = self.plugin.config.get('store_mobileread_dialog_sort_col', 0) self.results_view.model().sort_order = self.plugin.config.get('store_mobileread_dialog_sort_order', Qt.AscendingOrder) self.results_view.model().sort(self.results_view.model().sort_col, self.results_view.model().sort_order) @@ -189,7 +189,7 @@ class MobeReadStoreDialog(QDialog, Ui_Dialog): class BooksModel(QAbstractItemModel): - + total_changed = pyqtSignal(unicode) HEADERS = [_('Title'), _('Author(s)'), _('Format')] @@ -205,7 +205,7 @@ class BooksModel(QAbstractItemModel): def set_books(self, books): self.books = books self.all_books = books - + self.sort(self.sort_col, self.sort_order) def get_book(self, index): @@ -214,11 +214,11 @@ class BooksModel(QAbstractItemModel): return self.books[row] else: return None - + def set_filter(self, filter): #self.layoutAboutToBeChanged.emit() self.beginResetModel() - + self.filter = unicode(filter) self.books = [] if self.filter: @@ -241,7 +241,7 @@ class BooksModel(QAbstractItemModel): self.endResetModel() #self.layoutChanged.emit() - + def index(self, row, column, parent=QModelIndex()): return self.createIndex(row, column) @@ -255,7 +255,7 @@ class BooksModel(QAbstractItemModel): def columnCount(self, *args): return len(self.HEADERS) - + def headerData(self, section, orientation, role): if role != Qt.DisplayRole: return NONE @@ -295,7 +295,7 @@ class BooksModel(QAbstractItemModel): if not self.books: return - descending = order == Qt.DescendingOrder + descending = order == Qt.DescendingOrder self.books.sort(None, lambda x: sort_key(unicode(self.data_as_text(x, col))), descending) From 7a122f2b6e78ec7e149c9557f5bdb93aabf1e189 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 09:45:20 -0600 Subject: [PATCH 60/90] The Journal.ie by Phil Burns --- recipes/the_journal.recipe | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 recipes/the_journal.recipe diff --git a/recipes/the_journal.recipe b/recipes/the_journal.recipe new file mode 100644 index 0000000000..e65d7e272e --- /dev/null +++ b/recipes/the_journal.recipe @@ -0,0 +1,26 @@ +__license__ = 'GPL v3' +__copyright__ = '2011 Phil Burns' +''' +TheJournal.ie +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class TheJournal(BasicNewsRecipe): + + __author_ = 'Phil Burns' + title = u'TheJournal.ie' + oldest_article = 1 + max_articles_per_feed = 100 + encoding = 'utf8' + language = 'en_IE' + timefmt = ' (%A, %B %d, %Y)' + + no_stylesheets = True + remove_tags = [dict(name='div', attrs={'class':'footer'}), + dict(name=['script', 'noscript'])] + + extra_css = 'p, div { margin: 0pt; border: 0pt; text-indent: 0.5em }' + + feeds = [ + (u'Latest News', u'http://www.thejournal.ie/feed/')] From 1e246c89d31ecc41f073ee225c90c53b61630a81 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 09:51:10 -0600 Subject: [PATCH 61/90] ... --- src/calibre/ebooks/metadata/sources/overdrive.py | 2 +- src/calibre/gui2/metadata/bulk_download2.py | 3 ++- src/calibre/gui2/metadata/single.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 5f70802314..26c25b627f 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -43,7 +43,7 @@ class OverDrive(Source): def __init__(self, *args, **kwargs): Source.__init__(self, *args, **kwargs) - self.prefs.defaults['ignore_fields'] =['tags', 'pubdate', 'comments', 'identifier:isbn'] + self.prefs.defaults['ignore_fields'] =['tags', 'pubdate', 'comments'] def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 4aa4561078..2bbb177e14 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -310,7 +310,8 @@ def proceed(gui, job): def merge_result(oldmi, newmi): dummy = Metadata(_('Unknown')) for f in msprefs['ignore_fields']: - setattr(newmi, f, getattr(dummy, f)) + if ':' not in f: + setattr(newmi, f, getattr(dummy, f)) fields = set() for plugin in metadata_plugins(['identify']): fields |= plugin.touched_fields diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index d527dda022..63d4499966 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -326,7 +326,8 @@ class MetadataSingleDialogBase(ResizableDialog): mi = d.book dummy = Metadata(_('Unknown')) for f in msprefs['ignore_fields']: - setattr(mi, f, getattr(dummy, f)) + if ':' not in f: + setattr(mi, f, getattr(dummy, f)) if mi is not None: self.update_from_mi(mi) if d.cover_pixmap is not None: From dae6dcaa38a9fedcfd578068af89644af849f0a2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 10:01:11 -0600 Subject: [PATCH 62/90] Implement ignore_fields for individual plugins (I'd forgotten to do this earlier). --- src/calibre/ebooks/metadata/sources/identify.py | 4 ++++ src/calibre/ebooks/metadata/sources/overdrive.py | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index 6295efa0c0..4d21a0c210 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -382,7 +382,11 @@ def identify(log, abort, # {{{ log(plog) log('\n'+'*'*80) + dummy = Metadata(_('Unknown')) for i, result in enumerate(presults): + for f in plugin.prefs['ignore_fields']: + if ':' not in f: + setattr(result, f, getattr(dummy, f)) result.relevance_in_source = i result.has_cached_cover_url = (plugin.cached_cover_url_is_reliable and plugin.get_cached_cover_url(result.identifiers) is not diff --git a/src/calibre/ebooks/metadata/sources/overdrive.py b/src/calibre/ebooks/metadata/sources/overdrive.py index 26c25b627f..0affbdd805 100755 --- a/src/calibre/ebooks/metadata/sources/overdrive.py +++ b/src/calibre/ebooks/metadata/sources/overdrive.py @@ -41,10 +41,6 @@ class OverDrive(Source): supports_gzip_transfer_encoding = False cached_cover_url_is_reliable = True - def __init__(self, *args, **kwargs): - Source.__init__(self, *args, **kwargs) - self.prefs.defaults['ignore_fields'] =['tags', 'pubdate', 'comments'] - def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ identifiers={}, timeout=30): ovrdrv_id = identifiers.get('overdrive', None) @@ -390,7 +386,10 @@ class OverDrive(Source): if pub_date: from calibre.utils.date import parse_date - mi.pubdate = parse_date(pub_date[0].strip()) + try: + mi.pubdate = parse_date(pub_date[0].strip()) + except: + pass if lang: lang = lang[0].strip().lower() mi.language = {'english':'en', 'french':'fr', 'german':'de', From b09ae3027915e27e0c935030f87e1d2da96744d7 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 10:43:08 -0600 Subject: [PATCH 63/90] Fix #764129 (Fetch Annotations (Toolbar) button not working.) --- src/calibre/gui2/actions/annotate.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/calibre/gui2/actions/annotate.py b/src/calibre/gui2/actions/annotate.py index 48397936fb..f934a4a53c 100644 --- a/src/calibre/gui2/actions/annotate.py +++ b/src/calibre/gui2/actions/annotate.py @@ -22,7 +22,7 @@ class FetchAnnotationsAction(InterfaceAction): action_type = 'current' def genesis(self): - pass + self.qaction.triggered.connect(self.fetch_annotations) def fetch_annotations(self, *args): # Generate a path_map from selected ids @@ -52,6 +52,10 @@ class FetchAnnotationsAction(InterfaceAction): return path_map device = self.gui.device_manager.device + if not getattr(device, 'SUPPORTS_ANNOTATIONS', False): + return error_dialog(self.gui, _('Not supported'), + _('Fetching annotations is not supported for this device'), + show=True) if self.gui.current_view() is not self.gui.library_view: return error_dialog(self.gui, _('Use library only'), From f9ed8adb44e32991b885c9707de3c2bc07acc0b1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 11:20:08 -0600 Subject: [PATCH 64/90] Shave another 0.1 secs of worker launch time on non linux platforms --- src/calibre/__init__.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 0bf48a8a97..91e1b57b38 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -3,8 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' -import uuid, sys, os, re, logging, time, random, \ - __builtin__, warnings +import sys, os, re, logging, time, random, __builtin__, warnings from urllib import getproxies from urllib2 import unquote as urllib2_unquote __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) @@ -15,15 +14,20 @@ from functools import partial warnings.simplefilter('ignore', DeprecationWarning) -from calibre.constants import iswindows, isosx, islinux, isfreebsd, isfrozen, \ - terminal_controller, preferred_encoding, \ - __appname__, __version__, __author__, \ - win32event, win32api, winerror, fcntl, \ - filesystem_encoding, plugins, config_dir +from calibre.constants import (iswindows, isosx, islinux, isfreebsd, isfrozen, + terminal_controller, preferred_encoding, + __appname__, __version__, __author__, + win32event, win32api, winerror, fcntl, + filesystem_encoding, plugins, config_dir) from calibre.startup import winutil, winutilerror, guess_type if islinux and not getattr(sys, 'frozen', False): - # Imported before PyQt4 to workaround PyQt4 util-linux conflict on gentoo + # Imported before PyQt4 to workaround PyQt4 util-linux conflict discovered on gentoo + # See http://bugs.gentoo.org/show_bug.cgi?id=317557 + # Importing uuid is slow so get rid of this at some point, maybe in a few + # years when even Debian has caught up + # Also remember to remove it from site.py in the binary builds + import uuid uuid.uuid4() if False: From 2897f63a0f311b05815e4e2be82ca86506a98c8b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 12:48:22 -0600 Subject: [PATCH 65/90] More delay load optimizations to reduce worker startup time --- src/calibre/__init__.py | 44 +++++++++++++++++++----- src/calibre/constants.py | 26 ++++++++------ src/calibre/debug.py | 4 +-- src/calibre/ebooks/chm/reader.py | 2 +- src/calibre/ebooks/conversion/plumber.py | 4 ++- src/calibre/ebooks/fb2/fb2ml.py | 3 +- src/calibre/ebooks/metadata/__init__.py | 6 ++-- src/calibre/ebooks/metadata/fb2.py | 5 +-- src/calibre/ebooks/metadata/opf2.py | 8 ++--- src/calibre/ebooks/oeb/base.py | 5 ++- src/calibre/ebooks/oeb/reader.py | 2 +- src/calibre/ebooks/snb/snbfile.py | 5 +-- src/calibre/library/cli.py | 8 +++-- src/calibre/startup.py | 4 --- src/calibre/utils/config.py | 35 +++++++++++++------ 15 files changed, 104 insertions(+), 57 deletions(-) diff --git a/src/calibre/__init__.py b/src/calibre/__init__.py index 91e1b57b38..29c69a6799 100644 --- a/src/calibre/__init__.py +++ b/src/calibre/__init__.py @@ -3,9 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal <kovid@kovidgoyal.net>' __docformat__ = 'restructuredtext en' -import sys, os, re, logging, time, random, __builtin__, warnings -from urllib import getproxies -from urllib2 import unquote as urllib2_unquote +import sys, os, re, time, random, __builtin__, warnings __builtin__.__dict__['dynamic_property'] = lambda(func): func(None) from htmlentitydefs import name2codepoint from math import floor @@ -15,13 +13,12 @@ warnings.simplefilter('ignore', DeprecationWarning) from calibre.constants import (iswindows, isosx, islinux, isfreebsd, isfrozen, - terminal_controller, preferred_encoding, - __appname__, __version__, __author__, + preferred_encoding, __appname__, __version__, __author__, win32event, win32api, winerror, fcntl, filesystem_encoding, plugins, config_dir) -from calibre.startup import winutil, winutilerror, guess_type +from calibre.startup import winutil, winutilerror -if islinux and not getattr(sys, 'frozen', False): +if False and islinux and not getattr(sys, 'frozen', False): # Imported before PyQt4 to workaround PyQt4 util-linux conflict discovered on gentoo # See http://bugs.gentoo.org/show_bug.cgi?id=317557 # Importing uuid is slow so get rid of this at some point, maybe in a few @@ -33,8 +30,33 @@ if islinux and not getattr(sys, 'frozen', False): if False: # Prevent pyflakes from complaining winutil, winutilerror, __appname__, islinux, __version__ - fcntl, win32event, isfrozen, __author__, terminal_controller - winerror, win32api, isfreebsd, guess_type + fcntl, win32event, isfrozen, __author__ + winerror, win32api, isfreebsd + +_mt_inited = False +def _init_mimetypes(): + global _mt_inited + import mimetypes + mimetypes.init([P('mime.types')]) + _mt_inited = True + +def guess_type(*args, **kwargs): + import mimetypes + if not _mt_inited: + _init_mimetypes() + return mimetypes.guess_type(*args, **kwargs) + +def guess_all_extensions(*args, **kwargs): + import mimetypes + if not _mt_inited: + _init_mimetypes() + return mimetypes.guess_all_extensions(*args, **kwargs) + +def get_types_map(): + import mimetypes + if not _mt_inited: + _init_mimetypes() + return mimetypes.types_map def to_unicode(raw, encoding='utf-8', errors='strict'): if isinstance(raw, unicode): @@ -182,6 +204,7 @@ class CommandLineError(Exception): pass def setup_cli_handlers(logger, level): + import logging if os.environ.get('CALIBRE_WORKER', None) is not None and logger.handlers: return logger.setLevel(level) @@ -243,6 +266,7 @@ def extract(path, dir): extractor(path, dir) def get_proxies(debug=True): + from urllib import getproxies proxies = getproxies() for key, proxy in list(proxies.items()): if not proxy or '..' in proxy: @@ -552,6 +576,8 @@ def get_download_filename(url, cookie_file=None): Get a local filename for a URL using the content disposition header ''' from contextlib import closing + from urllib2 import unquote as urllib2_unquote + filename = '' br = browser() diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 6f26a63940..62612d9c66 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -1,23 +1,27 @@ +from future_builtins import map + __license__ = 'GPL v3' __copyright__ = '2008, Kovid Goyal kovid@kovidgoyal.net' __docformat__ = 'restructuredtext en' -__appname__ = 'calibre' -__version__ = '0.7.56' -__author__ = "Kovid Goyal <kovid@kovidgoyal.net>" - -import re, importlib -_ver = __version__.split('.') -_ver = [int(re.search(r'(\d+)', x).group(1)) for x in _ver] -numeric_version = tuple(_ver) +__appname__ = u'calibre' +numeric_version = (0, 7, 56) +__version__ = u'.'.join(map(unicode, numeric_version)) +__author__ = u"Kovid Goyal <kovid@kovidgoyal.net>" ''' Various run time constants. ''' -import sys, locale, codecs, os -from calibre.utils.terminfo import TerminalController +import sys, locale, codecs, os, importlib + +_tc = None +def terminal_controller(): + global _tc + if _tc is None: + from calibre.utils.terminfo import TerminalController + _tc = TerminalController(sys.stdout) + return _tc -terminal_controller = TerminalController(sys.stdout) iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower() isosx = 'darwin' in sys.platform.lower() diff --git a/src/calibre/debug.py b/src/calibre/debug.py index 86a0477811..8d65c37bbf 100644 --- a/src/calibre/debug.py +++ b/src/calibre/debug.py @@ -106,7 +106,7 @@ def migrate(old, new): from calibre.library.database import LibraryDatabase from calibre.library.database2 import LibraryDatabase2 from calibre.utils.terminfo import ProgressBar - from calibre import terminal_controller + from calibre.constants import terminal_controller class Dummy(ProgressBar): def setLabelText(self, x): pass def setAutoReset(self, y): pass @@ -119,7 +119,7 @@ def migrate(old, new): db = LibraryDatabase(old) db2 = LibraryDatabase2(new) - db2.migrate_old(db, Dummy(terminal_controller, 'Migrating database...')) + db2.migrate_old(db, Dummy(terminal_controller(), 'Migrating database...')) prefs['library_path'] = os.path.abspath(new) print 'Database migrated to', os.path.abspath(new) diff --git a/src/calibre/ebooks/chm/reader.py b/src/calibre/ebooks/chm/reader.py index 7c9a6bf48a..24814a34f9 100644 --- a/src/calibre/ebooks/chm/reader.py +++ b/src/calibre/ebooks/chm/reader.py @@ -5,8 +5,8 @@ __copyright__ = '2008, Kovid Goyal <kovid at kovidgoyal.net>,' \ ' and Alex Bramley <a.bramley at gmail.com>.' import os, re -from mimetypes import guess_type as guess_mimetype +from calibre import guess_type as guess_mimetype from calibre.ebooks.BeautifulSoup import BeautifulSoup, NavigableString from calibre.constants import iswindows, filesystem_encoding from calibre.utils.chm.chm import CHMFile diff --git a/src/calibre/ebooks/conversion/plumber.py b/src/calibre/ebooks/conversion/plumber.py index 8706d6eb2b..96ea3e5884 100644 --- a/src/calibre/ebooks/conversion/plumber.py +++ b/src/calibre/ebooks/conversion/plumber.py @@ -14,7 +14,8 @@ from calibre.ebooks.conversion.preprocess import HTMLPreProcessor from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.date import parse_date from calibre.utils.zipfile import ZipFile -from calibre import extract, walk, isbytestring, filesystem_encoding +from calibre import (extract, walk, isbytestring, filesystem_encoding, + get_types_map) from calibre.constants import __version__ DEBUG_README=u''' @@ -877,6 +878,7 @@ OptionRecommendation(name='sr3_replace', self.flush() import cssutils, logging cssutils.log.setLevel(logging.WARN) + get_types_map() # Ensure the mimetypes module is intialized if self.opts.debug_pipeline is not None: self.opts.verbose = max(self.opts.verbose, 4) diff --git a/src/calibre/ebooks/fb2/fb2ml.py b/src/calibre/ebooks/fb2/fb2ml.py index 385c4a5310..b45f8f9f9e 100644 --- a/src/calibre/ebooks/fb2/fb2ml.py +++ b/src/calibre/ebooks/fb2/fb2ml.py @@ -10,7 +10,6 @@ Transform OEB content into FB2 markup from base64 import b64encode from datetime import datetime -from mimetypes import types_map import re import uuid @@ -259,7 +258,7 @@ class FB2MLizer(object): continue if item.media_type in OEB_RASTER_IMAGES: try: - if not item.media_type == types_map['.jpeg'] or not item.media_type == types_map['.jpg']: + if item.media_type != 'image/jpeg': im = Image() im.load(item.data) im.set_compression_quality(70) diff --git a/src/calibre/ebooks/metadata/__init__.py b/src/calibre/ebooks/metadata/__init__.py index 6078a0aa94..2ae5f3ade5 100644 --- a/src/calibre/ebooks/metadata/__init__.py +++ b/src/calibre/ebooks/metadata/__init__.py @@ -6,11 +6,11 @@ __docformat__ = 'restructuredtext en' """ Provides abstraction for metadata reading.writing from a variety of ebook formats. """ -import os, mimetypes, sys, re +import os, sys, re from urllib import unquote, quote from urlparse import urlparse -from calibre import relpath +from calibre import relpath, guess_type from calibre.utils.config import tweaks @@ -118,7 +118,7 @@ class Resource(object): self.path = None self.fragment = '' try: - self.mime_type = mimetypes.guess_type(href_or_path)[0] + self.mime_type = guess_type(href_or_path)[0] except: self.mime_type = None if self.mime_type is None: diff --git a/src/calibre/ebooks/metadata/fb2.py b/src/calibre/ebooks/metadata/fb2.py index 2d6192f949..21f15b05ae 100644 --- a/src/calibre/ebooks/metadata/fb2.py +++ b/src/calibre/ebooks/metadata/fb2.py @@ -5,11 +5,12 @@ __copyright__ = '2008, Anatoly Shipitsin <norguhtar at gmail.com>' '''Read meta information from fb2 files''' -import mimetypes, os +import os from base64 import b64decode from lxml import etree from calibre.ebooks.metadata import MetaInformation from calibre.ebooks.chardet import xml_to_unicode +from calibre import guess_all_extensions XLINK_NS = 'http://www.w3.org/1999/xlink' def XLINK(name): @@ -71,7 +72,7 @@ def get_metadata(stream): binary = XPath('//fb2:binary[@id="%s"]'%id)(root) if binary: mt = binary[0].get('content-type', 'image/jpeg') - exts = mimetypes.guess_all_extensions(mt) + exts = guess_all_extensions(mt) if not exts: exts = ['.jpg'] cdata = (exts[0][1:], b64decode(tostring(binary[0]))) diff --git a/src/calibre/ebooks/metadata/opf2.py b/src/calibre/ebooks/metadata/opf2.py index d360451e2e..58c887bfdb 100644 --- a/src/calibre/ebooks/metadata/opf2.py +++ b/src/calibre/ebooks/metadata/opf2.py @@ -7,7 +7,7 @@ __docformat__ = 'restructuredtext en' lxml based OPF parser. ''' -import re, sys, unittest, functools, os, mimetypes, uuid, glob, cStringIO, json +import re, sys, unittest, functools, os, uuid, glob, cStringIO, json from urllib import unquote from urlparse import urlparse @@ -20,7 +20,7 @@ from calibre.ebooks.metadata import string_to_authors, MetaInformation, check_is from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.date import parse_date, isoformat from calibre.utils.localization import get_lang -from calibre import prints +from calibre import prints, guess_type from calibre.utils.cleantext import clean_ascii_chars class Resource(object): # {{{ @@ -42,7 +42,7 @@ class Resource(object): # {{{ self.path = None self.fragment = '' try: - self.mime_type = mimetypes.guess_type(href_or_path)[0] + self.mime_type = guess_type(href_or_path)[0] except: self.mime_type = None if self.mime_type is None: @@ -1000,7 +1000,7 @@ class OPF(object): # {{{ for t in ('cover', 'other.ms-coverimage-standard', 'other.ms-coverimage'): for item in self.guide: if item.type.lower() == t: - self.create_manifest_item(item.href(), mimetypes.guess_type(path)[0]) + self.create_manifest_item(item.href(), guess_type(path)[0]) return property(fget=fget, fset=fset) diff --git a/src/calibre/ebooks/oeb/base.py b/src/calibre/ebooks/oeb/base.py index ce75c97d78..f2c9696976 100644 --- a/src/calibre/ebooks/oeb/base.py +++ b/src/calibre/ebooks/oeb/base.py @@ -8,7 +8,6 @@ __copyright__ = '2008, Marshall T. Vandegrift <llasram@gmail.com>' __docformat__ = 'restructuredtext en' import os, re, uuid, logging -from mimetypes import types_map from collections import defaultdict from itertools import count from urlparse import urldefrag, urlparse, urlunparse, urljoin @@ -20,7 +19,7 @@ from calibre.translations.dynamic import translate from calibre.ebooks.chardet import xml_to_unicode from calibre.ebooks.oeb.entitydefs import ENTITYDEFS from calibre.ebooks.conversion.preprocess import CSSPreProcessor -from calibre import isbytestring, as_unicode +from calibre import isbytestring, as_unicode, get_types_map RECOVER_PARSER = etree.XMLParser(recover=True, no_network=True) @@ -247,7 +246,7 @@ def rewrite_links(root, link_repl_func, resolve_base_href=False): el.attrib['style'] = repl - +types_map = get_types_map() EPUB_MIME = types_map['.epub'] XHTML_MIME = types_map['.xhtml'] CSS_MIME = types_map['.css'] diff --git a/src/calibre/ebooks/oeb/reader.py b/src/calibre/ebooks/oeb/reader.py index 1c42a5a242..6c10436038 100644 --- a/src/calibre/ebooks/oeb/reader.py +++ b/src/calibre/ebooks/oeb/reader.py @@ -10,7 +10,6 @@ import sys, os, uuid, copy, re, cStringIO from itertools import izip from urlparse import urldefrag, urlparse from urllib import unquote as urlunquote -from mimetypes import guess_type from collections import defaultdict from lxml import etree @@ -29,6 +28,7 @@ from calibre.ebooks.oeb.entitydefs import ENTITYDEFS from calibre.utils.localization import get_lang from calibre.ptempfile import TemporaryDirectory from calibre.constants import __appname__, __version__ +from calibre import guess_type __all__ = ['OEBReader'] diff --git a/src/calibre/ebooks/snb/snbfile.py b/src/calibre/ebooks/snb/snbfile.py index 1a0986baf4..be4e537825 100644 --- a/src/calibre/ebooks/snb/snbfile.py +++ b/src/calibre/ebooks/snb/snbfile.py @@ -5,7 +5,8 @@ __copyright__ = '2010, Li Fanxi <lifanxi@freemindworld.com>' __docformat__ = 'restructuredtext en' import sys, struct, zlib, bz2, os -from mimetypes import types_map + +from calibre import guess_type class FileStream: def IsBinary(self): @@ -180,7 +181,7 @@ class SNBFile: file = open(os.path.join(path, fname), 'wb') file.write(f.fileBody) file.close() - fileNames.append((fname, types_map[ext])) + fileNames.append((fname, guess_type('a'+ext)[0])) return fileNames def Output(self, outputFile): diff --git a/src/calibre/library/cli.py b/src/calibre/library/cli.py index b1a8236151..61e7ec334d 100644 --- a/src/calibre/library/cli.py +++ b/src/calibre/library/cli.py @@ -10,8 +10,7 @@ Command line interface to the calibre database. import sys, os, cStringIO, re from textwrap import TextWrapper -from calibre import terminal_controller, preferred_encoding, prints, \ - isbytestring +from calibre import preferred_encoding, prints, isbytestring from calibre.utils.config import OptionParser, prefs, tweaks from calibre.ebooks.metadata.meta import get_metadata from calibre.library.database2 import LibraryDatabase2 @@ -53,6 +52,8 @@ def get_db(dbpath, options): def do_list(db, fields, afields, sort_by, ascending, search_text, line_width, separator, prefix, subtitle='Books in the calibre database'): + from calibre.constants import terminal_controller as tc + terminal_controller = tc() if sort_by: db.sort(sort_by, ascending) if search_text: @@ -1087,6 +1088,9 @@ def command_list_categories(args, dbpath): fields = ['category', 'tag_name', 'count', 'rating'] def do_list(): + from calibre.constants import terminal_controller as tc + terminal_controller = tc() + separator = ' ' widths = list(map(lambda x : 0, fields)) for i in data: diff --git a/src/calibre/startup.py b/src/calibre/startup.py index c883c43e8a..fd9ef01141 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -163,10 +163,6 @@ if not _run_once: __builtin__.__dict__['icu_upper'] = icu_upper __builtin__.__dict__['icu_title'] = title_case - import mimetypes - mimetypes.init([P('mime.types')]) - guess_type = mimetypes.guess_type - def test_lopen(): from calibre.ptempfile import TemporaryDirectory from calibre import CurrentDir diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 66316d051b..6f2840e95e 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -6,15 +6,15 @@ __docformat__ = 'restructuredtext en' ''' Manage application-wide preferences. ''' -import os, re, cPickle, textwrap, traceback, plistlib, json, base64, datetime +import os, re, cPickle, traceback, base64, datetime from copy import deepcopy from functools import partial from optparse import OptionParser as _OptionParser from optparse import IndentedHelpFormatter from collections import defaultdict -from calibre.constants import terminal_controller, config_dir, CONFIG_DIR_MODE, \ - __appname__, __version__, __author__ +from calibre.constants import (config_dir, CONFIG_DIR_MODE, __appname__, + __version__, __author__, terminal_controller) from calibre.utils.lock import LockError, ExclusiveFile plugin_dir = os.path.join(config_dir, 'plugins') @@ -29,23 +29,28 @@ def check_config_write_access(): class CustomHelpFormatter(IndentedHelpFormatter): def format_usage(self, usage): - return _("%sUsage%s: %s\n") % (terminal_controller.BLUE, terminal_controller.NORMAL, usage) + tc = terminal_controller() + return _("%sUsage%s: %s\n") % (tc.BLUE, tc.NORMAL, usage) def format_heading(self, heading): - return "%*s%s%s%s:\n" % (self.current_indent, terminal_controller.BLUE, - "", heading, terminal_controller.NORMAL) + tc = terminal_controller() + return "%*s%s%s%s:\n" % (self.current_indent, tc.BLUE, + "", heading, tc.NORMAL) def format_option(self, option): + import textwrap + tc = terminal_controller() + result = [] opts = self.option_strings[option] opt_width = self.help_position - self.current_indent - 2 if len(opts) > opt_width: opts = "%*s%s\n" % (self.current_indent, "", - terminal_controller.GREEN+opts+terminal_controller.NORMAL) + tc.GREEN+opts+tc.NORMAL) indent_first = self.help_position else: # start help on same line as opts - opts = "%*s%-*s " % (self.current_indent, "", opt_width + len(terminal_controller.GREEN + terminal_controller.NORMAL), - terminal_controller.GREEN + opts + terminal_controller.NORMAL) + opts = "%*s%-*s " % (self.current_indent, "", opt_width + + len(tc.GREEN + tc.NORMAL), tc.GREEN + opts + tc.NORMAL) indent_first = 0 result.append(opts) if option.help: @@ -71,9 +76,12 @@ class OptionParser(_OptionParser): gui_mode=False, conflict_handler='resolve', **kwds): + import textwrap + tc = terminal_controller() + usage = textwrap.dedent(usage) if epilog is None: - epilog = _('Created by ')+terminal_controller.RED+__author__+terminal_controller.NORMAL + epilog = _('Created by ')+tc.RED+__author__+tc.NORMAL usage += '\n\n'+_('''Whenever you pass arguments to %prog that have spaces in them, ''' '''enclose the arguments in quotation marks.''') _OptionParser.__init__(self, usage=usage, version=version, epilog=epilog, @@ -579,9 +587,11 @@ class XMLConfig(dict): self.refresh() def raw_to_object(self, raw): + import plistlib return plistlib.readPlistFromString(raw) def to_raw(self): + import plistlib return plistlib.writePlistToString(self) def refresh(self): @@ -601,6 +611,7 @@ class XMLConfig(dict): self.update(d) def __getitem__(self, key): + import plistlib try: ans = dict.__getitem__(self, key) if isinstance(ans, plistlib.Data): @@ -610,6 +621,7 @@ class XMLConfig(dict): return self.defaults.get(key, None) def get(self, key, default=None): + import plistlib try: ans = dict.__getitem__(self, key) if isinstance(ans, plistlib.Data): @@ -619,6 +631,7 @@ class XMLConfig(dict): return self.defaults.get(key, default) def __setitem__(self, key, val): + import plistlib if isinstance(val, (bytes, str)): val = plistlib.Data(val) dict.__setitem__(self, key, val) @@ -667,9 +680,11 @@ class JSONConfig(XMLConfig): EXTENSION = '.json' def raw_to_object(self, raw): + import json return json.loads(raw.decode('utf-8'), object_hook=from_json) def to_raw(self): + import json return json.dumps(self, indent=2, default=to_json) def __getitem__(self, key): From cdd47f4b10aecd9a55b13fc92e46e60e8dca6650 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 13:14:19 -0600 Subject: [PATCH 66/90] And some more delay load optimizations --- src/calibre/utils/config.py | 468 +----------------------------- src/calibre/utils/config_base.py | 467 +++++++++++++++++++++++++++++ src/calibre/utils/icu.py | 2 +- src/calibre/utils/localization.py | 2 +- 4 files changed, 480 insertions(+), 459 deletions(-) create mode 100644 src/calibre/utils/config_base.py diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 6f2840e95e..87682c9792 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -6,22 +6,25 @@ __docformat__ = 'restructuredtext en' ''' Manage application-wide preferences. ''' -import os, re, cPickle, traceback, base64, datetime +import os, re, cPickle, base64, datetime, json, plistlib from copy import deepcopy -from functools import partial from optparse import OptionParser as _OptionParser from optparse import IndentedHelpFormatter -from collections import defaultdict from calibre.constants import (config_dir, CONFIG_DIR_MODE, __appname__, __version__, __author__, terminal_controller) -from calibre.utils.lock import LockError, ExclusiveFile +from calibre.utils.lock import ExclusiveFile +from calibre.utils.config_base import (make_config_dir, Option, OptionValues, + OptionSet, ConfigInterface, Config, prefs, StringConfig, ConfigProxy, + read_raw_tweaks, read_tweaks, write_tweaks, tweaks) -plugin_dir = os.path.join(config_dir, 'plugins') +if False: + # Make pyflakes happy + Config, ConfigProxy, Option, OptionValues, StringConfig + OptionSet, ConfigInterface, read_tweaks, write_tweaks + read_raw_tweaks, tweaks -def make_config_dir(): - if not os.path.exists(plugin_dir): - os.makedirs(plugin_dir, mode=CONFIG_DIR_MODE) +test_eight_code = tweaks.get('test_eight_code', False) def check_config_write_access(): return os.access(config_dir, os.W_OK) and os.access(config_dir, os.X_OK) @@ -154,353 +157,6 @@ class OptionParser(_OptionParser): upper.__dict__[dest] = lower.__dict__[dest] - -class Option(object): - - def __init__(self, name, switches=[], help='', type=None, choices=None, - check=None, group=None, default=None, action=None, metavar=None): - if choices: - type = 'choice' - - self.name = name - self.switches = switches - self.help = help.replace('%default', repr(default)) if help else None - self.type = type - if self.type is None and action is None and choices is None: - if isinstance(default, float): - self.type = 'float' - elif isinstance(default, int) and not isinstance(default, bool): - self.type = 'int' - - self.choices = choices - self.check = check - self.group = group - self.default = default - self.action = action - self.metavar = metavar - - def __eq__(self, other): - return self.name == getattr(other, 'name', other) - - def __repr__(self): - return 'Option: '+self.name - - def __str__(self): - return repr(self) - -class OptionValues(object): - - def copy(self): - return deepcopy(self) - -class OptionSet(object): - - OVERRIDE_PAT = re.compile(r'#{3,100} Override Options #{15}(.*?)#{3,100} End Override #{3,100}', - re.DOTALL|re.IGNORECASE) - - def __init__(self, description=''): - self.description = description - self.defaults = {} - self.preferences = [] - self.group_list = [] - self.groups = {} - self.set_buffer = {} - - def has_option(self, name_or_option_object): - if name_or_option_object in self.preferences: - return True - for p in self.preferences: - if p.name == name_or_option_object: - return True - return False - - def get_option(self, name_or_option_object): - idx = self.preferences.index(name_or_option_object) - if idx > -1: - return self.preferences[idx] - for p in self.preferences: - if p.name == name_or_option_object: - return p - - def add_group(self, name, description=''): - if name in self.group_list: - raise ValueError('A group by the name %s already exists in this set'%name) - self.groups[name] = description - self.group_list.append(name) - return partial(self.add_opt, group=name) - - def update(self, other): - for name in other.groups.keys(): - self.groups[name] = other.groups[name] - if name not in self.group_list: - self.group_list.append(name) - for pref in other.preferences: - if pref in self.preferences: - self.preferences.remove(pref) - self.preferences.append(pref) - - def smart_update(self, opts1, opts2): - ''' - Updates the preference values in opts1 using only the non-default preference values in opts2. - ''' - for pref in self.preferences: - new = getattr(opts2, pref.name, pref.default) - if new != pref.default: - setattr(opts1, pref.name, new) - - def remove_opt(self, name): - if name in self.preferences: - self.preferences.remove(name) - - - def add_opt(self, name, switches=[], help=None, type=None, choices=None, - group=None, default=None, action=None, metavar=None): - ''' - Add an option to this section. - - :param name: The name of this option. Must be a valid Python identifier. - Must also be unique in this OptionSet and all its subsets. - :param switches: List of command line switches for this option - (as supplied to :module:`optparse`). If empty, this - option will not be added to the command line parser. - :param help: Help text. - :param type: Type checking of option values. Supported types are: - `None, 'choice', 'complex', 'float', 'int', 'string'`. - :param choices: List of strings or `None`. - :param group: Group this option belongs to. You must previously - have created this group with a call to :method:`add_group`. - :param default: The default value for this option. - :param action: The action to pass to optparse. Supported values are: - `None, 'count'`. For choices and boolean options, - action is automatically set correctly. - ''' - pref = Option(name, switches=switches, help=help, type=type, choices=choices, - group=group, default=default, action=action, metavar=None) - if group is not None and group not in self.groups.keys(): - raise ValueError('Group %s has not been added to this section'%group) - if pref in self.preferences: - raise ValueError('An option with the name %s already exists in this set.'%name) - self.preferences.append(pref) - self.defaults[name] = default - - def option_parser(self, user_defaults=None, usage='', gui_mode=False): - parser = OptionParser(usage, gui_mode=gui_mode) - groups = defaultdict(lambda : parser) - for group, desc in self.groups.items(): - groups[group] = parser.add_option_group(group.upper(), desc) - - for pref in self.preferences: - if not pref.switches: - continue - g = groups[pref.group] - action = pref.action - if action is None: - action = 'store' - if pref.default is True or pref.default is False: - action = 'store_' + ('false' if pref.default else 'true') - args = dict( - dest=pref.name, - help=pref.help, - metavar=pref.metavar, - type=pref.type, - choices=pref.choices, - default=getattr(user_defaults, pref.name, pref.default), - action=action, - ) - g.add_option(*pref.switches, **args) - - - return parser - - def get_override_section(self, src): - match = self.OVERRIDE_PAT.search(src) - if match: - return match.group() - return '' - - def parse_string(self, src): - options = {'cPickle':cPickle} - if src is not None: - try: - if not isinstance(src, unicode): - src = src.decode('utf-8') - exec src in options - except: - print 'Failed to parse options string:' - print repr(src) - traceback.print_exc() - opts = OptionValues() - for pref in self.preferences: - val = options.get(pref.name, pref.default) - formatter = __builtins__.get(pref.type, None) - if callable(formatter): - val = formatter(val) - setattr(opts, pref.name, val) - - return opts - - def render_group(self, name, desc, opts): - prefs = [pref for pref in self.preferences if pref.group == name] - lines = ['### Begin group: %s'%(name if name else 'DEFAULT')] - if desc: - lines += map(lambda x: '# '+x, desc.split('\n')) - lines.append(' ') - for pref in prefs: - lines.append('# '+pref.name.replace('_', ' ')) - if pref.help: - lines += map(lambda x: '# ' + x, pref.help.split('\n')) - lines.append('%s = %s'%(pref.name, - self.serialize_opt(getattr(opts, pref.name, pref.default)))) - lines.append(' ') - return '\n'.join(lines) - - def serialize_opt(self, val): - if val is val is True or val is False or val is None or \ - isinstance(val, (int, float, long, basestring)): - return repr(val) - from PyQt4.QtCore import QString - if isinstance(val, QString): - return repr(unicode(val)) - pickle = cPickle.dumps(val, -1) - return 'cPickle.loads(%s)'%repr(pickle) - - def serialize(self, opts): - src = '# %s\n\n'%(self.description.replace('\n', '\n# ')) - groups = [self.render_group(name, self.groups.get(name, ''), opts) \ - for name in [None] + self.group_list] - return src + '\n\n'.join(groups) - -class ConfigInterface(object): - - def __init__(self, description): - self.option_set = OptionSet(description=description) - self.add_opt = self.option_set.add_opt - self.add_group = self.option_set.add_group - self.remove_opt = self.remove = self.option_set.remove_opt - self.parse_string = self.option_set.parse_string - self.get_option = self.option_set.get_option - self.preferences = self.option_set.preferences - - def update(self, other): - self.option_set.update(other.option_set) - - def option_parser(self, usage='', gui_mode=False): - return self.option_set.option_parser(user_defaults=self.parse(), - usage=usage, gui_mode=gui_mode) - - def smart_update(self, opts1, opts2): - self.option_set.smart_update(opts1, opts2) - - -class Config(ConfigInterface): - ''' - A file based configuration. - ''' - - def __init__(self, basename, description=''): - ConfigInterface.__init__(self, description) - self.config_file_path = os.path.join(config_dir, basename+'.py') - - - def parse(self): - src = '' - if os.path.exists(self.config_file_path): - try: - with ExclusiveFile(self.config_file_path) as f: - try: - src = f.read().decode('utf-8') - except ValueError: - print "Failed to parse", self.config_file_path - traceback.print_exc() - except LockError: - raise IOError('Could not lock config file: %s'%self.config_file_path) - return self.option_set.parse_string(src) - - def as_string(self): - if not os.path.exists(self.config_file_path): - return '' - try: - with ExclusiveFile(self.config_file_path) as f: - return f.read().decode('utf-8') - except LockError: - raise IOError('Could not lock config file: %s'%self.config_file_path) - - def set(self, name, val): - if not self.option_set.has_option(name): - raise ValueError('The option %s is not defined.'%name) - try: - if not os.path.exists(config_dir): - make_config_dir() - with ExclusiveFile(self.config_file_path) as f: - src = f.read() - opts = self.option_set.parse_string(src) - setattr(opts, name, val) - footer = self.option_set.get_override_section(src) - src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n' - f.seek(0) - f.truncate() - if isinstance(src, unicode): - src = src.encode('utf-8') - f.write(src) - except LockError: - raise IOError('Could not lock config file: %s'%self.config_file_path) - -class StringConfig(ConfigInterface): - ''' - A string based configuration - ''' - - def __init__(self, src, description=''): - ConfigInterface.__init__(self, description) - self.src = src - - def parse(self): - return self.option_set.parse_string(self.src) - - def set(self, name, val): - if not self.option_set.has_option(name): - raise ValueError('The option %s is not defined.'%name) - opts = self.option_set.parse_string(self.src) - setattr(opts, name, val) - footer = self.option_set.get_override_section(self.src) - self.src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n' - -class ConfigProxy(object): - ''' - A Proxy to minimize file reads for widely used config settings - ''' - - def __init__(self, config): - self.__config = config - self.__opts = None - - @property - def defaults(self): - return self.__config.option_set.defaults - - def refresh(self): - self.__opts = self.__config.parse() - - def __getitem__(self, key): - return self.get(key) - - def __setitem__(self, key, val): - return self.set(key, val) - - def get(self, key): - if self.__opts is None: - self.refresh() - return getattr(self.__opts, key) - - def set(self, key, val): - if self.__opts is None: - self.refresh() - setattr(self.__opts, key, val) - return self.__config.set(key, val) - - def help(self, key): - return self.__config.get_option(key).help - class DynamicConfig(dict): ''' A replacement for QSettings that supports dynamic config keys. @@ -587,11 +243,9 @@ class XMLConfig(dict): self.refresh() def raw_to_object(self, raw): - import plistlib return plistlib.readPlistFromString(raw) def to_raw(self): - import plistlib return plistlib.writePlistToString(self) def refresh(self): @@ -611,7 +265,6 @@ class XMLConfig(dict): self.update(d) def __getitem__(self, key): - import plistlib try: ans = dict.__getitem__(self, key) if isinstance(ans, plistlib.Data): @@ -621,7 +274,6 @@ class XMLConfig(dict): return self.defaults.get(key, None) def get(self, key, default=None): - import plistlib try: ans = dict.__getitem__(self, key) if isinstance(ans, plistlib.Data): @@ -631,7 +283,6 @@ class XMLConfig(dict): return self.defaults.get(key, default) def __setitem__(self, key, val): - import plistlib if isinstance(val, (bytes, str)): val = plistlib.Data(val) dict.__setitem__(self, key, val) @@ -680,11 +331,9 @@ class JSONConfig(XMLConfig): EXTENSION = '.json' def raw_to_object(self, raw): - import json return json.loads(raw.decode('utf-8'), object_hook=from_json) def to_raw(self): - import json return json.dumps(self, indent=2, default=to_json) def __getitem__(self, key): @@ -705,101 +354,6 @@ class JSONConfig(XMLConfig): -def _prefs(): - c = Config('global', 'calibre wide preferences') - c.add_opt('database_path', - default=os.path.expanduser('~/library1.db'), - help=_('Path to the database in which books are stored')) - c.add_opt('filename_pattern', default=ur'(?P<title>.+) - (?P<author>[^_]+)', - help=_('Pattern to guess metadata from filenames')) - c.add_opt('isbndb_com_key', default='', - help=_('Access key for isbndb.com')) - c.add_opt('network_timeout', default=5, - help=_('Default timeout for network operations (seconds)')) - c.add_opt('library_path', default=None, - help=_('Path to directory in which your library of books is stored')) - c.add_opt('language', default=None, - help=_('The language in which to display the user interface')) - c.add_opt('output_format', default='EPUB', - help=_('The default output format for ebook conversions.')) - c.add_opt('input_format_order', default=['EPUB', 'MOBI', 'LIT', 'PRC', - 'FB2', 'HTML', 'HTM', 'XHTM', 'SHTML', 'XHTML', 'ZIP', 'ODT', 'RTF', 'PDF', - 'TXT'], - help=_('Ordered list of formats to prefer for input.')) - c.add_opt('read_file_metadata', default=True, - help=_('Read metadata from files')) - c.add_opt('worker_process_priority', default='normal', - help=_('The priority of worker processes. A higher priority ' - 'means they run faster and consume more resources. ' - 'Most tasks like conversion/news download/adding books/etc. ' - 'are affected by this setting.')) - c.add_opt('swap_author_names', default=False, - help=_('Swap author first and last names when reading metadata')) - c.add_opt('add_formats_to_existing', default=False, - help=_('Add new formats to existing book records')) - c.add_opt('installation_uuid', default=None, help='Installation UUID') - c.add_opt('new_book_tags', default=[], help=_('Tags to apply to books added to the library')) - - # these are here instead of the gui preferences because calibredb and - # calibre server can execute searches - c.add_opt('saved_searches', default={}, help=_('List of named saved searches')) - c.add_opt('user_categories', default={}, help=_('User-created tag browser categories')) - c.add_opt('manage_device_metadata', default='manual', - help=_('How and when calibre updates metadata on the device.')) - c.add_opt('limit_search_columns', default=False, - help=_('When searching for text without using lookup ' - 'prefixes, as for example, Red instead of title:Red, ' - 'limit the columns searched to those named below.')) - c.add_opt('limit_search_columns_to', - default=['title', 'authors', 'tags', 'series', 'publisher'], - help=_('Choose columns to be searched when not using prefixes, ' - 'as for example, when searching for Redd instead of ' - 'title:Red. Enter a list of search/lookup names ' - 'separated by commas. Only takes effect if you set the option ' - 'to limit search columns above.')) - - c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.') - return c - -prefs = ConfigProxy(_prefs()) -if prefs['installation_uuid'] is None: - import uuid - prefs['installation_uuid'] = str(uuid.uuid4()) - -# Read tweaks -def read_raw_tweaks(): - make_config_dir() - default_tweaks = P('default_tweaks.py', data=True, - allow_user_override=False) - tweaks_file = os.path.join(config_dir, 'tweaks.py') - if not os.path.exists(tweaks_file): - with open(tweaks_file, 'wb') as f: - f.write(default_tweaks) - with open(tweaks_file, 'rb') as f: - return default_tweaks, f.read() - -def read_tweaks(): - default_tweaks, tweaks = read_raw_tweaks() - l, g = {}, {} - try: - exec tweaks in g, l - except: - print 'Failed to load custom tweaks file' - traceback.print_exc() - dl, dg = {}, {} - exec default_tweaks in dg, dl - dl.update(l) - return dl - -def write_tweaks(raw): - make_config_dir() - tweaks_file = os.path.join(config_dir, 'tweaks.py') - with open(tweaks_file, 'wb') as f: - f.write(raw) - - -tweaks = read_tweaks() -test_eight_code = tweaks.get('test_eight_code', False) def migrate(): if hasattr(os, 'geteuid') and os.geteuid() == 0: diff --git a/src/calibre/utils/config_base.py b/src/calibre/utils/config_base.py new file mode 100644 index 0000000000..7660370353 --- /dev/null +++ b/src/calibre/utils/config_base.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal <kovid@kovidgoyal.net>' +__docformat__ = 'restructuredtext en' + +import os, re, cPickle, traceback +from functools import partial +from collections import defaultdict +from copy import deepcopy + +from calibre.utils.lock import LockError, ExclusiveFile +from calibre.constants import config_dir, CONFIG_DIR_MODE + +plugin_dir = os.path.join(config_dir, 'plugins') + +def make_config_dir(): + if not os.path.exists(plugin_dir): + os.makedirs(plugin_dir, mode=CONFIG_DIR_MODE) + +class Option(object): + + def __init__(self, name, switches=[], help='', type=None, choices=None, + check=None, group=None, default=None, action=None, metavar=None): + if choices: + type = 'choice' + + self.name = name + self.switches = switches + self.help = help.replace('%default', repr(default)) if help else None + self.type = type + if self.type is None and action is None and choices is None: + if isinstance(default, float): + self.type = 'float' + elif isinstance(default, int) and not isinstance(default, bool): + self.type = 'int' + + self.choices = choices + self.check = check + self.group = group + self.default = default + self.action = action + self.metavar = metavar + + def __eq__(self, other): + return self.name == getattr(other, 'name', other) + + def __repr__(self): + return 'Option: '+self.name + + def __str__(self): + return repr(self) + +class OptionValues(object): + + def copy(self): + return deepcopy(self) + +class OptionSet(object): + + OVERRIDE_PAT = re.compile(r'#{3,100} Override Options #{15}(.*?)#{3,100} End Override #{3,100}', + re.DOTALL|re.IGNORECASE) + + def __init__(self, description=''): + self.description = description + self.defaults = {} + self.preferences = [] + self.group_list = [] + self.groups = {} + self.set_buffer = {} + + def has_option(self, name_or_option_object): + if name_or_option_object in self.preferences: + return True + for p in self.preferences: + if p.name == name_or_option_object: + return True + return False + + def get_option(self, name_or_option_object): + idx = self.preferences.index(name_or_option_object) + if idx > -1: + return self.preferences[idx] + for p in self.preferences: + if p.name == name_or_option_object: + return p + + def add_group(self, name, description=''): + if name in self.group_list: + raise ValueError('A group by the name %s already exists in this set'%name) + self.groups[name] = description + self.group_list.append(name) + return partial(self.add_opt, group=name) + + def update(self, other): + for name in other.groups.keys(): + self.groups[name] = other.groups[name] + if name not in self.group_list: + self.group_list.append(name) + for pref in other.preferences: + if pref in self.preferences: + self.preferences.remove(pref) + self.preferences.append(pref) + + def smart_update(self, opts1, opts2): + ''' + Updates the preference values in opts1 using only the non-default preference values in opts2. + ''' + for pref in self.preferences: + new = getattr(opts2, pref.name, pref.default) + if new != pref.default: + setattr(opts1, pref.name, new) + + def remove_opt(self, name): + if name in self.preferences: + self.preferences.remove(name) + + + def add_opt(self, name, switches=[], help=None, type=None, choices=None, + group=None, default=None, action=None, metavar=None): + ''' + Add an option to this section. + + :param name: The name of this option. Must be a valid Python identifier. + Must also be unique in this OptionSet and all its subsets. + :param switches: List of command line switches for this option + (as supplied to :module:`optparse`). If empty, this + option will not be added to the command line parser. + :param help: Help text. + :param type: Type checking of option values. Supported types are: + `None, 'choice', 'complex', 'float', 'int', 'string'`. + :param choices: List of strings or `None`. + :param group: Group this option belongs to. You must previously + have created this group with a call to :method:`add_group`. + :param default: The default value for this option. + :param action: The action to pass to optparse. Supported values are: + `None, 'count'`. For choices and boolean options, + action is automatically set correctly. + ''' + pref = Option(name, switches=switches, help=help, type=type, choices=choices, + group=group, default=default, action=action, metavar=None) + if group is not None and group not in self.groups.keys(): + raise ValueError('Group %s has not been added to this section'%group) + if pref in self.preferences: + raise ValueError('An option with the name %s already exists in this set.'%name) + self.preferences.append(pref) + self.defaults[name] = default + + def option_parser(self, user_defaults=None, usage='', gui_mode=False): + from calibre.utils.config import OptionParser + parser = OptionParser(usage, gui_mode=gui_mode) + groups = defaultdict(lambda : parser) + for group, desc in self.groups.items(): + groups[group] = parser.add_option_group(group.upper(), desc) + + for pref in self.preferences: + if not pref.switches: + continue + g = groups[pref.group] + action = pref.action + if action is None: + action = 'store' + if pref.default is True or pref.default is False: + action = 'store_' + ('false' if pref.default else 'true') + args = dict( + dest=pref.name, + help=pref.help, + metavar=pref.metavar, + type=pref.type, + choices=pref.choices, + default=getattr(user_defaults, pref.name, pref.default), + action=action, + ) + g.add_option(*pref.switches, **args) + + + return parser + + def get_override_section(self, src): + match = self.OVERRIDE_PAT.search(src) + if match: + return match.group() + return '' + + def parse_string(self, src): + options = {'cPickle':cPickle} + if src is not None: + try: + if not isinstance(src, unicode): + src = src.decode('utf-8') + exec src in options + except: + print 'Failed to parse options string:' + print repr(src) + traceback.print_exc() + opts = OptionValues() + for pref in self.preferences: + val = options.get(pref.name, pref.default) + formatter = __builtins__.get(pref.type, None) + if callable(formatter): + val = formatter(val) + setattr(opts, pref.name, val) + + return opts + + def render_group(self, name, desc, opts): + prefs = [pref for pref in self.preferences if pref.group == name] + lines = ['### Begin group: %s'%(name if name else 'DEFAULT')] + if desc: + lines += map(lambda x: '# '+x, desc.split('\n')) + lines.append(' ') + for pref in prefs: + lines.append('# '+pref.name.replace('_', ' ')) + if pref.help: + lines += map(lambda x: '# ' + x, pref.help.split('\n')) + lines.append('%s = %s'%(pref.name, + self.serialize_opt(getattr(opts, pref.name, pref.default)))) + lines.append(' ') + return '\n'.join(lines) + + def serialize_opt(self, val): + if val is val is True or val is False or val is None or \ + isinstance(val, (int, float, long, basestring)): + return repr(val) + from PyQt4.QtCore import QString + if isinstance(val, QString): + return repr(unicode(val)) + pickle = cPickle.dumps(val, -1) + return 'cPickle.loads(%s)'%repr(pickle) + + def serialize(self, opts): + src = '# %s\n\n'%(self.description.replace('\n', '\n# ')) + groups = [self.render_group(name, self.groups.get(name, ''), opts) \ + for name in [None] + self.group_list] + return src + '\n\n'.join(groups) + +class ConfigInterface(object): + + def __init__(self, description): + self.option_set = OptionSet(description=description) + self.add_opt = self.option_set.add_opt + self.add_group = self.option_set.add_group + self.remove_opt = self.remove = self.option_set.remove_opt + self.parse_string = self.option_set.parse_string + self.get_option = self.option_set.get_option + self.preferences = self.option_set.preferences + + def update(self, other): + self.option_set.update(other.option_set) + + def option_parser(self, usage='', gui_mode=False): + return self.option_set.option_parser(user_defaults=self.parse(), + usage=usage, gui_mode=gui_mode) + + def smart_update(self, opts1, opts2): + self.option_set.smart_update(opts1, opts2) + + +class Config(ConfigInterface): + ''' + A file based configuration. + ''' + + def __init__(self, basename, description=''): + ConfigInterface.__init__(self, description) + self.config_file_path = os.path.join(config_dir, basename+'.py') + + + def parse(self): + src = '' + if os.path.exists(self.config_file_path): + try: + with ExclusiveFile(self.config_file_path) as f: + try: + src = f.read().decode('utf-8') + except ValueError: + print "Failed to parse", self.config_file_path + traceback.print_exc() + except LockError: + raise IOError('Could not lock config file: %s'%self.config_file_path) + return self.option_set.parse_string(src) + + def as_string(self): + if not os.path.exists(self.config_file_path): + return '' + try: + with ExclusiveFile(self.config_file_path) as f: + return f.read().decode('utf-8') + except LockError: + raise IOError('Could not lock config file: %s'%self.config_file_path) + + def set(self, name, val): + if not self.option_set.has_option(name): + raise ValueError('The option %s is not defined.'%name) + try: + if not os.path.exists(config_dir): + make_config_dir() + with ExclusiveFile(self.config_file_path) as f: + src = f.read() + opts = self.option_set.parse_string(src) + setattr(opts, name, val) + footer = self.option_set.get_override_section(src) + src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n' + f.seek(0) + f.truncate() + if isinstance(src, unicode): + src = src.encode('utf-8') + f.write(src) + except LockError: + raise IOError('Could not lock config file: %s'%self.config_file_path) + +class StringConfig(ConfigInterface): + ''' + A string based configuration + ''' + + def __init__(self, src, description=''): + ConfigInterface.__init__(self, description) + self.src = src + + def parse(self): + return self.option_set.parse_string(self.src) + + def set(self, name, val): + if not self.option_set.has_option(name): + raise ValueError('The option %s is not defined.'%name) + opts = self.option_set.parse_string(self.src) + setattr(opts, name, val) + footer = self.option_set.get_override_section(self.src) + self.src = self.option_set.serialize(opts)+ '\n\n' + footer + '\n' + +class ConfigProxy(object): + ''' + A Proxy to minimize file reads for widely used config settings + ''' + + def __init__(self, config): + self.__config = config + self.__opts = None + + @property + def defaults(self): + return self.__config.option_set.defaults + + def refresh(self): + self.__opts = self.__config.parse() + + def __getitem__(self, key): + return self.get(key) + + def __setitem__(self, key, val): + return self.set(key, val) + + def get(self, key): + if self.__opts is None: + self.refresh() + return getattr(self.__opts, key) + + def set(self, key, val): + if self.__opts is None: + self.refresh() + setattr(self.__opts, key, val) + return self.__config.set(key, val) + + def help(self, key): + return self.__config.get_option(key).help + + + +def _prefs(): + c = Config('global', 'calibre wide preferences') + c.add_opt('database_path', + default=os.path.expanduser('~/library1.db'), + help=_('Path to the database in which books are stored')) + c.add_opt('filename_pattern', default=ur'(?P<title>.+) - (?P<author>[^_]+)', + help=_('Pattern to guess metadata from filenames')) + c.add_opt('isbndb_com_key', default='', + help=_('Access key for isbndb.com')) + c.add_opt('network_timeout', default=5, + help=_('Default timeout for network operations (seconds)')) + c.add_opt('library_path', default=None, + help=_('Path to directory in which your library of books is stored')) + c.add_opt('language', default=None, + help=_('The language in which to display the user interface')) + c.add_opt('output_format', default='EPUB', + help=_('The default output format for ebook conversions.')) + c.add_opt('input_format_order', default=['EPUB', 'MOBI', 'LIT', 'PRC', + 'FB2', 'HTML', 'HTM', 'XHTM', 'SHTML', 'XHTML', 'ZIP', 'ODT', 'RTF', 'PDF', + 'TXT'], + help=_('Ordered list of formats to prefer for input.')) + c.add_opt('read_file_metadata', default=True, + help=_('Read metadata from files')) + c.add_opt('worker_process_priority', default='normal', + help=_('The priority of worker processes. A higher priority ' + 'means they run faster and consume more resources. ' + 'Most tasks like conversion/news download/adding books/etc. ' + 'are affected by this setting.')) + c.add_opt('swap_author_names', default=False, + help=_('Swap author first and last names when reading metadata')) + c.add_opt('add_formats_to_existing', default=False, + help=_('Add new formats to existing book records')) + c.add_opt('installation_uuid', default=None, help='Installation UUID') + c.add_opt('new_book_tags', default=[], help=_('Tags to apply to books added to the library')) + + # these are here instead of the gui preferences because calibredb and + # calibre server can execute searches + c.add_opt('saved_searches', default={}, help=_('List of named saved searches')) + c.add_opt('user_categories', default={}, help=_('User-created tag browser categories')) + c.add_opt('manage_device_metadata', default='manual', + help=_('How and when calibre updates metadata on the device.')) + c.add_opt('limit_search_columns', default=False, + help=_('When searching for text without using lookup ' + 'prefixes, as for example, Red instead of title:Red, ' + 'limit the columns searched to those named below.')) + c.add_opt('limit_search_columns_to', + default=['title', 'authors', 'tags', 'series', 'publisher'], + help=_('Choose columns to be searched when not using prefixes, ' + 'as for example, when searching for Redd instead of ' + 'title:Red. Enter a list of search/lookup names ' + 'separated by commas. Only takes effect if you set the option ' + 'to limit search columns above.')) + + c.add_opt('migrated', default=False, help='For Internal use. Don\'t modify.') + return c + +prefs = ConfigProxy(_prefs()) +if prefs['installation_uuid'] is None: + import uuid + prefs['installation_uuid'] = str(uuid.uuid4()) + +# Read tweaks +def read_raw_tweaks(): + make_config_dir() + default_tweaks = P('default_tweaks.py', data=True, + allow_user_override=False) + tweaks_file = os.path.join(config_dir, 'tweaks.py') + if not os.path.exists(tweaks_file): + with open(tweaks_file, 'wb') as f: + f.write(default_tweaks) + with open(tweaks_file, 'rb') as f: + return default_tweaks, f.read() + +def read_tweaks(): + default_tweaks, tweaks = read_raw_tweaks() + l, g = {}, {} + try: + exec tweaks in g, l + except: + import traceback + print 'Failed to load custom tweaks file' + traceback.print_exc() + dl, dg = {}, {} + exec default_tweaks in dg, dl + dl.update(l) + return dl + +def write_tweaks(raw): + make_config_dir() + tweaks_file = os.path.join(config_dir, 'tweaks.py') + with open(tweaks_file, 'wb') as f: + f.write(raw) + + +tweaks = read_tweaks() + + diff --git a/src/calibre/utils/icu.py b/src/calibre/utils/icu.py index f17ff1b17f..d5bef449c4 100644 --- a/src/calibre/utils/icu.py +++ b/src/calibre/utils/icu.py @@ -10,7 +10,7 @@ import sys from functools import partial from calibre.constants import plugins -from calibre.utils.config import tweaks +from calibre.utils.config_base import tweaks _icu = _collator = None _locale = None diff --git a/src/calibre/utils/localization.py b/src/calibre/utils/localization.py index f676b99e43..533fd03457 100644 --- a/src/calibre/utils/localization.py +++ b/src/calibre/utils/localization.py @@ -24,7 +24,7 @@ def available_translations(): def get_lang(): 'Try to figure out what language to display the interface in' - from calibre.utils.config import prefs + from calibre.utils.config_base import prefs lang = prefs['language'] lang = os.environ.get('CALIBRE_OVERRIDE_LANG', lang) if lang is not None: From 5884c1b21e9c0721774a53f946c28eb86afc5cce Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 13:46:02 -0600 Subject: [PATCH 67/90] ... --- setup/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/setup/__init__.py b/setup/__init__.py index 9e62fb377d..61bafd2282 100644 --- a/setup/__init__.py +++ b/setup/__init__.py @@ -24,8 +24,10 @@ def initialize_constants(): global __version__, __appname__, modules, functions, basenames, scripts src = open('src/calibre/constants.py', 'rb').read() - __version__ = re.search(r'__version__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1) - __appname__ = re.search(r'__appname__\s+=\s+[\'"]([^\'"]+)[\'"]', src).group(1) + nv = re.search(r'numeric_version\s+=\s+\((\d+), (\d+), (\d+)\)', src) + __version__ = '%s.%s.%s'%(nv.group(1), nv.group(2), nv.group(3)) + __appname__ = re.search(r'__appname__\s+=\s+(u{0,1})[\'"]([^\'"]+)[\'"]', + src).group(2) epsrc = re.compile(r'entry_points = (\{.*?\})', re.DOTALL).\ search(open('src/calibre/linux.py', 'rb').read()).group(1) entry_points = eval(epsrc, {'__appname__': __appname__}) From 303a7c6b5c0ded507fb8685ecb2989f0ab20b945 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 13:51:42 -0600 Subject: [PATCH 68/90] ... --- src/calibre/utils/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/utils/config.py b/src/calibre/utils/config.py index 87682c9792..8b23cf3071 100644 --- a/src/calibre/utils/config.py +++ b/src/calibre/utils/config.py @@ -16,13 +16,13 @@ from calibre.constants import (config_dir, CONFIG_DIR_MODE, __appname__, from calibre.utils.lock import ExclusiveFile from calibre.utils.config_base import (make_config_dir, Option, OptionValues, OptionSet, ConfigInterface, Config, prefs, StringConfig, ConfigProxy, - read_raw_tweaks, read_tweaks, write_tweaks, tweaks) + read_raw_tweaks, read_tweaks, write_tweaks, tweaks, plugin_dir) if False: # Make pyflakes happy Config, ConfigProxy, Option, OptionValues, StringConfig OptionSet, ConfigInterface, read_tweaks, write_tweaks - read_raw_tweaks, tweaks + read_raw_tweaks, tweaks, plugin_dir test_eight_code = tweaks.get('test_eight_code', False) From 7fc7478c979751606da53b7f9d5fbed0225f971d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 14:06:56 -0600 Subject: [PATCH 69/90] Delay load calibre C extensions. Worker launch is now under 0.07 seconds (only about 3 times the time taken to launch bare python) --- src/calibre/constants.py | 64 +++++++++++++++++++++++----------- src/calibre/devices/scanner.py | 2 +- src/calibre/gui2/main.py | 6 ++-- 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 62612d9c66..a15ffbf967 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -12,7 +12,7 @@ __author__ = u"Kovid Goyal <kovid@kovidgoyal.net>" Various run time constants. ''' -import sys, locale, codecs, os, importlib +import sys, locale, codecs, os, importlib, collections _tc = None def terminal_controller(): @@ -52,15 +52,12 @@ def debug(): DEBUG = True # plugins {{{ -plugins = None -if plugins is None: - # Load plugins - def load_plugins(): - plugins = {} - plugin_path = sys.extensions_location - sys.path.insert(0, plugin_path) - for plugin in [ +class Plugins(collections.Mapping): + + def __init__(self): + self._plugins = {} + plugins = [ 'pictureflow', 'lzx', 'msdes', @@ -74,19 +71,44 @@ if plugins is None: 'chm_extra', 'icu', 'speedup', - ] + \ - (['winutil'] if iswindows else []) + \ - (['usbobserver'] if isosx else []): - try: - p, err = importlib.import_module(plugin), '' - except Exception as err: - p = None - err = str(err) - plugins[plugin] = (p, err) - sys.path.remove(plugin_path) - return plugins + ] + if iswindows: + plugins.append('winutil') + if isosx: + plugins.append(['usbobserver']) + self.plugins = frozenset(plugins) - plugins = load_plugins() + def load_plugin(self, name): + if name in self._plugins: + return + sys.path.insert(0, sys.extensions_location) + try: + p, err = importlib.import_module(name), '' + except Exception as err: + p = None + err = str(err) + self._plugins[name] = (p, err) + sys.path.remove(sys.extensions_location) + + def __iter__(self): + return iter(self.plugins) + + def __len__(self): + return len(self.plugins) + + def __contains__(self, name): + return name in self.plugins + + def __getitem__(self, name): + if name not in self.plugins: + raise KeyError('No plugin named %r'%name) + self.load_plugin(name) + return self._plugins[name] + + +plugins = None +if plugins is None: + plugins = Plugins() # }}} # config_dir {{{ diff --git a/src/calibre/devices/scanner.py b/src/calibre/devices/scanner.py index c63eada0c8..9b729a3561 100644 --- a/src/calibre/devices/scanner.py +++ b/src/calibre/devices/scanner.py @@ -8,7 +8,7 @@ manner. import sys, os, re from threading import RLock -from calibre import iswindows, isosx, plugins, islinux +from calibre.constants import iswindows, isosx, plugins, islinux osx_scanner = win_scanner = linux_scanner = None diff --git a/src/calibre/gui2/main.py b/src/calibre/gui2/main.py index c67ec8c2b4..ee18d8e9ca 100644 --- a/src/calibre/gui2/main.py +++ b/src/calibre/gui2/main.py @@ -19,6 +19,9 @@ from calibre.utils.config import prefs, dynamic from calibre.library.database2 import LibraryDatabase2 from calibre.library.sqlite import sqlite, DatabaseException +if iswindows: + winutil = plugins['winutil'][0] + def option_parser(): parser = _option_parser('''\ %prog [opts] [path_to_ebook] @@ -80,8 +83,7 @@ def get_library_path(parent=None): if library_path is None: # Need to migrate to new database layout base = os.path.expanduser('~') if iswindows: - base = plugins['winutil'][0].special_folder_path( - plugins['winutil'][0].CSIDL_PERSONAL) + base = winutil.special_folder_path(winutil.CSIDL_PERSONAL) if not base or not os.path.exists(base): from PyQt4.Qt import QDir base = unicode(QDir.homePath()).replace('/', os.sep) From 7b2ea64d43bcce5281ea270b6f68a3e2c28dfe93 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 14:11:48 -0600 Subject: [PATCH 70/90] ... --- src/calibre/constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index a15ffbf967..e673f8878a 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -75,7 +75,7 @@ class Plugins(collections.Mapping): if iswindows: plugins.append('winutil') if isosx: - plugins.append(['usbobserver']) + plugins.append('usbobserver') self.plugins = frozenset(plugins) def load_plugin(self, name): From 6502429c6de42504beaef7c6c2ee10d6dfb5bfc0 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 15:35:40 -0600 Subject: [PATCH 71/90] Fix Il Sole 24 Ore --- recipes/ilsole24ore.recipe | 98 ++++++++++++++++++-------------------- 1 file changed, 46 insertions(+), 52 deletions(-) diff --git a/recipes/ilsole24ore.recipe b/recipes/ilsole24ore.recipe index 920c703222..0cf1ddc6bf 100644 --- a/recipes/ilsole24ore.recipe +++ b/recipes/ilsole24ore.recipe @@ -1,71 +1,65 @@ -#!/usr/bin/env python -__license__ = 'GPL v3' -__author__ = 'Lorenzo Vigentini & Edwin van Maastrigt' -__copyright__ = '2009, Lorenzo Vigentini <l.vigentini at gmail.com> and Edwin van Maastrigt <evanmaastrigt at gmail.com>' -__description__ = 'Financial news daily paper - v1.02 (30, January 2010)' +__author__ = 'Marco Saraceno' +__copyright__ = '2010, Marco Saraceno <marcosaraceno at gmail.com>' +description = 'Italian daily newspaper - v 1.1 (Mar14,2011)' ''' -http://www.ilsole24ore.com/ +http://www.ilsole24ore.com ''' from calibre.web.feeds.news import BasicNewsRecipe +class IlSole24Ore(BasicNewsRecipe): + __author__ = 'Marco Saraceno' + description = 'Italian financial daily newspaper' -class ilsole24Ore(BasicNewsRecipe): - author = 'Lorenzo Vigentini & Edwin van Maastrigt' - description = 'Financial news daily paper' - - cover_url = 'http://www.ilsole24ore.com/img2007/print_header.gif' - - title = u'il Sole 24 Ore New' - publisher = 'italiaNews' - category = 'News, finance, economy, politics' + cover_url = 'http://www.shopping24.ilsole24ore.com/ProductRelated/rds/img/logo_sole.gif' + title = u'Il Sole 24 Ore' + publisher = 'Gruppo editoriale GRUPPO 24ORE' + category = 'News, politics, culture, economy, financial, Italian' language = 'it' timefmt = '[%a, %d %b, %Y]' oldest_article = 2 - max_articles_per_feed = 50 + max_articles_per_feed = 100 use_embedded_content = False + extra_css = '.headline {font-size: x-large;} \n .fact { padding-top: 10pt }' + + + remove_tags = [ + dict(name='div', attrs={'class':['header','titolo']}), + dict(name='table', attrs={'class':['footer1024','footerdown']}), + ] - remove_javascript = True - no_stylesheets = True def get_article_url(self, article): - return article.get('id', article.get('guid', None)) + link = article.get('link', None) + if link is None: + return article + if link.split('/')[-1]=="story01.htm": + link=link.split('/')[-2] + a=['0B','0C','0D','0E','0F','0G','0N' ,'0L0S','0A'] + b=['.' ,'/' ,'?' ,'-' ,'=' ,'&' ,'.com','www.','0'] + for i in range(0,len(a)): + link=link.replace(a[i],b[i]) + link="http://"+link + return link + + feeds = [ + (u'Notizie Italia', u'http://www.ilsole24ore.com/rss/notizie/italia.xml'), + (u'Notizie Europa', u'http://www.ilsole24ore.com/rss/notizie/europa.xml'), + (u'Notizie USA', u'http://www.ilsole24ore.com/rss/notizie/usa.xml'), + (u'Notizie Americhe', u'http://www.ilsole24ore.com/rss/notizie/americhe.xml'), + (u'Notizie Medio Oriente e Africa', u'http://www.ilsole24ore.com/rss/notizie/medio-oriente-e-africa.xml'), + (u'Notizie Asia e Oceania', u'http://www.ilsole24ore.com/rss/notizie/asia-e-oceania.xml'), + (u'Commenti', u'http://www.ilsole24ore.com/rss/commenti-e-idee.xml'), + (u'Norme e tributi', u'http://www.ilsole24ore.com/rss/norme-e-tributi.xml'), + (u'Finanza', u'http://www.ilsole24ore.com/rss/finanza-e-mercati.xml'), + (u'Economia', u'http://www.ilsole24ore.com/rss/economia.xml'), + (u'Tecnologia', u'http://www.ilsole24ore.com/rss/tecnologie.xml'), + (u'Cultura', u'http://www.ilsole24ore.com/rss/cultura.xml'), + ] def print_version(self, url): - link, sep, params = url.rpartition('?') - if link is None: - return link.replace('_1.php', '_php') - return link.replace('.shtml', '_PRN.shtml') - - keep_only_tags = [ - dict(name='div', attrs={'class':'txt'}) - ] -# remove_tags = [dict(name='br')] - - feeds = [ - (u'Prima pagina', u'http://www.ilsole24ore.com/rss/primapagina.xml'), - (u'Norme e tributi', u'http://www.ilsole24ore.com/rss/norme-tributi.xml'), - (u'Finanza e mercati', u'http://www.ilsole24ore.com/rss/finanza-mercati.xml'), - (u'Economia e lavoro', u'http://www.ilsole24ore.com/rss/economia-lavoro.xml'), - (u'Italia', u'http://www.ilsole24ore.com/rss/italia.xml'), - (u'Mondo', u'http://www.ilsole24ore.com/rss/mondo.xml'), - (u'Tecnologia e business', u'http://www.ilsole24ore.com/rss/tecnologia-business.xml'), - (u'Cultura e tempo libero', u'http://www.ilsole24ore.com/rss/tempolibero-cultura.xml'), - (u'Sport', u'http://www.ilsole24ore.com/rss/sport.xml'), - (u'Professionisti 24', u'http://www.ilsole24ore.com/rss/prof_home.xml'), - (u'Ambiente e Sicurezza',u'http://www.ilsole24ore.com/rss/prof_as.xml') - ] - - extra_css = ''' - html, body, table, tr, td, h1, h2, h3, h4, h5, h6, p, a, span, br, img {margin:0;padding:0;border:0;font-size:12px;font-family:"Georgia","Times New Roman";} - .linkHighlight {color:#0292c6;} - .txt {border-bottom:1px solid #7c7c7c;padding-bottom:20px};text-align:justify;font-family:"serif"} - .txt p {line-height:18px;} - .txt span {line-height:22px;} - .title h3 {color:#7b7b7b;} - .title h4 {color:#08526e;font-size:26px;font-family:"Times New Roman";font-weight:normal;} - ''' + return url.replace('.shtml', '_PRN.shtml') From f05d6d063fb22ed17cd4f890f1797130c5e00019 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 16:51:59 -0600 Subject: [PATCH 72/90] ... --- resources/template-functions.json | 5 +++-- src/calibre/constants.py | 10 +++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/resources/template-functions.json b/resources/template-functions.json index c19627c6c7..cf858c7691 100644 --- a/resources/template-functions.json +++ b/resources/template-functions.json @@ -5,6 +5,7 @@ "strcat": "def evaluate(self, formatter, kwargs, mi, locals, *args):\n i = 0\n res = ''\n for i in range(0, len(args)):\n res += args[i]\n return res\n", "substr": "def evaluate(self, formatter, kwargs, mi, locals, str_, start_, end_):\n return str_[int(start_): len(str_) if int(end_) == 0 else int(end_)]\n", "ifempty": "def evaluate(self, formatter, kwargs, mi, locals, val, value_if_empty):\n if val:\n return val\n else:\n return value_if_empty\n", + "booksize": "def evaluate(self, formatter, kwargs, mi, locals):\n if mi.book_size is not None:\n try:\n return str(mi.book_size)\n except:\n pass\n return ''\n", "select": "def evaluate(self, formatter, kwargs, mi, locals, val, key):\n if not val:\n return ''\n vals = [v.strip() for v in val.split(',')]\n for v in vals:\n if v.startswith(key+':'):\n return v[len(key)+1:]\n return ''\n", "field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n return formatter.get_value(name, [], kwargs)\n", "subtract": "def evaluate(self, formatter, kwargs, mi, locals, x, y):\n x = float(x if x else 0)\n y = float(y if y else 0)\n return unicode(x - y)\n", @@ -25,9 +26,9 @@ "capitalize": "def evaluate(self, formatter, kwargs, mi, locals, val):\n return capitalize(val)\n", "count": "def evaluate(self, formatter, kwargs, mi, locals, val, sep):\n return unicode(len(val.split(sep)))\n", "lowercase": "def evaluate(self, formatter, kwargs, mi, locals, val):\n return val.lower()\n", - "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n locals[target] = value\n return value\n", - "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n if (len(args) % 2) != 1:\n raise ValueError(_('switch requires an odd number of arguments'))\n i = 0\n while i < len(args):\n if i + 1 >= len(args):\n return args[i]\n if re.search(args[i], val):\n return args[i+1]\n i += 2\n", "strcmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n v = strcmp(x, y)\n if v < 0:\n return lt\n if v == 0:\n return eq\n return gt\n", + "switch": "def evaluate(self, formatter, kwargs, mi, locals, val, *args):\n if (len(args) % 2) != 1:\n raise ValueError(_('switch requires an odd number of arguments'))\n i = 0\n while i < len(args):\n if i + 1 >= len(args):\n return args[i]\n if re.search(args[i], val):\n return args[i+1]\n i += 2\n", + "assign": "def evaluate(self, formatter, kwargs, mi, locals, target, value):\n locals[target] = value\n return value\n", "raw_field": "def evaluate(self, formatter, kwargs, mi, locals, name):\n return unicode(getattr(mi, name, None))\n", "cmp": "def evaluate(self, formatter, kwargs, mi, locals, x, y, lt, eq, gt):\n x = float(x if x else 0)\n y = float(y if y else 0)\n if x < y:\n return lt\n if x == y:\n return eq\n return gt\n" } \ No newline at end of file diff --git a/src/calibre/constants.py b/src/calibre/constants.py index e673f8878a..28dbcc4299 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -22,11 +22,11 @@ def terminal_controller(): _tc = TerminalController(sys.stdout) return _tc - -iswindows = 'win32' in sys.platform.lower() or 'win64' in sys.platform.lower() -isosx = 'darwin' in sys.platform.lower() -isnewosx = isosx and getattr(sys, 'new_app_bundle', False) -isfreebsd = 'freebsd' in sys.platform.lower() +_plat = sys.platform.lower() +iswindows = 'win32' in _plat or 'win64' in _plat +isosx = 'darwin' in _plat +isnewosx = isosx and getattr(sys, 'new_app_bundle', False) +isfreebsd = 'freebsd' in _plat islinux = not(iswindows or isosx or isfreebsd) isfrozen = hasattr(sys, 'frozen') isunix = isosx or islinux From f15e78aa3eac06ade474078928a5b581cad30ecd Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 17:03:49 -0600 Subject: [PATCH 73/90] Fix Handelsblatt --- recipes/handelsblatt.recipe | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/recipes/handelsblatt.recipe b/recipes/handelsblatt.recipe index 945dac0560..056fcfb26b 100644 --- a/recipes/handelsblatt.recipe +++ b/recipes/handelsblatt.recipe @@ -1,4 +1,3 @@ - from calibre.web.feeds.news import BasicNewsRecipe class Handelsblatt(BasicNewsRecipe): @@ -7,14 +6,11 @@ class Handelsblatt(BasicNewsRecipe): oldest_article = 7 max_articles_per_feed = 100 no_stylesheets = True - cover_url = 'http://www.handelsblatt.com/images/logo/logo_handelsblatt.com.png' +# cover_url = 'http://www.handelsblatt.com/images/logo/logo_handelsblatt.com.png' language = 'de' - # keep_only_tags = [] - keep_only_tags = (dict(name = 'div', attrs = {'class': ['hcf-detail-abstract hcf-teaser ajaxify','hcf-detail','hcf-author-wrapper']})) - # keep_only_tags.append(dict(name = 'div', attrs = {'id': 'fullText'})) - remove_tags = [dict(name='img', attrs = {'src': 'http://www.handelsblatt.com/images/icon/loading.gif'}) - ,dict(name='ul' , attrs={'class':['hcf-detail-tools']}) - ] + + remove_tags_before = dict(attrs={'class':'hcf-overline'}) + remove_tags_after = dict(attrs={'class':'hcf-footer'}) feeds = [ (u'Handelsblatt Exklusiv',u'http://www.handelsblatt.com/rss/exklusiv'), @@ -28,17 +24,16 @@ class Handelsblatt(BasicNewsRecipe): (u'Handelsblatt Magazin',u'http://www.handelsblatt.com/rss/magazin/'), (u'Handelsblatt Weblogs',u'http://www.handelsblatt.com/rss/blogs') ] + extra_css = ''' - .hcf-headline {font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:x-large;} - .hcf-overline {font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:x-large;} - .hcf-exclusive {font-family:Arial,Helvetica,sans-serif; font-style:italic;font-weight:bold; margin-right:5pt;} - p{font-family:Arial,Helvetica,sans-serif;} - .hcf-location-mark{font-weight:bold; margin-right:5pt;} - .MsoNormal{font-family:Helvetica,Arial,sans-serif;} - .hcf-author-wrapper{font-style:italic;} - .hcf-article-date{font-size:x-small;} - .hcf-caption {font-style:italic;font-size:small;} - img {align:left;} + h1{font-family:Arial,Helvetica,sans-serif; font-weight:bold;font-size:large;} + h2{font-family:Arial,Helvetica,sans-serif; font-weight:normal;font-size:small;} + p{font-family:Arial,Helvetica,sans-serif;font-size:small;} + body{font-family:Helvetica,Arial,sans-serif;font-size:small;} ''' - + def print_version(self, url): + url = url.split('/') + url[-1] = 'v_detail_tab_print,'+url[-1] + url = '/'.join(url) + return url From f48de860fa1a83abd5873e1327e18cf9ae35deaa Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 20:05:41 -0600 Subject: [PATCH 74/90] ... --- src/calibre/gui2/metadata/bulk_download2.py | 53 +++++++++++++-------- 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 2bbb177e14..4b59ce68e9 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -242,7 +242,7 @@ def apply_metadata(job, gui, q, result): q.finished.disconnect() if result != q.Accepted: return - id_map, failed_ids, failed_covers, title_map = job.result + id_map, failed_ids, failed_covers, title_map, all_failed = job.result id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in failed_ids]) if not id_map: @@ -279,31 +279,39 @@ def apply_metadata(job, gui, q, result): def proceed(gui, job): gui.status_bar.show_message(_('Metadata download completed'), 3000) - id_map, failed_ids, failed_covers, title_map = job.result - fmsg = det_msg = '' - if failed_ids or failed_covers: - fmsg = '<p>'+_('Could not download metadata and/or covers for %d of the books. Click' - ' "Show details" to see which books.')%len(failed_ids) - det_msg = [] - for i in failed_ids | failed_covers: - title = title_map[i] - if i in failed_ids: - title += (' ' + _('(Failed metadata)')) - if i in failed_covers: - title += (' ' + _('(Failed cover)')) - det_msg.append(title) - msg = '<p>' + _('Finished downloading metadata for <b>%d book(s)</b>. ' - 'Proceed with updating the metadata in your library?')%len(id_map) - q = MessageBox(MessageBox.QUESTION, _('Download complete'), - msg + fmsg, det_msg='\n'.join(det_msg), show_copy_button=bool(failed_ids), - parent=gui) + id_map, failed_ids, failed_covers, title_map, all_failed = job.result + if all_failed: + q = error_dialog(gui, _('Download failed'), + _('Failed to download metadata or covers for any of the %d' + ' book(s).') % len(id_map)) + else: + fmsg = det_msg = '' + if failed_ids or failed_covers: + fmsg = '<p>'+_('Could not download metadata and/or covers for %d of the books. Click' + ' "Show details" to see which books.')%len(failed_ids) + det_msg = [] + for i in failed_ids | failed_covers: + title = title_map[i] + if i in failed_ids: + title += (' ' + _('(Failed metadata)')) + if i in failed_covers: + title += (' ' + _('(Failed cover)')) + det_msg.append(title) + msg = '<p>' + _('Finished downloading metadata for <b>%d book(s)</b>. ' + 'Proceed with updating the metadata in your library?')%len(id_map) + q = MessageBox(MessageBox.QUESTION, _('Download complete'), + msg + fmsg, det_msg='\n'.join(det_msg), show_copy_button=bool(failed_ids), + parent=gui) + q.finished.connect(partial(apply_metadata, job, gui, q)) + q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole) q.vlb.setIcon(QIcon(I('debug.png'))) q.vlb.clicked.connect(partial(view_log, job, q)) q.det_msg_toggle.setVisible(bool(failed_ids | failed_covers)) q.setModal(False) + if all_failed: + q.det_msg_toggle.setVisible(False) q.show() - q.finished.connect(partial(apply_metadata, job, gui, q)) # }}} @@ -336,6 +344,7 @@ def download(ids, db, do_identify, covers, title_map = {} ans = {} count = 0 + all_failed = True for i, mi in izip(ids, metadata): if abort.is_set(): log.error('Aborting...') @@ -350,6 +359,7 @@ def download(ids, db, do_identify, covers, except: pass if results: + all_failed = False mi = merge_result(mi, results[0]) identifiers = mi.identifiers if not mi.is_null('rating'): @@ -367,6 +377,7 @@ def download(ids, db, do_identify, covers, with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: f.write(cdata[-1]) mi.cover = f.name + all_failed = False else: failed_covers.add(i) ans[i] = mi @@ -374,7 +385,7 @@ def download(ids, db, do_identify, covers, notifications.put((count/len(ids), _('Downloaded %d of %d')%(count, len(ids)))) log('Download complete, with %d failures'%len(failed_ids)) - return (ans, failed_ids, failed_covers, title_map) + return (ans, failed_ids, failed_covers, title_map, all_failed) From ac8df2f09962c58f2d7ed8c8696967203c132efb Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Wed, 20 Apr 2011 20:09:47 -0600 Subject: [PATCH 75/90] ... --- src/calibre/gui2/metadata/bulk_download2.py | 24 ++++++++++----------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 4b59ce68e9..82804b9c96 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -280,27 +280,29 @@ def apply_metadata(job, gui, q, result): def proceed(gui, job): gui.status_bar.show_message(_('Metadata download completed'), 3000) id_map, failed_ids, failed_covers, title_map, all_failed = job.result + det_msg = [] + for i in failed_ids | failed_covers: + title = title_map[i] + if i in failed_ids: + title += (' ' + _('(Failed metadata)')) + if i in failed_covers: + title += (' ' + _('(Failed cover)')) + det_msg.append(title) + det_msg = '\n'.join(det_msg) + if all_failed: q = error_dialog(gui, _('Download failed'), _('Failed to download metadata or covers for any of the %d' - ' book(s).') % len(id_map)) + ' book(s).') % len(id_map), det_msg=det_msg) else: fmsg = det_msg = '' if failed_ids or failed_covers: fmsg = '<p>'+_('Could not download metadata and/or covers for %d of the books. Click' ' "Show details" to see which books.')%len(failed_ids) - det_msg = [] - for i in failed_ids | failed_covers: - title = title_map[i] - if i in failed_ids: - title += (' ' + _('(Failed metadata)')) - if i in failed_covers: - title += (' ' + _('(Failed cover)')) - det_msg.append(title) msg = '<p>' + _('Finished downloading metadata for <b>%d book(s)</b>. ' 'Proceed with updating the metadata in your library?')%len(id_map) q = MessageBox(MessageBox.QUESTION, _('Download complete'), - msg + fmsg, det_msg='\n'.join(det_msg), show_copy_button=bool(failed_ids), + msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids), parent=gui) q.finished.connect(partial(apply_metadata, job, gui, q)) @@ -309,8 +311,6 @@ def proceed(gui, job): q.vlb.clicked.connect(partial(view_log, job, q)) q.det_msg_toggle.setVisible(bool(failed_ids | failed_covers)) q.setModal(False) - if all_failed: - q.det_msg_toggle.setVisible(False) q.show() # }}} From 04cea0cef5df8c146dea032d9d779e0d8d6ced7c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 08:34:05 -0600 Subject: [PATCH 76/90] Fix #768066 (Daily Telegraph Australia Sport) --- recipes/daily_telegraph.recipe | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/recipes/daily_telegraph.recipe b/recipes/daily_telegraph.recipe index 5e1a2f7bfb..5ee48f3f79 100644 --- a/recipes/daily_telegraph.recipe +++ b/recipes/daily_telegraph.recipe @@ -61,6 +61,12 @@ class DailyTelegraph(BasicNewsRecipe): (u'Entertainment News', u'http://feeds.news.com.au/public/rss/2.0/dtele_entertainment_news_201.xml'), (u'Lifestyle News', u'http://feeds.news.com.au/public/rss/2.0/dtele_lifestyle_227.xml'), (u'Music', u'http://feeds.news.com.au/public/rss/2.0/dtele_music_441.xml'), + (u'Sport', + u'http://feeds.news.com.au/public/rss/2.0/dtele_sport_203.xml'), + (u'Soccer', + u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_soccer_344.xml'), + (u'Rugby Union', + u'http://feeds.news.com.au/public/rss/2.0/dtele_sports_rugby_union_342.xml'), (u'Property Confidential', u'http://feeds.news.com.au/public/rss/2.0/dtele_property_confidential_463.xml'), (u'Property - Your Space', u'http://feeds.news.com.au/public/rss/2.0/dtele_property_yourspace_462.xml'), (u'Confidential News', u'http://feeds.news.com.au/public/rss/2.0/dtele_entertainment_confidential_252.xml'), From 1e2782c613cc568334f694d8add50a74449f4bcd Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 08:41:21 -0600 Subject: [PATCH 77/90] ... --- src/calibre/gui2/metadata/bulk_download2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 82804b9c96..017635c6fb 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -295,7 +295,7 @@ def proceed(gui, job): _('Failed to download metadata or covers for any of the %d' ' book(s).') % len(id_map), det_msg=det_msg) else: - fmsg = det_msg = '' + fmsg = '' if failed_ids or failed_covers: fmsg = '<p>'+_('Could not download metadata and/or covers for %d of the books. Click' ' "Show details" to see which books.')%len(failed_ids) From e910505e4e29ce57fff33be3fa24792bb79d84c9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 09:31:19 -0600 Subject: [PATCH 78/90] Fix search parsing broken on windows, though I have no idea why it broke --- src/calibre/utils/search_query_parser.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index a50ca20fc1..3c41498107 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -16,14 +16,14 @@ methods :method:`SearchQueryParser.universal_set` and If this module is run, it will perform a series of unit tests. ''' -import sys, string, operator +import sys, operator from calibre.utils.pyparsing import CaselessKeyword, Group, Forward, \ CharsNotIn, Suppress, OneOrMore, MatchFirst, CaselessLiteral, \ Optional, NoMatch, ParseException, QuotedString from calibre.constants import preferred_encoding from calibre.utils.icu import sort_key - +from calibre import prints ''' @@ -109,7 +109,7 @@ class SearchQueryParser(object): def run_tests(parser, result, tests): failed = [] for test in tests: - print '\tTesting:', test[0], + prints('\tTesting:', test[0], end=' ') res = parser.parseString(test[0]) if list(res.get(result, None)) == test[1]: print 'OK' @@ -134,7 +134,7 @@ class SearchQueryParser(object): for l in standard_locations: location |= l location = Optional(location, default='all') - word_query = CharsNotIn(string.whitespace + '()') + word_query = CharsNotIn(u'\t\r\n\u00a0 ' + u'()') #quoted_query = Suppress('"')+CharsNotIn('"')+Suppress('"') quoted_query = QuotedString('"', escChar='\\') query = quoted_query | word_query @@ -617,7 +617,7 @@ class Tester(SearchQueryParser): def run_tests(self): failed = [] for query in self.tests.keys(): - print 'Testing query:', query, + prints('Testing query:', query, end=' ') res = self.parse(query) if res != self.tests[query]: print 'FAILED', 'Expected:', self.tests[query], 'Got:', res From deb05bae3392d600cdff5d12074ccc7ebfed7a87 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 09:39:41 -0600 Subject: [PATCH 79/90] BabyOnline.ro by Silviu Cotoara and update ecuisine.ro, egirl.ro and tabu.ro --- recipes/babyonline.recipe | 59 +++++++++++++++++++++++++++++++++++ recipes/ecuisine.recipe | 8 ++--- recipes/egirl.recipe | 4 +-- recipes/icons/babyonline.png | Bin 0 -> 256 bytes recipes/tabu.recipe | 4 ++- 5 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 recipes/babyonline.recipe create mode 100644 recipes/icons/babyonline.png diff --git a/recipes/babyonline.recipe b/recipes/babyonline.recipe new file mode 100644 index 0000000000..0b892ed673 --- /dev/null +++ b/recipes/babyonline.recipe @@ -0,0 +1,59 @@ +# -*- coding: utf-8 -*- +#!/usr/bin/env python + +__license__ = 'GPL v3' +__copyright__ = u'2011, Silviu Cotoar\u0103' +''' +babyonline.ro +''' + +from calibre.web.feeds.news import BasicNewsRecipe + +class BabyOnline(BasicNewsRecipe): + title = u'Baby Online' + __author__ = u'Silviu Cotoar\u0103' + description = u'De la p\u0103rinte la p\u0103rinte' + publisher = u'Baby Online' + oldest_article = 50 + language = 'ro' + max_articles_per_feed = 100 + no_stylesheets = True + use_embedded_content = False + category = 'Ziare,Reviste,Copii,Mame' + encoding = 'utf-8' + cover_url = 'http://www.babyonline.ro/images/default/logo.gif' + + conversion_options = { + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } + + keep_only_tags = [ + dict(name='div', attrs={'id':'article_container'}) + ] + + remove_tags = [ + dict(name='div', attrs={'id':'bar_nav'}), + dict(name='div', attrs={'id':'service_send'}), + dict(name='div', attrs={'id':'other_videos'}), + dict(name='div', attrs={'class':'dot_line_yellow'}), + dict(name='a', attrs={'class':'print'}), + dict(name='a', attrs={'class':'email'}), + dict(name='a', attrs={'class':'YM'}), + dict(name='a', attrs={'class':'comment'}), + dict(name='div', attrs={'class':'tombstone_cross'}), + dict(name='span', attrs={'class':'liketext'}) + ] + + remove_tags_after = [ + dict(name='div', attrs={'id':'service_send'}) + ] + + feeds = [ + (u'Feeds', u'http://www.babyonline.ro/rss_homepage.xml') + ] + + def preprocess_html(self, soup): + return self.adeify_images(soup) diff --git a/recipes/ecuisine.recipe b/recipes/ecuisine.recipe index 53631e0b14..77d761e653 100644 --- a/recipes/ecuisine.recipe +++ b/recipes/ecuisine.recipe @@ -14,14 +14,14 @@ class EcuisineRo(BasicNewsRecipe): __author__ = u'Silviu Cotoar\u0103' description = u'Reinventeaz\u0103 pl\u0103cerea de a g\u0103ti' publisher = 'eCuisine' - oldest_article = 5 + oldest_article = 50 language = 'ro' max_articles_per_feed = 100 no_stylesheets = True use_embedded_content = False category = 'Ziare,Retete,Bucatarie' encoding = 'utf-8' - cover_url = '' + cover_url = 'http://www.ecuisine.ro/sites/all/themes/ecuisine/images/logo.gif' conversion_options = { 'comments' : description @@ -31,8 +31,8 @@ class EcuisineRo(BasicNewsRecipe): } keep_only_tags = [ - dict(name='div', attrs={'class':'page-title'}) - , dict(name='div', attrs={'class':'content clearfix'}) + dict(name='h1', attrs={'id':'page-title'}) + , dict(name='div', attrs={'class':'field-item even'}) ] remove_tags = [ diff --git a/recipes/egirl.recipe b/recipes/egirl.recipe index b456323db9..56d515669d 100644 --- a/recipes/egirl.recipe +++ b/recipes/egirl.recipe @@ -31,8 +31,8 @@ class EgirlRo(BasicNewsRecipe): } keep_only_tags = [ - dict(name='div', attrs={'id':'title_art'}) - , dict(name='div', attrs={'class':'content_style'}) + dict(name='div', attrs={'id':'content_art'}) + , dict(name='div', attrs={'class':'content_articol'}) ] feeds = [ diff --git a/recipes/icons/babyonline.png b/recipes/icons/babyonline.png new file mode 100644 index 0000000000000000000000000000000000000000..030c611d8894f55a620faf17b9e365e8acb03943 GIT binary patch literal 256 zcmV+b0ssDqP)<h;3K|Lk000e1NJLTq000mG000mO0{{R3C@l|D0000pP)t-s0002| z?&SR1wEVto^j>uQ*4*@xr2OvG{K|y%YJB|0%>4fR^n#T9(wzL_y!^IQ^ipiG@(hds z0004WQchC<K<3zH0001cNkl<Z7?owu;SPf^2n66FD%hi?@Bg%;bkl6R_``hZT|&W6 z%Kg`w`xk7ku7q-$)r7_%)(N|P0OCC=W{`*mKsBNJjONgKK|~<LC!Ap8I%0s`8xo}2 zAaa%FvMQia+dX<-3EKp20mnF_f~SJIuTl57=DFqX+IL=4w+awBtLUx(0000<MNUMn GLSTZ)#CM(m literal 0 HcmV?d00001 diff --git a/recipes/tabu.recipe b/recipes/tabu.recipe index f98ed8a155..941c491c79 100644 --- a/recipes/tabu.recipe +++ b/recipes/tabu.recipe @@ -37,10 +37,12 @@ class TabuRo(BasicNewsRecipe): ] remove_tags = [ - dict(name='div', attrs={'class':'asemanatoare'}) + dict(name='div', attrs={'class':'asemanatoare'}), + dict(name='div', attrs={'class':'social'}) ] remove_tags_after = [ + dict(name='div', attrs={'class':'social'}), dict(name='div', attrs={'id':'comments'}), dict(name='div', attrs={'class':'asemanatoare'}) ] From 17a6ff32f0269f9c87b6913da0e9092cbfd216f2 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 09:58:53 -0600 Subject: [PATCH 80/90] Ignore a plugin with the same name as a builting plugin --- src/calibre/customize/ui.py | 12 +++++++++++- src/calibre/gui2/preferences/plugins.py | 12 ++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index c58f36524e..e1b68feace 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -22,6 +22,11 @@ from calibre.utils.config import make_config_dir, Config, ConfigProxy, \ from calibre.ebooks.epub.fix import ePubFixer from calibre.ebooks.metadata.sources.base import Source +builtin_names = frozenset([p.name for p in builtin_plugins]) + +class NameConflict(ValueError): + pass + def _config(): c = Config('customize') c.add_opt('plugins', default={}, help=_('Installed plugins')) @@ -355,6 +360,9 @@ def set_file_type_metadata(stream, mi, ftype): def add_plugin(path_to_zip_file): make_config_dir() plugin = load_plugin(path_to_zip_file) + if plugin.name in builtin_names: + raise NameConflict( + 'A builtin plugin with the name %r already exists' % plugin.name) plugin = initialize_plugin(plugin, path_to_zip_file) plugins = config['plugins'] zfp = os.path.join(plugin_dir, plugin.name+'.zip') @@ -506,7 +514,9 @@ def initialize_plugin(plugin, path_to_zip_file): def initialize_plugins(): global _initialized_plugins _initialized_plugins = [] - for zfp in list(config['plugins'].values()) + builtin_plugins: + user_plugins = [p for name, p in config['plugins'].iteritems() if name not + in builtin_names] + for zfp in user_plugins + builtin_plugins: try: try: plugin = load_plugin(zfp) if not isinstance(zfp, type) else zfp diff --git a/src/calibre/gui2/preferences/plugins.py b/src/calibre/gui2/preferences/plugins.py index 79cd2b1ce4..8888a64e84 100644 --- a/src/calibre/gui2/preferences/plugins.py +++ b/src/calibre/gui2/preferences/plugins.py @@ -13,9 +13,9 @@ from PyQt4.Qt import Qt, QModelIndex, QAbstractItemModel, QVariant, QIcon, \ from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.plugins_ui import Ui_Form -from calibre.customize.ui import initialized_plugins, is_disabled, enable_plugin, \ - disable_plugin, plugin_customization, add_plugin, \ - remove_plugin +from calibre.customize.ui import (initialized_plugins, is_disabled, enable_plugin, + disable_plugin, plugin_customization, add_plugin, + remove_plugin, NameConflict) from calibre.gui2 import NONE, error_dialog, info_dialog, choose_files, \ question_dialog, gprefs from calibre.utils.search_query_parser import SearchQueryParser @@ -279,7 +279,11 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): ' Are you sure you want to proceed?'), show_copy_button=False): return - plugin = add_plugin(path) + try: + plugin = add_plugin(path) + except NameConflict as e: + return error_dialog(self, _('Already exists'), + unicode(e), show=True) self._plugin_model.populate() self._plugin_model.reset() self.changed_signal.emit() From 60bbc8d0021dbd1a935f9268ec5dc78bf485226f Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 10:05:22 -0600 Subject: [PATCH 81/90] Remove user plugins with name conflicts automatically --- src/calibre/customize/ui.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/calibre/customize/ui.py b/src/calibre/customize/ui.py index e1b68feace..d3ecab7f16 100644 --- a/src/calibre/customize/ui.py +++ b/src/calibre/customize/ui.py @@ -514,9 +514,11 @@ def initialize_plugin(plugin, path_to_zip_file): def initialize_plugins(): global _initialized_plugins _initialized_plugins = [] - user_plugins = [p for name, p in config['plugins'].iteritems() if name not - in builtin_names] - for zfp in user_plugins + builtin_plugins: + conflicts = [name for name in config['plugins'] if name in + builtin_names] + for p in conflicts: + remove_plugin(p) + for zfp in list(config['plugins'].itervalues()) + builtin_plugins: try: try: plugin = load_plugin(zfp) if not isinstance(zfp, type) else zfp From 510693d26153292297cd996d5ae884b67eecb2a9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 10:37:01 -0600 Subject: [PATCH 82/90] Change icon for identifiers --- resources/images/id_card.png | Bin 6483 -> 0 bytes resources/images/identifiers.png | Bin 0 -> 705 bytes src/calibre/library/field_metadata.py | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 resources/images/id_card.png create mode 100644 resources/images/identifiers.png diff --git a/resources/images/id_card.png b/resources/images/id_card.png deleted file mode 100644 index 80ac5fda116e91d0c4ad9315381c7e2fd360976c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6483 zcmchc_di?z+s6;JDWR&VQZZ`OilSyeLRD>2n<BQFrLk-8U8=P9Divx3HDXq2N$sNc zDr(Qz<9qrS+`ru?kCT(f*}1OkeVy0yb)vO3pI)b8rUC%qx{9*mGjJL7Z>Asyztx`~ zW`YYc^fOHz;OgH;!tjnixQ6njve7H>h_Qb&kpgdqHvmBQRTSlQy(czJeO~H*&$=Yw zma_+gY1Jr#!?+*}%kU`C1u`oqtHsdHuBF;BT$SV7&rV6+ix`d166?kWF+JFf!Uayx zagrv_+t-YUHQ`S?f46(Sfus$P{!LY4?tQdxJ2TiESmSe-s~6534}H1J|8A`}W6Rtt z(44JxtO7?)uK<hx6|CYxb~%y61nX3$eq&iksw^@@TnK$Z^enCh;8QXVV&oJ=#QeW( zez{A|%FK+N+zbs3&C})`8E?_Bs4{ea^=f}_Pk>1+6S=_9wzAJE7tn;m$&8RTS!tV^ zHltUcCE%o*{}$o6OiWC)vSs;H7%uD|eE4bm3PTOOxj1$3!4S<H3t1re+87l{I6s{1 zioa7`U2XNfIT{d3TfaD(HPE}su4`f<W7QSst=l~P68~kvGA^`zZY3vhW^|Mc7#|-` zd~^uKV@2}6F$6*O2A{GKj*hAYaQ<h9lMM^A`miuD2?;hCPa0skKd04ucL5q~rMHrh zkZ^e!0Q|0ZvvL?u)K{|{%QkVk&i)Nf^WBXiFD53YudgpPHTCrL^kjd9Is_u<D?mg> zMg|}%#%|rZ#gj2)7$x9bv;K!nE#D?#QJwpSg8k~yM?pcsus7}P?O(rswGWRcupmg^ z|L1?ID0<h3<}+8)7^j*{HtL4mZNS=yHnIaii80eFFvmwz%<XyuFgdv^CY>C3Vt2lb z1^%D_g8_md`M;3N1@p^7*E`gYhxY`x?r~2%=sI?EU)+JZhHKs4q3wRgYc<4sF;^F~ zW#7K=O_36^L+ii$-Ag9@a%kFjRBC_{;C9qUjyH7=ugp|CI0o$glnMXtrqsa4tr+yG znD+HCK5);EB}w9%nVrsbRZ*iAkK9D9%VcYrj}v)^J72M%LU<gx3KNX+AYf<q%KK33 z3<#2s>$R;lRA;KbEi$)H+Vo#*|1E9Clf`Md=Ht$Jp1@;$O^fl({>5(ux$ax2{{C!m zwFOZ5UQqAdu&A>w8M^{-qwJ4$H+F%JY;wYHO32GV?V`fG@#<;orK(-M#llN#%9~Hz zBw4`JPP{J)FDWgRaa;J7z$&xA{$u;->>>cDH2&<{93YGSp1`omJ)Kfz?JF55^(75! zZfICpxV$x`@<FNDtl}10mI2DKlc~w`UZd;7aBOU`x)MbLIgYO7(z2$^aqO0`S>^+q zo{ws2N~xIga+j69EY5;B)gH%+W_{y-Q~KMtI(m8@J9F*7r{ed6VVYbTKwW{|m7@&M zW?7}%FCfBv;2Uh`7Nz%^h2DrkAvq&#TK1ak_a&Q2?j`GMZ3a2#Uv^97;!0|oo0~5# zE|S{M0|HG=Sv@Wlku{T>!HC?Jmb+H6KaE{nw%Xd-wga!uZj0K=?*A0k(9k$&>6rh! zu@R^Kv7n%1)03x2+kr%q5Lgi%K&PPYv4qZo?Fit1m^B031?|>iP#WoG&F|I9lnszi z0^e^=zbhDCCz|`#aup-*_2+3$zt`2pk+5N9E(Wu<TAWjK)G`0k*?F_)9lxWaBPILU z;U+&nze%mb$l6FLr(L_44q52D%<M^DUzW_xCsLjp@AG-J{yX_|ezKn?V6^vxGw}zO zg)01u)A-);Vj_6nxh$`l2EJ*LJr=s;hh*sIms9|^joo;eZGr`2cVD)6Y@IawJ5)5P z{eJ4rR2!%YnscTLmLkK4E5#M#2m<;yzr7Z<yYpF@oR;P9+M0ul%Uxz>Oj+6a<xwjg zkf;5(`DkW+W8?K)F!{vfBoP9cA~E}Vcopq3RbSQ6(9qN*`yl6+v74J47|4Nvf#@4N zE$4gqhzPPcHxCcO?&5bFBNu1q>1w-v@b|3m`sd<oUkB@h-(!4`Ajr_jC;&VxjV6Kx z5#a?K=cM{j19+?eKNTKJ(*dWsMq+i3q%A$p7kG28s-n3nw~#rX#^r7zIeSV*##uK{ zj(~B^59<fF<GU6X7QibI2aPxg^7B_4e+J-iID%qu6|JYI=ZRwlHaul+q70{H{SM2f zT`E3(k_4+dgp}Ip;T=F%PfuJ-OfpLo$!P9R&;#Csb!k|_=E$j>0E9mJY%xd<aKB9g zP)|4z6x<k$YH}gweaQ3vtoxC{qRa=x3N-BX)gQ}q3w^Ci&I7JGV{grRFUI@qFE6dE z%=&FN%bj#<37NM{&CJXU4pzMwE}FgA^5P^7{o5Rd!GN_^teJJ%>l&bAWOOnYO1J#y zk1XMr4mcdB!xabwg51?f|K#N4(bhE1>U%<2nSgY>D1E<qH)9Zl;h<qv9RJ{c7Zno1 zn=HT?fz*A6;^<B;<BiG3U}CB;pYG#{I)Dy1Cgw@2qwZMmlKW?)>Y}2e<#z^VL_xh7 z5)xu!WhQmyxw)vZa#JZODe0^}+iH3Rh=P9UyG6<Dl@}hi&@P8stU&4QU#n=WK!qvp z3%K*i-jZ<hK{AY@wzl@)pQ&@1c6gX!?z{T&gQ#6<N{Z8Li~suQ=MY%a@j@&Ze~FKN zvK@#r2}{4DEsuuh;HEpH@d;(b7K2xdf?o6?c-3O2Tg$sFmW-{MJQTA1l>+RF$dpR1 za0LqyMuyHeF(mr%Sc|2hB_`k)mv)raZz}Gv^o%ilvHnMeuHMKp`C9@iT*Ma2ZA3>B z0<*TZ2HM&~$#FPbDF&loW=J33_RR!gJrOdKVxsG^?X-7#4Vw?shhgI=k&NL?>c}bq z{Cp*s3YH6>*-i|!uj5Uq)K(Pv^{j6e-73J!^{xE2*qH61j8-IpidS`ie6^9Kakoy3 zMva2~8`Ng7qe?(917rCF$#sei5r+%V0WOJj0CvClw%~{E`<u?<)r@pqhb)i|(rlA6 z3LxxjUU_z~wq)5S<tM&K8gk-xgf>l#r8Z5pB?YHE+z@Enq^!>r0zR4TA)N5La=F~Q zQl6QqJc>cTebU|fr5%x^z`-QBX?75liZ6__Nsqg!5MkIwHE;O12=x;{B!|c}o&jws zA6nOT*=Ag16GX4wxQB?74ew}AGe60Y^D;}OWGAUJ+k`^*dSW<A-&gu_8A?9pcCEyM z0?j3G#~rU{lu8l2UmkoK@wVinj8W<hRG<^FqT27(wX62T(btD7q4yyby<{|@dbQQ1 zID%>#{ssm9J-`lNV;)_i{0y$%@v~yNh!DaSO%BJ!760w-4ti}#Hd@tX92>E2!!|tW z0=CI(4})pLv|qeXaZ^c{ou_-~^+0}bec#9bKVVLhwF72E>qsy1eg11mzEj38_ulna z7yT72EBjJx{>S81MmuL0=yv%U2JVNAT~Qnq#HLmc0`T4RC~HgC=Cg{F3(VRFubUIl zFmL|FZk+*GRk#tC6`-B6Bf)K0XiKG;AoxSX&U&qEpI^wVURq3Sr_a?q+wZ7nC>i&3 z35iTOJoJ2mTv!ph8@>?T=k^?lO#0FUSGr&40vSy=ahx2NW0}4Vbm~y=Q9DrtLk}CL z1Vc2xzJ+Ojb)&BKm&#Ow31uIwSNbz_2&Z$cRJW0Sd&$F1C7g>{V`!d1P+Sr+<O+vx zCh-j=q6Zymj0*@fyAVbkn*rmKj5o@;8LA<`AeuYue6Kc;n`E7ES~ARi4Z-`Tn(hm# zYz_UxTFLNwwe%B_(s>odpB$3c;1xSf%(SGv`doZ1EtlL1aD2h4%`h<MKAgrQcY!L) zOb|D8T3e(S?nAO`a{Wh<b9#<gToMLQzT;vERzYvt^JqgTiC`p#wCc8u2WJOlP7gS% zNsXL;q_)>*xokS;a>WX1u|a-x*V<_H5CwU8UYYPRLX8HU9!^#+y>n}1qkJh%Bildh z6aE{`GkS#}4y{am=HEdH46G9xd~O4ChX?rNC@2s|aV>}l0-)p(A%O19kYJQL1po`3 zVPf=m@yZ!|)BEc5#c)gnOQ@znpnu11r}z;aKSKw6xJZMVgjhtHBMKLq5Qtco?6Vi+ zw)IgUiBiw4uBHUa^ltzV;{UGR<P%2%L8v$gaTpNP10aGRFo5J5yR}d6zL&uxF-bHb z@W{V^b2TSLe8hT`QaX!~Rzm#BX7H;<K@lj57ygK3qoVmz6tC=l64Tr<k;Y{ZwONZW z5WhzuX+#82G!#S%B<ujf+;PW402ZcwuSaz>J)YFos=a{jd~J@+oQs5lD5F<e68?M2 zo!gj_k{vy6<L*Negv`ne)BDi20`#I`pwwgmU^~R@Fg%vaBI~Z{EdT<*Lgv<)5R3GC z9y;DvZPW+|Q5#89`COs;I}5|uPDx3?+2D?n;OA+Vqm$P$`BP%9Rjth8Q2$tjIagfd zsYsHPEfL2JIP|Ad=sn(3MdUXqW4MhJt(AC)e^l8e0FY3;77Uske(Ly$9Tov&;UMGW z#n1pOJaOp7^MKaLP0f5SIhtgy>$J4#>O~gYcaw*jEIHd7U}5jESRY_+_Y{2KM~&Ox zgTl$mh)0DuE_GjD1b$;<M~S7M2dL<|8KwX*;W}??5#r#sPajpwBk#vp5@%m}pXSt) zM5X@z850i75spKHA>T#DjdUOjoa2dqj7P$dt$B9=Xhd&+4g(dRnbwcZL*h2mlVPP) z?N=mEdkvlR7HGQQG#=hvie6&j3%6jq4Y{4M5B)1$dD`ngo=j{=i9#a;?`JL@9;hYr zcIW3SiwIQG6c1+{NLP1e_PR;m@|KzO>#(2%QU!cb)*>YGy!Y?B<EAIL2kg(gRta=x zsug1<pGTqFohZxG91&{Bi{>))R*_$h4EoRYH2n1Nw2rRW2b+_Pvizt@WOc~Eb;17a zBj09P;Js9V?dk&gQ^CA;6dX!(W9@0ehz~0dfib%E4N*|5|Jo96ueeI!Ic~z^a>^!6 z6owB&t?h7<sMpBB5b&gX8|YIZQ+rnEUk=29@35TgS@o{O6(!X}`U|?a6%{CemR>Bq za`Rp(Z*6_KD_S?X>4~FP2tf_3(#Z0}y$o#$#@x@-hOy8(j8ajl1~yq8{rS3wj7{1j zFO7C2p=`{XAQWzVy*fX7{Pc3?Fj~%kV8}SKikX(YV;*>deWfxCM=m@X)$*uq+d` zW&H@kWur$jQ9nN(e<z!8lId*Fe{!*f7>KQ6_LCL+t1&)KdeYE2J{~?OIBS9C8Gv=2 zDRA#iIQ3<U`gO9ZYP*X<<QQ_O8~s)Na5gAQT(0Yl#2e}Tk|Hzm$DX-oo;vpGPy_0E zR#m@ATbSn5^B_u&CCzy^g)=s2KTN%Nx%R~5G|6l@3myjX)$yCo>s`K~&@v-<4RI^O z9;vH$FRW7FP6IQ!mAckueo@3{tvrd^`Xq^3#oG~iWUKsH7RbM3hUq=<YGw}eCj%@F zOiE<g5F!roznhz&;G_d&cfZ{rB_+L^xI!2+9}AiGKbm$NE7xPR0rd`0=}TCXdbKQ> z)mJwgU_veGvGyyHEdUC5ZqIC@rR_j=vh8ICjyr_|f<??gxedz28pT8wsiC2v<eZ#V zk9B?Ec&CHz?DSMjr~lDWX1BHUtL2||IsQe{&l}vAHpZ(>g~mYLM#-Kb;VL8~w6(EO zh|{Taoz4F6!Ds$!_*)(=;x=mGrZ%^!)#2jxwY3_>u?q7FXIJ8&i?bsz@$tcS**;zN zVxwbYIsf^mTf7IkJK!s6lpQzMCGRd5qa7|GD_D}pQ%OTCddpTU&I^+8HpTl}Y;}!@ z0MDCXH>KNh3{FID?#vwuG@9(HoKYIuvYXs}Wb_5URGu;{hjbYaTbU$OU#aB!+|$On z%5y5e7UK0Z`)=HjtT6g%mU{rUpoKnH9*42mC5q_ZwK7r8HI9p%@`leez)lCPA3VA| zUYsznt^Uw>_|FCaK;;Ed5MaU=B>so#{7BvfQF!R|JC~O#o)|Fs;*=N6c>_~YQc_-C zgnvxQ&aYpV`;>1O{!91ClP6|o*-YVEY+nooe!(6oND2#Y=SAKn27K#1Ra8`LZMmP% z`WP7*l{m-qgbh@P{Tdy0Sdq9Eq*<&M3=BFz@K`XEcbUl83#Und>=jNq2d2IKeNK&= zOiT%7q2PhwWSlZJHI)?lW!$N#7*C5gl$V$1a-5*g4ucd7eiFRk3LB_38g%g2)YN<i zWlIA(coY5%&cA$px>^ikg^P=et&!)&DIt=M&8)_L5Cj=DRn_I*3?BSm-s4CN1~V}+ z0s1N`D&1fv(*#XMhlbpjdfpZ11Q@3-HTxZb`d=2LM3#hmYN;$z?uGgJ#O%K&CVXf8 zkBHi`ANg%HtYm|}$I*<R>*N>bNLtqP2R1Riwlaa2=OBIIu^TP^XFYg4Sd$m?k!&tA zO((I!Ed#6Qn0)W%8F9x^IL)iIUnQWgLRkM-D!yQKnVOpBY5zdMC0P=d@M)losAxG| z+grke=a=z4u=nur@bdC9F`1mGb?nc6gnK#vmHL(tG`=f6J>8lT{^f#5gsgF6!zoPL zz+enChWvbe8yg!z0K68AfI)IsbWI!4C5?@zAeDlbStqC_Gtx$a))06FQu*A}$9ZpN zw{yfD^#cO~t*orT6kAy>f}yIBia^2t?w|B!N{LEJ{+^t4tR621N6Yb!g)@^p;}ay~ zojZ4Mjg4F?r+?mRajE1>tZpUR;8WdLj!sr`TqkN)hqdG`K<-^dmge&gi3@`_4e~b# zN{=5u&eOK<PQJ(TePm=rBU|?5?_dEdD=Uass!V5Yl<;TKNLI4M@1kB?&vDb2Te$7M z52D2t6<Um)VBAJZ4eDxY0EDJyA81KH6k=})Z~guI<LA%kP2Rf=uU5dQ{GD%tE}m`S zD)RE$*9jmh{rvf}sE9jm_fPlRj`_;5b+D~;L|ji3Hur3JP8<1{v9qsF(<l|R=xE3u zYiZE|WrkJxB_-YSFG+}lK^Vkhv0yJ6+zlnG2ZLW`RPFT81WcNelG2M8FANL}e0+Rd zT!bF&HPzKg$jQm^^E*z|yhsx=ZS>x?1i>1kS1K(nt*WZ(;NZZK_9lT%LXO{Z1f%!v z-8%qbU^ldb!JJ+MOzaQ&@V3qd64pj~dV0Qmd8nWa<^l$p^JxLuy}tfEt#6H!LHYBl zaVN6SJkTk@Vuu!Y2f|&Gxx^nn)YsQ<d|n<L9K5&Z9@&wUmR5{$FgG_xBI&!17dm5Y zY;1g+XEr_UtgU}<9)fHdVFA|2-{XJHEjgLtZTFvDZviH1(@*R%mXa(*JSd$PM7hPq z#ksj{+egD}ByPCrn!Dc3W^cQ{VV%Hg3<zkM-R}SKBe!`Wg--_@Qbn3sm@G3Mm1n$+ z3KUN+ts@5Q3+<M-W?Ngqm>e3>TS3~Em@m}d9_PH_;r)Fq$a9PPZS<)XoBsjQ+<X=c zsebwM+k!8D$s6kFt<6YfY5GRmj3=EjZYwtOOyI_*Y$7{|0NMV?4vU0Er7Y}S8=5xg zF#Q95erllHtXXwC@%e4&iq#+Q5RJ?wN@6_r;8@66w{F}C^k7+N)shQY`9E8DQ@26! z4<1>m=S4J`ZMU@{z@$x=GtovaPTfmt<8*nJ9kWRJP#zqtOtImk2m34i!x`XRKGT|p zgRQFnajQI_5arH_70^l->qfz&^1Y9D=A9?MtfHk;ZED4*UVKBix^9DD4eBi6L>r^V zI_=4-n<eenP2c!(LL>6XJxPg!ANd^~93CpQRH<2U{Vh;8HJuK=Mh*n6oW#UX8Rvlr z&PZw=$+Sc!teg<UrE-v8RfUxK^~+vtd1+~|oU&`rox8z2p6{pNwXWZ*=*H(l69iEj zQSf1Qe*XGWi6R6cY+rV#5yn_mM{oMn$ETs!*3rc!m5UEbV}|n-9J$wg4fQYs-_ry3 ze)xmIsGExt6m4)CJQiwoYW`SLGojORb^D)s0$YbW5lJ&*&>%lBR{r)tPXc7YXU6)0 z(dW_3OqT4GlJ#ykmRFn{7lTL=82x~{D2<M;ZXv-2{boSPQyy|T5kVedVQ=ubJ8uA5 zoXb2A`(yo+83fR7nDZ?iWGwI$U*jAvE+0{*JDQAFCgO;WNeR9j`!u$SmhGoo7tGgu zqOGO7uNo<(oV3v)c#@Yhd7XlN;-+JkpEcbLJ^+smfl-9O@2g3i2&(Am*%;kb@izv5 z0p@0I;6iIs-kOlu-rpOjp{M5bp68B~W}I|OAWp<7F78**iq8ba!aSE5k&;Hhd$N+{ xKdF{nXVD{HNl2ABjgB2RaIXCS$NPhGIWn{vqo@~(0(>U|s65eBELT8<{0~i~U(Wyl diff --git a/resources/images/identifiers.png b/resources/images/identifiers.png new file mode 100644 index 0000000000000000000000000000000000000000..17906dd2d270c31afcf999ec9f90f5adabd79dc8 GIT binary patch literal 705 zcmeAS@N?(olHy`uVBq!ia0vp^`#_k34M=vEY@G|F7>k44ofy`glX(f`uqAoByD<C* z!3BGlPX>x`7I;J!Gca%qgD@k*tT_@uLG}_)Usv|qETRI+QkCslUJMLOj-D=#Ar-gY zUNy|SZ6MM1Q2*>D!({h<S=+QL7fX!4a#%L)N-f!*^U8_Yp2MqfwnF!Ob_1Q$V!lD^ zC$j)eBn<XldHmo`@TBJ|*^`2+_iTG!Y&l(Jx=L{M-z#?4`vd1ruG#-^SMcvM(Le8G zKHsgMd*=DOy(Pu_`of=o+Pvrc;<>M{-#x!mK7Grzx^vkxPyaes`uE4*eed5E@4vR! z)^hvSeSFU%_g723`Fxwb+IrXj(m2U40nab3e{Pkwclq-Axu2(`WFIe{v{WYlsp0m| z*RP*GCvmmtacOkq1G9=NhRfdV+Sy~ye{6N}%J*ljU&pcCpI(0L)_L>0Kisc>ef<6R zKAAhK-?rICo|5=$vpi4F{q;Y2_wV&DuT7h4+H=jA|J}=nbN2sSe%w~b?hyOK<`0fP z6lw(QIOJRSv2g`)${^`;$7=*kAwjzyngvf<D|_B^n)RLLUGJ9e-De&h`Tg3i58}vf z`w`V|{rtbm&#Qg)>XkbW@7=lX_d?tC<=g(XUpo-hr(b78IBppj=D*QDb7JNa<1X>* OAmyH}elF{r5}E);^eL|Z literal 0 HcmV?d00001 diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index 33929ac2e4..374505c467 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -35,7 +35,7 @@ category_icon_map = { 'custom:' : 'column.png', 'user:' : 'tb_folder.png', 'search' : 'search.png', - 'identifiers': 'id_card.png' + 'identifiers': 'identifiers.png' } From 643baeb3301cb8a6dec4b2c5a2c422ddb94a7f21 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 10:57:13 -0600 Subject: [PATCH 83/90] ... --- src/calibre/constants.py | 1 + src/calibre/startup.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/src/calibre/constants.py b/src/calibre/constants.py index 28dbcc4299..86dd1ada3b 100644 --- a/src/calibre/constants.py +++ b/src/calibre/constants.py @@ -45,6 +45,7 @@ fcntl = None if iswindows else importlib.import_module('fcntl') filesystem_encoding = sys.getfilesystemencoding() if filesystem_encoding is None: filesystem_encoding = 'utf-8' + DEBUG = False def debug(): diff --git a/src/calibre/startup.py b/src/calibre/startup.py index fd9ef01141..308308e1f2 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -61,6 +61,12 @@ if not _run_once: ################################################################################ # Initialize locale + # Import string as we do not want locale specific + # string.whitespace/printable, on windows especially, this causes problems + # Before the delay load optimizations, string was loaded before this point + # anyway, so we preserve the old behavior, explicitly. + import string + string try: locale.setlocale(locale.LC_ALL, '') except: From aaa9bcf645bab5ba68ad28338e98d6493d71a07c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 11:06:58 -0600 Subject: [PATCH 84/90] ... --- src/calibre/startup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/calibre/startup.py b/src/calibre/startup.py index 308308e1f2..78f8aff7e3 100644 --- a/src/calibre/startup.py +++ b/src/calibre/startup.py @@ -62,9 +62,9 @@ if not _run_once: ################################################################################ # Initialize locale # Import string as we do not want locale specific - # string.whitespace/printable, on windows especially, this causes problems + # string.whitespace/printable, on windows especially, this causes problems. # Before the delay load optimizations, string was loaded before this point - # anyway, so we preserve the old behavior, explicitly. + # anyway, so we preserve the old behavior explicitly. import string string try: From a7dd8eb33ce3d34b3fcca3b71e73414381bec8be Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 11:37:31 -0600 Subject: [PATCH 85/90] Fix #768332 (Drag/drop image does not resize in book details) --- src/calibre/gui2/book_details.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/book_details.py b/src/calibre/gui2/book_details.py index 6c3dae3c94..4e75a42e89 100644 --- a/src/calibre/gui2/book_details.py +++ b/src/calibre/gui2/book_details.py @@ -418,6 +418,7 @@ class BookDetails(QWidget): # {{{ if y is None: # Local image self.cover_view.paste_from_clipboard(x) + self.update_layout() else: self.remote_file_dropped.emit(x, y) # We do not support setting cover *and* adding formats for @@ -449,6 +450,7 @@ class BookDetails(QWidget): # {{{ self.setAcceptDrops(True) self._layout = DetailsLayout(vertical, self) self.setLayout(self._layout) + self.current_path = '' self.cover_view = CoverView(vertical, self) self.cover_view.cover_changed.connect(self.cover_changed.emit) @@ -482,15 +484,19 @@ class BookDetails(QWidget): # {{{ def show_data(self, data): self.book_info.show_data(data) self.cover_view.show_data(data) + self.current_path = data.get(_('Path'), '') + self.update_layout() + + def update_layout(self): self._layout.do_layout(self.rect()) try: sz = self.cover_view.pixmap.size() except: sz = QSize(0, 0) self.setToolTip( - '<p>'+_('Double-click to open Book Details window') + - '<br><br>' + _('Path') + ': ' + data.get(_('Path'), '') + - '<br><br>' + _('Cover size: %dx%d')%(sz.width(), sz.height()) + '<p>'+_('Double-click to open Book Details window') + + '<br><br>' + _('Path') + ': ' + self.current_path + + '<br><br>' + _('Cover size: %dx%d')%(sz.width(), sz.height()) ) def reset_info(self): From 549b89c82f22cb02b44f4e175f8a67ae7ca4032d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 12:47:11 -0600 Subject: [PATCH 86/90] Edit metadata dialog: Have the cover view be a constant default size --- src/calibre/gui2/metadata/basic_widgets.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index 593a3839ac..d34be6c564 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -24,7 +24,7 @@ from calibre.ebooks.metadata.meta import get_metadata from calibre.gui2 import file_icon_provider, UNDEFINED_QDATE, UNDEFINED_DATE, \ choose_files, error_dialog, choose_images, question_dialog from calibre.utils.date import local_tz, qt_to_dt -from calibre import strftime, fit_image +from calibre import strftime from calibre.ebooks import BOOK_EXTENSIONS from calibre.customize.ui import run_plugins_on_import from calibre.utils.date import utcfromtimestamp @@ -672,12 +672,7 @@ class Cover(ImageView): # {{{ self.frame_size = (sz.width()//3, sz.height()) def sizeHint(self): - sz = ImageView.sizeHint(self) - w, h = sz.width(), sz.height() - resized, nw, nh = fit_image(w, h, self.frame_size[0], - self.frame_size[1]) - if resized: - sz = QSize(nw, nh) + sz = QSize(self.frame_size[0], self.frame_size[1]) return sz def select_cover(self, *args): From bacd84c21d45aaf5012711c5492b0f7c9cb9ac2c Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 13:40:56 -0600 Subject: [PATCH 87/90] MOBI Input: Use the viasual formatting of the Table of Contents to try to automatically create a multi-level TOC when converting/viewing MOBI files. Fixes #763681 (Private bug) --- src/calibre/ebooks/__init__.py | 41 +++++++++++++++++- src/calibre/ebooks/mobi/reader.py | 69 ++++++++++++++++++++++++++++-- src/calibre/ebooks/oeb/stylizer.py | 44 +++---------------- 3 files changed, 112 insertions(+), 42 deletions(-) diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py index a56abb907e..d5b214884e 100644 --- a/src/calibre/ebooks/__init__.py +++ b/src/calibre/ebooks/__init__.py @@ -7,7 +7,7 @@ Code for the conversion of ebook formats and the reading of metadata from various formats. ''' -import traceback, os +import traceback, os, re from calibre import CurrentDir class ConversionError(Exception): @@ -169,3 +169,42 @@ def calibre_cover(title, author_string, series_string=None, lines.append(TextLine(series_string, author_size)) return create_cover_page(lines, I('library.png'), output_format='jpg') +UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|ex|en|px|mm|cm|in|pt|pc)$') + +def unit_convert(value, base, font, dpi): + ' Return value in pts' + if isinstance(value, (int, long, float)): + return value + try: + return float(value) * 72.0 / dpi + except: + pass + result = value + m = UNIT_RE.match(value) + if m is not None and m.group(1): + value = float(m.group(1)) + unit = m.group(2) + if unit == '%': + result = (value / 100.0) * base + elif unit == 'px': + result = value * 72.0 / dpi + elif unit == 'in': + result = value * 72.0 + elif unit == 'pt': + result = value + elif unit == 'em': + result = value * font + elif unit in ('ex', 'en'): + # This is a hack for ex since we have no way to know + # the x-height of the font + font = font + result = value * font * 0.5 + elif unit == 'pc': + result = value * 12.0 + elif unit == 'mm': + result = value * 0.04 + elif unit == 'cm': + result = value * 0.40 + return result + + diff --git a/src/calibre/ebooks/mobi/reader.py b/src/calibre/ebooks/mobi/reader.py index a65649dfd2..d9c6853795 100644 --- a/src/calibre/ebooks/mobi/reader.py +++ b/src/calibre/ebooks/mobi/reader.py @@ -20,7 +20,7 @@ from calibre.utils.filenames import ascii_filename from calibre.utils.date import parse_date from calibre.utils.cleantext import clean_ascii_chars from calibre.ptempfile import TemporaryDirectory -from calibre.ebooks import DRMError +from calibre.ebooks import DRMError, unit_convert from calibre.ebooks.chardet import ENCODING_PATS from calibre.ebooks.mobi import MobiError from calibre.ebooks.mobi.huffcdic import HuffReader @@ -258,6 +258,8 @@ class MobiReader(object): } ''') self.tag_css_rules = {} + self.left_margins = {} + self.text_indents = {} if hasattr(filename_or_stream, 'read'): stream = filename_or_stream @@ -567,9 +569,21 @@ class MobiReader(object): elif tag.tag == 'img': tag.set('width', width) else: - styles.append('text-indent: %s' % self.ensure_unit(width)) + ewidth = self.ensure_unit(width) + styles.append('text-indent: %s' % ewidth) + try: + ewidth_val = unit_convert(ewidth, 12, 500, 166) + self.text_indents[tag] = ewidth_val + except: + pass if width.startswith('-'): styles.append('margin-left: %s' % self.ensure_unit(width[1:])) + try: + ewidth_val = unit_convert(ewidth[1:], 12, 500, 166) + self.left_margins[tag] = ewidth_val + except: + pass + if attrib.has_key('align'): align = attrib.pop('align').strip() if align: @@ -661,6 +675,26 @@ class MobiReader(object): if hasattr(parent, 'remove'): parent.remove(tag) + def get_left_whitespace(self, tag): + + def whitespace(tag): + lm = ti = 0.0 + if tag.tag == 'p': + ti = unit_convert('1.5em', 12, 500, 166) + if tag.tag == 'blockquote': + lm = unit_convert('2em', 12, 500, 166) + lm = self.left_margins.get(tag, lm) + ti = self.text_indents.get(tag, ti) + return lm + ti + + parent = tag + ans = 0.0 + while parent is not None: + ans += whitespace(parent) + parent = parent.getparent() + + return ans + def create_opf(self, htmlfile, guide=None, root=None): mi = getattr(self.book_header.exth, 'mi', self.embedded_mi) if mi is None: @@ -731,16 +765,45 @@ class MobiReader(object): except: text = '' text = ent_pat.sub(entity_to_unicode, text) - tocobj.add_item(toc.partition('#')[0], href[1:], + item = tocobj.add_item(toc.partition('#')[0], href[1:], text) + item.left_space = int(self.get_left_whitespace(x)) found = True if reached and found and x.get('class', None) == 'mbp_pagebreak': break if tocobj is not None: + tocobj = self.structure_toc(tocobj) opf.set_toc(tocobj) return opf, ncx_manifest_entry + def structure_toc(self, toc): + indent_vals = set() + for item in toc: + indent_vals.add(item.left_space) + if len(indent_vals) > 6 or len(indent_vals) < 2: + # Too many or too few levels, give up + return toc + indent_vals = sorted(indent_vals) + + last_found = [None for i in indent_vals] + + newtoc = TOC() + + def find_parent(level): + candidates = last_found[:level] + for x in reversed(candidates): + if x is not None: + return x + return newtoc + + for item in toc: + level = indent_vals.index(item.left_space) + parent = find_parent(level) + last_found[level] = parent.add_item(item.href, item.fragment, + item.text) + + return newtoc def sizeof_trailing_entries(self, data): def sizeof_trailing_entry(ptr, psize): diff --git a/src/calibre/ebooks/oeb/stylizer.py b/src/calibre/ebooks/oeb/stylizer.py index 39ab41eede..4f06efba9f 100644 --- a/src/calibre/ebooks/oeb/stylizer.py +++ b/src/calibre/ebooks/oeb/stylizer.py @@ -18,6 +18,7 @@ from cssutils import profile as cssprofiles from lxml import etree from lxml.cssselect import css_to_xpath, ExpressionError, SelectorSyntaxError from calibre import force_unicode +from calibre.ebooks import unit_convert from calibre.ebooks.oeb.base import XHTML, XHTML_NS, CSS_MIME, OEB_STYLES from calibre.ebooks.oeb.base import XPNSMAP, xpath, urlnormalize from calibre.ebooks.oeb.profile import PROFILES @@ -444,7 +445,6 @@ class Stylizer(object): class Style(object): - UNIT_RE = re.compile(r'^(-*[0-9]*[.]?[0-9]*)\s*(%|em|ex|en|px|mm|cm|in|pt|pc)$') MS_PAT = re.compile(r'^\s*(mso-|panose-|text-underline|tab-interval)') def __init__(self, element, stylizer): @@ -507,43 +507,11 @@ class Style(object): return result def _unit_convert(self, value, base=None, font=None): - ' Return value in pts' - if isinstance(value, (int, long, float)): - return value - try: - return float(value) * 72.0 / self._profile.dpi - except: - pass - result = value - m = self.UNIT_RE.match(value) - if m is not None and m.group(1): - value = float(m.group(1)) - unit = m.group(2) - if unit == '%': - if base is None: - base = self.width - result = (value / 100.0) * base - elif unit == 'px': - result = value * 72.0 / self._profile.dpi - elif unit == 'in': - result = value * 72.0 - elif unit == 'pt': - result = value - elif unit == 'em': - font = font or self.fontSize - result = value * font - elif unit in ('ex', 'en'): - # This is a hack for ex since we have no way to know - # the x-height of the font - font = font or self.fontSize - result = value * font * 0.5 - elif unit == 'pc': - result = value * 12.0 - elif unit == 'mm': - result = value * 0.04 - elif unit == 'cm': - result = value * 0.40 - return result + 'Return value in pts' + if base is None: + base = self.width + font = font or self.fontSize + return unit_convert(value, base, font, self._profile.dpi) def pt_to_px(self, value): return (self._profile.dpi / 72.0) * value From aed8f4b31c18d01a0d34cdfa434731cb147c87d5 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 16:13:48 -0600 Subject: [PATCH 88/90] ... --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index d8bd0267ee..1424d83137 100644 --- a/setup.py +++ b/setup.py @@ -15,9 +15,9 @@ from setup import prints, get_warnings def check_version_info(): vi = sys.version_info - if vi[0] == 2 and vi[1] > 5: + if vi[0] == 2 and vi[1] > 6: return None - return 'calibre requires python >= 2.6' + return 'calibre requires python >= 2.7 and < 3' def option_parser(): parser = optparse.OptionParser() From 460cb932af73939899c100cf51dbcdc4e423439d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 19:11:25 -0600 Subject: [PATCH 89/90] ... --- src/calibre/gui2/store/web_control.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/web_control.py b/src/calibre/gui2/store/web_control.py index 5ce385bf59..17b42c5643 100644 --- a/src/calibre/gui2/store/web_control.py +++ b/src/calibre/gui2/store/web_control.py @@ -70,7 +70,7 @@ class NPWebView(QWebView): if ext not in BOOK_EXTENSIONS: if ext == 'acsm': from calibre.gui2.dialogs.confirm_delete import confirm - confirm('<p>' + _('This ebook is a DRMed EPUB file. ' + if not confirm('<p>' + _('This ebook is a DRMed EPUB file. ' 'You will be prompted to save this file to your ' 'computer. Once it is saved, open it with ' '<a href="http://www.adobe.com/products/digitaleditions/">' @@ -79,7 +79,8 @@ class NPWebView(QWebView): '.epub file. You can add this book to calibre ' 'using "Add Books" and selecting the file from ' 'the ADE library folder.'), - 'acsm_download', self) + 'acsm_download', self): + return home = os.path.expanduser('~') name = QFileDialog.getSaveFileName(self, _('File is not a supported ebook type. Save to disk?'), From 79f2d3bd4daa305eeca3132627a633d18a8ae25f Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Thu, 21 Apr 2011 19:32:11 -0600 Subject: [PATCH 90/90] Bettwe cover thumbnails for the Amazon store plugin --- src/calibre/gui2/store/amazon_plugin.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/calibre/gui2/store/amazon_plugin.py b/src/calibre/gui2/store/amazon_plugin.py index d5d8b54600..a68e4611f0 100644 --- a/src/calibre/gui2/store/amazon_plugin.py +++ b/src/calibre/gui2/store/amazon_plugin.py @@ -154,6 +154,13 @@ class AmazonKindleStore(StorePlugin): cover_img = data.xpath('//div[@class="productImage"]/a[@href="%s"]/img/@src' % asin_href) if cover_img: cover_url = cover_img[0] + parts = cover_url.split('/') + bn = parts[-1] + f, _, ext = bn.rpartition('.') + if '_' in f: + bn = f.partition('_')[0]+'_SL160_.'+ext + parts[-1] = bn + cover_url = '/'.join(parts) title = ''.join(data.xpath('div[@class="productTitle"]/a/text()')) author = ''.join(data.xpath('div[@class="productTitle"]/span[@class="ptBrand"]/text()')) @@ -174,7 +181,7 @@ class AmazonKindleStore(StorePlugin): def get_details(self, search_result, timeout): url = 'http://amazon.com/dp/' - + br = browser() with closing(br.open(url + search_result.detail_item, timeout=timeout)) as nf: idata = html.fromstring(nf.read()) @@ -187,4 +194,4 @@ class AmazonKindleStore(StorePlugin): search_result.drm = SearchResult.DRM_LOCKED return True - \ No newline at end of file +