diff --git a/recipes/novosti.recipe b/recipes/novosti.recipe index d66e7d28b7..a0a573d7ba 100644 --- a/recipes/novosti.recipe +++ b/recipes/novosti.recipe @@ -1,6 +1,6 @@ __license__ = 'GPL v3' -__copyright__ = '2008-2010, Darko Miletic ' +__copyright__ = '2008-2011, Darko Miletic ' ''' novosti.rs ''' @@ -21,34 +21,71 @@ class Novosti(BasicNewsRecipe): encoding = 'utf-8' language = 'sr' publication_type = 'newspaper' + masthead_url = 'http://www.novosti.rs/images/basic/logo-print.png' extra_css = """ @font-face {font-family: "sans1";src:url(res:///opt/sony/ebook/FONT/tt0003m_.ttf)} .article_description,body{font-family: Arial,Helvetica,sans1,sans-serif} .author{font-size: small} .articleLead{font-size: large; font-weight: bold} + img{display: block; margin-bottom: 1em; margin-top: 1em} """ conversion_options = { - 'comment' : description - , 'tags' : category - , 'publisher' : publisher - , 'language' : language + 'comment' : description + , 'tags' : category + , 'publisher' : publisher + , 'language' : language + , 'pretty_print' : True } preprocess_regexps = [(re.compile(u'\u0110'), lambda match: u'\u00D0')] - keep_only_tags = [dict(attrs={'class':['articleTitle','author','articleLead','articleBody']})] - remove_tags = [dict(name=['embed','object','iframe','base','link','meta'])] - feeds = [(u'Vesti', u'http://www.novosti.rs/rss/rss-vesti')] + keep_only_tags = [dict(attrs={'class':['articleTitle','articleInfo','articleLead','singlePhoto fl','articleBody']})] + remove_tags = [ + dict(name=['embed','object','iframe','base','link','meta']) + ,dict(name='a', attrs={'class':'loadComments topCommentsLink'}) + ] + remove_attributes = ['lang','xmlns:fb'] + + feeds = [ + (u'Politika' , u'http://www.novosti.rs/rss/2-Sve%20vesti') + ,(u'Drustvo' , u'http://www.novosti.rs/rss/1-Sve%20vesti') + ,(u'Ekonomija' , u'http://www.novosti.rs/rss/3-Sve%20vesti') + ,(u'Hronika' , u'http://www.novosti.rs/rss/4-Sve%20vesti') + ,(u'Dosije' , u'http://www.novosti.rs/rss/5-Sve%20vesti') + ,(u'Reportaze' , u'http://www.novosti.rs/rss/6-Sve%20vesti') + ,(u'Tehnologije' , u'http://www.novosti.rs/rss/35-Sve%20vesti') + ,(u'Zanimljivosti', u'http://www.novosti.rs/rss/26-Sve%20vesti') + ,(u'Auto' , u'http://www.novosti.rs/rss/50-Sve%20vesti') + ,(u'Sport' , u'http://www.novosti.rs/rss/11|47|12|14|13-Sve%20vesti') + ,(u'Svet' , u'http://www.novosti.rs/rss/7-Sve%20vesti') + ,(u'Region' , u'http://www.novosti.rs/rss/8-Sve%20vesti') + ,(u'Dijaspora' , u'http://www.novosti.rs/rss/9-Sve%20vesti') + ,(u'Spektakl' , u'http://www.novosti.rs/rss/10-Sve%20vesti') + ,(u'Kultura' , u'http://www.novosti.rs/rss/31-Sve%20vesti') + ,(u'Srbija' , u'http://www.novosti.rs/rss/15-Sve%20vesti') + ,(u'Beograd' , u'http://www.novosti.rs/rss/16-Sve%20vesti') + ,(u'Zivot+' , u'http://www.novosti.rs/rss/24|33|34|25|20|18|32|19-Sve%20vesti') + ,(u'Turizam' , u'http://www.novosti.rs/rss/36-Sve%20vesti') + ] def preprocess_html(self, soup): for item in soup.findAll(style=True): del item['style'] - for item in soup.findAll('span', attrs={'class':'author'}): - item.name='p' + for item in soup.findAll('a'): + limg = item.find('img') + if item.string is not None: + str = item.string + item.replaceWith(str) + else: + if limg: + item.name = 'div' + item.attrs = [] + else: + str = self.tag_to_string(item) + item.replaceWith(str) for item in soup.findAll('img'): if not item.has_key('alt'): item['alt'] = 'image' return soup - diff --git a/recipes/tabu.recipe b/recipes/tabu.recipe index d0ede613fd..f98ed8a155 100644 --- a/recipes/tabu.recipe +++ b/recipes/tabu.recipe @@ -24,30 +24,29 @@ class TabuRo(BasicNewsRecipe): cover_url = 'http://www.tabu.ro/img/tabu-logo2.png' conversion_options = { - 'comments' : description - ,'tags' : category - ,'language' : language - ,'publisher' : publisher - } + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } keep_only_tags = [ - dict(name='div', attrs={'id':'Article'}), - ] + dict(name='h2', attrs={'class':'articol_titlu'}), + dict(name='div', attrs={'class':'poza_articol_featured'}), + dict(name='div', attrs={'class':'articol_text'}) + ] remove_tags = [ - dict(name='div', attrs={'id':['advertisementArticle']}), - dict(name='div', attrs={'class':'voting_number'}), - dict(name='div', attrs={'id':'number_votes'}), - dict(name='div', attrs={'id':'rating_one'}), - dict(name='div', attrs={'class':'float: right;'}) + dict(name='div', attrs={'class':'asemanatoare'}) ] remove_tags_after = [ dict(name='div', attrs={'id':'comments'}), - ] + dict(name='div', attrs={'class':'asemanatoare'}) + ] feeds = [ - (u'Feeds', u'http://www.tabu.ro/rss_all.xml') + (u'Feeds', u'http://www.tabu.ro/feed/') ] def preprocess_html(self, soup): diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 470b4f5a35..8956780e2c 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -625,8 +625,9 @@ if test_eight_code: from calibre.ebooks.metadata.sources.google import GoogleBooks from calibre.ebooks.metadata.sources.amazon import Amazon from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary + from calibre.ebooks.metadata.sources.isbndb import ISBNDB - plugins += [GoogleBooks, Amazon, OpenLibrary] + plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB] # }}} else: diff --git a/src/calibre/devices/misc.py b/src/calibre/devices/misc.py index 01eba48a30..b9710d1958 100644 --- a/src/calibre/devices/misc.py +++ b/src/calibre/devices/misc.py @@ -244,7 +244,7 @@ class EEEREADER(USBMS): FORMATS = ['epub', 'fb2', 'txt', 'pdf'] VENDOR_ID = [0x0b05] - PRODUCT_ID = [0x178f] + PRODUCT_ID = [0x178f, 0x17a1] BCD = [0x0319] EBOOK_DIR_MAIN = EBOOK_DIR_CARD_A = 'Book' diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py index 7776be5e28..a56abb907e 100644 --- a/src/calibre/ebooks/__init__.py +++ b/src/calibre/ebooks/__init__.py @@ -26,7 +26,7 @@ class ParserError(ValueError): pass BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'text', 'htm', 'xhtm', - 'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc', + 'html', 'htmlz', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc', 'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip', 'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb'] diff --git a/src/calibre/ebooks/htmlz/output.py b/src/calibre/ebooks/htmlz/output.py index 7cdf04bcdb..03fe12c89e 100644 --- a/src/calibre/ebooks/htmlz/output.py +++ b/src/calibre/ebooks/htmlz/output.py @@ -12,7 +12,7 @@ from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.oeb.base import OEB_IMAGES +from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile @@ -71,9 +71,13 @@ class HTMLZOutput(OutputFormatPlugin): os.makedirs(os.path.join(tdir, 'images')) for item in oeb_book.manifest: if item.media_type in OEB_IMAGES and item.href in images: + if item.media_type == SVG_MIME: + data = unicode(etree.tostring(item.data, encoding=unicode)) + else: + data = item.data fname = os.path.join(tdir, 'images', images[item.href]) with open(fname, 'wb') as img: - img.write(item.data) + img.write(data) # Metadata with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: diff --git a/src/calibre/ebooks/metadata/extz.py b/src/calibre/ebooks/metadata/extz.py index 0ecdbe9ea6..18069b2a6a 100644 --- a/src/calibre/ebooks/metadata/extz.py +++ b/src/calibre/ebooks/metadata/extz.py @@ -8,12 +8,13 @@ Read meta information from extZ (TXTZ, HTMLZ...) files. ''' import os +import posixpath from cStringIO import StringIO from calibre.ebooks.metadata import MetaInformation -from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf -from calibre.ptempfile import TemporaryDirectory +from calibre.ebooks.metadata.opf2 import OPF +from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.zipfile import ZipFile, safe_replace def get_metadata(stream, extract_cover=True): @@ -23,16 +24,75 @@ def get_metadata(stream, extract_cover=True): mi = MetaInformation(_('Unknown'), [_('Unknown')]) stream.seek(0) - with TemporaryDirectory('_untxtz_mdata') as tdir: - try: - zf = ZipFile(stream) - zf.extract('metadata.opf', tdir) - with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff: - mi = OPF(opff).to_book_metadata() - except: - return mi + try: + with ZipFile(stream) as zf: + opf_name = get_first_opf_name(zf) + opf_stream = StringIO(zf.read(opf_name)) + opf = OPF(opf_stream) + mi = opf.to_book_metadata() + if extract_cover: + cover_name = opf.raster_cover + if cover_name: + mi.cover_data = ('jpg', zf.read(cover_name)) + except: + return mi return mi def set_metadata(stream, mi): - opf = StringIO(metadata_to_opf(mi)) - safe_replace(stream, 'metadata.opf', opf) + replacements = {} + + # Get the OPF in the archive. + with ZipFile(stream) as zf: + opf_path = get_first_opf_name(zf) + opf_stream = StringIO(zf.read(opf_path)) + opf = OPF(opf_stream) + + # Cover. + new_cdata = None + try: + new_cdata = mi.cover_data[1] + if not new_cdata: + raise Exception('no cover') + except: + try: + new_cdata = open(mi.cover, 'rb').read() + except: + pass + if new_cdata: + raster_cover = opf.raster_cover + if not raster_cover: + raster_cover = 'cover.jpg' + cpath = posixpath.join(posixpath.dirname(opf_path), raster_cover) + new_cover = _write_new_cover(new_cdata, cpath) + replacements[cpath] = open(new_cover.name, 'rb') + + # Update the metadata. + opf.smart_update(mi, replace_metadata=True) + newopf = StringIO(opf.render()) + safe_replace(stream, opf_path, newopf, extra_replacements=replacements) + + # Cleanup temporary files. + try: + if cpath is not None: + replacements[cpath].close() + os.remove(replacements[cpath].name) + except: + pass + +def get_first_opf_name(zf): + names = zf.namelist() + opfs = [] + for n in names: + if n.endswith('.opf') and '/' not in n: + opfs.append(n) + if not opfs: + raise Exception('No OPF found') + opfs.sort() + return opfs[0] + +def _write_new_cover(new_cdata, cpath): + from calibre.utils.magick.draw import save_cover_data_to + new_cover = PersistentTemporaryFile(suffix=os.path.splitext(cpath)[1]) + new_cover.close() + save_cover_data_to(new_cdata, new_cover.name) + return new_cover diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 00c6ea5e09..af519909f8 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -181,6 +181,10 @@ class Source(Plugin): #: construct the configuration widget for this plugin options = () + #: A string that is displayed at the top of the config widget for this + #: plugin + config_help_message = None + def __init__(self, *args, **kwargs): Plugin.__init__(self, *args, **kwargs) diff --git a/src/calibre/ebooks/metadata/sources/covers.py b/src/calibre/ebooks/metadata/sources/covers.py index cf6ec90c54..d28ce146c6 100644 --- a/src/calibre/ebooks/metadata/sources/covers.py +++ b/src/calibre/ebooks/metadata/sources/covers.py @@ -76,6 +76,11 @@ def run_download(log, results, abort, (plugin, width, height, fmt, bytes) ''' + if title == _('Unknown'): + title = None + if authors == [_('Unknown')]: + authors = None + plugins = [p for p in metadata_plugins(['cover']) if p.is_configured()] rq = Queue() @@ -145,7 +150,7 @@ def download_cover(log, Synchronous cover download. Returns the "best" cover as per user prefs/cover resolution. - Return cover is a tuple: (plugin, width, height, fmt, data) + Returned cover is a tuple: (plugin, width, height, fmt, data) Returns None if no cover is found. ''' diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index fad810c26e..b494e05e1a 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -253,6 +253,10 @@ def merge_identify_results(result_map, log): def identify(log, abort, # {{{ title=None, authors=None, identifiers={}, timeout=30): + if title == _('Unknown'): + title = None + if authors == [_('Unknown')]: + authors = None start_time = time.time() plugins = [p for p in metadata_plugins(['identify']) if p.is_configured()] diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index ab9342c6cb..b8deea56df 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -7,7 +7,19 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from calibre.ebooks.metadata.sources.base import Source +from urllib import quote + +from lxml import etree + +from calibre.ebooks.metadata import check_isbn +from calibre.ebooks.metadata.sources.base import Source, Option +from calibre.ebooks.chardet import xml_to_unicode +from calibre.utils.cleantext import clean_ascii_chars +from calibre.utils.icu import lower +from calibre.ebooks.metadata.book.base import Metadata + +BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%s&page_number=1&results=subjects,authors,texts&' + class ISBNDB(Source): @@ -18,6 +30,20 @@ class ISBNDB(Source): touched_fields = frozenset(['title', 'authors', 'identifier:isbn', 'comments', 'publisher']) supports_gzip_transfer_encoding = True + # Shortcut, since we have no cached cover URLS + cached_cover_url_is_reliable = False + + options = ( + Option('isbndb_key', 'string', None, _('IsbnDB key:'), + _('To use isbndb.com you have to sign up for a free account' + 'at isbndb.com and get an access key.')), + ) + + config_help_message = '

'+_('To use metadata from isbndb.com you must sign' + ' up for a free account and get an isbndb key and enter it below.' + ' Instructions to get the key are ' + 'here.') + def __init__(self, *args, **kwargs): Source.__init__(self, *args, **kwargs) @@ -35,9 +61,186 @@ class ISBNDB(Source): except: pass - self.isbndb_key = prefs['isbndb_key'] + @property + def isbndb_key(self): + return self.prefs['isbndb_key'] def is_configured(self): return self.isbndb_key is not None + def create_query(self, title=None, authors=None, identifiers={}): # {{{ + base_url = BASE_URL%self.isbndb_key + isbn = check_isbn(identifiers.get('isbn', None)) + q = '' + if isbn is not None: + q = 'index1=isbn&value1='+isbn + elif title or authors: + tokens = [] + title_tokens = list(self.get_title_tokens(title)) + tokens += title_tokens + author_tokens = self.get_author_tokens(authors, + only_first_author=True) + tokens += author_tokens + tokens = [quote(t) for t in tokens] + q = '+'.join(tokens) + q = 'index1=combined&value1='+q + + if not q: + return None + if isinstance(q, unicode): + q = q.encode('utf-8') + return base_url + q + # }}} + + def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ + identifiers={}, timeout=30): + if not self.is_configured(): + return + query = self.create_query(title=title, authors=authors, + identifiers=identifiers) + if not query: + err = 'Insufficient metadata to construct query' + log.error(err) + return err + + results = [] + try: + results = self.make_query(query, abort, title=title, authors=authors, + identifiers=identifiers, timeout=timeout) + except: + err = 'Failed to make query to ISBNDb, aborting.' + log.exception(err) + return err + + if not results and identifiers.get('isbn', False) and title and authors and \ + not abort.is_set(): + return self.identify(log, result_queue, abort, title=title, + authors=authors, timeout=timeout) + + for result in results: + self.clean_downloaded_metadata(result) + result_queue.put(result) + + def parse_feed(self, feed, seen, orig_title, orig_authors, identifiers): + + def tostring(x): + if x is None: + return '' + return etree.tostring(x, method='text', encoding=unicode).strip() + + orig_isbn = identifiers.get('isbn', None) + title_tokens = list(self.get_title_tokens(orig_title)) + author_tokens = list(self.get_author_tokens(orig_authors)) + results = [] + + def ismatch(title, authors): + authors = lower(' '.join(authors)) + title = lower(title) + match = not title_tokens + for t in title_tokens: + if lower(t) in title: + match = True + break + amatch = not author_tokens + for a in author_tokens: + if lower(a) in authors: + amatch = True + break + if not author_tokens: amatch = True + return match and amatch + + bl = feed.find('BookList') + if bl is None: + err = tostring(etree.find('errormessage')) + raise ValueError('ISBNDb query failed:' + err) + total_results = int(bl.get('total_results')) + shown_results = int(bl.get('shown_results')) + for bd in bl.xpath('.//BookData'): + isbn = check_isbn(bd.get('isbn13', bd.get('isbn', None))) + if not isbn: + continue + if orig_isbn and isbn != orig_isbn: + continue + title = tostring(bd.find('Title')) + if not title: + continue + authors = [] + for au in bd.xpath('.//Authors/Person'): + au = tostring(au) + if au: + if ',' in au: + ln, _, fn = au.partition(',') + au = fn.strip() + ' ' + ln.strip() + authors.append(au) + if not authors: + continue + comments = tostring(bd.find('Summary')) + if not comments: + # Require comments, since without them the result is useless + # anyway + continue + id_ = (title, tuple(authors)) + if id_ in seen: + continue + seen.add(id_) + if not ismatch(title, authors): + continue + publisher = tostring(bd.find('PublisherText')) + if not publisher: publisher = None + if publisher and 'audio' in publisher.lower(): + continue + mi = Metadata(title, authors) + mi.isbn = isbn + mi.publisher = publisher + mi.comments = comments + results.append(mi) + return total_results, shown_results, results + + def make_query(self, q, abort, title=None, authors=None, identifiers={}, + max_pages=10, timeout=30): + page_num = 1 + parser = etree.XMLParser(recover=True, no_network=True) + br = self.browser + + seen = set() + + candidates = [] + total_found = 0 + while page_num <= max_pages and not abort.is_set(): + url = q.replace('&page_number=1&', '&page_number=%d&'%page_num) + page_num += 1 + raw = br.open_novisit(url, timeout=timeout).read() + feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw), + strip_encoding_pats=True)[0], parser=parser) + total, found, results = self.parse_feed( + feed, seen, title, authors, identifiers) + total_found += found + candidates += results + if total_found >= total or len(candidates) > 9: + break + + return candidates + # }}} + +if __name__ == '__main__': + # To run these test use: + # calibre-debug -e src/calibre/ebooks/metadata/sources/isbndb.py + from calibre.ebooks.metadata.sources.test import (test_identify_plugin, + title_test, authors_test) + test_identify_plugin(ISBNDB.name, + [ + + + ( + {'title':'Great Gatsby', + 'authors':['Fitzgerald']}, + [title_test('The great gatsby', exact=True), + authors_test(['F. Scott Fitzgerald'])] + ), + + ( + {'title': 'Flatland', 'authors':['Abbott']}, + [title_test('Flatland', exact=False)] + ), + ]) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 3feef7b6f5..8c7b740cdb 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -496,6 +496,10 @@ class MobiMLizer(object): vtag.append(child) return + if tag == 'blockquote': + old_mim = self.opts.mobi_ignore_margins + self.opts.mobi_ignore_margins = False + if text or tag in CONTENT_TAGS or tag in NESTABLE_TAGS: self.mobimlize_content(tag, text, bstate, istates) for child in elem: @@ -511,6 +515,8 @@ class MobiMLizer(object): if tail: self.mobimlize_content(tag, tail, bstate, istates) + if tag == 'blockquote': + self.opts.mobi_ignore_margins = old_mim if bstate.content and style['page-break-after'] in PAGE_BREAKS: bstate.pbreak = True diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index 5f4c47cdf3..fc47b26c02 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -310,10 +310,11 @@ class Serializer(object): if href not in id_offsets: self.logger.warn('Hyperlink target %r not found' % href) href, _ = urldefrag(href) - ioff = self.id_offsets[href] - for hoff in hoffs: - buffer.seek(hoff) - buffer.write('%010d' % ioff) + if href in self.id_offsets: + ioff = self.id_offsets[href] + for hoff in hoffs: + buffer.seek(hoff) + buffer.write('%010d' % ioff) class MobiWriter(object): COLLAPSE_RE = re.compile(r'[ \t\r\n\v]+') diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 22aaabf592..e39427021e 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -357,6 +357,7 @@ class FileIconProvider(QFileIconProvider): 'bmp' : 'bmp', 'svg' : 'svg', 'html' : 'html', + 'htmlz' : 'html', 'htm' : 'html', 'xhtml' : 'html', 'xhtm' : 'html', diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index cf67cd6cfa..f8dd0693ea 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -22,6 +22,8 @@ from calibre.constants import preferred_encoding, filesystem_encoding from calibre.gui2.actions import InterfaceAction from calibre.gui2 import config, question_dialog from calibre.ebooks.metadata import MetaInformation +from calibre.utils.config import test_eight_code +from calibre.ebooks.metadata.sources.base import msprefs def get_filters(): return [ @@ -178,13 +180,26 @@ class AddAction(InterfaceAction): except IndexError: self.gui.library_view.model().books_added(self.isbn_add_dialog.value) self.isbn_add_dialog.accept() - orig = config['overwrite_author_title_metadata'] - config['overwrite_author_title_metadata'] = True - try: - self.gui.iactions['Edit Metadata'].do_download_metadata( - self.add_by_isbn_ids) - finally: - config['overwrite_author_title_metadata'] = orig + if test_eight_code: + orig = msprefs['ignore_fields'] + new = list(orig) + for x in ('title', 'authors'): + if x in new: + new.remove(x) + msprefs['ignore_fields'] = new + try: + self.gui.iactions['Edit Metadata'].download_metadata( + ids=self.add_by_isbn_ids) + finally: + msprefs['ignore_fields'] = orig + else: + orig = config['overwrite_author_title_metadata'] + config['overwrite_author_title_metadata'] = True + try: + self.gui.iactions['Edit Metadata'].do_download_metadata( + self.add_by_isbn_ids) + finally: + config['overwrite_author_title_metadata'] = orig return diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index e29e6c344d..18a73fb282 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -10,7 +10,7 @@ from functools import partial from PyQt4.Qt import Qt, QMenu, QModelIndex -from calibre.gui2 import error_dialog, config +from calibre.gui2 import error_dialog, config, Dispatcher from calibre.gui2.dialogs.metadata_single import MetadataSingleDialog from calibre.gui2.dialogs.metadata_bulk import MetadataBulkDialog from calibre.gui2.dialogs.confirm_delete import confirm @@ -35,16 +35,23 @@ class EditMetadataAction(InterfaceAction): md.addAction(_('Edit metadata in bulk'), partial(self.edit_metadata, False, bulk=True)) md.addSeparator() - md.addAction(_('Download metadata and covers'), - partial(self.download_metadata, False, covers=True), + if test_eight_code: + dall = self.download_metadata + dident = partial(self.download_metadata, covers=False) + dcovers = partial(self.download_metadata, identify=False) + else: + dall = partial(self.download_metadata_old, False, covers=True) + dident = partial(self.download_metadata_old, False, covers=False) + dcovers = partial(self.download_metadata_old, False, covers=True, + set_metadata=False, set_social_metadata=False) + + md.addAction(_('Download metadata and covers'), dall, Qt.ControlModifier+Qt.Key_D) - md.addAction(_('Download only metadata'), - partial(self.download_metadata, False, covers=False)) - md.addAction(_('Download only covers'), - partial(self.download_metadata, False, covers=True, - set_metadata=False, set_social_metadata=False)) - md.addAction(_('Download only social metadata'), - partial(self.download_metadata, False, covers=False, + md.addAction(_('Download only metadata'), dident) + md.addAction(_('Download only covers'), dcovers) + if not test_eight_code: + md.addAction(_('Download only social metadata'), + partial(self.download_metadata_old, False, covers=False, set_metadata=False, set_social_metadata=True)) self.metadata_menu = md @@ -73,7 +80,26 @@ class EditMetadataAction(InterfaceAction): self.qaction.setEnabled(enabled) self.action_merge.setEnabled(enabled) - def download_metadata(self, checked, covers=True, set_metadata=True, + def download_metadata(self, identify=True, covers=True, ids=None): + if ids is None: + rows = self.gui.library_view.selectionModel().selectedRows() + if not rows or len(rows) == 0: + return error_dialog(self.gui, _('Cannot download metadata'), + _('No books selected'), show=True) + db = self.gui.library_view.model().db + ids = [db.id(row.row()) for row in rows] + from calibre.gui2.metadata.bulk_download2 import start_download + start_download(self.gui, ids, + Dispatcher(self.bulk_metadata_downloaded), identify, covers) + + def bulk_metadata_downloaded(self, job): + if job.failed: + self.gui.job_exception(job, dialog_title=_('Failed to download metadata')) + return + from calibre.gui2.metadata.bulk_download2 import proceed + proceed(self.gui, job) + + def download_metadata_old(self, checked, covers=True, set_metadata=True, set_social_metadata=None): rows = self.gui.library_view.selectionModel().selectedRows() if not rows or len(rows) == 0: diff --git a/src/calibre/gui2/dialogs/job_view.ui b/src/calibre/gui2/dialogs/job_view.ui index 8b54c23573..1e854c0f29 100644 --- a/src/calibre/gui2/dialogs/job_view.ui +++ b/src/calibre/gui2/dialogs/job_view.ui @@ -1,7 +1,8 @@ - + + Dialog - - + + 0 0 @@ -9,38 +10,41 @@ 462 - + Details of job - - + + :/images/view.png:/images/view.png - - - - + + + + false - + QPlainTextEdit::NoWrap - + true - - - + + + QDialogButtonBox::Ok + + + - + @@ -49,11 +53,11 @@ Dialog accept() - + 617 442 - + 206 -5 diff --git a/src/calibre/gui2/email.py b/src/calibre/gui2/email.py index c6d58fa340..c8adeb7d31 100644 --- a/src/calibre/gui2/email.py +++ b/src/calibre/gui2/email.py @@ -6,9 +6,7 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, socket, time, cStringIO -from threading import Thread -from Queue import Queue +import os, socket, time from binascii import unhexlify from functools import partial from itertools import repeat @@ -16,67 +14,20 @@ from itertools import repeat from calibre.utils.smtp import compose_mail, sendmail, extract_email_address, \ config as email_config from calibre.utils.filenames import ascii_filename -from calibre.utils.ipc.job import BaseJob -from calibre.ptempfile import PersistentTemporaryFile from calibre.customize.ui import available_input_formats, available_output_formats from calibre.ebooks.metadata import authors_to_string from calibre.constants import preferred_encoding from calibre.gui2 import config, Dispatcher, warning_dialog from calibre.library.save_to_disk import get_components from calibre.utils.config import tweaks +from calibre.gui2.threaded_jobs import ThreadedJob -class EmailJob(BaseJob): # {{{ - - def __init__(self, callback, description, attachment, aname, to, subject, text, job_manager): - BaseJob.__init__(self, description) - self.exception = None - self.job_manager = job_manager - self.email_args = (attachment, aname, to, subject, text) - self.email_sent_callback = callback - self.log_path = None - self._log_file = cStringIO.StringIO() - self._log_file.write(self.description.encode('utf-8') + '\n') - - @property - def log_file(self): - if self.log_path is not None: - return open(self.log_path, 'rb') - return cStringIO.StringIO(self._log_file.getvalue()) - - def start_work(self): - self.start_time = time.time() - self.job_manager.changed_queue.put(self) - - def job_done(self): - self.duration = time.time() - self.start_time - self.percent = 1 - # Dump log onto disk - lf = PersistentTemporaryFile('email_log') - lf.write(self._log_file.getvalue()) - lf.close() - self.log_path = lf.name - self._log_file.close() - self._log_file = None - - self.job_manager.changed_queue.put(self) - - def log_write(self, what): - self._log_file.write(what) - -# }}} - -class Emailer(Thread): # {{{ +class Sendmail(object): MAX_RETRIES = 1 - def __init__(self, job_manager): - Thread.__init__(self) - self.daemon = True - self.jobs = Queue() - self.job_manager = job_manager - self._run = True + def __init__(self): self.calculate_rate_limit() - self.last_send_time = time.time() - self.rate_limit def calculate_rate_limit(self): @@ -87,70 +38,28 @@ class Emailer(Thread): # {{{ 'gmail.com' in rh or 'live.com' in rh): self.rate_limit = tweaks['public_smtp_relay_delay'] - def stop(self): - self._run = False - self.jobs.put(None) + def __call__(self, attachment, aname, to, subject, text, log=None, + abort=None, notifications=None): - def run(self): - while self._run: + try_count = 0 + while try_count <= self.MAX_RETRIES: + if try_count > 0: + log('\nRetrying in %d seconds...\n' % + self.rate_limit) try: - job = self.jobs.get() + self.sendmail(attachment, aname, to, subject, text, log) + try_count = self.MAX_RETRIES + log('Email successfully sent') except: - break - if job is None or not self._run: - break - try_count = 0 - failed, exc = False, None - job.start_work() - if job.kill_on_start: - job.log_write('Aborted\n') - job.failed = failed - job.killed = True - job.job_done() - continue + if abort.is_set(): + return + if try_count == self.MAX_RETRIES: + raise + log.exception('\nSending failed...\n') - while try_count <= self.MAX_RETRIES: - failed = False - if try_count > 0: - job.log_write('\nRetrying in %d seconds...\n' % - self.rate_limit) - try: - self.sendmail(job) - break - except Exception as e: - if not self._run: - return - import traceback - failed = True - exc = e - job.log_write('\nSending failed...\n') - job.log_write(traceback.format_exc()) + try_count += 1 - try_count += 1 - - if not self._run: - break - - job.failed = failed - job.exception = exc - job.job_done() - try: - job.email_sent_callback(job) - except: - import traceback - traceback.print_exc() - - def send_mails(self, jobnames, callback, attachments, to_s, subjects, - texts, attachment_names): - for name, attachment, to, subject, text, aname in zip(jobnames, - attachments, to_s, subjects, texts, attachment_names): - description = _('Email %s to %s') % (name, to) - job = EmailJob(callback, description, attachment, aname, to, - subject, text, self.job_manager) - self.job_manager.add_job(job) - self.jobs.put(job) - - def sendmail(self, job): + def sendmail(self, attachment, aname, to, subject, text, log): while time.time() - self.last_send_time <= self.rate_limit: time.sleep(1) try: @@ -158,7 +67,6 @@ class Emailer(Thread): # {{{ from_ = opts.from_ if not from_: from_ = 'calibre ' - attachment, aname, to, subject, text = job.email_args msg = compose_mail(from_, to, text, subject, open(attachment, 'rb'), aname) efrom, eto = map(extract_email_address, (from_, to)) @@ -169,48 +77,56 @@ class Emailer(Thread): # {{{ username=opts.relay_username, password=unhexlify(opts.relay_password), port=opts.relay_port, encryption=opts.encryption, - debug_output=partial(print, file=job._log_file)) + debug_output=log.debug) finally: self.last_send_time = time.time() - def email_news(self, mi, remove, get_fmts, done): - opts = email_config().parse() - accounts = [(account, [x.strip().lower() for x in x[0].split(',')]) - for account, x in opts.accounts.items() if x[1]] - sent_mails = [] - for i, x in enumerate(accounts): - account, fmts = x - files = get_fmts(fmts) - files = [f for f in files if f is not None] - if not files: - continue - attachment = files[0] - to_s = [account] - subjects = [_('News:')+' '+mi.title] - texts = [ - _('Attached is the %s periodical downloaded by calibre.') - % (mi.title,) - ] - attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]] - attachments = [attachment] - jobnames = [mi.title] - do_remove = [] - if i == len(accounts) - 1: - do_remove = remove - self.send_mails(jobnames, - Dispatcher(partial(done, remove=do_remove)), - attachments, to_s, subjects, texts, attachment_names) - sent_mails.append(to_s[0]) - return sent_mails +gui_sendmail = Sendmail() -# }}} +def send_mails(jobnames, callback, attachments, to_s, subjects, + texts, attachment_names, job_manager): + for name, attachment, to, subject, text, aname in zip(jobnames, + attachments, to_s, subjects, texts, attachment_names): + description = _('Email %s to %s') % (name, to) + job = ThreadedJob('email', description, gui_sendmail, (attachment, aname, to, + subject, text), {}, callback, killable=False) + job_manager.run_threaded_job(job) + + +def email_news(mi, remove, get_fmts, done, job_manager): + opts = email_config().parse() + accounts = [(account, [x.strip().lower() for x in x[0].split(',')]) + for account, x in opts.accounts.items() if x[1]] + sent_mails = [] + for i, x in enumerate(accounts): + account, fmts = x + files = get_fmts(fmts) + files = [f for f in files if f is not None] + if not files: + continue + attachment = files[0] + to_s = [account] + subjects = [_('News:')+' '+mi.title] + texts = [ + _('Attached is the %s periodical downloaded by calibre.') + % (mi.title,) + ] + attachment_names = [ascii_filename(mi.title)+os.path.splitext(attachment)[1]] + attachments = [attachment] + jobnames = [mi.title] + do_remove = [] + if i == len(accounts) - 1: + do_remove = remove + send_mails(jobnames, + Dispatcher(partial(done, remove=do_remove)), + attachments, to_s, subjects, texts, attachment_names, + job_manager) + sent_mails.append(to_s[0]) + return sent_mails class EmailMixin(object): # {{{ - def __init__(self): - self.emailer = Emailer(self.job_manager) - def send_by_mail(self, to, fmts, delete_from_library, subject='', send_ids=None, do_auto_convert=True, specific_format=None): ids = [self.library_view.model().id(r) for r in self.library_view.selectionModel().selectedRows()] if send_ids is None else send_ids @@ -246,8 +162,7 @@ class EmailMixin(object): # {{{ components = get_components(subject, mi, id) if not components: components = [mi.title] - subject = os.path.join(*components) - subjects.append(subject) + subjects.append(os.path.join(*components)) a = authors_to_string(mi.authors if mi.authors else \ [_('Unknown')]) texts.append(_('Attached, you will find the e-book') + \ @@ -262,11 +177,10 @@ class EmailMixin(object): # {{{ to_s = list(repeat(to, len(attachments))) if attachments: - if not self.emailer.is_alive(): - self.emailer.start() - self.emailer.send_mails(jobnames, + send_mails(jobnames, Dispatcher(partial(self.email_sent, remove=remove)), - attachments, to_s, subjects, texts, attachment_names) + attachments, to_s, subjects, texts, attachment_names, + self.job_manager) self.status_bar.show_message(_('Sending email to')+' '+to, 3000) auto = [] @@ -334,10 +248,8 @@ class EmailMixin(object): # {{{ files, auto = self.library_view.model().\ get_preferred_formats_from_ids([id_], fmts) return files - if not self.emailer.is_alive(): - self.emailer.start() - sent_mails = self.emailer.email_news(mi, remove, - get_fmts, self.email_sent) + sent_mails = email_news(mi, remove, + get_fmts, self.email_sent, self.job_manager) if sent_mails: self.status_bar.show_message(_('Sent news to')+' '+\ ', '.join(sent_mails), 3000) diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index 80f1f1c2cf..a75ff01b21 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -247,6 +247,11 @@ class LayoutMixin(object): # {{{ for x in ('cb', 'tb', 'bd'): button = getattr(self, x+'_splitter').button button.setIconSize(QSize(24, 24)) + if isosx: + button.setStyleSheet(''' + QToolButton { background: none; border:none; padding: 0px; } + QToolButton:checked { background: rgba(0, 0, 0, 25%); } + ''') self.status_bar.addPermanentWidget(button) self.status_bar.addPermanentWidget(self.jobs_button) self.setStatusBar(self.status_bar) diff --git a/src/calibre/gui2/jobs.py b/src/calibre/gui2/jobs.py index dbde030e81..34eef4406a 100644 --- a/src/calibre/gui2/jobs.py +++ b/src/calibre/gui2/jobs.py @@ -8,14 +8,13 @@ Job management. ''' import re - from Queue import Empty, Queue -from PyQt4.Qt import QAbstractTableModel, QVariant, QModelIndex, Qt, \ - QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, \ - QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, \ - QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, \ - QByteArray +from PyQt4.Qt import (QAbstractTableModel, QVariant, QModelIndex, Qt, + QTimer, pyqtSignal, QIcon, QDialog, QAbstractItemDelegate, QApplication, + QSize, QStyleOptionProgressBarV2, QString, QStyle, QToolTip, QFrame, + QHBoxLayout, QVBoxLayout, QSizePolicy, QLabel, QCoreApplication, QAction, + QByteArray) from calibre.utils.ipc.server import Server from calibre.utils.ipc.job import ParallelJob @@ -25,8 +24,9 @@ from calibre.gui2.dialogs.jobs_ui import Ui_JobsDialog from calibre import __appname__ from calibre.gui2.dialogs.job_view_ui import Ui_Dialog from calibre.gui2.progress_indicator import ProgressIndicator +from calibre.gui2.threaded_jobs import ThreadedJobServer, ThreadedJob -class JobManager(QAbstractTableModel): +class JobManager(QAbstractTableModel): # {{{ job_added = pyqtSignal(int) job_done = pyqtSignal(int) @@ -42,6 +42,7 @@ class JobManager(QAbstractTableModel): self.add_job = Dispatcher(self._add_job) self.server = Server(limit=int(config['worker_limit']/2.0), enforce_cpu_limit=config['enforce_cpu_limit']) + self.threaded_server = ThreadedJobServer() self.changed_queue = Queue() self.timer = QTimer(self) @@ -146,12 +147,21 @@ class JobManager(QAbstractTableModel): jobs.add(self.server.changed_jobs_queue.get_nowait()) except Empty: break + + # Update device jobs while True: try: jobs.add(self.changed_queue.get_nowait()) except Empty: break + # Update threaded jobs + while True: + try: + jobs.add(self.threaded_server.changed_jobs.get_nowait()) + except Empty: + break + if jobs: needs_reset = False for job in jobs: @@ -207,11 +217,22 @@ class JobManager(QAbstractTableModel): self.server.add_job(job) return job + def run_threaded_job(self, job): + self.add_job(job) + self.threaded_server.add_job(job) + def launch_gui_app(self, name, args=[], kwargs={}, description=''): job = ParallelJob(name, description, lambda x: x, args=args, kwargs=kwargs) self.server.run_job(job, gui=True, redirect_output=False) + def _kill_job(self, job): + if isinstance(job, ParallelJob): + self.server.kill_job(job) + elif isinstance(job, ThreadedJob): + self.threaded_server.kill_job(job) + else: + job.kill_on_start = True def kill_job(self, row, view): job = self.jobs[row] @@ -221,29 +242,29 @@ class JobManager(QAbstractTableModel): if job.duration is not None: return error_dialog(view, _('Cannot kill job'), _('Job has already run')).exec_() - if isinstance(job, ParallelJob): - self.server.kill_job(job) - else: - job.kill_on_start = True + if not getattr(job, 'killable', True): + return error_dialog(view, _('Cannot kill job'), + _('This job cannot be stopped'), show=True) + self._kill_job(job) def kill_all_jobs(self): for job in self.jobs: - if isinstance(job, DeviceJob) or job.duration is not None: + if (isinstance(job, DeviceJob) or job.duration is not None or + not getattr(job, 'killable', True)): continue - if isinstance(job, ParallelJob): - self.server.kill_job(job) - else: - job.kill_on_start = True + self._kill_job(job) def terminate_all_jobs(self): self.server.killall() for job in self.jobs: - if isinstance(job, DeviceJob) or job.duration is not None: + if (isinstance(job, DeviceJob) or job.duration is not None or + not getattr(job, 'killable', True)): continue if not isinstance(job, ParallelJob): - job.kill_on_start = True - + self._kill_job(job) +# }}} +# Jobs UI {{{ class ProgressBarDelegate(QAbstractItemDelegate): def sizeHint(self, option, index): @@ -269,6 +290,11 @@ class DetailView(QDialog, Ui_Dialog): self.setupUi(self) self.setWindowTitle(job.description) self.job = job + self.html_view = hasattr(job, 'html_details') + if self.html_view: + self.log.setVisible(False) + else: + self.tb.setVisible(False) self.next_pos = 0 self.update() self.timer = QTimer(self) @@ -277,12 +303,19 @@ class DetailView(QDialog, Ui_Dialog): def update(self): - f = self.job.log_file - f.seek(self.next_pos) - more = f.read() - self.next_pos = f.tell() - if more: - self.log.appendPlainText(more.decode('utf-8', 'replace')) + if self.html_view: + html = self.job.html_details + if len(html) > self.next_pos: + self.next_pos = len(html) + self.tb.setHtml( + '

%s
'%html) + else: + f = self.job.log_file + f.seek(self.next_pos) + more = f.read() + self.next_pos = f.tell() + if more: + self.log.appendPlainText(more.decode('utf-8', 'replace')) class JobsButton(QFrame): @@ -441,3 +474,5 @@ class JobsDialog(QDialog, Ui_JobsDialog): def hide(self, *args): self.save_state() return QDialog.hide(self, *args) +# }}} + diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index e98817a02f..c01e708933 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -408,6 +408,7 @@ class ToolBar(BaseToolBar): # {{{ self.d_widget.layout().addWidget(self.donate_button) if isosx: self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }') + self.d_widget.layout().addWidget(QLabel(u'\u00a0')) bar.addWidget(self.d_widget) self.showing_donate = True elif what in self.gui.iactions: diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py new file mode 100644 index 0000000000..5f0af1b316 --- /dev/null +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -0,0 +1,320 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os +from functools import partial +from itertools import izip + +from PyQt4.Qt import (QIcon, QDialog, QVBoxLayout, QTextBrowser, QSize, + QDialogButtonBox, QApplication, QTimer, QLabel, QProgressBar) + +from calibre.gui2.dialogs.message_box import MessageBox +from calibre.gui2.threaded_jobs import ThreadedJob +from calibre.utils.icu import lower +from calibre.ebooks.metadata import authors_to_string +from calibre.gui2 import question_dialog, error_dialog +from calibre.ebooks.metadata.sources.identify import identify, msprefs +from calibre.ebooks.metadata.sources.covers import download_cover +from calibre.ebooks.metadata.book.base import Metadata +from calibre.customize.ui import metadata_plugins +from calibre.ptempfile import PersistentTemporaryFile + +def show_config(gui, parent): + from calibre.gui2.preferences import show_config_widget + show_config_widget('Sharing', 'Metadata download', parent=parent, + gui=gui, never_shutdown=True) + +def start_download(gui, ids, callback, identify, covers): + q = MessageBox(MessageBox.QUESTION, _('Schedule download?'), + '

'+_('The download of metadata for the %d selected book(s) will' + ' run in the background. Proceed?')%len(ids) + + '

'+_('You can monitor the progress of the download ' + 'by clicking the rotating spinner in the bottom right ' + 'corner.') + + '

'+_('When the download completes you will be asked for' + ' confirmation before calibre applies the downloaded metadata.'), + show_copy_button=False, parent=gui) + b = q.bb.addButton(_('Configure download'), q.bb.ActionRole) + b.setIcon(QIcon(I('config.png'))) + b.clicked.connect(partial(show_config, gui, q)) + q.det_msg_toggle.setVisible(False) + + ret = q.exec_() + b.clicked.disconnect() + if ret != q.Accepted: + return + + job = ThreadedJob('metadata bulk download', + _('Download metadata for %d books')%len(ids), + download, (ids, gui.current_db, identify, covers), {}, callback) + gui.job_manager.run_threaded_job(job) + gui.status_bar.show_message(_('Metadata download started'), 3000) + + +class ViewLog(QDialog): # {{{ + + def __init__(self, html, parent=None): + QDialog.__init__(self, parent) + self.l = l = QVBoxLayout() + self.setLayout(l) + + self.tb = QTextBrowser(self) + self.tb.setHtml('

%s
' % html) + l.addWidget(self.tb) + + self.bb = QDialogButtonBox(QDialogButtonBox.Ok) + self.bb.accepted.connect(self.accept) + self.bb.rejected.connect(self.reject) + self.copy_button = self.bb.addButton(_('Copy to clipboard'), + self.bb.ActionRole) + self.copy_button.setIcon(QIcon(I('edit-copy.png'))) + self.copy_button.clicked.connect(self.copy_to_clipboard) + l.addWidget(self.bb) + self.setModal(False) + self.resize(QSize(700, 500)) + self.setWindowTitle(_('Download log')) + self.setWindowIcon(QIcon(I('debug.png'))) + self.show() + + def copy_to_clipboard(self): + txt = self.tb.toPlainText() + QApplication.clipboard().setText(txt) + +_vl = None +def view_log(job, parent): + global _vl + _vl = ViewLog(job.html_details, parent) + +# }}} + +class ApplyDialog(QDialog): + + def __init__(self, id_map, gui): + QDialog.__init__(self, gui) + + self.l = l = QVBoxLayout() + self.setLayout(l) + l.addWidget(QLabel(_('Applying downloaded metadata to your library'))) + + self.pb = QProgressBar(self) + l.addWidget(self.pb) + self.pb.setMinimum(0) + self.pb.setMaximum(len(id_map)) + + self.bb = QDialogButtonBox(QDialogButtonBox.Cancel) + self.bb.rejected.connect(self.reject) + self.bb.accepted.connect(self.accept) + l.addWidget(self.bb) + + self.gui = gui + self.id_map = list(id_map.iteritems()) + self.current_idx = 0 + + self.failures = [] + self.ids = [] + self.canceled = False + + QTimer.singleShot(20, self.do_one) + + def do_one(self): + if self.canceled: + return + i, mi = self.id_map[self.current_idx] + db = self.gui.current_db + try: + set_title = not mi.is_null('title') + set_authors = not mi.is_null('authors') + db.set_metadata(i, mi, commit=False, set_title=set_title, + set_authors=set_authors) + self.ids.append(i) + except: + import traceback + self.failures.append((i, traceback.format_exc())) + + try: + if mi.cover: + os.remove(mi.cover) + except: + pass + + self.pb.setValue(self.pb.value()+1) + + if self.current_idx >= len(self.id_map) - 1: + self.finalize() + else: + self.current_idx += 1 + QTimer.singleShot(20, self.do_one) + + def reject(self): + self.canceled = True + QDialog.reject(self) + + def finalize(self): + if self.canceled: + return + if self.failures: + msg = [] + db = self.gui.current_db + for i, tb in self.failures: + title = db.title(i, index_is_id=True) + authors = db.authors(i, index_is_id=True) + if authors: + authors = [x.replace('|', ',') for x in authors.split(',')] + title += ' - ' + authors_to_string(authors) + msg.append(title+'\n\n'+tb+'\n'+('*'*80)) + + error_dialog(self, _('Some failures'), + _('Failed to apply updated metadata for some books' + ' in your library. Click "Show Details" to see ' + 'details.'), det_msg='\n\n'.join(msg), show=True) + self.accept() + if self.ids: + cr = self.gui.library_view.currentIndex().row() + self.gui.library_view.model().refresh_ids( + self.ids, cr) + if self.gui.cover_flow: + self.gui.cover_flow.dataChanged() + +_amd = None +def apply_metadata(job, gui, q, result): + global _amd + q.vlb.clicked.disconnect() + q.finished.disconnect() + if result != q.Accepted: + return + id_map, failed_ids, failed_covers, title_map = job.result + id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in + failed_ids]) + if not id_map: + return + + modified = set() + db = gui.current_db + + for i, mi in id_map.iteritems(): + lm = db.metadata_last_modified(i, index_is_id=True) + if lm > mi.last_modified: + title = db.title(i, index_is_id=True) + authors = db.authors(i, index_is_id=True) + if authors: + authors = [x.replace('|', ',') for x in authors.split(',')] + title += ' - ' + authors_to_string(authors) + modified.add(title) + + if modified: + modified = sorted(modified, key=lower) + if not question_dialog(gui, _('Some books changed'), '

'+ + _('The metadata for some books in your library has' + ' changed since you started the download. If you' + ' proceed, some of those changes may be overwritten. ' + 'Click "Show details" to see the list of changed books. ' + 'Do you want to proceed?'), det_msg='\n'.join(modified)): + return + + _amd = ApplyDialog(id_map, gui) + _amd.exec_() + +def proceed(gui, job): + gui.status_bar.show_message(_('Metadata download completed'), 3000) + id_map, failed_ids, failed_covers, title_map = job.result + fmsg = det_msg = '' + if failed_ids or failed_covers: + fmsg = '

'+_('Could not download metadata and/or covers for %d of the books. Click' + ' "Show details" to see which books.')%len(failed_ids) + det_msg = [] + for i in failed_ids | failed_covers: + title = title_map[i] + if i in failed_ids: + title += (' ' + _('(Failed metadata)')) + if i in failed_covers: + title += (' ' + _('(Failed cover)')) + det_msg.append(title) + msg = '

' + _('Finished downloading metadata for %d book(s). ' + 'Proceed with updating the metadata in your library?')%len(id_map) + q = MessageBox(MessageBox.QUESTION, _('Download complete'), + msg + fmsg, det_msg='\n'.join(det_msg), show_copy_button=bool(failed_ids), + parent=gui) + q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole) + q.vlb.setIcon(QIcon(I('debug.png'))) + q.vlb.clicked.connect(partial(view_log, job, q)) + q.det_msg_toggle.setVisible(bool(failed_ids | failed_covers)) + q.setModal(False) + q.show() + q.finished.connect(partial(apply_metadata, job, gui, q)) + +def merge_result(oldmi, newmi): + dummy = Metadata(_('Unknown')) + for f in msprefs['ignore_fields']: + setattr(newmi, f, getattr(dummy, f)) + fields = set() + for plugin in metadata_plugins(['identify']): + fields |= plugin.touched_fields + + for f in fields: + # Optimize so that set_metadata does not have to do extra work later + if not f.startswith('identifier:'): + if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): + setattr(newmi, f, getattr(dummy, f)) + + newmi.last_modified = oldmi.last_modified + + return newmi + +def download(ids, db, do_identify, covers, + log=None, abort=None, notifications=None): + ids = list(ids) + metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) + for i in ids] + failed_ids = set() + failed_covers = set() + title_map = {} + ans = {} + count = 0 + for i, mi in izip(ids, metadata): + if abort.is_set(): + log.error('Aborting...') + break + title, authors, identifiers = mi.title, mi.authors, mi.identifiers + title_map[i] = title + if do_identify: + results = [] + try: + results = identify(log, abort, title=title, authors=authors, + identifiers=identifiers) + except: + pass + if results: + mi = merge_result(mi, results[0]) + identifiers = mi.identifiers + if not mi.is_null('rating'): + # set_metadata expects a rating out of 10 + mi.rating *= 2 + else: + log.error('Failed to download metadata for', title) + failed_ids.add(i) + # We don't want set_metadata operating on anything but covers + mi = merge_result(mi, mi) + if covers: + cdata = download_cover(log, title=title, authors=authors, + identifiers=identifiers) + if cdata is not None: + with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: + f.write(cdata[-1]) + mi.cover = f.name + else: + failed_covers.add(i) + ans[i] = mi + count += 1 + notifications.put((count/len(ids), + _('Downloaded %d of %d')%(count, len(ids)))) + log('Download complete, with %d failures'%len(failed_ids)) + return (ans, failed_ids, failed_covers, title_map) + + + diff --git a/src/calibre/gui2/metadata/config.py b/src/calibre/gui2/metadata/config.py index 68c935061d..abb45faa46 100644 --- a/src/calibre/gui2/metadata/config.py +++ b/src/calibre/gui2/metadata/config.py @@ -56,7 +56,12 @@ class ConfigWidget(QWidget): self.setLayout(l) self.gb = QGroupBox(_('Downloaded metadata fields'), self) - l.addWidget(self.gb, 0, 0, 1, 2) + if plugin.config_help_message: + self.pchm = QLabel(plugin.config_help_message) + self.pchm.setWordWrap(True) + self.pchm.setOpenExternalLinks(True) + l.addWidget(self.pchm, 0, 0, 1, 2) + l.addWidget(self.gb, l.rowCount(), 0, 1, 2) self.gb.l = QGridLayout() self.gb.setLayout(self.gb.l) self.fields_view = v = QListView(self) @@ -81,7 +86,7 @@ class ConfigWidget(QWidget): widget.setValue(val) elif opt.type == 'string': widget = QLineEdit(self) - widget.setText(val) + widget.setText(val if val else '') elif opt.type == 'bool': widget = QCheckBox(opt.label, self) widget.setChecked(bool(val)) diff --git a/src/calibre/gui2/metadata/single.py b/src/calibre/gui2/metadata/single.py index 4ce76d8cc8..2e5b43ceba 100644 --- a/src/calibre/gui2/metadata/single.py +++ b/src/calibre/gui2/metadata/single.py @@ -612,6 +612,10 @@ class MetadataSingleDialogAlt1(MetadataSingleDialogBase): # {{{ self.button_box.addButton(self.fetch_metadata_button, QDialogButtonBox.ActionRole) + self.config_metadata_button.setToolButtonStyle(Qt.ToolButtonTextOnly) + self.config_metadata_button.setText(_('Configure metadata downloading')) + self.button_box.addButton(self.config_metadata_button, + QDialogButtonBox.ActionRole) sto(self.button_box, self.title) def create_row(row, widget, tab_to, button=None, icon=None, span=1): diff --git a/src/calibre/gui2/preferences/emailp.py b/src/calibre/gui2/preferences/emailp.py index ded6891387..1644dc6b73 100644 --- a/src/calibre/gui2/preferences/emailp.py +++ b/src/calibre/gui2/preferences/emailp.py @@ -84,11 +84,11 @@ class EmailAccounts(QAbstractTableModel): # {{{ account = self.account_order[row] if col == 3: self.accounts[account][1] ^= True - if col == 2: + elif col == 2: self.subjects[account] = unicode(value.toString()) elif col == 1: self.accounts[account][0] = unicode(value.toString()).upper() - else: + elif col == 0: na = unicode(value.toString()) from email.utils import parseaddr addr = parseaddr(na)[-1] @@ -100,7 +100,7 @@ class EmailAccounts(QAbstractTableModel): # {{{ self.accounts[na][0] = 'AZW, MOBI, TPZ, PRC, AZW1' self.dataChanged.emit( - self.index(index.row(), 0), self.index(index.row(), 2)) + self.index(index.row(), 0), self.index(index.row(), 3)) return True def make_default(self, index): diff --git a/src/calibre/gui2/preferences/metadata_sources.py b/src/calibre/gui2/preferences/metadata_sources.py index 4500a03b30..17a70bcc33 100644 --- a/src/calibre/gui2/preferences/metadata_sources.py +++ b/src/calibre/gui2/preferences/metadata_sources.py @@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en' from operator import attrgetter from PyQt4.Qt import (QAbstractTableModel, Qt, QAbstractListModel, QWidget, - pyqtSignal, QVBoxLayout, QDialogButtonBox, QFrame, QLabel) + pyqtSignal, QVBoxLayout, QDialogButtonBox, QFrame, QLabel, QIcon) from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.metadata_sources_ui import Ui_Form @@ -67,6 +67,13 @@ class SourcesModel(QAbstractTableModel): # {{{ return self.enabled_overrides.get(plugin, orig) elif role == Qt.UserRole: return plugin + elif (role == Qt.DecorationRole and col == 0 and not + plugin.is_configured()): + return QIcon(I('list_remove.png')) + elif role == Qt.ToolTipRole: + if plugin.is_configured(): + return _('This source is configured and ready to go') + return _('This source needs configuration') return NONE def setData(self, index, val, role): diff --git a/src/calibre/gui2/preferences/metadata_sources.ui b/src/calibre/gui2/preferences/metadata_sources.ui index 546120f628..b515f13ba1 100644 --- a/src/calibre/gui2/preferences/metadata_sources.ui +++ b/src/calibre/gui2/preferences/metadata_sources.ui @@ -48,6 +48,16 @@ + + + + Sources with a red X next to their names must be configured before they will be used. + + + true + + + diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index 8936493290..ea7cab95d0 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -436,14 +436,15 @@ class SavedSearchBoxMixin(object): # {{{ b = getattr(self, x+'_search_button') b.setStatusTip(b.toolTip()) - def saved_searches_changed(self, set_restriction=None): + def saved_searches_changed(self, set_restriction=None, recount=True): p = sorted(saved_searches().names(), key=sort_key) if set_restriction is None: set_restriction = unicode(self.search_restriction.currentText()) # rebuild the restrictions combobox using current saved searches self.search_restriction.clear() self.search_restriction.addItem('') - self.tags_view.recount() + if recount: + self.tags_view.recount() for s in p: self.search_restriction.addItem(s) if set_restriction: # redo the search restriction if there was one diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 73c191101c..74e448da6e 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -25,8 +25,9 @@ class SearchRestrictionMixin(object): r = self.search_restriction.findText(name) if r < 0: r = 0 - self.search_restriction.setCurrentIndex(r) - self.apply_search_restriction(r) + if r != self.search_restriction.currentIndex(): + self.search_restriction.setCurrentIndex(r) + self.apply_search_restriction(r) def apply_search_restriction(self, i): r = unicode(self.search_restriction.currentText()) diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py new file mode 100644 index 0000000000..9c791c5b0d --- /dev/null +++ b/src/calibre/gui2/threaded_jobs.py @@ -0,0 +1,245 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai +from __future__ import (unicode_literals, division, absolute_import, + print_function) + +__license__ = 'GPL v3' +__copyright__ = '2011, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +import os, time, tempfile, json +from threading import Thread, RLock, Event +from Queue import Queue + +from calibre.utils.ipc.job import BaseJob +from calibre.utils.logging import GUILog +from calibre.ptempfile import base_dir + +class ThreadedJob(BaseJob): + + def __init__(self, + type_, description, + + func, args, kwargs, + + callback, + + max_concurrent_count=1, + killable=True, + log=None): + ''' + A job that is run in its own thread in the calibre main process + + :param type_: The type of this job (a string). The type is used in + conjunction with max_concurrent_count to prevent too many jobs of the + same type from running + + :param description: A user viewable job description + + :func: The function that actually does the work. This function *must* + accept at least three keyword arguments: abort, log and notifications. abort is + An Event object. func should periodically check abort.is_set(0 and if + it is True, it should stop processing as soon as possible. notifications + is a Queue. func should put progress notifications into it in the form + of a tuple (frac, msg). frac is a number between 0 and 1 indicating + progress and msg is a string describing the progress. log is a Log + object which func should use for all debugging output. func should + raise an Exception to indicate failure. This exception is stored in + job.exception and can thus be used to pass arbitrary information to + callback. + + :param args,kwargs: These are passed to func when it is called + + :param callback: A callable that is called on completion of this job. + Note that it is not called if the user kills the job. Check job.failed + to see if the job succeeded or not. And use job.log to get the job log. + + :param killable: If False the GUI wont let the user kill this job + + :param log: Must be a subclass of GUILog or None. If None a default + GUILog is created. + ''' + BaseJob.__init__(self, description) + + self.type = type_ + self.max_concurrent_count = max_concurrent_count + self.killable = killable + self.callback = callback + self.abort = Event() + self.exception = None + + kwargs['notifications'] = self.notifications + kwargs['abort'] = self.abort + self.log = GUILog() if log is None else log + kwargs['log'] = self.log + + self.func, self.args, self.kwargs = func, args, kwargs + self.consolidated_log = None + + def start_work(self): + self.start_time = time.time() + self.log('Starting job:', self.description) + try: + self.result = self.func(*self.args, **self.kwargs) + except Exception as e: + self.exception = e + self.failed = True + self.log.exception('Job: "%s" failed with error:'%self.description) + self.log.debug('Called with args:', self.args, self.kwargs) + + self.duration = time.time() - self.start_time + try: + self.callback(self) + except: + import traceback + traceback.print_exc() + self._cleanup() + + def _cleanup(self): + + try: + self.consolidate_log() + except: + self.log.exception('Log consolidation failed') + + # No need to keep references to these around anymore + self.func = self.args = self.kwargs = self.notifications = None + # We can't delete self.callback as it might be a Dispatch object and if + # it is garbage collected it won't work + + def kill(self): + if self.start_time is None: + self.start_time = time.time() + self.duration = 0.0001 + else: + self.duration = time.time() - self.start_time() + self.abort.set() + + self.log('Aborted job:', self.description) + self.killed = True + self.failed = True + self._cleanup() + + def consolidate_log(self): + logs = [self.log.html, self.log.plain_text] + bdir = base_dir() + log_dir = os.path.join(bdir, 'threaded_job_logs') + if not os.path.exists(log_dir): + os.makedirs(log_dir) + fd, path = tempfile.mkstemp(suffix='.json', prefix='log-', dir=log_dir) + with os.fdopen(fd, 'wb') as f: + f.write(json.dumps(logs, ensure_ascii=False, + indent=2).encode('utf-8')) + self.consolidated_log = path + self.log = None + + def read_consolidated_log(self): + with open(self.consolidated_log, 'rb') as f: + return json.loads(f.read().decode('utf-8')) + + @property + def details(self): + if self.consolidated_log is None: + return self.log.plain_text + return self.read_consolidated_log()[1] + + @property + def html_details(self): + if self.consolidated_log is None: + return self.log.html + return self.read_consolidated_log()[0] + +class ThreadedJobWorker(Thread): + + def __init__(self, job): + Thread.__init__(self) + self.daemon = True + self.job = job + + def run(self): + try: + self.job.start_work() + except: + import traceback + from calibre import prints + prints('Job had unhandled exception:', self.job.description) + traceback.print_exc() + +class ThreadedJobServer(Thread): + + def __init__(self): + Thread.__init__(self) + self.daemon = True + self.lock = RLock() + + self.queued_jobs = [] + self.running_jobs = set() + self.changed_jobs = Queue() + self.keep_going = True + + def close(self): + self.keep_going = False + + def add_job(self, job): + with self.lock: + self.queued_jobs.append(job) + + if not self.is_alive(): + self.start() + + def run(self): + while self.keep_going: + try: + self.run_once() + except: + import traceback + traceback.print_exc() + time.sleep(0.1) + + def run_once(self): + with self.lock: + remove = set() + for worker in self.running_jobs: + if worker.is_alive(): + # Get progress notifications + if worker.job.consume_notifications(): + self.changed_jobs.put(worker.job) + else: + remove.add(worker) + self.changed_jobs.put(worker.job) + + for worker in remove: + self.running_jobs.remove(worker) + + jobs = self.get_startable_jobs() + for job in jobs: + w = ThreadedJobWorker(job) + w.start() + self.running_jobs.add(w) + self.changed_jobs.put(job) + self.queued_jobs.remove(job) + + def kill_job(self, job): + with self.lock: + if job in self.queued_jobs: + self.queued_jobs.remove(job) + elif job in self.running_jobs: + self.running_jobs.remove(job) + job.kill() + self.changed_jobs.put(job) + + def running_jobs_of_type(self, type_): + return len([w for w in self.running_jobs if w.job.type == type_]) + + def get_startable_jobs(self): + queued_types = [] + ans = [] + for job in self.queued_jobs: + num = self.running_jobs_of_type(job.type) + num += queued_types.count(job.type) + if num < job.max_concurrent_count: + queued_types.append(job.type) + ans.append(job) + return ans + + diff --git a/src/calibre/gui2/ui.py b/src/calibre/gui2/ui.py index 4a9d76418c..e7853b9491 100644 --- a/src/calibre/gui2/ui.py +++ b/src/calibre/gui2/ui.py @@ -446,12 +446,13 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.search.clear() self.saved_search.clear() self.book_details.reset_info() - self.library_view.model().count_changed() prefs['library_path'] = self.library_path + #self.library_view.model().count_changed() db = self.library_view.model().db + self.iactions['Choose Library'].count_changed(db.count()) self.set_window_title() self.apply_named_search_restriction('') # reset restriction to null - self.saved_searches_changed() # reload the search restrictions combo box + self.saved_searches_changed(recount=False) # reload the search restrictions combo box self.apply_named_search_restriction(db.prefs['gui_restriction']) for action in self.iactions.values(): action.library_changed(db) @@ -607,6 +608,7 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ self.update_checker.terminate() self.listener.close() self.job_manager.server.close() + self.job_manager.threaded_server.close() while self.spare_servers: self.spare_servers.pop().close() self.device_manager.keep_going = False @@ -615,8 +617,6 @@ class Main(MainWindow, MainWindowMixin, DeviceMixin, EmailMixin, # {{{ mb.stop() self.hide_windows() - if self.emailer.is_alive(): - self.emailer.stop() try: try: if self.content_server is not None: diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index 74ddb2bc41..5f4cfcba07 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -191,7 +191,8 @@ class CacheRow(list): # {{{ if is_comp: id = list.__getitem__(self, 0) self._must_do = False - mi = self.db.get_metadata(id, index_is_id=True) + mi = self.db.get_metadata(id, index_is_id=True, + get_user_categories=False) for c in self._composites: self[c] = mi.get(self._composites[c]) return list.__getitem__(self, col) @@ -390,58 +391,68 @@ class ResultCache(SearchQueryParser): # {{{ def build_numeric_relop_dict(self): self.numeric_search_relops = { '=':[1, lambda r, q: r == q], - '>':[1, lambda r, q: r > q], - '<':[1, lambda r, q: r < q], + '>':[1, lambda r, q: r is not None and r > q], + '<':[1, lambda r, q: r is not None and r < q], '!=':[2, lambda r, q: r != q], - '>=':[2, lambda r, q: r >= q], - '<=':[2, lambda r, q: r <= q] + '>=':[2, lambda r, q: r is not None and r >= q], + '<=':[2, lambda r, q: r is not None and r <= q] } def get_numeric_matches(self, location, query, candidates, val_func = None): matches = set([]) if len(query) == 0: return matches - if query == 'false': - query = '0' - elif query == 'true': - query = '!=0' - relop = None - for k in self.numeric_search_relops.keys(): - if query.startswith(k): - (p, relop) = self.numeric_search_relops[k] - query = query[p:] - if relop is None: - (p, relop) = self.numeric_search_relops['='] if val_func is None: loc = self.field_metadata[location]['rec_index'] val_func = lambda item, loc=loc: item[loc] - dt = self.field_metadata[location]['datatype'] - if dt == 'int': - cast = (lambda x: int (x)) - adjust = lambda x: x - elif dt == 'rating': - cast = (lambda x: int (x)) - adjust = lambda x: x/2 - elif dt in ('float', 'composite'): - cast = lambda x : float (x) - adjust = lambda x: x - else: # count operation - cast = (lambda x: int (x)) - adjust = lambda x: x - if len(query) > 1: - mult = query[-1:].lower() - mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) - if mult != 1.0: - query = query[:-1] + q = '' + val_func = lambda item, loc=loc: item[loc] + cast = adjust = lambda x: x + + if query == 'false': + if dt == 'rating': + relop = lambda x,y: not bool(x) + else: + relop = lambda x,y: x is None + elif query == 'true': + if dt == 'rating': + relop = lambda x,y: bool(x) + else: + relop = lambda x,y: x is not None else: - mult = 1.0 - try: - q = cast(query) * mult - except: - return matches + relop = None + for k in self.numeric_search_relops.keys(): + if query.startswith(k): + (p, relop) = self.numeric_search_relops[k] + query = query[p:] + if relop is None: + (p, relop) = self.numeric_search_relops['='] + + if dt == 'int': + cast = lambda x: int (x) + elif dt == 'rating': + cast = lambda x: 0 if x is None else int (x) + adjust = lambda x: x/2 + elif dt in ('float', 'composite'): + cast = lambda x : float (x) + else: # count operation + cast = (lambda x: int (x)) + + if len(query) > 1: + mult = query[-1:].lower() + mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) + if mult != 1.0: + query = query[:-1] + else: + mult = 1.0 + try: + q = cast(query) * mult + except: + raise ParseException(query, len(query), + 'Non-numeric value in query', self) for id_ in candidates: item = self._data[id_] @@ -450,10 +461,8 @@ class ResultCache(SearchQueryParser): # {{{ try: v = cast(val_func(item)) except: - v = 0 - if not v: - v = 0 - else: + v = None + if v: v = adjust(v) if relop(v, q): matches.add(item[0]) @@ -583,8 +592,7 @@ class ResultCache(SearchQueryParser): # {{{ candidates = self.universal_set() if len(candidates) == 0: return matches - if location not in self.all_search_locations: - return matches + self.test_location_is_valid(location, query) if len(location) > 2 and location.startswith('@') and \ location[1:] in self.db_prefs['grouped_search_terms']: diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 50ecc4f1e5..7b4d52dbcd 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -823,7 +823,8 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): pass return (path, mi, sequence) - def get_metadata(self, idx, index_is_id=False, get_cover=False): + def get_metadata(self, idx, index_is_id=False, get_cover=False, + get_user_categories=True): ''' Convenience method to return metadata as a :class:`Metadata` object. Note that the list of formats is not verified. @@ -853,6 +854,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.uuid = row[fm['uuid']] mi.title_sort = row[fm['sort']] mi.last_modified = row[fm['last_modified']] + mi.size = row[fm['size']] formats = row[fm['formats']] if not formats: formats = None @@ -882,16 +884,17 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): user_cats = self.prefs['user_categories'] user_cat_vals = {} - for ucat in user_cats: - res = [] - for name,cat,ign in user_cats[ucat]: - v = mi.get(cat, None) - if isinstance(v, list): - if name in v: + if get_user_categories: + for ucat in user_cats: + res = [] + for name,cat,ign in user_cats[ucat]: + v = mi.get(cat, None) + if isinstance(v, list): + if name in v: + res.append([name,cat]) + elif name == v: res.append([name,cat]) - elif name == v: - res.append([name,cat]) - user_cat_vals[ucat] = res + user_cat_vals[ucat] = res mi.user_categories = user_cat_vals if get_cover: @@ -1779,7 +1782,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): path_changed = True if set_authors: if not mi.authors: - mi.authors = [_('Unknown')] + mi.authors = [_('Unknown')] authors = [] for a in mi.authors: authors += string_to_authors(a) diff --git a/src/calibre/manual/server.rst b/src/calibre/manual/server.rst index 82ec5c2927..aa98ba57df 100644 --- a/src/calibre/manual/server.rst +++ b/src/calibre/manual/server.rst @@ -22,6 +22,8 @@ First start the |app| content server as shown below:: calibre-server --url-prefix /calibre --port 8080 +The key parameter here is ``--url-prefix /calibre``. This causes the content server to serve all URLs prefixed by calibre. To see this in action, visit ``http://localhost:8080/calibre`` in your browser. You should see the normal content server website, but now it will run under /calibre. + Now suppose you are using Apache as your main server. First enable the proxy modules in apache, by adding the following to :file:`httpd.conf`:: LoadModule proxy_module modules/mod_proxy.so diff --git a/src/calibre/utils/ipc/job.py b/src/calibre/utils/ipc/job.py index 91db333791..f4b54aee95 100644 --- a/src/calibre/utils/ipc/job.py +++ b/src/calibre/utils/ipc/job.py @@ -75,12 +75,20 @@ class BaseJob(object): self._run_state = self.RUNNING self._status_text = _('Working...') - while consume_notifications: + if consume_notifications: + return self.consume_notifications() + return False + + def consume_notifications(self): + got_notification = False + while self.notifications is not None: try: self.percent, self._message = self.notifications.get_nowait() self.percent *= 100. + got_notification = True except Empty: break + return got_notification @property def status_text(self): diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index a50ca20fc1..387ad1487e 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -20,7 +20,7 @@ import sys, string, operator from calibre.utils.pyparsing import CaselessKeyword, Group, Forward, \ CharsNotIn, Suppress, OneOrMore, MatchFirst, CaselessLiteral, \ - Optional, NoMatch, ParseException, QuotedString + Optional, NoMatch, ParseException, QuotedString, Word from calibre.constants import preferred_encoding from calibre.utils.icu import sort_key @@ -128,12 +128,8 @@ class SearchQueryParser(object): self._tests_failed = False self.optimize = optimize # Define a token - standard_locations = map(lambda x : CaselessLiteral(x)+Suppress(':'), - locations) - location = NoMatch() - for l in standard_locations: - location |= l - location = Optional(location, default='all') + self.standard_locations = locations + location = Optional(Word(string.ascii_letters+'#')+Suppress(':'), default='all') word_query = CharsNotIn(string.whitespace + '()') #quoted_query = Suppress('"')+CharsNotIn('"')+Suppress('"') quoted_query = QuotedString('"', escChar='\\') @@ -250,7 +246,14 @@ class SearchQueryParser(object): raise ParseException(query, len(query), 'undefined saved search', self) return self._get_matches(location, query, candidates) + def test_location_is_valid(self, location, query): + if location not in self.standard_locations: + raise ParseException(query, len(query), + _('No column exists with lookup name ') + location, self) + def _get_matches(self, location, query, candidates): + location = location.lower() + self.test_location_is_valid(location, query) if self.optimize: return self.get_matches(location, query, candidates=candidates) else: