diff --git a/recipes/hallo_assen.recipe b/recipes/hallo_assen.recipe new file mode 100644 index 0000000000..423cd86df1 --- /dev/null +++ b/recipes/hallo_assen.recipe @@ -0,0 +1,36 @@ +from calibre.web.feeds.news import BasicNewsRecipe + +class AdvancedUserRecipe1302341394(BasicNewsRecipe): + title = u'Hallo Assen' + oldest_article = 180 + max_articles_per_feed = 100 + + __author__ = 'Reijndert' + no_stylesheets = True + cover_url = 'http://www.halloassen.nl/multimedia/halloassen/archive/00002/HalloAssen_2518a.gif' + language = 'nl' + country = 'NL' + version = 1 + category = u'Nieuws' + timefmt = ' %Y-%m-%d (%a)' + + + + keep_only_tags = [dict(name='div', attrs={'class':'photoFrame'}) + ,dict(name='div', attrs={'class':'textContent'}) + ] + + remove_tags = [ + dict(name='div',attrs={'id':'articleLinks'}) + ,dict(name='div',attrs={'class':'categories clearfix'}) + ,dict(name='div',attrs={'id':'rating'}) + ,dict(name='div',attrs={'id':'comments'}) + ] + + feeds = [(u'Ons Nieuws', u'http://feeds.feedburner.com/halloassen/onsnieuws'), (u'Politie', u'http://www.halloassen.nl/rss/?c=37'), (u'Rechtbank', u'http://www.halloassen.nl/rss/?c=39'), (u'Justitie', u'http://www.halloassen.nl/rss/?c=36'), (u'Evenementen', u'http://www.halloassen.nl/rss/?c=34'), (u'Cultuur', u'http://www.halloassen.nl/rss/?c=32'), (u'Politiek', u'http://www.halloassen.nl/rss/?c=38'), (u'Economie', u'http://www.halloassen.nl/rss/?c=33')] + + + extra_css = ''' + body {font-family: verdana, arial, helvetica, geneva, sans-serif;} + ''' + diff --git a/recipes/tabu.recipe b/recipes/tabu.recipe index d0ede613fd..f98ed8a155 100644 --- a/recipes/tabu.recipe +++ b/recipes/tabu.recipe @@ -24,30 +24,29 @@ class TabuRo(BasicNewsRecipe): cover_url = 'http://www.tabu.ro/img/tabu-logo2.png' conversion_options = { - 'comments' : description - ,'tags' : category - ,'language' : language - ,'publisher' : publisher - } + 'comments' : description + ,'tags' : category + ,'language' : language + ,'publisher' : publisher + } keep_only_tags = [ - dict(name='div', attrs={'id':'Article'}), - ] + dict(name='h2', attrs={'class':'articol_titlu'}), + dict(name='div', attrs={'class':'poza_articol_featured'}), + dict(name='div', attrs={'class':'articol_text'}) + ] remove_tags = [ - dict(name='div', attrs={'id':['advertisementArticle']}), - dict(name='div', attrs={'class':'voting_number'}), - dict(name='div', attrs={'id':'number_votes'}), - dict(name='div', attrs={'id':'rating_one'}), - dict(name='div', attrs={'class':'float: right;'}) + dict(name='div', attrs={'class':'asemanatoare'}) ] remove_tags_after = [ dict(name='div', attrs={'id':'comments'}), - ] + dict(name='div', attrs={'class':'asemanatoare'}) + ] feeds = [ - (u'Feeds', u'http://www.tabu.ro/rss_all.xml') + (u'Feeds', u'http://www.tabu.ro/feed/') ] def preprocess_html(self, soup): diff --git a/recipes/weblogs_sl.recipe b/recipes/weblogs_sl.recipe index c23c6c5093..5205a94a02 100644 --- a/recipes/weblogs_sl.recipe +++ b/recipes/weblogs_sl.recipe @@ -3,7 +3,7 @@ __license__ = 'GPL v3' __copyright__ = '4 February 2011, desUBIKado' __author__ = 'desUBIKado' __version__ = 'v0.05' -__date__ = '9, February 2011' +__date__ = '13, April 2011' ''' http://www.weblogssl.com/ ''' @@ -19,7 +19,7 @@ class weblogssl(BasicNewsRecipe): category = 'Gadgets, Tech news, Product reviews, mobiles, science, cinema, entertainment, culture, tv, food, recipes, life style, motor, F1, sports, economy' language = 'es' timefmt = '[%a, %d %b, %Y]' - oldest_article = 1.5 + oldest_article = 1 max_articles_per_feed = 100 encoding = 'utf-8' use_embedded_content = False @@ -28,50 +28,52 @@ class weblogssl(BasicNewsRecipe): no_stylesheets = True # Si no se quiere recuperar todos los blogs se puede suprimir la descarga del que se desee poniendo - # un caracter # por delante, es decir, # (u'Applesfera', u'http://feeds.weblogssl.com/applesfera'), - # haría que no se descargase Applesfera. OJO: El último feed no debe llevar la coma al final + # un caracter # por delante, es decir, # ,(u'Applesfera', u'http://feeds.weblogssl.com/applesfera') + # haría que no se descargase Applesfera. feeds = [ - (u'Xataka', u'http://feeds.weblogssl.com/xataka2'), - (u'Xataka M\xf3vil', u'http://feeds.weblogssl.com/xatakamovil'), - (u'Xataka Android', u'http://feeds.weblogssl.com/xatakandroid'), - (u'Xataka Foto', u'http://feeds.weblogssl.com/xatakafoto'), - (u'Xataka ON', u'http://feeds.weblogssl.com/xatakaon'), - (u'Xataka Ciencia', u'http://feeds.weblogssl.com/xatakaciencia'), - (u'Genbeta', u'http://feeds.weblogssl.com/genbeta'), - (u'Applesfera', u'http://feeds.weblogssl.com/applesfera'), - (u'Vida Extra', u'http://feeds.weblogssl.com/vidaextra'), - (u'Naci\xf3n Red', u'http://feeds.weblogssl.com/nacionred'), - (u'Blog de Cine', u'http://feeds.weblogssl.com/blogdecine'), - (u'Vaya tele', u'http://feeds.weblogssl.com/vayatele2'), - (u'Hipers\xf3nica', u'http://feeds.weblogssl.com/hipersonica'), - (u'Diario del viajero', u'http://feeds.weblogssl.com/diariodelviajero'), - (u'Papel en blanco', u'http://feeds.weblogssl.com/papelenblanco'), - (u'Pop rosa', u'http://feeds.weblogssl.com/poprosa'), - (u'Zona FandoM', u'http://feeds.weblogssl.com/zonafandom'), - (u'Fandemia', u'http://feeds.weblogssl.com/fandemia'), - (u'Noctamina', u'http://feeds.weblogssl.com/noctamina'), - (u'Tendencias', u'http://feeds.weblogssl.com/trendencias'), - (u'Beb\xe9s y m\xe1s', u'http://feeds.weblogssl.com/bebesymas'), - (u'Directo al paladar', u'http://feeds.weblogssl.com/directoalpaladar'), - (u'Compradicci\xf3n', u'http://feeds.weblogssl.com/compradiccion'), - (u'Decoesfera', u'http://feeds.weblogssl.com/decoesfera'), - (u'Embelezzia', u'http://feeds.weblogssl.com/embelezzia'), - (u'Vit\xf3nica', u'http://feeds.weblogssl.com/vitonica'), - (u'Ambiente G', u'http://feeds.weblogssl.com/ambienteg'), - (u'Arrebatadora', u'http://feeds.weblogssl.com/arrebatadora'), - (u'Mensencia', u'http://feeds.weblogssl.com/mensencia'), - (u'Peques y m\xe1s', u'http://feeds.weblogssl.com/pequesymas'), - (u'Motorpasi\xf3n', u'http://feeds.weblogssl.com/motorpasion'), - (u'Motorpasi\xf3n F1', u'http://feeds.weblogssl.com/motorpasionf1'), - (u'Motorpasi\xf3n Moto', u'http://feeds.weblogssl.com/motorpasionmoto'), - (u'Notas de futbol', u'http://feeds.weblogssl.com/notasdefutbol'), - (u'Fuera de l\xedmites', u'http://feeds.weblogssl.com/fueradelimites'), - (u'Salir a ganar', u'http://feeds.weblogssl.com/saliraganar'), - (u'El blog salm\xf3n', u'http://feeds.weblogssl.com/elblogsalmon2'), - (u'Pymes y aut\xf3nomos', u'http://feeds.weblogssl.com/pymesyautonomos'), - (u'Tecnolog\xeda Pyme', u'http://feeds.weblogssl.com/tecnologiapyme'), - (u'Ahorro diario', u'http://feeds.weblogssl.com/ahorrodiario') + (u'Xataka', u'http://feeds.weblogssl.com/xataka2') + ,(u'Xataka M\xf3vil', u'http://feeds.weblogssl.com/xatakamovil') + ,(u'Xataka Android', u'http://feeds.weblogssl.com/xatakandroid') + ,(u'Xataka Foto', u'http://feeds.weblogssl.com/xatakafoto') + ,(u'Xataka ON', u'http://feeds.weblogssl.com/xatakaon') + ,(u'Xataka Ciencia', u'http://feeds.weblogssl.com/xatakaciencia') + ,(u'Genbeta', u'http://feeds.weblogssl.com/genbeta') + ,(u'Genbeta Dev', u'http://feeds.weblogssl.com/genbetadev') + ,(u'Applesfera', u'http://feeds.weblogssl.com/applesfera') + ,(u'Vida Extra', u'http://feeds.weblogssl.com/vidaextra') + ,(u'Naci\xf3n Red', u'http://feeds.weblogssl.com/nacionred') + ,(u'Blog de Cine', u'http://feeds.weblogssl.com/blogdecine') + ,(u'Vaya tele', u'http://feeds.weblogssl.com/vayatele2') + ,(u'Hipers\xf3nica', u'http://feeds.weblogssl.com/hipersonica') + ,(u'Diario del viajero', u'http://feeds.weblogssl.com/diariodelviajero') + ,(u'Papel en blanco', u'http://feeds.weblogssl.com/papelenblanco') + ,(u'Pop rosa', u'http://feeds.weblogssl.com/poprosa') + ,(u'Zona FandoM', u'http://feeds.weblogssl.com/zonafandom') + ,(u'Fandemia', u'http://feeds.weblogssl.com/fandemia') + ,(u'Noctamina', u'http://feeds.weblogssl.com/noctamina') + ,(u'Tendencias', u'http://feeds.weblogssl.com/trendencias') + ,(u'Beb\xe9s y m\xe1s', u'http://feeds.weblogssl.com/bebesymas') + ,(u'Directo al paladar', u'http://feeds.weblogssl.com/directoalpaladar') + ,(u'Compradicci\xf3n', u'http://feeds.weblogssl.com/compradiccion') + ,(u'Decoesfera', u'http://feeds.weblogssl.com/decoesfera') + ,(u'Embelezzia', u'http://feeds.weblogssl.com/embelezzia') + ,(u'Vit\xf3nica', u'http://feeds.weblogssl.com/vitonica') + ,(u'Ambiente G', u'http://feeds.weblogssl.com/ambienteg') + ,(u'Arrebatadora', u'http://feeds.weblogssl.com/arrebatadora') + ,(u'Mensencia', u'http://feeds.weblogssl.com/mensencia') + ,(u'Peques y m\xe1s', u'http://feeds.weblogssl.com/pequesymas') + ,(u'Motorpasi\xf3n', u'http://feeds.weblogssl.com/motorpasion') + ,(u'Motorpasi\xf3n F1', u'http://feeds.weblogssl.com/motorpasionf1') + ,(u'Motorpasi\xf3n Moto', u'http://feeds.weblogssl.com/motorpasionmoto') + ,(u'Motorpasi\xf3n Futuro', u'http://feeds.weblogssl.com/motorpasionfuturo') + ,(u'Notas de futbol', u'http://feeds.weblogssl.com/notasdefutbol') + ,(u'Fuera de l\xedmites', u'http://feeds.weblogssl.com/fueradelimites') + ,(u'Salir a ganar', u'http://feeds.weblogssl.com/saliraganar') + ,(u'El blog salm\xf3n', u'http://feeds.weblogssl.com/elblogsalmon2') + ,(u'Pymes y aut\xf3nomos', u'http://feeds.weblogssl.com/pymesyautonomos') + ,(u'Tecnolog\xeda Pyme', u'http://feeds.weblogssl.com/tecnologiapyme') + ,(u'Ahorro diario', u'http://feeds.weblogssl.com/ahorrodiario') ] @@ -102,3 +104,4 @@ class weblogssl(BasicNewsRecipe): video_yt['src'] = fuente3 + '/0.jpg' return soup + diff --git a/recipes/wsj.recipe b/recipes/wsj.recipe index f2854e65ca..cf84722bac 100644 --- a/recipes/wsj.recipe +++ b/recipes/wsj.recipe @@ -81,6 +81,11 @@ class WallStreetJournal(BasicNewsRecipe): feeds.append((title, articles)) return feeds + def abs_wsj_url(self, href): + if not href.startswith('http'): + href = 'http://online.wsj.com' + href + return href + def parse_index(self): soup = self.wsj_get_index() @@ -99,14 +104,14 @@ class WallStreetJournal(BasicNewsRecipe): pageone = a['href'].endswith('pageone') if pageone: title = 'Front Section' - url = 'http://online.wsj.com' + a['href'] + url = self.abs_wsj_url(a['href']) feeds = self.wsj_add_feed(feeds,title,url) title = "What's News" url = url.replace('pageone','whatsnews') feeds = self.wsj_add_feed(feeds,title,url) else: title = self.tag_to_string(a) - url = 'http://online.wsj.com' + a['href'] + url = self.abs_wsj_url(a['href']) feeds = self.wsj_add_feed(feeds,title,url) return feeds @@ -163,7 +168,7 @@ class WallStreetJournal(BasicNewsRecipe): title = self.tag_to_string(a).strip() + ' [%s]'%meta else: title = self.tag_to_string(a).strip() - url = 'http://online.wsj.com'+a['href'] + url = self.abs_wsj_url(a['href']) desc = '' for p in container.findAll('p'): desc = self.tag_to_string(p) diff --git a/resources/images/highlight_only_off.png b/resources/images/highlight_only_off.png new file mode 100644 index 0000000000..603d60a686 Binary files /dev/null and b/resources/images/highlight_only_off.png differ diff --git a/resources/images/highlight_only_on.png b/resources/images/highlight_only_on.png new file mode 100644 index 0000000000..8d679e56e4 Binary files /dev/null and b/resources/images/highlight_only_on.png differ diff --git a/src/calibre/customize/builtins.py b/src/calibre/customize/builtins.py index 8f6c597ee5..d5957eb70a 100644 --- a/src/calibre/customize/builtins.py +++ b/src/calibre/customize/builtins.py @@ -625,8 +625,9 @@ if test_eight_code: from calibre.ebooks.metadata.sources.google import GoogleBooks from calibre.ebooks.metadata.sources.amazon import Amazon from calibre.ebooks.metadata.sources.openlibrary import OpenLibrary + from calibre.ebooks.metadata.sources.isbndb import ISBNDB - plugins += [GoogleBooks, Amazon, OpenLibrary] + plugins += [GoogleBooks, Amazon, OpenLibrary, ISBNDB] # }}} else: diff --git a/src/calibre/customize/profiles.py b/src/calibre/customize/profiles.py index 346adf4737..5c29f1e79b 100644 --- a/src/calibre/customize/profiles.py +++ b/src/calibre/customize/profiles.py @@ -344,6 +344,7 @@ class iPadOutput(OutputProfile): border-spacing:1px; margin-left: 5%; margin-right: 5%; + page-break-inside:avoid; width: 90%; -webkit-border-radius:4px; } diff --git a/src/calibre/devices/android/driver.py b/src/calibre/devices/android/driver.py index a63ce8c581..1fca46f766 100644 --- a/src/calibre/devices/android/driver.py +++ b/src/calibre/devices/android/driver.py @@ -54,6 +54,9 @@ class ANDROID(USBMS): 0x6877 : [0x0400], }, + # Viewsonic + 0x0489 : { 0xc001 : [0x0226] }, + # Acer 0x502 : { 0x3203 : [0x0100]}, diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 213f74f816..2cc478603a 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -349,7 +349,7 @@ class ITUNES(DriverBase): break break if self.report_progress is not None: - self.report_progress(j+1/task_count, _('Updating device metadata listing...')) + self.report_progress((j+1)/task_count, _('Updating device metadata listing...')) if self.report_progress is not None: self.report_progress(1.0, _('Updating device metadata listing...')) @@ -428,7 +428,7 @@ class ITUNES(DriverBase): } if self.report_progress is not None: - self.report_progress(i+1/book_count, _('%d of %d') % (i+1, book_count)) + self.report_progress((i+1)/book_count, _('%d of %d') % (i+1, book_count)) self._purge_orphans(library_books, cached_books) elif iswindows: @@ -466,7 +466,7 @@ class ITUNES(DriverBase): } if self.report_progress is not None: - self.report_progress(i+1/book_count, + self.report_progress((i+1)/book_count, _('%d of %d') % (i+1, book_count)) self._purge_orphans(library_books, cached_books) @@ -916,6 +916,8 @@ class ITUNES(DriverBase): """ if DEBUG: self.log.info("ITUNES.reset()") + if report_progress: + self.set_progress_reporter(report_progress) def set_progress_reporter(self, report_progress): ''' @@ -924,6 +926,9 @@ class ITUNES(DriverBase): If it is called with -1 that means that the task does not have any progress information ''' + if DEBUG: + self.log.info("ITUNES.set_progress_reporter()") + self.report_progress = report_progress def set_plugboards(self, plugboards, pb_func): @@ -1041,7 +1046,7 @@ class ITUNES(DriverBase): # Report progress if self.report_progress is not None: - self.report_progress(i+1/file_count, _('%d of %d') % (i+1, file_count)) + self.report_progress((i+1)/file_count, _('%d of %d') % (i+1, file_count)) elif iswindows: try: @@ -1081,7 +1086,7 @@ class ITUNES(DriverBase): # Report progress if self.report_progress is not None: - self.report_progress(i+1/file_count, _('%d of %d') % (i+1, file_count)) + self.report_progress((i+1)/file_count, _('%d of %d') % (i+1, file_count)) finally: pythoncom.CoUninitialize() @@ -3065,7 +3070,7 @@ class ITUNES_ASYNC(ITUNES): } if self.report_progress is not None: - self.report_progress(i+1/book_count, _('%d of %d') % (i+1, book_count)) + self.report_progress((i+1)/book_count, _('%d of %d') % (i+1, book_count)) elif iswindows: try: @@ -3104,7 +3109,7 @@ class ITUNES_ASYNC(ITUNES): } if self.report_progress is not None: - self.report_progress(i+1/book_count, + self.report_progress((i+1)/book_count, _('%d of %d') % (i+1, book_count)) finally: diff --git a/src/calibre/devices/usbms/books.py b/src/calibre/devices/usbms/books.py index 8c92aa8a6e..cfebe796a3 100644 --- a/src/calibre/devices/usbms/books.py +++ b/src/calibre/devices/usbms/books.py @@ -203,6 +203,8 @@ class CollectionsBookList(BookList): val = [orig_val] elif fm['datatype'] == 'text' and fm['is_multiple']: val = orig_val + elif fm['datatype'] == 'composite' and fm['is_multiple']: + val = [v.strip() for v in val.split(fm['is_multiple'])] else: val = [val] diff --git a/src/calibre/ebooks/__init__.py b/src/calibre/ebooks/__init__.py index 7776be5e28..a56abb907e 100644 --- a/src/calibre/ebooks/__init__.py +++ b/src/calibre/ebooks/__init__.py @@ -26,7 +26,7 @@ class ParserError(ValueError): pass BOOK_EXTENSIONS = ['lrf', 'rar', 'zip', 'rtf', 'lit', 'txt', 'txtz', 'text', 'htm', 'xhtm', - 'html', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc', + 'html', 'htmlz', 'xhtml', 'pdf', 'pdb', 'pdr', 'prc', 'mobi', 'azw', 'doc', 'epub', 'fb2', 'djvu', 'lrx', 'cbr', 'cbz', 'cbc', 'oebzip', 'rb', 'imp', 'odt', 'chm', 'tpz', 'azw1', 'pml', 'pmlz', 'mbp', 'tan', 'snb'] diff --git a/src/calibre/ebooks/chm/input.py b/src/calibre/ebooks/chm/input.py index f55a76d67e..61160e8dac 100644 --- a/src/calibre/ebooks/chm/input.py +++ b/src/calibre/ebooks/chm/input.py @@ -51,6 +51,7 @@ class CHMInput(InputFormatPlugin): mainpath = os.path.join(tdir, mainname) metadata = get_metadata_from_reader(self._chm_reader) + self._chm_reader.CloseCHM() odi = options.debug_pipeline options.debug_pipeline = None diff --git a/src/calibre/ebooks/htmlz/output.py b/src/calibre/ebooks/htmlz/output.py index 7cdf04bcdb..03fe12c89e 100644 --- a/src/calibre/ebooks/htmlz/output.py +++ b/src/calibre/ebooks/htmlz/output.py @@ -12,7 +12,7 @@ from lxml import etree from calibre.customize.conversion import OutputFormatPlugin, \ OptionRecommendation -from calibre.ebooks.oeb.base import OEB_IMAGES +from calibre.ebooks.oeb.base import OEB_IMAGES, SVG_MIME from calibre.ptempfile import TemporaryDirectory from calibre.utils.zipfile import ZipFile @@ -71,9 +71,13 @@ class HTMLZOutput(OutputFormatPlugin): os.makedirs(os.path.join(tdir, 'images')) for item in oeb_book.manifest: if item.media_type in OEB_IMAGES and item.href in images: + if item.media_type == SVG_MIME: + data = unicode(etree.tostring(item.data, encoding=unicode)) + else: + data = item.data fname = os.path.join(tdir, 'images', images[item.href]) with open(fname, 'wb') as img: - img.write(item.data) + img.write(data) # Metadata with open(os.path.join(tdir, 'metadata.opf'), 'wb') as mdataf: diff --git a/src/calibre/ebooks/lrf/input.py b/src/calibre/ebooks/lrf/input.py index e354bee562..9777a8a998 100644 --- a/src/calibre/ebooks/lrf/input.py +++ b/src/calibre/ebooks/lrf/input.py @@ -6,8 +6,8 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, textwrap, sys -from copy import deepcopy +import os, textwrap, sys, operator +from copy import deepcopy, copy from lxml import etree @@ -149,9 +149,65 @@ class TextBlock(etree.XSLTExtension): self.root = root self.parent = root self.add_text_to = (self.parent, 'text') + self.fix_deep_nesting(node) for child in node: self.process_child(child) + def fix_deep_nesting(self, node): + deepest = 1 + + def depth(node): + parent = node.getparent() + ans = 1 + while parent is not None: + ans += 1 + parent = parent.getparent() + return ans + + for span in node.xpath('descendant::Span'): + d = depth(span) + if d > deepest: + deepest = d + if d > 500: + break + + if deepest < 500: + return + + self.log.warn('Found deeply nested spans. Flattening.') + #with open('/t/before.xml', 'wb') as f: + # f.write(etree.tostring(node, method='xml')) + + spans = [(depth(span), span) for span in node.xpath('descendant::Span')] + spans.sort(key=operator.itemgetter(0), reverse=True) + + for depth, span in spans: + if depth < 3: + continue + p = span.getparent() + gp = p.getparent() + idx = p.index(span) + pidx = gp.index(p) + children = list(p)[idx:] + t = children[-1].tail + t = t if t else '' + children[-1].tail = t + (p.tail if p.tail else '') + p.tail = '' + pattrib = dict(**p.attrib) if p.tag == 'Span' else {} + for child in children: + p.remove(child) + if pattrib and child.tag == "Span": + attrib = copy(pattrib) + attrib.update(child.attrib) + child.attrib.update(attrib) + + + for child in reversed(children): + gp.insert(pidx+1, child) + + #with open('/t/after.xml', 'wb') as f: + # f.write(etree.tostring(node, method='xml')) + def add_text(self, text): if text: if getattr(self.add_text_to[0], self.add_text_to[1]) is None: diff --git a/src/calibre/ebooks/metadata/book/base.py b/src/calibre/ebooks/metadata/book/base.py index ff22cd3608..167ae52fa3 100644 --- a/src/calibre/ebooks/metadata/book/base.py +++ b/src/calibre/ebooks/metadata/book/base.py @@ -483,7 +483,7 @@ class Metadata(object): self_tags = self.get(x, []) self.set_user_metadata(x, meta) # get... did the deepcopy other_tags = other.get(x, []) - if meta['is_multiple']: + if meta['datatype'] == 'text' and meta['is_multiple']: # Case-insensitive but case preserving merging lotags = [t.lower() for t in other_tags] lstags = [t.lower() for t in self_tags] diff --git a/src/calibre/ebooks/metadata/extz.py b/src/calibre/ebooks/metadata/extz.py index 0ecdbe9ea6..18069b2a6a 100644 --- a/src/calibre/ebooks/metadata/extz.py +++ b/src/calibre/ebooks/metadata/extz.py @@ -8,12 +8,13 @@ Read meta information from extZ (TXTZ, HTMLZ...) files. ''' import os +import posixpath from cStringIO import StringIO from calibre.ebooks.metadata import MetaInformation -from calibre.ebooks.metadata.opf2 import OPF, metadata_to_opf -from calibre.ptempfile import TemporaryDirectory +from calibre.ebooks.metadata.opf2 import OPF +from calibre.ptempfile import PersistentTemporaryFile from calibre.utils.zipfile import ZipFile, safe_replace def get_metadata(stream, extract_cover=True): @@ -23,16 +24,75 @@ def get_metadata(stream, extract_cover=True): mi = MetaInformation(_('Unknown'), [_('Unknown')]) stream.seek(0) - with TemporaryDirectory('_untxtz_mdata') as tdir: - try: - zf = ZipFile(stream) - zf.extract('metadata.opf', tdir) - with open(os.path.join(tdir, 'metadata.opf'), 'rb') as opff: - mi = OPF(opff).to_book_metadata() - except: - return mi + try: + with ZipFile(stream) as zf: + opf_name = get_first_opf_name(zf) + opf_stream = StringIO(zf.read(opf_name)) + opf = OPF(opf_stream) + mi = opf.to_book_metadata() + if extract_cover: + cover_name = opf.raster_cover + if cover_name: + mi.cover_data = ('jpg', zf.read(cover_name)) + except: + return mi return mi def set_metadata(stream, mi): - opf = StringIO(metadata_to_opf(mi)) - safe_replace(stream, 'metadata.opf', opf) + replacements = {} + + # Get the OPF in the archive. + with ZipFile(stream) as zf: + opf_path = get_first_opf_name(zf) + opf_stream = StringIO(zf.read(opf_path)) + opf = OPF(opf_stream) + + # Cover. + new_cdata = None + try: + new_cdata = mi.cover_data[1] + if not new_cdata: + raise Exception('no cover') + except: + try: + new_cdata = open(mi.cover, 'rb').read() + except: + pass + if new_cdata: + raster_cover = opf.raster_cover + if not raster_cover: + raster_cover = 'cover.jpg' + cpath = posixpath.join(posixpath.dirname(opf_path), raster_cover) + new_cover = _write_new_cover(new_cdata, cpath) + replacements[cpath] = open(new_cover.name, 'rb') + + # Update the metadata. + opf.smart_update(mi, replace_metadata=True) + newopf = StringIO(opf.render()) + safe_replace(stream, opf_path, newopf, extra_replacements=replacements) + + # Cleanup temporary files. + try: + if cpath is not None: + replacements[cpath].close() + os.remove(replacements[cpath].name) + except: + pass + +def get_first_opf_name(zf): + names = zf.namelist() + opfs = [] + for n in names: + if n.endswith('.opf') and '/' not in n: + opfs.append(n) + if not opfs: + raise Exception('No OPF found') + opfs.sort() + return opfs[0] + +def _write_new_cover(new_cdata, cpath): + from calibre.utils.magick.draw import save_cover_data_to + new_cover = PersistentTemporaryFile(suffix=os.path.splitext(cpath)[1]) + new_cover.close() + save_cover_data_to(new_cdata, new_cover.name) + return new_cover diff --git a/src/calibre/ebooks/metadata/sources/base.py b/src/calibre/ebooks/metadata/sources/base.py index 5089d8951b..d9144fdf34 100644 --- a/src/calibre/ebooks/metadata/sources/base.py +++ b/src/calibre/ebooks/metadata/sources/base.py @@ -181,6 +181,10 @@ class Source(Plugin): #: construct the configuration widget for this plugin options = () + #: A string that is displayed at the top of the config widget for this + #: plugin + config_help_message = None + def __init__(self, *args, **kwargs): Plugin.__init__(self, *args, **kwargs) diff --git a/src/calibre/ebooks/metadata/sources/covers.py b/src/calibre/ebooks/metadata/sources/covers.py index cf6ec90c54..d28ce146c6 100644 --- a/src/calibre/ebooks/metadata/sources/covers.py +++ b/src/calibre/ebooks/metadata/sources/covers.py @@ -76,6 +76,11 @@ def run_download(log, results, abort, (plugin, width, height, fmt, bytes) ''' + if title == _('Unknown'): + title = None + if authors == [_('Unknown')]: + authors = None + plugins = [p for p in metadata_plugins(['cover']) if p.is_configured()] rq = Queue() @@ -145,7 +150,7 @@ def download_cover(log, Synchronous cover download. Returns the "best" cover as per user prefs/cover resolution. - Return cover is a tuple: (plugin, width, height, fmt, data) + Returned cover is a tuple: (plugin, width, height, fmt, data) Returns None if no cover is found. ''' diff --git a/src/calibre/ebooks/metadata/sources/identify.py b/src/calibre/ebooks/metadata/sources/identify.py index fad810c26e..170ceb6c7a 100644 --- a/src/calibre/ebooks/metadata/sources/identify.py +++ b/src/calibre/ebooks/metadata/sources/identify.py @@ -253,6 +253,10 @@ def merge_identify_results(result_map, log): def identify(log, abort, # {{{ title=None, authors=None, identifiers={}, timeout=30): + if title == _('Unknown'): + title = None + if authors == [_('Unknown')]: + authors = None start_time = time.time() plugins = [p for p in metadata_plugins(['identify']) if p.is_configured()] @@ -391,8 +395,8 @@ if __name__ == '__main__': # tests {{{ # unknown to Amazon {'identifiers':{'isbn': '9780307459671'}, 'title':'Invisible Gorilla', 'authors':['Christopher Chabris']}, - [title_test('The Invisible Gorilla: And Other Ways Our Intuitions Deceive Us', - exact=True), authors_test(['Christopher Chabris', 'Daniel Simons'])] + [title_test('The Invisible Gorilla', + exact=True), authors_test(['Christopher F. Chabris', 'Daniel Simons'])] ), @@ -400,7 +404,7 @@ if __name__ == '__main__': # tests {{{ {'title':'Learning Python', 'authors':['Lutz']}, [title_test('Learning Python', - exact=True), authors_test(['Mark Lutz']) + exact=True), authors_test(['Mark J. Lutz', 'David Ascher']) ] ), diff --git a/src/calibre/ebooks/metadata/sources/isbndb.py b/src/calibre/ebooks/metadata/sources/isbndb.py index ab9342c6cb..b8deea56df 100644 --- a/src/calibre/ebooks/metadata/sources/isbndb.py +++ b/src/calibre/ebooks/metadata/sources/isbndb.py @@ -7,7 +7,19 @@ __license__ = 'GPL v3' __copyright__ = '2011, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from calibre.ebooks.metadata.sources.base import Source +from urllib import quote + +from lxml import etree + +from calibre.ebooks.metadata import check_isbn +from calibre.ebooks.metadata.sources.base import Source, Option +from calibre.ebooks.chardet import xml_to_unicode +from calibre.utils.cleantext import clean_ascii_chars +from calibre.utils.icu import lower +from calibre.ebooks.metadata.book.base import Metadata + +BASE_URL = 'http://isbndb.com/api/books.xml?access_key=%s&page_number=1&results=subjects,authors,texts&' + class ISBNDB(Source): @@ -18,6 +30,20 @@ class ISBNDB(Source): touched_fields = frozenset(['title', 'authors', 'identifier:isbn', 'comments', 'publisher']) supports_gzip_transfer_encoding = True + # Shortcut, since we have no cached cover URLS + cached_cover_url_is_reliable = False + + options = ( + Option('isbndb_key', 'string', None, _('IsbnDB key:'), + _('To use isbndb.com you have to sign up for a free account' + 'at isbndb.com and get an access key.')), + ) + + config_help_message = '

'+_('To use metadata from isbndb.com you must sign' + ' up for a free account and get an isbndb key and enter it below.' + ' Instructions to get the key are ' + 'here.') + def __init__(self, *args, **kwargs): Source.__init__(self, *args, **kwargs) @@ -35,9 +61,186 @@ class ISBNDB(Source): except: pass - self.isbndb_key = prefs['isbndb_key'] + @property + def isbndb_key(self): + return self.prefs['isbndb_key'] def is_configured(self): return self.isbndb_key is not None + def create_query(self, title=None, authors=None, identifiers={}): # {{{ + base_url = BASE_URL%self.isbndb_key + isbn = check_isbn(identifiers.get('isbn', None)) + q = '' + if isbn is not None: + q = 'index1=isbn&value1='+isbn + elif title or authors: + tokens = [] + title_tokens = list(self.get_title_tokens(title)) + tokens += title_tokens + author_tokens = self.get_author_tokens(authors, + only_first_author=True) + tokens += author_tokens + tokens = [quote(t) for t in tokens] + q = '+'.join(tokens) + q = 'index1=combined&value1='+q + + if not q: + return None + if isinstance(q, unicode): + q = q.encode('utf-8') + return base_url + q + # }}} + + def identify(self, log, result_queue, abort, title=None, authors=None, # {{{ + identifiers={}, timeout=30): + if not self.is_configured(): + return + query = self.create_query(title=title, authors=authors, + identifiers=identifiers) + if not query: + err = 'Insufficient metadata to construct query' + log.error(err) + return err + + results = [] + try: + results = self.make_query(query, abort, title=title, authors=authors, + identifiers=identifiers, timeout=timeout) + except: + err = 'Failed to make query to ISBNDb, aborting.' + log.exception(err) + return err + + if not results and identifiers.get('isbn', False) and title and authors and \ + not abort.is_set(): + return self.identify(log, result_queue, abort, title=title, + authors=authors, timeout=timeout) + + for result in results: + self.clean_downloaded_metadata(result) + result_queue.put(result) + + def parse_feed(self, feed, seen, orig_title, orig_authors, identifiers): + + def tostring(x): + if x is None: + return '' + return etree.tostring(x, method='text', encoding=unicode).strip() + + orig_isbn = identifiers.get('isbn', None) + title_tokens = list(self.get_title_tokens(orig_title)) + author_tokens = list(self.get_author_tokens(orig_authors)) + results = [] + + def ismatch(title, authors): + authors = lower(' '.join(authors)) + title = lower(title) + match = not title_tokens + for t in title_tokens: + if lower(t) in title: + match = True + break + amatch = not author_tokens + for a in author_tokens: + if lower(a) in authors: + amatch = True + break + if not author_tokens: amatch = True + return match and amatch + + bl = feed.find('BookList') + if bl is None: + err = tostring(etree.find('errormessage')) + raise ValueError('ISBNDb query failed:' + err) + total_results = int(bl.get('total_results')) + shown_results = int(bl.get('shown_results')) + for bd in bl.xpath('.//BookData'): + isbn = check_isbn(bd.get('isbn13', bd.get('isbn', None))) + if not isbn: + continue + if orig_isbn and isbn != orig_isbn: + continue + title = tostring(bd.find('Title')) + if not title: + continue + authors = [] + for au in bd.xpath('.//Authors/Person'): + au = tostring(au) + if au: + if ',' in au: + ln, _, fn = au.partition(',') + au = fn.strip() + ' ' + ln.strip() + authors.append(au) + if not authors: + continue + comments = tostring(bd.find('Summary')) + if not comments: + # Require comments, since without them the result is useless + # anyway + continue + id_ = (title, tuple(authors)) + if id_ in seen: + continue + seen.add(id_) + if not ismatch(title, authors): + continue + publisher = tostring(bd.find('PublisherText')) + if not publisher: publisher = None + if publisher and 'audio' in publisher.lower(): + continue + mi = Metadata(title, authors) + mi.isbn = isbn + mi.publisher = publisher + mi.comments = comments + results.append(mi) + return total_results, shown_results, results + + def make_query(self, q, abort, title=None, authors=None, identifiers={}, + max_pages=10, timeout=30): + page_num = 1 + parser = etree.XMLParser(recover=True, no_network=True) + br = self.browser + + seen = set() + + candidates = [] + total_found = 0 + while page_num <= max_pages and not abort.is_set(): + url = q.replace('&page_number=1&', '&page_number=%d&'%page_num) + page_num += 1 + raw = br.open_novisit(url, timeout=timeout).read() + feed = etree.fromstring(xml_to_unicode(clean_ascii_chars(raw), + strip_encoding_pats=True)[0], parser=parser) + total, found, results = self.parse_feed( + feed, seen, title, authors, identifiers) + total_found += found + candidates += results + if total_found >= total or len(candidates) > 9: + break + + return candidates + # }}} + +if __name__ == '__main__': + # To run these test use: + # calibre-debug -e src/calibre/ebooks/metadata/sources/isbndb.py + from calibre.ebooks.metadata.sources.test import (test_identify_plugin, + title_test, authors_test) + test_identify_plugin(ISBNDB.name, + [ + + + ( + {'title':'Great Gatsby', + 'authors':['Fitzgerald']}, + [title_test('The great gatsby', exact=True), + authors_test(['F. Scott Fitzgerald'])] + ), + + ( + {'title': 'Flatland', 'authors':['Abbott']}, + [title_test('Flatland', exact=False)] + ), + ]) diff --git a/src/calibre/ebooks/metadata/sources/test.py b/src/calibre/ebooks/metadata/sources/test.py index 2e72f86c47..284a7ba45e 100644 --- a/src/calibre/ebooks/metadata/sources/test.py +++ b/src/calibre/ebooks/metadata/sources/test.py @@ -218,11 +218,11 @@ def test_identify_plugin(name, tests): # {{{ '')+'-%s-cover.jpg'%sanitize_file_name2(mi.title.replace(' ', '_'))) with open(cover, 'wb') as f: - f.write(cdata) + f.write(cdata[-1]) prints('Cover downloaded to:', cover) - if len(cdata) < 10240: + if len(cdata[-1]) < 10240: prints('Downloaded cover too small') raise SystemExit(1) diff --git a/src/calibre/ebooks/mobi/mobiml.py b/src/calibre/ebooks/mobi/mobiml.py index 1e626cf916..3c36a6166d 100644 --- a/src/calibre/ebooks/mobi/mobiml.py +++ b/src/calibre/ebooks/mobi/mobiml.py @@ -463,9 +463,9 @@ class MobiMLizer(object): text = COLLAPSE.sub(' ', elem.text) valign = style['vertical-align'] not_baseline = valign in ('super', 'sub', 'text-top', - 'text-bottom') or ( + 'text-bottom', 'top', 'bottom') or ( isinstance(valign, (float, int)) and abs(valign) != 0) - issup = valign in ('super', 'text-top') or ( + issup = valign in ('super', 'text-top', 'top') or ( isinstance(valign, (float, int)) and valign > 0) vtag = 'sup' if issup else 'sub' if not_baseline and not ignore_valign and tag not in NOT_VTAGS and not isblock: @@ -484,6 +484,7 @@ class MobiMLizer(object): parent = bstate.para if bstate.inline is None else bstate.inline if parent is not None: vtag = etree.SubElement(parent, XHTML(vtag)) + vtag = etree.SubElement(vtag, XHTML('small')) # Add anchors for child in vbstate.body: if child is not vbstate.para: diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index 5f4c47cdf3..fc47b26c02 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -310,10 +310,11 @@ class Serializer(object): if href not in id_offsets: self.logger.warn('Hyperlink target %r not found' % href) href, _ = urldefrag(href) - ioff = self.id_offsets[href] - for hoff in hoffs: - buffer.seek(hoff) - buffer.write('%010d' % ioff) + if href in self.id_offsets: + ioff = self.id_offsets[href] + for hoff in hoffs: + buffer.seek(hoff) + buffer.write('%010d' % ioff) class MobiWriter(object): COLLAPSE_RE = re.compile(r'[ \t\r\n\v]+') diff --git a/src/calibre/ebooks/oeb/transforms/page_margin.py b/src/calibre/ebooks/oeb/transforms/page_margin.py index bc1925e343..d7c99d24c6 100644 --- a/src/calibre/ebooks/oeb/transforms/page_margin.py +++ b/src/calibre/ebooks/oeb/transforms/page_margin.py @@ -20,8 +20,9 @@ class RemoveAdobeMargins(object): self.oeb, self.opts, self.log = oeb, opts, log for item in self.oeb.manifest: - if item.media_type in ('application/vnd.adobe-page-template+xml', - 'application/vnd.adobe.page-template+xml'): + if (item.media_type in ('application/vnd.adobe-page-template+xml', + 'application/vnd.adobe.page-template+xml') and + hasattr(item.data, 'xpath')): self.log('Removing page margins specified in the' ' Adobe page template') for elem in item.data.xpath( diff --git a/src/calibre/gui2/__init__.py b/src/calibre/gui2/__init__.py index 22aaabf592..e39427021e 100644 --- a/src/calibre/gui2/__init__.py +++ b/src/calibre/gui2/__init__.py @@ -357,6 +357,7 @@ class FileIconProvider(QFileIconProvider): 'bmp' : 'bmp', 'svg' : 'svg', 'html' : 'html', + 'htmlz' : 'html', 'htm' : 'html', 'xhtml' : 'html', 'xhtm' : 'html', diff --git a/src/calibre/gui2/actions/edit_metadata.py b/src/calibre/gui2/actions/edit_metadata.py index 9f2cacb177..18a73fb282 100644 --- a/src/calibre/gui2/actions/edit_metadata.py +++ b/src/calibre/gui2/actions/edit_metadata.py @@ -94,7 +94,7 @@ class EditMetadataAction(InterfaceAction): def bulk_metadata_downloaded(self, job): if job.failed: - self.job_exception(job, dialog_title=_('Failed to download metadata')) + self.gui.job_exception(job, dialog_title=_('Failed to download metadata')) return from calibre.gui2.metadata.bulk_download2 import proceed proceed(self.gui, job) diff --git a/src/calibre/gui2/actions/view.py b/src/calibre/gui2/actions/view.py index a606ca09bc..48d10e660a 100644 --- a/src/calibre/gui2/actions/view.py +++ b/src/calibre/gui2/actions/view.py @@ -6,9 +6,8 @@ __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' import os, time -from functools import partial -from PyQt4.Qt import Qt, QMenu +from PyQt4.Qt import Qt, QMenu, QAction, pyqtSignal from calibre.constants import isosx from calibre.gui2 import error_dialog, Dispatcher, question_dialog, config, \ @@ -18,6 +17,19 @@ from calibre.utils.config import prefs from calibre.ptempfile import PersistentTemporaryFile from calibre.gui2.actions import InterfaceAction +class HistoryAction(QAction): + + view_historical = pyqtSignal(object) + + def __init__(self, id_, title, parent): + QAction.__init__(self, title, parent) + self.id = id_ + self.triggered.connect(self._triggered) + + def _triggered(self): + self.view_historical.emit(self.id) + + class ViewAction(InterfaceAction): name = 'View' @@ -28,18 +40,51 @@ class ViewAction(InterfaceAction): self.persistent_files = [] self.qaction.triggered.connect(self.view_book) self.view_menu = QMenu() - self.view_menu.addAction(_('View'), partial(self.view_book, False)) - ac = self.view_menu.addAction(_('View specific format')) - ac.setShortcut((Qt.ControlModifier if isosx else Qt.AltModifier)+Qt.Key_V) + ac = self.view_specific_action = QAction(_('View specific format'), + self.gui) self.qaction.setMenu(self.view_menu) + ac.setShortcut((Qt.ControlModifier if isosx else Qt.AltModifier)+Qt.Key_V) ac.triggered.connect(self.view_specific_format, type=Qt.QueuedConnection) - - self.view_menu.addSeparator() + ac = self.view_action = QAction(self.qaction.icon(), + self.qaction.text(), self.gui) + ac.triggered.connect(self.view_book) ac = self.create_action(spec=(_('Read a random book'), 'catalog.png', None, None), attr='action_pick_random') ac.triggered.connect(self.view_random) - self.view_menu.addAction(ac) + ac = self.clear_history_action = QAction( + _('Clear recently viewed list'), self.gui) + ac.triggered.connect(self.clear_history) + def initialization_complete(self): + self.build_menus(self.gui.current_db) + + def build_menus(self, db): + self.view_menu.clear() + self.view_menu.addAction(self.qaction) + self.view_menu.addAction(self.view_specific_action) + self.view_menu.addSeparator() + self.view_menu.addAction(self.action_pick_random) + self.history_actions = [] + history = db.prefs.get('gui_view_history', []) + if history: + self.view_menu.addSeparator() + for id_, title in history: + ac = HistoryAction(id_, title, self.view_menu) + self.view_menu.addAction(ac) + ac.view_historical.connect(self.view_historical) + self.view_menu.addSeparator() + self.view_menu.addAction(self.clear_history_action) + + def clear_history(self): + db = self.gui.current_db + db.prefs['gui_view_history'] = [] + self.build_menus(db) + + def view_historical(self, id_): + self._view_calibre_books([id_]) + + def library_changed(self, db): + self.build_menus(db) def location_selected(self, loc): enabled = loc == 'library' @@ -47,15 +92,17 @@ class ViewAction(InterfaceAction): action.setEnabled(enabled) def view_format(self, row, format): - fmt_path = self.gui.library_view.model().db.format_abspath(row, format) - if fmt_path: - self._view_file(fmt_path) + id_ = self.gui.library_view.model().id(row) + self.view_format_by_id(id_, format) def view_format_by_id(self, id_, format): - fmt_path = self.gui.library_view.model().db.format_abspath(id_, format, + db = self.gui.current_db + fmt_path = db.format_abspath(id_, format, index_is_id=True) if fmt_path: + title = db.title(id_, index_is_id=True) self._view_file(fmt_path) + self.update_history([(id_, title)]) def book_downloaded_for_viewing(self, job): if job.failed: @@ -162,6 +209,54 @@ class ViewAction(InterfaceAction): self.gui.iactions['Choose Library'].pick_random() self._view_books([self.gui.library_view.currentIndex()]) + def _view_calibre_books(self, ids): + db = self.gui.current_db + views = [] + for id_ in ids: + try: + formats = db.formats(id_, index_is_id=True) + except: + error_dialog(self.gui, _('Cannot view'), + _('This book no longer exists in your library'), show=True) + self.update_history([], remove=set([id_])) + continue + + title = db.title(id_, index_is_id=True) + if not formats: + error_dialog(self.gui, _('Cannot view'), + _('%s has no available formats.')%(title,), show=True) + continue + + formats = formats.upper().split(',') + + fmt = formats[0] + for format in prefs['input_format_order']: + if format in formats: + fmt = format + break + views.append((id_, title)) + self.view_format_by_id(id_, fmt) + + self.update_history(views) + + def update_history(self, views, remove=frozenset()): + db = self.gui.current_db + if views: + seen = set() + history = [] + for id_, title in views + db.prefs.get('gui_view_history', []): + if title not in seen: + seen.add(title) + history.append((id_, title)) + + db.prefs['gui_view_history'] = history[:10] + self.build_menus(db) + if remove: + history = db.prefs.get('gui_view_history', []) + history = [x for x in history if x[0] not in remove] + db.prefs['gui_view_history'] = history[:10] + self.build_menus(db) + def _view_books(self, rows): if not rows or len(rows) == 0: self._launch_viewer() @@ -171,28 +266,8 @@ class ViewAction(InterfaceAction): return if self.gui.current_view() is self.gui.library_view: - for row in rows: - if hasattr(row, 'row'): - row = row.row() - - formats = self.gui.library_view.model().db.formats(row) - title = self.gui.library_view.model().db.title(row) - if not formats: - error_dialog(self.gui, _('Cannot view'), - _('%s has no available formats.')%(title,), show=True) - continue - - formats = formats.upper().split(',') - - - in_prefs = False - for format in prefs['input_format_order']: - if format in formats: - in_prefs = True - self.view_format(row, format) - break - if not in_prefs: - self.view_format(row, formats[0]) + ids = list(map(self.gui.library_view.model().id, rows)) + self._view_calibre_books(ids) else: paths = self.gui.current_view().model().paths(rows) for path in paths: diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 0683f2cb91..8a97183ffe 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -519,6 +519,8 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): val = [] if fm['is_multiple'] else [''] elif not fm['is_multiple']: val = [val] + elif fm['datatype'] == 'composite': + val = [v.strip() for v in val.split(fm['is_multiple'])] elif field == 'authors': val = [v.replace('|', ',') for v in val] else: diff --git a/src/calibre/gui2/init.py b/src/calibre/gui2/init.py index 80f1f1c2cf..a75ff01b21 100644 --- a/src/calibre/gui2/init.py +++ b/src/calibre/gui2/init.py @@ -247,6 +247,11 @@ class LayoutMixin(object): # {{{ for x in ('cb', 'tb', 'bd'): button = getattr(self, x+'_splitter').button button.setIconSize(QSize(24, 24)) + if isosx: + button.setStyleSheet(''' + QToolButton { background: none; border:none; padding: 0px; } + QToolButton:checked { background: rgba(0, 0, 0, 25%); } + ''') self.status_bar.addPermanentWidget(button) self.status_bar.addPermanentWidget(self.jobs_button) self.setStatusBar(self.status_bar) diff --git a/src/calibre/gui2/layout.py b/src/calibre/gui2/layout.py index e98817a02f..ebd2acfe1d 100644 --- a/src/calibre/gui2/layout.py +++ b/src/calibre/gui2/layout.py @@ -196,6 +196,10 @@ class SearchBar(QWidget): # {{{ l.addWidget(x) x.setToolTip(_("Reset Quick Search")) + x = parent.highlight_only_button = QToolButton(self) + x.setIcon(QIcon(I('arrow-down.png'))) + l.addWidget(x) + x = parent.search_options_button = QToolButton(self) x.setIcon(QIcon(I('config.png'))) x.setObjectName("search_option_button") @@ -408,6 +412,7 @@ class ToolBar(BaseToolBar): # {{{ self.d_widget.layout().addWidget(self.donate_button) if isosx: self.d_widget.setStyleSheet('QWidget, QToolButton {background-color: none; border: none; }') + self.d_widget.layout().addWidget(QLabel(u'\u00a0')) bar.addWidget(self.d_widget) self.showing_donate = True elif what in self.gui.iactions: diff --git a/src/calibre/gui2/library/models.py b/src/calibre/gui2/library/models.py index f7074a6fee..0a4b7a26ba 100644 --- a/src/calibre/gui2/library/models.py +++ b/src/calibre/gui2/library/models.py @@ -314,6 +314,13 @@ class BooksModel(QAbstractTableModel): # {{{ if not isinstance(order, bool): order = order == Qt.AscendingOrder label = self.column_map[col] + self._sort(label, order, reset) + + def sort_by_named_field(self, field, order, reset=True): + if field in self.db.field_metadata.keys(): + self._sort(field, order, reset) + + def _sort(self, label, order, reset): self.db.sort(label, order) if reset: self.reset() diff --git a/src/calibre/gui2/library/views.py b/src/calibre/gui2/library/views.py index 0cce33da9e..e87e7226e1 100644 --- a/src/calibre/gui2/library/views.py +++ b/src/calibre/gui2/library/views.py @@ -236,6 +236,16 @@ class BooksView(QTableView): # {{{ sm.select(idx, sm.Select|sm.Rows) self.scroll_to_row(indices[0].row()) self.selected_ids = [] + + def sort_by_named_field(self, field, order, reset=True): + if field in self.column_map: + idx = self.column_map.index(field) + if order: + self.sortByColumn(idx, Qt.AscendingOrder) + else: + self.sortByColumn(idx, Qt.DescendingOrder) + else: + self._model.sort_by_named_field(field, order, reset) # }}} # Ondevice column {{{ diff --git a/src/calibre/gui2/metadata/basic_widgets.py b/src/calibre/gui2/metadata/basic_widgets.py index b2ee79c9c0..73913ba58f 100644 --- a/src/calibre/gui2/metadata/basic_widgets.py +++ b/src/calibre/gui2/metadata/basic_widgets.py @@ -846,7 +846,7 @@ class RatingEdit(QSpinBox): # {{{ class TagsEdit(MultiCompleteLineEdit): # {{{ LABEL = _('Ta&gs:') TOOLTIP = '

'+_('Tags categorize the book. This is particularly ' - 'useful while searching.

They can be any words' + 'useful while searching.

They can be any words ' 'or phrases, separated by commas.') def __init__(self, parent): diff --git a/src/calibre/gui2/metadata/bulk_download2.py b/src/calibre/gui2/metadata/bulk_download2.py index 19cd3df9d4..5f0af1b316 100644 --- a/src/calibre/gui2/metadata/bulk_download2.py +++ b/src/calibre/gui2/metadata/bulk_download2.py @@ -54,6 +54,8 @@ def start_download(gui, ids, callback, identify, covers): _('Download metadata for %d books')%len(ids), download, (ids, gui.current_db, identify, covers), {}, callback) gui.job_manager.run_threaded_job(job) + gui.status_bar.show_message(_('Metadata download started'), 3000) + class ViewLog(QDialog): # {{{ @@ -75,7 +77,7 @@ class ViewLog(QDialog): # {{{ self.copy_button.clicked.connect(self.copy_to_clipboard) l.addWidget(self.bb) self.setModal(False) - self.resize(QSize(500, 400)) + self.resize(QSize(700, 500)) self.setWindowTitle(_('Download log')) self.setWindowIcon(QIcon(I('debug.png'))) self.show() @@ -110,25 +112,27 @@ class ApplyDialog(QDialog): self.bb.accepted.connect(self.accept) l.addWidget(self.bb) - self.db = gui.current_db + self.gui = gui self.id_map = list(id_map.iteritems()) self.current_idx = 0 self.failures = [] + self.ids = [] self.canceled = False QTimer.singleShot(20, self.do_one) - self.exec_() def do_one(self): if self.canceled: return i, mi = self.id_map[self.current_idx] + db = self.gui.current_db try: set_title = not mi.is_null('title') set_authors = not mi.is_null('authors') - self.db.set_metadata(i, mi, commit=False, set_title=set_title, + db.set_metadata(i, mi, commit=False, set_title=set_title, set_authors=set_authors) + self.ids.append(i) except: import traceback self.failures.append((i, traceback.format_exc())) @@ -156,9 +160,10 @@ class ApplyDialog(QDialog): return if self.failures: msg = [] + db = self.gui.current_db for i, tb in self.failures: - title = self.db.title(i, index_is_id=True) - authors = self.db.authors(i, index_is_id=True) + title = db.title(i, index_is_id=True) + authors = db.authors(i, index_is_id=True) if authors: authors = [x.replace('|', ',') for x in authors.split(',')] title += ' - ' + authors_to_string(authors) @@ -169,6 +174,12 @@ class ApplyDialog(QDialog): ' in your library. Click "Show Details" to see ' 'details.'), det_msg='\n\n'.join(msg), show=True) self.accept() + if self.ids: + cr = self.gui.library_view.currentIndex().row() + self.gui.library_view.model().refresh_ids( + self.ids, cr) + if self.gui.cover_flow: + self.gui.cover_flow.dataChanged() _amd = None def apply_metadata(job, gui, q, result): @@ -177,7 +188,7 @@ def apply_metadata(job, gui, q, result): q.finished.disconnect() if result != q.Accepted: return - id_map, failed_ids = job.result + id_map, failed_ids, failed_covers, title_map = job.result id_map = dict([(k, v) for k, v in id_map.iteritems() if k not in failed_ids]) if not id_map: @@ -207,23 +218,32 @@ def apply_metadata(job, gui, q, result): return _amd = ApplyDialog(id_map, gui) + _amd.exec_() def proceed(gui, job): - id_map, failed_ids = job.result + gui.status_bar.show_message(_('Metadata download completed'), 3000) + id_map, failed_ids, failed_covers, title_map = job.result fmsg = det_msg = '' - if failed_ids: - fmsg = _('Could not download metadata for %d of the books. Click' + if failed_ids or failed_covers: + fmsg = '

'+_('Could not download metadata and/or covers for %d of the books. Click' ' "Show details" to see which books.')%len(failed_ids) - det_msg = '\n'.join([id_map[i].title for i in failed_ids]) + det_msg = [] + for i in failed_ids | failed_covers: + title = title_map[i] + if i in failed_ids: + title += (' ' + _('(Failed metadata)')) + if i in failed_covers: + title += (' ' + _('(Failed cover)')) + det_msg.append(title) msg = '

' + _('Finished downloading metadata for %d book(s). ' 'Proceed with updating the metadata in your library?')%len(id_map) q = MessageBox(MessageBox.QUESTION, _('Download complete'), - msg + fmsg, det_msg=det_msg, show_copy_button=bool(failed_ids), + msg + fmsg, det_msg='\n'.join(det_msg), show_copy_button=bool(failed_ids), parent=gui) q.vlb = q.bb.addButton(_('View log'), q.bb.ActionRole) q.vlb.setIcon(QIcon(I('debug.png'))) q.vlb.clicked.connect(partial(view_log, job, q)) - q.det_msg_toggle.setVisible(bool(failed_ids)) + q.det_msg_toggle.setVisible(bool(failed_ids | failed_covers)) q.setModal(False) q.show() q.finished.connect(partial(apply_metadata, job, gui, q)) @@ -242,12 +262,18 @@ def merge_result(oldmi, newmi): if (not newmi.is_null(f) and getattr(newmi, f) == getattr(oldmi, f)): setattr(newmi, f, getattr(dummy, f)) + newmi.last_modified = oldmi.last_modified + + return newmi + def download(ids, db, do_identify, covers, log=None, abort=None, notifications=None): ids = list(ids) metadata = [db.get_metadata(i, index_is_id=True, get_user_categories=False) for i in ids] failed_ids = set() + failed_covers = set() + title_map = {} ans = {} count = 0 for i, mi in izip(ids, metadata): @@ -255,6 +281,7 @@ def download(ids, db, do_identify, covers, log.error('Aborting...') break title, authors, identifiers = mi.title, mi.authors, mi.identifiers + title_map[i] = title if do_identify: results = [] try: @@ -265,22 +292,29 @@ def download(ids, db, do_identify, covers, if results: mi = merge_result(mi, results[0]) identifiers = mi.identifiers + if not mi.is_null('rating'): + # set_metadata expects a rating out of 10 + mi.rating *= 2 else: log.error('Failed to download metadata for', title) - failed_ids.add(mi) + failed_ids.add(i) + # We don't want set_metadata operating on anything but covers + mi = merge_result(mi, mi) if covers: cdata = download_cover(log, title=title, authors=authors, identifiers=identifiers) - if cdata: + if cdata is not None: with PersistentTemporaryFile('.jpg', 'downloaded-cover-') as f: - f.write(cdata) + f.write(cdata[-1]) mi.cover = f.name + else: + failed_covers.add(i) ans[i] = mi count += 1 notifications.put((count/len(ids), _('Downloaded %d of %d')%(count, len(ids)))) log('Download complete, with %d failures'%len(failed_ids)) - return (ans, failed_ids) + return (ans, failed_ids, failed_covers, title_map) diff --git a/src/calibre/gui2/metadata/config.py b/src/calibre/gui2/metadata/config.py index 68c935061d..abb45faa46 100644 --- a/src/calibre/gui2/metadata/config.py +++ b/src/calibre/gui2/metadata/config.py @@ -56,7 +56,12 @@ class ConfigWidget(QWidget): self.setLayout(l) self.gb = QGroupBox(_('Downloaded metadata fields'), self) - l.addWidget(self.gb, 0, 0, 1, 2) + if plugin.config_help_message: + self.pchm = QLabel(plugin.config_help_message) + self.pchm.setWordWrap(True) + self.pchm.setOpenExternalLinks(True) + l.addWidget(self.pchm, 0, 0, 1, 2) + l.addWidget(self.gb, l.rowCount(), 0, 1, 2) self.gb.l = QGridLayout() self.gb.setLayout(self.gb.l) self.fields_view = v = QListView(self) @@ -81,7 +86,7 @@ class ConfigWidget(QWidget): widget.setValue(val) elif opt.type == 'string': widget = QLineEdit(self) - widget.setText(val) + widget.setText(val if val else '') elif opt.type == 'bool': widget = QCheckBox(opt.label, self) widget.setChecked(bool(val)) diff --git a/src/calibre/gui2/preferences/columns.py b/src/calibre/gui2/preferences/columns.py index 03a50e6f3a..92aafccce0 100644 --- a/src/calibre/gui2/preferences/columns.py +++ b/src/calibre/gui2/preferences/columns.py @@ -163,8 +163,9 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): elif '*edited' in self.custcols[c]: cc = self.custcols[c] db.set_custom_column_metadata(cc['colnum'], name=cc['name'], - label=cc['label'], - display = self.custcols[c]['display']) + label=cc['label'], + display = self.custcols[c]['display'], + notify=False) if '*must_restart' in self.custcols[c]: must_restart = True return must_restart diff --git a/src/calibre/gui2/preferences/create_custom_column.py b/src/calibre/gui2/preferences/create_custom_column.py index f476845f8b..fcbaaf181f 100644 --- a/src/calibre/gui2/preferences/create_custom_column.py +++ b/src/calibre/gui2/preferences/create_custom_column.py @@ -41,6 +41,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): 'text':_('Yes/No'), 'is_multiple':False}, 10:{'datatype':'composite', 'text':_('Column built from other columns'), 'is_multiple':False}, + 11:{'datatype':'*composite', + 'text':_('Column built from other columns, behaves like tags'), 'is_multiple':True}, } def __init__(self, parent, editing, standard_colheads, standard_colnames): @@ -99,7 +101,9 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): c = parent.custcols[col] self.column_name_box.setText(c['label']) self.column_heading_box.setText(c['name']) - ct = c['datatype'] if not c['is_multiple'] else '*text' + ct = c['datatype'] + if c['is_multiple']: + ct = '*' + ct self.orig_column_number = c['colnum'] self.orig_column_name = col column_numbers = dict(map(lambda x:(self.column_types[x]['datatype'], x), @@ -109,7 +113,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): if ct == 'datetime': if c['display'].get('date_format', None): self.date_format_box.setText(c['display'].get('date_format', '')) - elif ct == 'composite': + elif ct in ['composite', '*composite']: self.composite_box.setText(c['display'].get('composite_template', '')) sb = c['display'].get('composite_sort', 'text') vals = ['text', 'number', 'date', 'bool'] @@ -167,7 +171,7 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): getattr(self, 'date_format_'+x).setVisible(col_type == 'datetime') for x in ('box', 'default_label', 'label', 'sort_by', 'sort_by_label', 'make_category'): - getattr(self, 'composite_'+x).setVisible(col_type == 'composite') + getattr(self, 'composite_'+x).setVisible(col_type in ['composite', '*composite']) for x in ('box', 'default_label', 'label'): getattr(self, 'enum_'+x).setVisible(col_type == 'enumeration') self.use_decorations.setVisible(col_type in ['text', 'composite', 'enumeration']) @@ -187,8 +191,8 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): 'because these names are reserved for the index of a series column.')) col_heading = unicode(self.column_heading_box.text()).strip() col_type = self.column_types[self.column_type_box.currentIndex()]['datatype'] - if col_type == '*text': - col_type='text' + if col_type[0] == '*': + col_type = col_type[1:] is_multiple = True else: is_multiple = False @@ -249,11 +253,10 @@ class CreateCustomColumn(QDialog, Ui_QCreateCustomColumn): elif col_type == 'text' and is_multiple: display_dict = {'is_names': self.is_names.isChecked()} - if col_type in ['text', 'composite', 'enumeration']: + if col_type in ['text', 'composite', 'enumeration'] and not is_multiple: display_dict['use_decorations'] = self.use_decorations.checkState() if not self.editing_col: - db.field_metadata self.parent.custcols[key] = { 'label':col, 'name':col_heading, diff --git a/src/calibre/gui2/preferences/emailp.py b/src/calibre/gui2/preferences/emailp.py index 1644dc6b73..1256691c22 100644 --- a/src/calibre/gui2/preferences/emailp.py +++ b/src/calibre/gui2/preferences/emailp.py @@ -202,7 +202,8 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): self.changed_signal.emit() def refresh_gui(self, gui): - gui.emailer.calculate_rate_limit() + from calibre.gui2.email import gui_sendmail + gui_sendmail.calculate_rate_limit() if __name__ == '__main__': diff --git a/src/calibre/gui2/preferences/metadata_sources.py b/src/calibre/gui2/preferences/metadata_sources.py index 4500a03b30..17a70bcc33 100644 --- a/src/calibre/gui2/preferences/metadata_sources.py +++ b/src/calibre/gui2/preferences/metadata_sources.py @@ -10,7 +10,7 @@ __docformat__ = 'restructuredtext en' from operator import attrgetter from PyQt4.Qt import (QAbstractTableModel, Qt, QAbstractListModel, QWidget, - pyqtSignal, QVBoxLayout, QDialogButtonBox, QFrame, QLabel) + pyqtSignal, QVBoxLayout, QDialogButtonBox, QFrame, QLabel, QIcon) from calibre.gui2.preferences import ConfigWidgetBase, test_widget from calibre.gui2.preferences.metadata_sources_ui import Ui_Form @@ -67,6 +67,13 @@ class SourcesModel(QAbstractTableModel): # {{{ return self.enabled_overrides.get(plugin, orig) elif role == Qt.UserRole: return plugin + elif (role == Qt.DecorationRole and col == 0 and not + plugin.is_configured()): + return QIcon(I('list_remove.png')) + elif role == Qt.ToolTipRole: + if plugin.is_configured(): + return _('This source is configured and ready to go') + return _('This source needs configuration') return NONE def setData(self, index, val, role): diff --git a/src/calibre/gui2/preferences/metadata_sources.ui b/src/calibre/gui2/preferences/metadata_sources.ui index 546120f628..b515f13ba1 100644 --- a/src/calibre/gui2/preferences/metadata_sources.ui +++ b/src/calibre/gui2/preferences/metadata_sources.ui @@ -48,6 +48,16 @@ + + + + Sources with a red X next to their names must be configured before they will be used. + + + true + + + diff --git a/src/calibre/gui2/preferences/search.py b/src/calibre/gui2/preferences/search.py index db93cbd525..7bdb12ec55 100644 --- a/src/calibre/gui2/preferences/search.py +++ b/src/calibre/gui2/preferences/search.py @@ -171,10 +171,10 @@ class ConfigWidget(ConfigWidgetBase, Ui_Form): return ConfigWidgetBase.commit(self) def refresh_gui(self, gui): + gui.set_highlight_only_button_icon() if self.muc_changed: gui.tags_view.set_new_model() gui.search.search_as_you_type(config['search_as_you_type']) - gui.library_view.model().set_highlight_only(config['highlight_search_matches']) gui.search.do_search() def clear_histories(self, *args): diff --git a/src/calibre/gui2/search_box.py b/src/calibre/gui2/search_box.py index ea7cab95d0..359cb0b2f6 100644 --- a/src/calibre/gui2/search_box.py +++ b/src/calibre/gui2/search_box.py @@ -10,7 +10,7 @@ import re from PyQt4.Qt import QComboBox, Qt, QLineEdit, QStringList, pyqtSlot, QDialog, \ pyqtSignal, QCompleter, QAction, QKeySequence, QTimer, \ - QString + QString, QIcon from calibre.gui2 import config from calibre.gui2.dialogs.confirm_delete import confirm @@ -383,6 +383,22 @@ class SearchBoxMixin(object): # {{{ self.advanced_search_button.setStatusTip(self.advanced_search_button.toolTip()) self.clear_button.setStatusTip(self.clear_button.toolTip()) self.search_options_button.clicked.connect(self.search_options_button_clicked) + self.set_highlight_only_button_icon() + self.highlight_only_button.clicked.connect(self.highlight_only_clicked) + tt = _('Enable or disable search highlighting.') + '

' + tt += config.help('highlight_search_matches') + self.highlight_only_button.setToolTip(tt) + + def highlight_only_clicked(self, state): + config['highlight_search_matches'] = not config['highlight_search_matches'] + self.set_highlight_only_button_icon() + + def set_highlight_only_button_icon(self): + if config['highlight_search_matches']: + self.highlight_only_button.setIcon(QIcon(I('highlight_only_on.png'))) + else: + self.highlight_only_button.setIcon(QIcon(I('highlight_only_off.png'))) + self.library_view.model().set_highlight_only(config['highlight_search_matches']) def focus_search_box(self, *args): self.search.setFocus(Qt.OtherFocusReason) @@ -443,6 +459,7 @@ class SavedSearchBoxMixin(object): # {{{ # rebuild the restrictions combobox using current saved searches self.search_restriction.clear() self.search_restriction.addItem('') + self.search_restriction.addItem(_('*Current search')) if recount: self.tags_view.recount() for s in p: diff --git a/src/calibre/gui2/search_restriction_mixin.py b/src/calibre/gui2/search_restriction_mixin.py index 74e448da6e..8ef02b34b0 100644 --- a/src/calibre/gui2/search_restriction_mixin.py +++ b/src/calibre/gui2/search_restriction_mixin.py @@ -29,13 +29,32 @@ class SearchRestrictionMixin(object): self.search_restriction.setCurrentIndex(r) self.apply_search_restriction(r) - def apply_search_restriction(self, i): - r = unicode(self.search_restriction.currentText()) - if r is not None and r != '': - restriction = 'search:"%s"'%(r) + def apply_text_search_restriction(self, search): + if not search: + self.search_restriction.setItemText(1, _('*Current search')) + self.search_restriction.setCurrentIndex(0) else: - restriction = '' + self.search_restriction.setCurrentIndex(1) + self.search_restriction.setItemText(1, search) + self._apply_search_restriction(search) + def apply_search_restriction(self, i): + self.search_restriction.setItemText(1, _('*Current search')) + if i == 1: + restriction = unicode(self.search.currentText()) + if not restriction: + self.search_restriction.setCurrentIndex(0) + else: + self.search_restriction.setItemText(1, restriction) + else: + r = unicode(self.search_restriction.currentText()) + if r is not None and r != '': + restriction = 'search:"%s"'%(r) + else: + restriction = '' + self._apply_search_restriction(restriction) + + def _apply_search_restriction(self, restriction): self.saved_search.clear() # The order below is important. Set the restriction, force a '' search # to apply it, reset the tag browser to take it into account, then set diff --git a/src/calibre/gui2/tag_view.py b/src/calibre/gui2/tag_view.py index 83695b86c1..6ad6f053cb 100644 --- a/src/calibre/gui2/tag_view.py +++ b/src/calibre/gui2/tag_view.py @@ -86,6 +86,7 @@ class TagsView(QTreeView): # {{{ tag_item_renamed = pyqtSignal() search_item_renamed = pyqtSignal() drag_drop_finished = pyqtSignal(object) + restriction_error = pyqtSignal() def __init__(self, parent=None): QTreeView.__init__(self, parent=None) @@ -1117,9 +1118,13 @@ class TagsModel(QAbstractItemModel): # {{{ # Get the categories if self.search_restriction: - data = self.db.get_categories(sort=sort, + try: + data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map, ids=self.db.search('', return_matches=True)) + except: + data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map) + self.tags_view.restriction_error.emit() else: data = self.db.get_categories(sort=sort, icon_map=self.category_icon_map) @@ -1822,9 +1827,15 @@ class TagBrowserMixin(object): # {{{ self.tags_view.tag_item_renamed.connect(self.do_tag_item_renamed) self.tags_view.search_item_renamed.connect(self.saved_searches_changed) self.tags_view.drag_drop_finished.connect(self.drag_drop_finished) + self.tags_view.restriction_error.connect(self.do_restriction_error, + type=Qt.QueuedConnection) self.edit_categories.clicked.connect(lambda x: self.do_edit_user_categories()) + def do_restriction_error(self): + error_dialog(self.tags_view, _('Invalid search restriction'), + _('The current search restriction is invalid'), show=True) + def do_add_subcategory(self, on_category_key, new_category_name=None): ''' Add a subcategory to the category 'on_category'. If new_category_name is diff --git a/src/calibre/gui2/threaded_jobs.py b/src/calibre/gui2/threaded_jobs.py index f98488da79..9c791c5b0d 100644 --- a/src/calibre/gui2/threaded_jobs.py +++ b/src/calibre/gui2/threaded_jobs.py @@ -189,7 +189,11 @@ class ThreadedJobServer(Thread): def run(self): while self.keep_going: - self.run_once() + try: + self.run_once() + except: + import traceback + traceback.print_exc() time.sleep(0.1) def run_once(self): diff --git a/src/calibre/library/caches.py b/src/calibre/library/caches.py index a108feb388..4d696afe91 100644 --- a/src/calibre/library/caches.py +++ b/src/calibre/library/caches.py @@ -391,58 +391,68 @@ class ResultCache(SearchQueryParser): # {{{ def build_numeric_relop_dict(self): self.numeric_search_relops = { '=':[1, lambda r, q: r == q], - '>':[1, lambda r, q: r > q], - '<':[1, lambda r, q: r < q], + '>':[1, lambda r, q: r is not None and r > q], + '<':[1, lambda r, q: r is not None and r < q], '!=':[2, lambda r, q: r != q], - '>=':[2, lambda r, q: r >= q], - '<=':[2, lambda r, q: r <= q] + '>=':[2, lambda r, q: r is not None and r >= q], + '<=':[2, lambda r, q: r is not None and r <= q] } def get_numeric_matches(self, location, query, candidates, val_func = None): matches = set([]) if len(query) == 0: return matches - if query == 'false': - query = '0' - elif query == 'true': - query = '!=0' - relop = None - for k in self.numeric_search_relops.keys(): - if query.startswith(k): - (p, relop) = self.numeric_search_relops[k] - query = query[p:] - if relop is None: - (p, relop) = self.numeric_search_relops['='] if val_func is None: loc = self.field_metadata[location]['rec_index'] val_func = lambda item, loc=loc: item[loc] - dt = self.field_metadata[location]['datatype'] - if dt == 'int': - cast = (lambda x: int (x)) - adjust = lambda x: x - elif dt == 'rating': - cast = (lambda x: int (x)) - adjust = lambda x: x/2 - elif dt in ('float', 'composite'): - cast = lambda x : float (x) - adjust = lambda x: x - else: # count operation - cast = (lambda x: int (x)) - adjust = lambda x: x - if len(query) > 1: - mult = query[-1:].lower() - mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) - if mult != 1.0: - query = query[:-1] + q = '' + val_func = lambda item, loc=loc: item[loc] + cast = adjust = lambda x: x + + if query == 'false': + if dt == 'rating': + relop = lambda x,y: not bool(x) + else: + relop = lambda x,y: x is None + elif query == 'true': + if dt == 'rating': + relop = lambda x,y: bool(x) + else: + relop = lambda x,y: x is not None else: - mult = 1.0 - try: - q = cast(query) * mult - except: - return matches + relop = None + for k in self.numeric_search_relops.keys(): + if query.startswith(k): + (p, relop) = self.numeric_search_relops[k] + query = query[p:] + if relop is None: + (p, relop) = self.numeric_search_relops['='] + + if dt == 'int': + cast = lambda x: int (x) + elif dt == 'rating': + cast = lambda x: 0 if x is None else int (x) + adjust = lambda x: x/2 + elif dt in ('float', 'composite'): + cast = lambda x : float (x) + else: # count operation + cast = (lambda x: int (x)) + + if len(query) > 1: + mult = query[-1:].lower() + mult = {'k':1024.,'m': 1024.**2, 'g': 1024.**3}.get(mult, 1.0) + if mult != 1.0: + query = query[:-1] + else: + mult = 1.0 + try: + q = cast(query) * mult + except: + raise ParseException(query, len(query), + 'Non-numeric value in query', self) for id_ in candidates: item = self._data[id_] @@ -451,10 +461,8 @@ class ResultCache(SearchQueryParser): # {{{ try: v = cast(val_func(item)) except: - v = 0 - if not v: - v = 0 - else: + v = None + if v: v = adjust(v) if relop(v, q): matches.add(item[0]) @@ -584,8 +592,7 @@ class ResultCache(SearchQueryParser): # {{{ candidates = self.universal_set() if len(candidates) == 0: return matches - if location not in self.all_search_locations: - return matches + self.test_location_is_valid(location, query) if len(location) > 2 and location.startswith('@') and \ location[1:] in self.db_prefs['grouped_search_terms']: @@ -744,7 +751,7 @@ class ResultCache(SearchQueryParser): # {{{ if loc not in exclude_fields: # time for text matching if is_multiple_cols[loc] is not None: - vals = item[loc].split(is_multiple_cols[loc]) + vals = [v.strip() for v in item[loc].split(is_multiple_cols[loc])] else: vals = [item[loc]] ### make into list to make _match happy if _match(q, vals, matchkind): diff --git a/src/calibre/library/custom_columns.py b/src/calibre/library/custom_columns.py index 8eed121b21..187d718a39 100644 --- a/src/calibre/library/custom_columns.py +++ b/src/calibre/library/custom_columns.py @@ -182,7 +182,7 @@ class CustomColumns(object): else: is_category = False if v['is_multiple']: - is_m = '|' + is_m = ',' if v['datatype'] == 'composite' else '|' else: is_m = None tn = 'custom_column_{0}'.format(v['num']) @@ -318,7 +318,7 @@ class CustomColumns(object): self.conn.commit() def set_custom_column_metadata(self, num, name=None, label=None, - is_editable=None, display=None): + is_editable=None, display=None, notify=True): changed = False if name is not None: self.conn.execute('UPDATE custom_columns SET name=? WHERE id=?', @@ -340,6 +340,9 @@ class CustomColumns(object): if changed: self.conn.commit() + if notify: + self.notify('metadata', []) + return changed def set_custom_bulk_multiple(self, ids, add=[], remove=[], @@ -595,7 +598,7 @@ class CustomColumns(object): raise ValueError('%r is not a supported data type'%datatype) normalized = datatype not in ('datetime', 'comments', 'int', 'bool', 'float', 'composite') - is_multiple = is_multiple and datatype in ('text',) + is_multiple = is_multiple and datatype in ('text', 'composite') num = self.conn.execute( ('INSERT INTO ' 'custom_columns(label,name,datatype,is_multiple,editable,display,normalized)' diff --git a/src/calibre/library/database2.py b/src/calibre/library/database2.py index 50b404b4be..b5155368c7 100644 --- a/src/calibre/library/database2.py +++ b/src/calibre/library/database2.py @@ -854,6 +854,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): mi.uuid = row[fm['uuid']] mi.title_sort = row[fm['sort']] mi.last_modified = row[fm['last_modified']] + mi.size = row[fm['size']] formats = row[fm['formats']] if not formats: formats = None @@ -1223,7 +1224,12 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): if field['datatype'] == 'composite': dex = field['rec_index'] for book in self.data.iterall(): - if book[dex] == id_: + if field['is_multiple']: + vals = [v.strip() for v in book[dex].split(field['is_multiple']) + if v.strip()] + if id_ in vals: + ans.add(book[0]) + elif book[dex] == id_: ans.add(book[0]) return ans @@ -1353,6 +1359,7 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): cat = tb_cats[category] if cat['datatype'] == 'composite' and \ cat['display'].get('make_category', False): + tids[category] = {} tcategories[category] = {} md.append((category, cat['rec_index'], cat['is_multiple'], cat['datatype'] == 'composite')) @@ -1401,8 +1408,18 @@ class LibraryDatabase2(LibraryDatabase, SchemaUpgrade, CustomColumns): prints('get_categories: item', val, 'is not in', cat, 'list!') else: vals = book[dex].split(mult) + if is_comp: + vals = [v.strip() for v in vals if v.strip()] + for val in vals: + if val not in tids: + tids[cat][val] = (val, val) + item = tcategories[cat].get(val, None) + if not item: + item = tag_class(val, val) + tcategories[cat][val] = item + item.c += 1 + item.id = val for val in vals: - if not val: continue try: (item_id, sort_val) = tids[cat][val] # let exceptions fly item = tcategories[cat].get(val, None) diff --git a/src/calibre/library/field_metadata.py b/src/calibre/library/field_metadata.py index ae91283523..33929ac2e4 100644 --- a/src/calibre/library/field_metadata.py +++ b/src/calibre/library/field_metadata.py @@ -364,11 +364,11 @@ class FieldMetadata(dict): self._tb_cats[k]['display'] = {} self._tb_cats[k]['is_editable'] = True self._add_search_terms_to_map(k, v['search_terms']) - for x in ('timestamp', 'last_modified'): - self._tb_cats[x]['display'] = { + self._tb_cats['timestamp']['display'] = { 'date_format': tweaks['gui_timestamp_display_format']} self._tb_cats['pubdate']['display'] = { 'date_format': tweaks['gui_pubdate_display_format']} + self._tb_cats['last_modified']['display'] = {'date_format': 'iso'} self.custom_field_prefix = '#' self.get = self._tb_cats.get diff --git a/src/calibre/manual/faq.rst b/src/calibre/manual/faq.rst index 44e3665131..5a2b2669bb 100644 --- a/src/calibre/manual/faq.rst +++ b/src/calibre/manual/faq.rst @@ -20,9 +20,9 @@ What formats does |app| support conversion to/from? |app| supports the conversion of many input formats to many output formats. It can convert every input format in the following list, to every output format. -*Input Formats:* CBZ, CBR, CBC, CHM, EPUB, FB2, HTML, LIT, LRF, MOBI, ODT, PDF, PRC**, PDB, PML, RB, RTF, SNB, TCR, TXT +*Input Formats:* CBZ, CBR, CBC, CHM, EPUB, FB2, HTML, HTMLZ, LIT, LRF, MOBI, ODT, PDF, PRC**, PDB, PML, RB, RTF, SNB, TCR, TXT, TXTZ -*Output Formats:* EPUB, FB2, OEB, LIT, LRF, MOBI, PDB, PML, RB, PDF, SNB, TCR, TXT +*Output Formats:* EPUB, FB2, OEB, LIT, LRF, MOBI, HTMLZ, PDB, PML, RB, PDF, SNB, TCR, TXT, TXTZ ** PRC is a generic format, |app| supports PRC files with TextRead and MOBIBook headers @@ -30,7 +30,7 @@ It can convert every input format in the following list, to every output format. What are the best source formats to convert? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -In order of decreasing preference: LIT, MOBI, EPUB, HTML, PRC, RTF, PDB, TXT, PDF +In order of decreasing preference: LIT, MOBI, EPUB, FB2, HTML, PRC, RTF, PDB, TXT, PDF Why does the PDF conversion lose some images/tables? ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/src/calibre/manual/server.rst b/src/calibre/manual/server.rst index 82ec5c2927..aa98ba57df 100644 --- a/src/calibre/manual/server.rst +++ b/src/calibre/manual/server.rst @@ -22,6 +22,8 @@ First start the |app| content server as shown below:: calibre-server --url-prefix /calibre --port 8080 +The key parameter here is ``--url-prefix /calibre``. This causes the content server to serve all URLs prefixed by calibre. To see this in action, visit ``http://localhost:8080/calibre`` in your browser. You should see the normal content server website, but now it will run under /calibre. + Now suppose you are using Apache as your main server. First enable the proxy modules in apache, by adding the following to :file:`httpd.conf`:: LoadModule proxy_module modules/mod_proxy.so diff --git a/src/calibre/manual/template_lang.rst b/src/calibre/manual/template_lang.rst index c6e29e3915..cdb8df2e2b 100644 --- a/src/calibre/manual/template_lang.rst +++ b/src/calibre/manual/template_lang.rst @@ -236,15 +236,16 @@ The following functions are available in addition to those described in single-f * ``format_date(x, date_format)`` -- format_date(val, format_string) -- format the value, which must be a date field, using the format_string, returning a string. The formatting codes are:: d : the day as number without a leading zero (1 to 31) - dd : the day as number with a leading zero (01 to 31) ' - ddd : the abbreviated localized day name (e.g. "Mon" to "Sun"). ' - dddd : the long localized day name (e.g. "Monday" to "Sunday"). ' - M : the month as number without a leading zero (1 to 12). ' - MM : the month as number with a leading zero (01 to 12) ' - MMM : the abbreviated localized month name (e.g. "Jan" to "Dec"). ' - MMMM : the long localized month name (e.g. "January" to "December"). ' - yy : the year as two digit number (00 to 99). ' - yyyy : the year as four digit number.' + dd : the day as number with a leading zero (01 to 31) + ddd : the abbreviated localized day name (e.g. "Mon" to "Sun"). + dddd : the long localized day name (e.g. "Monday" to "Sunday"). + M : the month as number without a leading zero (1 to 12). + MM : the month as number with a leading zero (01 to 12) + MMM : the abbreviated localized month name (e.g. "Jan" to "Dec"). + MMMM : the long localized month name (e.g. "January" to "December"). + yy : the year as two digit number (00 to 99). + yyyy : the year as four digit number. + iso : the date with time and timezone. Must be the only format present. * ``eval(string)`` -- evaluates the string as a program, passing the local variables (those ``assign`` ed to). This permits using the template processor to construct complex results from local variables. * ``multiply(x, y)`` -- returns x * y. Throws an exception if either x or y are not numbers. diff --git a/src/calibre/utils/date.py b/src/calibre/utils/date.py index 9b76a5a71a..c35e8ee2ab 100644 --- a/src/calibre/utils/date.py +++ b/src/calibre/utils/date.py @@ -142,6 +142,10 @@ def format_date(dt, format, assume_utc=False, as_utc=False): dt = dt.replace(tzinfo=_utc_tz if assume_utc else _local_tz) dt = dt.astimezone(_utc_tz if as_utc else _local_tz) + + if format == 'iso': + return isoformat(dt, assume_utc=assume_utc, as_utc=as_utc) + strf = partial(strftime, t=dt.timetuple()) def format_day(mo): diff --git a/src/calibre/utils/formatter_functions.py b/src/calibre/utils/formatter_functions.py index 015a639af1..7957bd0749 100644 --- a/src/calibre/utils/formatter_functions.py +++ b/src/calibre/utils/formatter_functions.py @@ -504,7 +504,8 @@ class BuiltinFormat_date(BuiltinFormatterFunction): 'MMM : the abbreviated localized month name (e.g. "Jan" to "Dec"). ' 'MMMM : the long localized month name (e.g. "January" to "December"). ' 'yy : the year as two digit number (00 to 99). ' - 'yyyy : the year as four digit number.') + 'yyyy : the year as four digit number. ' + 'iso : the date with time and timezone. Must be the only format present') def evaluate(self, formatter, kwargs, mi, locals, val, format_string): if not val: diff --git a/src/calibre/utils/search_query_parser.py b/src/calibre/utils/search_query_parser.py index a50ca20fc1..10d8b64a0d 100644 --- a/src/calibre/utils/search_query_parser.py +++ b/src/calibre/utils/search_query_parser.py @@ -19,8 +19,8 @@ If this module is run, it will perform a series of unit tests. import sys, string, operator from calibre.utils.pyparsing import CaselessKeyword, Group, Forward, \ - CharsNotIn, Suppress, OneOrMore, MatchFirst, CaselessLiteral, \ - Optional, NoMatch, ParseException, QuotedString + CharsNotIn, Suppress, OneOrMore, MatchFirst, alphas, alphanums, \ + Optional, ParseException, QuotedString, Word from calibre.constants import preferred_encoding from calibre.utils.icu import sort_key @@ -128,12 +128,9 @@ class SearchQueryParser(object): self._tests_failed = False self.optimize = optimize # Define a token - standard_locations = map(lambda x : CaselessLiteral(x)+Suppress(':'), - locations) - location = NoMatch() - for l in standard_locations: - location |= l - location = Optional(location, default='all') + self.standard_locations = locations + location = Optional(Word(alphas+'#', bodyChars=alphanums+'_')+Suppress(':'), + default='all') word_query = CharsNotIn(string.whitespace + '()') #quoted_query = Suppress('"')+CharsNotIn('"')+Suppress('"') quoted_query = QuotedString('"', escChar='\\') @@ -250,7 +247,14 @@ class SearchQueryParser(object): raise ParseException(query, len(query), 'undefined saved search', self) return self._get_matches(location, query, candidates) + def test_location_is_valid(self, location, query): + if location not in self.standard_locations: + raise ParseException(query, len(query), + _('No column exists with lookup name ') + location, self) + def _get_matches(self, location, query, candidates): + location = location.lower() + self.test_location_is_valid(location, query) if self.optimize: return self.get_matches(location, query, candidates=candidates) else: