From c1183e5aefac35f2484f5d46b69d4c138bb05413 Mon Sep 17 00:00:00 2001 From: GRiker Date: Fri, 12 Nov 2010 15:12:25 -0700 Subject: [PATCH 1/2] GwR wip catalog thumb caching --- src/calibre/library/catalog.py | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 4861c8c75c..21711fe3ff 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -16,6 +16,7 @@ from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString from calibre.ptempfile import PersistentTemporaryDirectory +from calibre.utils.config import config_dir from calibre.utils.date import isoformat, now as nowf from calibre.utils.logging import default_log as log @@ -862,6 +863,8 @@ class EPUB_MOBI(CatalogPlugin): self.__booksByDateRead = None self.__booksByTitle = None self.__booksByTitle_noSeriesPrefix = None + self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') + self.__archive_path = os.path.join(self.__cache_dir, "thumbs.zip") self.__catalogPath = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') self.__contentDir = os.path.join(self.catalogPath, "content") self.__currentStep = 0.0 @@ -902,6 +905,22 @@ class EPUB_MOBI(CatalogPlugin): self.__output_profile = profile break + if False: + # Confirm/create thumbs archive + if not os.path.exists(self.cache_dir): + if DEBUG: + self.log.info(" creating thumb cache '%s'" % self.__cache_dir) + os.makedirs(self.__cache_dir) + + if not os.path.exists(self.__archive_path): + self.log.info(" creating thumbnail archive") + zfw = ZipFile(self.archive_path, mode='w') + zfw.writestr("Catalog Thumbs Archive",'') + zfw.close() + else: + if DEBUG: + self.log.info(" existing thumb cache at '%s'" % self.archive_path) + # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX if self.opts.generate_titles: self.__totalSteps += 2 @@ -1723,14 +1742,18 @@ class EPUB_MOBI(CatalogPlugin): else: pubdateTag.insert(0,NavigableString('
')) - # Insert the rating + # Insert the rating, remove if unrated # Render different ratings chars for epub/mobi stars = int(title['rating']) / 2 - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) - ratingTag = body.find(attrs={'class':'rating'}) - ratingTag.insert(0,NavigableString('%s%s
' % (star_string,empty_stars))) + if stars: + star_string = self.FULL_RATING_SYMBOL * stars + empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + ratingTag.insert(0,NavigableString('%s%s
' % (star_string,empty_stars))) + else: + ratingLabel = body.find('td',text="Rating").replaceWith("Unrated") + ratingTag.insert(0,NavigableString('
')) + # Insert user notes or remove Notes label. Notes > 1 line will push formatting down if 'notes' in title: From 66aa330f125e9b3915fc9bc1c1087b5d8505972e Mon Sep 17 00:00:00 2001 From: GRiker Date: Sat, 13 Nov 2010 14:12:49 -0700 Subject: [PATCH 2/2] GwR catalog enhancements - Wishlist, improved description layout --- resources/catalog/stylesheet.css | 13 +- src/calibre/gui2/catalog/catalog_epub_mobi.py | 4 +- src/calibre/gui2/catalog/catalog_epub_mobi.ui | 30 ++- src/calibre/library/catalog.py | 235 +++++++++++------- 4 files changed, 183 insertions(+), 99 deletions(-) diff --git a/resources/catalog/stylesheet.css b/resources/catalog/stylesheet.css index c1f1a1c55d..f57aafdd44 100644 --- a/resources/catalog/stylesheet.css +++ b/resources/catalog/stylesheet.css @@ -81,7 +81,7 @@ p.unread_book { text-indent:-2em; } -p.missing_book { +p.wishlist_item { text-align:left; margin-top:0px; margin-bottom:0px; @@ -112,3 +112,14 @@ hr.annotations_divider { margin-top:0em; margin-bottom:0em; } + +td.publisher, td.date { + font-weight:bold; + text-align:center; + } +td.rating { + text-align: center; + } +td.thumbnail img { + -webkit-box-shadow: 6px 6px 6px #888; + } \ No newline at end of file diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 1f0fdae7e2..ea4edb10b9 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -23,7 +23,9 @@ class PluginWidget(QWidget,Ui_Form): ('generate_recently_added', True), ('note_tag','*'), ('numbers_as_text', False), - ('read_tag','+')] + ('read_tag','+'), + ('wishlist_tag','Wishlist'), + ] # Output synced to the connected device? diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.ui b/src/calibre/gui2/catalog/catalog_epub_mobi.ui index fa6b53e3a4..3956886c4a 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.ui +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.ui @@ -42,28 +42,28 @@ - + Additional note tag prefix: - + - + - + Regex pattern describing tags to exclude as genres: @@ -76,7 +76,7 @@ - + Regex tips: @@ -88,7 +88,7 @@ - + Qt::Vertical @@ -101,34 +101,44 @@ - + Include 'Titles' Section - + Include 'Recently Added' Section - + Sort numbers as text - + Include 'Series' Section + + + + + + + Wishlist tag: + + + diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 21711fe3ff..270311b7a8 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -3,14 +3,14 @@ __license__ = 'GPL v3' __copyright__ = '2010, Greg Riker ' -import datetime, htmlentitydefs, os, re, shutil, codecs +import base64, codecs, cStringIO, datetime, htmlentitydefs, os, re, shutil, time, zlib from collections import namedtuple from copy import deepcopy - +from PIL import Image as PILImage from xml.sax.saxutils import escape -from calibre import prints, prepare_string_for_xml, strftime +from calibre import fit_image, prints, prepare_string_for_xml, strftime from calibre.constants import preferred_encoding from calibre.customize import CatalogPlugin from calibre.customize.conversion import OptionRecommendation, DummyReporter @@ -19,6 +19,7 @@ from calibre.ptempfile import PersistentTemporaryDirectory from calibre.utils.config import config_dir from calibre.utils.date import isoformat, now as nowf from calibre.utils.logging import default_log as log +from calibre.utils.zipfile import ZipFile, ZipInfo FIELDS = ['all', 'author_sort', 'authors', 'comments', 'cover', 'formats', 'id', 'isbn', 'ondevice', 'pubdate', 'publisher', 'rating', @@ -609,6 +610,12 @@ class EPUB_MOBI(CatalogPlugin): action = None, help=_("Tag indicating book has been read.\n" "Default: '%default'\n" "Applies to: ePub, MOBI output formats")), + Option('--wishlist-tag', + default='Wishlist', + dest='wishlist_tag', + action = None, + help=_("Tag indicating book to be displayed as wishlist item.\n" "Default: '%default'\n" + "Applies to: ePub, MOBI output formats")), ] class NumberToText(object): @@ -905,21 +912,17 @@ class EPUB_MOBI(CatalogPlugin): self.__output_profile = profile break - if False: - # Confirm/create thumbs archive - if not os.path.exists(self.cache_dir): - if DEBUG: - self.log.info(" creating thumb cache '%s'" % self.__cache_dir) - os.makedirs(self.__cache_dir) - - if not os.path.exists(self.__archive_path): - self.log.info(" creating thumbnail archive") - zfw = ZipFile(self.archive_path, mode='w') - zfw.writestr("Catalog Thumbs Archive",'') - zfw.close() - else: - if DEBUG: - self.log.info(" existing thumb cache at '%s'" % self.archive_path) + # Confirm/create thumbs archive + if not os.path.exists(self.__cache_dir): + self.opts.log.info(" creating new thumb cache '%s'" % self.__cache_dir) + os.makedirs(self.__cache_dir) + if not os.path.exists(self.__archive_path): + self.opts.log.info(" creating thumbnail archive") + zfw = ZipFile(self.__archive_path, mode='w') + zfw.writestr("Catalog Thumbs Archive",'') + zfw.close() + else: + self.opts.log.info(" existing thumb cache at '%s'" % self.__archive_path) # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX if self.opts.generate_titles: @@ -1341,6 +1344,7 @@ class EPUB_MOBI(CatalogPlugin): this_title = {} this_title['id'] = record['id'] + this_title['uuid'] = record['uuid'] this_title['title'] = self.convertHTMLEntities(record['title']) if record['series']: @@ -1654,15 +1658,15 @@ class EPUB_MOBI(CatalogPlugin): aTag.insert(0, title['author']) # Prefix author with read|reading|none symbol or missing symbol - if 'formats' in title and title['formats']: + if self.opts.wishlist_tag in title['tags']: + authorTag.insert(0, NavigableString(self.MISSING_SYMBOL + " by ")) + else: if title['read']: authorTag.insert(0, NavigableString(self.READ_SYMBOL + " by ")) elif self.opts.connected_kindle and title['id'] in self.bookmarked_books: authorTag.insert(0, NavigableString(self.READING_SYMBOL + " by ")) else: authorTag.insert(0, NavigableString(self.NOT_READ_SYMBOL + " by ")) - else: - authorTag.insert(0, NavigableString(self.MISSING_SYMBOL + " by ")) authorTag.insert(1, aTag) ''' @@ -1751,7 +1755,7 @@ class EPUB_MOBI(CatalogPlugin): empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) ratingTag.insert(0,NavigableString('%s%s
' % (star_string,empty_stars))) else: - ratingLabel = body.find('td',text="Rating").replaceWith("Unrated") + #ratingLabel = body.find('td',text="Rating").replaceWith("Unrated") ratingTag.insert(0,NavigableString('
')) @@ -1760,10 +1764,11 @@ class EPUB_MOBI(CatalogPlugin): notesTag = body.find(attrs={'class':'notes'}) notesTag.insert(0,NavigableString(title['notes'] + '
')) else: - notes_labelTag = body.find(attrs={'class':'notes_label'}) - empty_labelTag = Tag(soup, "td") - empty_labelTag.insert(0,NavigableString('
')) - notes_labelTag.replaceWith(empty_labelTag) + pass +# notes_labelTag = body.find(attrs={'class':'notes_label'}) +# empty_labelTag = Tag(soup, "td") +# empty_labelTag.insert(0,NavigableString('
')) +# notes_labelTag.replaceWith(empty_labelTag) # Insert the blurb if 'description' in title and title['description'] > '': @@ -1853,8 +1858,12 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # book with read|reading|unread symbol or missing symbol - if 'formats' in book and book['formats']: + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book['tags']: + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: if book['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) @@ -1869,11 +1878,6 @@ class EPUB_MOBI(CatalogPlugin): pBookTag['class'] = "unread_book" pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) ptc += 1 - else: - # missing formats - pBookTag['class'] = "missing_book" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 # Link to book aTag = Tag(soup, "a") @@ -2028,8 +2032,12 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # book with read|reading|unread symbol or missing symbol - if 'formats' in book and book['formats']: + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book['tags']: + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: if book['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) @@ -2044,11 +2052,6 @@ class EPUB_MOBI(CatalogPlugin): pBookTag['class'] = "unread_book" pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) ptc += 1 - else: - # missing book - pBookTag['class'] = "missing_book" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 aTag = Tag(soup, "a") aTag['href'] = "book_%d.html" % (int(float(book['id']))) @@ -2162,8 +2165,12 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # book with read|reading|unread symbol or missing symbol - if 'formats' in new_entry and new_entry['formats']: + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in new_entry['tags']: + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: if new_entry['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) @@ -2178,11 +2185,6 @@ class EPUB_MOBI(CatalogPlugin): pBookTag['class'] = "unread_book" pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) ptc += 1 - else: - # missing book - pBookTag['class'] = "missing_book" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 aTag = Tag(soup, "a") aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) @@ -2214,8 +2216,12 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # book with read|reading|unread symbol or missing symbol - if 'formats' in new_entry and new_entry['formats']: + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in new_entry['tags']: + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: if new_entry['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) @@ -2230,11 +2236,6 @@ class EPUB_MOBI(CatalogPlugin): pBookTag['class'] = "unread_book" pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) ptc += 1 - else: - # missing book - pBookTag['class'] = "missing_book" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 aTag = Tag(soup, "a") aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) @@ -2669,8 +2670,12 @@ class EPUB_MOBI(CatalogPlugin): else: book['read'] = False - # book with read|reading|unread symbol or missing symbol - if 'formats' in book and book['formats']: + # book with read|reading|unread symbol or wishlist item + if self.opts.wishlist_tag in book['tags']: + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: if book['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) @@ -2685,11 +2690,6 @@ class EPUB_MOBI(CatalogPlugin): pBookTag['class'] = "unread_book" pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) ptc += 1 - else: - # missing book - pBookTag['class'] = "missing_book" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 aTag = Tag(soup, "a") aTag['href'] = "book_%d.html" % (int(float(book['id']))) @@ -2767,8 +2767,7 @@ class EPUB_MOBI(CatalogPlugin): this_book['title'] = book['title'] this_book['author_sort'] = book['author_sort'].capitalize() this_book['read'] = book['read'] - if 'formats' in book: - this_book['formats'] = book['formats'] + this_book['tags'] = book['tags'] this_book['id'] = book['id'] this_book['series'] = book['series'] normalized_tag = self.genre_tags_dict[friendly_tag] @@ -4170,8 +4169,15 @@ class EPUB_MOBI(CatalogPlugin): pBookTag = Tag(soup, "p") ptc = 0 - # book with read|reading|unread symbol or missing symbol - if 'formats' in book and book['formats']: + # book with read|reading|unread symbol or wishlist item + # If this is the wishlist_tag genre, don't show missing symbols + # normalized_wishlist_tag = self.genre_tags_dict[self.opts.wishlist_tag] + if self.opts.wishlist_tag in book['tags'] and \ + self.genre_tags_dict[self.opts.wishlist_tag] != genre: + pBookTag['class'] = "wishlist_item" + pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) + ptc += 1 + else: if book['read']: # check mark pBookTag.insert(ptc,NavigableString(self.READ_SYMBOL)) @@ -4186,11 +4192,6 @@ class EPUB_MOBI(CatalogPlugin): pBookTag['class'] = "unread_book" pBookTag.insert(ptc,NavigableString(self.NOT_READ_SYMBOL)) ptc += 1 - else: - # missing book - pBookTag['class'] = "missing_book" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 # Add the book title aTag = Tag(soup, "a") @@ -4240,31 +4241,31 @@ class EPUB_MOBI(CatalogPlugin): - + - + - + - + - + - + - +
   
   
Publisher
Published
Rating
Notes
   
@@ -4452,15 +4453,74 @@ class EPUB_MOBI(CatalogPlugin): return ' '.join(translated) def generateThumbnail(self, title, image_dir, thumb_file): - from calibre.utils.magick import Image - try: - img = Image() - img.open(title['cover']) - # img, width, height - img.thumbnail(self.thumbWidth, self.thumbHeight) - img.save(os.path.join(image_dir, thumb_file)) - except: - self.opts.log.error("generateThumbnail(): Error with %s" % title['title']) + ''' + Thumbs are cached with the full cover's crc. If the crc doesn't + match, the cover has been changed since the thumb was cached and needs + to be replaced. + 2010-11-13 switched to PIL library for faster thumb generation + + ''' + + if True: + # Generate crc for current cover + #self.opts.log.info(" generateThumbnail():") + data = open(title['cover'], 'rb') + stream = cStringIO.StringIO() + stream.write(data.read()) + cover_crc = hex(zlib.crc32(stream.getvalue())) + stream.close() + + # Test cache for uuid + try: + zfr = ZipFile(self.__archive_path, mode='r') + t_info = zfr.getinfo(title['uuid']) + if t_info.comment == cover_crc: + # uuid found in cache with matching crc + thumb_data = zfr.read(title['uuid']) + zfr.extract(title['uuid'],image_dir) + zfr.close() + os.rename(os.path.join(image_dir,title['uuid']), + os.path.join(image_dir,thumb_file)) + return + else: + # uuid found in cache, but crc mismatch + zfr.close() + + except: + # uuid not found in cache + zfr.close() + + # Create a new thumb + im = PILImage.open(title['cover']) + scaled, width, height = fit_image(im.size[0],im.size[1], + self.thumbWidth, self.thumbHeight) + im = im.resize((int(width),int(height)), PILImage.ANTIALIAS) + + # Save thumb for catalog + im.convert('RGB').save(os.path.join(image_dir, thumb_file),'JPEG') + + # Save thumb to archive + thumb_stream = cStringIO.StringIO() + im.convert('RGB').save(thumb_stream,'JPEG') + thumb_data = thumb_stream.getvalue() + thumb_stream.close() + t_info = ZipInfo(title['uuid'],time.localtime()[0:6]) + t_info.comment = cover_crc + zfw = ZipFile(self.__archive_path, mode='a') + zfw.writestr(t_info, thumb_data) + zfw.close() + + else: + # non-caching code using ImageMagick + from calibre.utils.magick import Image + try: + img = Image() + img.open(title['cover']) + # img, width, height + img.thumbnail(self.thumbWidth, self.thumbHeight) + img.save(os.path.join(image_dir, thumb_file)) + except: + self.opts.log.error("generateThumbnail(): Error with %s" % title['title']) def getFriendlyGenreTag(self, genre): # Find the first instance of friendly_tag matching genre @@ -4714,7 +4774,8 @@ class EPUB_MOBI(CatalogPlugin): if key in ['catalog_title','authorClip','connected_kindle','descriptionClip', 'exclude_genre','exclude_tags','note_tag','numbers_as_text', 'output_profile','read_tag', - 'search_text','sort_by','sort_descriptions_by_author','sync']: + 'search_text','sort_by','sort_descriptions_by_author','sync', + 'wishlist_tag']: build_log.append(" %s: %s" % (key, opts_dict[key])) if opts.verbose: