From 9013a5d97dc73e6f3ea416b6b30a2cdf3522fcc0 Mon Sep 17 00:00:00 2001 From: GRiker Date: Mon, 17 Jan 2011 08:53:18 -0700 Subject: [PATCH 01/10] GwR catalog 1.0 revisions --- resources/catalog/section_list_templates.py | 39 ++ src/calibre/gui2/actions/catalog.py | 18 +- src/calibre/library/catalog.py | 550 ++++++++------------ 3 files changed, 279 insertions(+), 328 deletions(-) create mode 100644 resources/catalog/section_list_templates.py diff --git a/resources/catalog/section_list_templates.py b/resources/catalog/section_list_templates.py new file mode 100644 index 0000000000..de73147fcf --- /dev/null +++ b/resources/catalog/section_list_templates.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +# vim:fileencoding=UTF-8:ts=4:sw=4:sta:et:sts=4:ai + +__license__ = 'GPL v3' +__copyright__ = '2010, Kovid Goyal ' +__docformat__ = 'restructuredtext en' + +''' + Available fields: + {title} Title of the book + {series} Series name + {series_index} Number of the book in the series + {rating} Rating + {rating_parens} Rating, in parentheses + {pubyear} Year the book was published + {pubyear_parens} Year the book was published, in parentheses +''' +# Books by Author +by_authors_normal_title_template = '{title} {pubyear_parens}' +by_authors_series_title_template = '[{series_index}] {title} {pubyear_parens}' + +# Books by Title +by_titles_normal_title_template = '{title}' +by_titles_series_title_template = '{title} ({series} [{series_index}])' + +# Books by Series +by_series_title_template = '[{series_index}] {title} {pubyear_parens}' + +# Books by Genre +by_genres_normal_title_template = '{title} {pubyear_parens}' +by_genres_series_title_template = '{series_index}. {title} {pubyear_parens}' + +# Recently Added +by_recently_added_normal_title_template = '{title}' +by_recently_added_series_title_template = '{title} ({series} [{series_index}])' + +# By Month added +by_month_added_normal_title_template = '{title} {pubyear_parens}' +by_month_added_series_title_template = '[{series_index}] {title} {pubyear_parens}' \ No newline at end of file diff --git a/src/calibre/gui2/actions/catalog.py b/src/calibre/gui2/actions/catalog.py index 6d3bb539a2..1650c80d70 100644 --- a/src/calibre/gui2/actions/catalog.py +++ b/src/calibre/gui2/actions/catalog.py @@ -5,11 +5,11 @@ __license__ = 'GPL v3' __copyright__ = '2010, Kovid Goyal ' __docformat__ = 'restructuredtext en' -import os, shutil +import re, os, shutil from PyQt4.Qt import QModelIndex -from calibre.gui2 import error_dialog, choose_dir +from calibre.gui2 import choose_dir, error_dialog, info_dialog, warning_dialog from calibre.gui2.tools import generate_catalog from calibre.utils.config import dynamic from calibre.gui2.actions import InterfaceAction @@ -55,10 +55,16 @@ class GenerateCatalogAction(InterfaceAction): def catalog_generated(self, job): if job.result: - # Error during catalog generation - return error_dialog(self.gui, _('Catalog generation terminated'), - job.result, - show=True) + # Problems during catalog generation + dialog_title = job.result.pop(0) + if re.match('warning:', job.result[0].lower()): + job.result.append("Catalog generation complete.") + warning_dialog(self.gui, dialog_title, '\n'.join(job.result), show=True) + else: + job.result.append("Catalog generation terminated.") + error_dialog(self.gui, dialog_title,'\n'.join(job.result),show=True) + return + if job.failed: return self.gui.job_exception(job) id = self.gui.library_view.model().add_catalog(job.catalog_file_path, job.catalog_title) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 087d40c4eb..f1c5e3ae65 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -546,9 +546,9 @@ class EPUB_MOBI(CatalogPlugin): name = 'Catalog_EPUB_MOBI' description = 'EPUB/MOBI catalog generator' supported_platforms = ['windows', 'osx', 'linux'] - minimum_calibre_version = (0, 6, 34) + minimum_calibre_version = (0, 7, 40) author = 'Greg Riker' - version = (0, 0, 1) + version = (1, 0, 0) file_types = set(['epub','mobi']) THUMB_SMALLEST = "1.0" @@ -900,15 +900,7 @@ class EPUB_MOBI(CatalogPlugin): ''' Generates catalog source files from calibre database - Implementation notes - - 'Marker tags' in a book's metadata are used to flag special conditions: - (Defaults) - '~' : Do not catalog this book - '+' : Mark this book as read (check mark) in lists - '*' : Display trailing text as 'Note: ' in top frame next to cover - '[] : Source of content (e.g., Amazon, Project Gutenberg). Do not create genre - - - Program flow + Flow of control: gui2.actions.catalog:generate_catalog() gui2.tools:generate_catalog() or library.cli:command_catalog() called from gui2.convert.gui_conversion:gui_catalog() @@ -953,7 +945,7 @@ class EPUB_MOBI(CatalogPlugin): self.__creator = opts.creator self.__db = db self.__descriptionClip = opts.descriptionClip - self.__error = None + self.__error = [] self.__generateForKindle = True if (self.opts.fmt == 'mobi' and \ self.opts.output_profile and \ self.opts.output_profile.startswith("kindle")) else False @@ -1033,6 +1025,22 @@ class EPUB_MOBI(CatalogPlugin): # +1 thumbs self.__totalSteps += 3 + # Load section list templates + templates = ['by_authors_normal_title_template', + 'by_authors_series_title_template', + 'by_titles_normal_title_template', + 'by_titles_series_title_template', + 'by_series_title_template', + 'by_genres_normal_title_template', + 'by_genres_series_title_template', + 'by_recently_added_normal_title_template', + 'by_recently_added_series_title_template', + 'by_month_added_normal_title_template', + 'by_month_added_series_title_template'] + execfile(P(os.path.join('catalog','section_list_templates.py')),locals()) + for t in templates: + setattr(self,t,eval(t)) + # Accessors if True: ''' @@ -1420,26 +1428,12 @@ class EPUB_MOBI(CatalogPlugin): ''' self.updateProgressFullStep("Sorting database") - - ''' - # Sort titles case-insensitive, by author - self.booksByAuthor = sorted(self.booksByTitle, - key=lambda x:(x['author_sort'].upper(), x['author_sort'].upper())) - ''' - self.booksByAuthor = list(self.booksByTitle) - self.booksByAuthor.sort(self.author_compare) - - if False and self.verbose: - self.opts.log.info("fetchBooksByAuthor(): %d books" % len(self.booksByAuthor)) - self.opts.log.info(" %-30s %-20s %s" % ('title', 'series', 'series_index')) - for title in self.booksByAuthor: - self.opts.log.info((u" %-30s %-20s%5s " % \ - (title['title'][:30], - title['series'][:20] if title['series'] else '', - title['series_index'], - )).encode('utf-8')) - raise SystemExit + self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) +# for book in self.booksByAuthor: +# print "{0:<30} {1:<30} {2:<30}".format(book['title'],book['author'],book['author_sort']) +# print +# stop # Build the unique_authors set from existing data authors = [(record['author'], record['author_sort'].capitalize()) for record in self.booksByAuthor] @@ -1457,16 +1451,17 @@ class EPUB_MOBI(CatalogPlugin): multiple_authors = True if author != current_author and i: - # Warn, exit if friendly matches previous, but sort doesn't + # Exit if author matches previous, but author_sort doesn't match if author[0] == current_author[0]: error_msg = _(''' -\n*** Metadata error *** -Inconsistent Author Sort values for Author '{0}', unable to continue building catalog. +Inconsistent Author Sort values for Author '{0}', unable to continue building catalog.\n Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, then rebuild the catalog.\n''').format(author[0]) - + self.opts.log.warn('\n*** Metadata error ***') self.opts.log.warn(error_msg) - self.error = error_msg + + self.error.append('Metadata error') + self.error.append(error_msg) return False # New author, save the previous author/sort/count @@ -1496,16 +1491,8 @@ then rebuild the catalog.\n''').format(author[0]) return True def fetchBooksByTitle(self): - self.updateProgressFullStep("Fetching database") - # Get the database as a dictionary - # Sort by title - # Search is a string like this: - # not tag: author:"Riker" - # So we need to merge opts.exclude_tag with opts.search_text - # not tag:"~" author:"Riker" - self.opts.sort_by = 'title' # Merge opts.exclude_tags with opts.search_text @@ -1528,7 +1515,6 @@ then rebuild the catalog.\n''').format(author[0]) else: self.opts.search_text = search_phrase - #print "fetchBooksByTitle(): opts.search_text: %s" % self.opts.search_text # Fetch the database as a dictionary data = self.plugin.search_sort_db(self.db, self.opts) data = self.processExclusions(data) @@ -1536,8 +1522,6 @@ then rebuild the catalog.\n''').format(author[0]) # Populate this_title{} from data[{},{}] titles = [] for record in data: - if False: - print "available record metadata:\n%s" % sorted(record.keys()) this_title = {} this_title['id'] = record['id'] @@ -1547,7 +1531,6 @@ then rebuild the catalog.\n''').format(author[0]) if record['series']: this_title['series'] = record['series'] this_title['series_index'] = record['series_index'] - this_title['title'] = self.generateSeriesTitle(this_title) else: this_title['series'] = None this_title['series_index'] = 0.0 @@ -1572,7 +1555,12 @@ then rebuild the catalog.\n''').format(author[0]) this_title['publisher'] = re.sub('&', '&', record['publisher']) this_title['rating'] = record['rating'] if record['rating'] else 0 - this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) + + if re.match('0100-01-01',str(record['pubdate'].date())): + this_title['date'] = None + else: + this_title['date'] = strftime(u'%B %Y', record['pubdate'].timetuple()) + this_title['timestamp'] = record['timestamp'] if record['comments']: @@ -1646,7 +1634,7 @@ then rebuild the catalog.\n''').format(author[0]) title['title_sort'][0:40])).decode('mac-roman')) return True else: - self.error = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.") + self.error.append( _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.")) return False def fetchBookmarks(self): @@ -1748,13 +1736,12 @@ then rebuild the catalog.\n''').format(author[0]) self.bookmarked_books = {} def generateHTMLDescriptions(self): - # Write each title to a separate HTML file in contentdir + ''' + Write each title to a separate HTML file in contentdir + ''' self.updateProgressFullStep("'Descriptions'") for (title_num, title) in enumerate(self.booksByTitle): - if False: - self.opts.log.info("%3s: %s - %s" % (title['id'], title['title'], title['author'])) - self.updateProgressMicroStep("Description %d of %d" % \ (title_num, len(self.booksByTitle)), float(title_num*100/len(self.booksByTitle))/100) @@ -1768,8 +1755,9 @@ then rebuild the catalog.\n''').format(author[0]) outfile.close() def generateHTMLByTitle(self): - # Write books by title A-Z to HTML file - + ''' + Write books by title A-Z to HTML file + ''' self.updateProgressFullStep("'Titles'") soup = self.generateHTMLEmptyHeader("Books By Alpha Title") @@ -1807,22 +1795,11 @@ then rebuild the catalog.\n''').format(author[0]) current_letter = "" # Re-sort title list without leading series/series_index + # Incoming title : if not self.useSeriesPrefixInTitlesSection: nspt = deepcopy(self.booksByTitle) - for book in nspt: - if book['series']: - tokens = book['title'].partition(':') - book['title'] = '%s (%s)' % (tokens[2].strip(), tokens[0]) - book['title_sort'] = self.generateSortTitle(book['title']) - nspt = sorted(nspt, - key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) + nspt = sorted(nspt, key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) self.booksByTitle_noSeriesPrefix = nspt - if False and self.verbose: - self.opts.log.info("no_series_prefix_titles: %d books" % len(nspt)) - self.opts.log.info(" %-40s %-40s" % ('title', 'title_sort')) - for title in nspt: - self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], - title['title_sort'][0:40])).encode('utf-8')) # Loop through the books by title title_list = self.booksByTitle @@ -1878,7 +1855,14 @@ then rebuild the catalog.\n''').format(author[0]) aTag = Tag(soup, "a") if self.opts.generate_descriptions: aTag['href'] = "book_%d.html" % (int(float(book['id']))) - aTag.insert(0,escape(book['title'])) + + # Generate the title from the template + args = self.generateFormatArgs(book) + if book['series']: + formatted_title = self.by_titles_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) pBookTag.insert(ptc, aTag) ptc += 1 @@ -1916,7 +1900,9 @@ then rebuild the catalog.\n''').format(author[0]) self.htmlFileList_1.append("content/ByAlphaTitle.html") def generateHTMLByAuthor(self): - # Write books by author A-Z + ''' + Write books by author A-Z + ''' self.updateProgressFullStep("'Authors'") friendly_name = "Authors" @@ -1953,7 +1939,8 @@ then rebuild the catalog.\n''').format(author[0]) current_author = '' current_letter = '' current_series = None - for book in self.booksByAuthor: + for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): + book_count += 1 if self.letter_or_symbol(book['author_sort'][0].upper()) != current_letter : # Start a new letter with Index letter @@ -2067,14 +2054,18 @@ then rebuild the catalog.\n''').format(author[0]) aTag = Tag(soup, "a") if self.opts.generate_descriptions: aTag['href'] = "book_%d.html" % (int(float(book['id']))) - # Use series, series index if avail else title, + year of publication + + # Generate the title from the template + args = self.generateFormatArgs(book) if current_series: - aTag.insert(0,'%s (%s)' % (escape(book['title'][len(book['series'])+1:]), - book['date'].split()[1])) + #aTag.insert(0,'%s%s' % (escape(book['title'][len(book['series'])+1:]),pubyear)) + formatted_title = self.by_authors_series_title_template.format(**args).rstrip() else: - aTag.insert(0,'%s (%s)' % (escape(book['title']), - book['date'].split()[1])) + #aTag.insert(0,'%s%s' % (escape(book['title']), pubyear)) + formatted_title = self.by_authors_normal_title_template.format(**args).rstrip() non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) ptc += 1 @@ -2111,7 +2102,6 @@ then rebuild the catalog.\n''').format(author[0]) # Add the divTag to the body body.insert(btc, divTag) - # Write the generated file to contentdir outfile_spec = "%s/ByAlphaAuthor.html" % (self.contentDir) outfile = open(outfile_spec, 'w') @@ -2120,13 +2110,15 @@ then rebuild the catalog.\n''').format(author[0]) self.htmlFileList_1.append("content/ByAlphaAuthor.html") def generateHTMLByDateAdded(self): - # Write books by reverse chronological order + ''' + Write books by reverse chronological order + ''' self.updateProgressFullStep("'Recently Added'") def add_books_to_HTML_by_month(this_months_list, dtc): if len(this_months_list): - this_months_list.sort(self.author_compare) + this_months_list = sorted(this_months_list, key=self.booksByAuthorSorter_author_sort) # Create a new month anchor date_string = strftime(u'%B %Y', current_date.timetuple()) @@ -2156,16 +2148,6 @@ then rebuild the catalog.\n''').format(author[0]) divTag.insert(dtc,pAuthorTag) dtc += 1 - ''' - # Insert an <hr /> between non-series and series - if not current_series and non_series_books and new_entry['series']: - # Insert an <hr /> - hrTag = Tag(soup,'hr') - hrTag['class'] = "series_divider" - divTag.insert(dtc,hrTag) - dtc += 1 - ''' - # Check for series if new_entry['series'] and new_entry['series'] != current_series: # Start a new series @@ -2213,11 +2195,15 @@ then rebuild the catalog.\n''').format(author[0]) aTag = Tag(soup, "a") if self.opts.generate_descriptions: aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) + + # Generate the title from the template + args = self.generateFormatArgs(new_entry) if current_series: - aTag.insert(0,escape(new_entry['title'][len(new_entry['series'])+1:])) + formatted_title = self.by_month_added_series_title_template.format(**args).rstrip() else: - aTag.insert(0,escape(new_entry['title'])) + formatted_title = self.by_month_added_normal_title_template.format(**args).rstrip() non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) pBookTag.insert(ptc, aTag) ptc += 1 @@ -2265,7 +2251,14 @@ then rebuild the catalog.\n''').format(author[0]) aTag = Tag(soup, "a") if self.opts.generate_descriptions: aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) - aTag.insert(0,escape(new_entry['title'])) + + # Generate the title from the template + args = self.generateFormatArgs(new_entry) + if new_entry['series']: + formatted_title = self.by_recently_added_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_recently_added_normal_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) pBookTag.insert(ptc, aTag) ptc += 1 @@ -2323,17 +2316,12 @@ then rebuild the catalog.\n''').format(author[0]) divTag = Tag(soup, "div") dtc = 0 - # Add books by date range + # >>> Books by date range <<< if self.useSeriesPrefixInTitlesSection: self.booksByDateRange = sorted(self.booksByTitle, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) else: nspt = deepcopy(self.booksByTitle) - for book in nspt: - if book['series']: - tokens = book['title'].partition(':') - book['title'] = '%s (%s)' % (tokens[2].strip(), tokens[0]) - book['title_sort'] = self.generateSortTitle(book['title']) self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) date_range_list = [] @@ -2356,15 +2344,6 @@ then rebuild the catalog.\n''').format(author[0]) dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) date_range_list = [book] - ''' - if books_added_in_date_range: - # Add an <hr> separating date ranges from months - hrTag = Tag(soup,'hr') - hrTag['class'] = "description_divider" - divTag.insert(dtc,hrTag) - dtc += 1 - ''' - # >>>> Books by month <<<< # Sort titles case-insensitive for by month using series prefix self.booksByMonth = sorted(self.booksByTitle, @@ -2395,7 +2374,9 @@ then rebuild the catalog.\n''').format(author[0]) self.htmlFileList_2.append("content/ByDateAdded.html") def generateHTMLByDateRead(self): - # Write books by active bookmarks + ''' + Write books by active bookmarks + ''' friendly_name = 'Recently Read' self.updateProgressFullStep("'%s'" % friendly_name) if not self.bookmarked_books: @@ -2533,32 +2514,6 @@ then rebuild the catalog.\n''').format(author[0]) self.booksByDateRead = sorted(bookmarked_books, key=lambda x:(x['bookmark_timestamp'], x['bookmark_timestamp']),reverse=True) - ''' - # >>>> Recently by date range <<<< - date_range_list = [] - today_time = datetime.datetime.utcnow() - today_time.replace(hour=23, minute=59, second=59) - books_added_in_date_range = False - for (i, date) in enumerate(self.DATE_RANGE): - date_range_limit = self.DATE_RANGE[i] - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - delta = today_time-bookmark_time - if delta.days <= date_range_limit: - date_range_list.append(book) - books_added_in_date_range = True - else: - break - - dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) - date_range_list = [book] - ''' - # >>>> Recently read by day <<<< current_date = datetime.date.fromordinal(1) todays_list = [] @@ -2713,10 +2668,15 @@ then rebuild the catalog.\n''').format(author[0]) # Use series, series index if avail else just title #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) - # Link to book - aTag.insert(0,'%d. %s (%s)' % (book['series_index'], - escape(book['title']), - strftime(u'%Y', book['pubdate'].timetuple()))) + # Reassert 'date' since this is the result of a new search + if re.match('0100-01-01',str(book['pubdate'].date())): + book['date'] = None + else: + book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) + + args = self.generateFormatArgs(book) + formatted_title = self.by_series_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) pBookTag.insert(ptc, aTag) ptc += 1 @@ -2760,10 +2720,11 @@ then rebuild the catalog.\n''').format(author[0]) self.htmlFileList_1.append("content/BySeries.html") def generateHTMLByTags(self): - # Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... - # Note that special tags - ~+*[] - have already been filtered from books[] - # There may be synonomous tags - + ''' + Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... + Note that special tags - have already been filtered from books[] + There may be synonomous tags + ''' self.updateProgressFullStep("'Genres'") self.genre_tags_dict = self.filterDbTags(self.db.all_tags()) @@ -2787,6 +2748,8 @@ then rebuild the catalog.\n''').format(author[0]) this_book['tags'] = book['tags'] this_book['id'] = book['id'] this_book['series'] = book['series'] + this_book['series_index'] = book['series_index'] + this_book['date'] = book['date'] normalized_tag = self.genre_tags_dict[friendly_tag] genre_tag_list = [key for genre in genre_list for key in genre] if normalized_tag in genre_tag_list: @@ -2843,13 +2806,7 @@ then rebuild the catalog.\n''').format(author[0]) unique_authors.append((current_author[0], current_author[1], books_by_current_author)) else: books_by_current_author += 1 - ''' - # Extract the unique entries - unique_authors = [] - for author in authors: - if not author in unique_authors: - unique_authors.append(author) - ''' + # Write the genre book list as an article titles_spanned = self.generateHTMLByGenre(genre, True if index==0 else False, genre_tag_set[genre], @@ -2863,18 +2820,14 @@ then rebuild the catalog.\n''').format(author[0]) 'books':genre_tag_set[genre], 'titles_spanned':titles_spanned}) - if False and self.opts.verbose: - for genre in master_genre_list: - print "genre['tag']: %s" % genre['tag'] - for book in genre['books']: - print book['title'] self.genres = master_genre_list def generateThumbnails(self): - # Generate a thumbnail per cover. If a current thumbnail exists, skip - # If a cover doesn't exist, use default - # Return list of active thumbs - + ''' + Generate a thumbnail per cover. If a current thumbnail exists, skip + If a cover doesn't exist, use default + Return list of active thumbs + ''' self.updateProgressFullStep("'Thumbnails'") thumbs = ['thumbnail_default.jpg'] image_dir = "%s/images" % self.catalogPath @@ -2886,45 +2839,51 @@ then rebuild the catalog.\n''').format(author[0]) thumb_file = 'thumbnail_%d.jpg' % int(title['id']) thumb_generated = True + valid_cover = True try: - self.generateThumbnail(title, image_dir, thumb_file) thumbs.append("thumbnail_%d.jpg" % int(title['id'])) + self.generateThumbnail(title, image_dir, thumb_file) except: + if 'cover' in title and os.path.exists(title['cover']): + valid_cover = False + self.opts.log.warn(" *** Invalid cover file for '%s' ***" % (title['title'])) + if not self.error: + self.error.append('Invalid cover files') + self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) thumb_generated = False - if not thumb_generated: - # Use default cover - if False and self.verbose: - self.opts.log.warn(" using default cover for '%s'" % \ - (title['title'])) + self.opts.log.warn(" using default cover for '%s'" % (title['title'])) # Check to make sure default is current # Check to see if thumbnail exists - thumb_fp = "%s/thumbnail_default.jpg" % (image_dir) - cover = "%s/DefaultCover.png" % (self.catalogPath) + default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") + cover = os.path.join(self.catalogPath, "DefaultCover.png") if not os.path.exists(cover): shutil.copyfile(I('book.png'), cover) - if os.path.isfile(thumb_fp): + if os.path.isfile(default_thumb_fp): # Check to see if default cover is newer than thumbnail # os.path.getmtime() = modified time # os.path.ctime() = creation time cover_timestamp = os.path.getmtime(cover) - thumb_timestamp = os.path.getmtime(thumb_fp) + thumb_timestamp = os.path.getmtime(default_thumb_fp) if thumb_timestamp < cover_timestamp: if False and self.verbose: self.opts.log.warn("updating thumbnail_default for %s" % title['title']) - #title['cover'] = "%s/DefaultCover.jpg" % self.catalogPath + #title['cover'] = os.path.join(self.catalogPath,"DefaultCover.jpg") title['cover'] = cover - self.generateThumbnail(title, image_dir, "thumbnail_default.jpg") + self.generateThumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) else: if False and self.verbose: self.opts.log.warn(" generating new thumbnail_default.jpg") - #title['cover'] = "%s/DefaultCover.jpg" % self.catalogPath + #title['cover'] = os.path.join(self.catalogPath,"DefaultCover.jpg") title['cover'] = cover - self.generateThumbnail(title, image_dir, "thumbnail_default.jpg") + self.generateThumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) - # Write the thumb_width to the file validating cache contents + + # Write thumb_width to the file, validating cache contents # Allows detection of aborted catalog builds with ZipFile(self.__archive_path, mode='a') as zfw: zfw.writestr('thumb_width', self.opts.thumb_width) @@ -3162,15 +3121,17 @@ then rebuild the catalog.\n''').format(author[0]) navLabelTag = Tag(ncx_soup, "navLabel") textTag = Tag(ncx_soup, "text") if book['series']: - tokens = list(book['title'].partition(':')) + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] if self.generateForKindle: # Don't include Author for Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s)' % \ - (tokens[2].strip(), tokens[0]), dest='title'))) + textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s])' % + (book['title'], book['series'], series_index), dest='title'))) else: # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s · %s (%s)' % \ - (tokens[2].strip(), book['author'], tokens[0]), dest='title'))) + textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s]) · %s ' % + (book['title'], book['series'], series_index, book['author']), dest='title'))) else: if self.generateForKindle: # Don't include Author for Kindle @@ -3725,43 +3686,6 @@ then rebuild the catalog.\n''').format(author[0]) add_to_master_date_range_list(current_titles_list) current_titles_list = [book['title']] - ''' - # Add *article* entries for each populated date range - # master_date_range_list{}: [0]:titles list [1]:datestr - for books_by_date_range in master_date_range_list: - navPointByDateRangeTag = Tag(soup, 'navPoint') - navPointByDateRangeTag['class'] = "article" - navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(books_by_date_range[1])) - navLabelTag.insert(0, textTag) - navPointByDateRangeTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bdr_%s" % (HTML_file, - books_by_date_range[1].replace(' ','')) - - navPointByDateRangeTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_date_range[0])) - navPointByDateRangeTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ - '%d title' % books_by_date_range[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDateRangeTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDateRangeTag) - nptc += 1 - ''' - # Create an NCX article entry for each populated day # Loop over the booksByDate list, find start of each month, # add description_preview_count titles @@ -3944,7 +3868,8 @@ then rebuild the catalog.\n''').format(author[0]) outfile = open("%s/%s.ncx" % (self.catalogPath, self.basename), 'w') outfile.write(self.ncxSoup.prettify()) - # Helpers + + # --------------- Helpers --------------- def author_to_author_sort(self, author): tokens = author.split() tokens = tokens[-1:] + tokens[:-1] @@ -3952,45 +3877,39 @@ then rebuild the catalog.\n''').format(author[0]) tokens[0] += ',' return ' '.join(tokens).capitalize() - def author_compare(self,x,y): - # Return -1 if x<y - # Return 0 if x==y - # Return 1 if x>y - - # Different authors - sort by author_sort - if x['author_sort'].capitalize() > y['author_sort'].capitalize(): - return 1 - elif x['author_sort'].capitalize() < y['author_sort'].capitalize(): - return -1 + def booksByAuthorSorter_author_sort(self, book): + ''' + Sort non-series books before series books + ''' + if not book['series']: + key = '%s %s' % (book['author_sort'], + book['title_sort'].capitalize()) else: - # Same author - if x['series'] != y['series']: - # One title is a series, the other is not - if not x['series']: - # Sort regular titles < series titles - return -1 - elif not y['series']: - return 1 + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s ~%s %s' % (book['author_sort'], + self.generateSortTitle(book['series']), + series_index) + return key - # Different series - if x['title_sort'].lstrip() > y['title_sort'].lstrip(): - return 1 - else: - return -1 - else: - # Same series - if x['series'] == y['series']: - if float(x['series_index']) > float(y['series_index']): - return 1 - elif float(x['series_index']) < float(y['series_index']): - return -1 - else: - return 0 - else: - if x['series'] > y['series']: - return 1 - else: - return -1 + def booksByAuthorSorter_author(self, book): + ''' + Sort non-series books before series books + ''' + if not book['series']: + key = '%s %s' % (self.author_to_author_sort(book['author']), + book['title_sort'].capitalize()) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s ~%s %s' % (self.author_to_author_sort(book['author']), + self.generateSortTitle(book['series']), + series_index) + return key def calculateThumbnailSize(self): ''' Calculate thumbnail dimensions based on device DPI. Scale Kindle by 50% ''' @@ -4155,6 +4074,20 @@ then rebuild the catalog.\n''').format(author[0]) # Strip white space to '' return re.sub("\W","", author) + def generateFormatArgs(self, book): + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + args = dict( + title = book['title'], + series = book['series'], + series_index = series_index, + rating = self.generateRatingString(book), + rating_parens = '(%s)' % self.generateRatingString(book) if 'rating' in book else '', + pubyear = book['date'].split()[1] if book['date'] else '', + pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') + return args + def generateHTMLByGenre(self, genre, section_head, books, outfile): # Write an HTML file of this genre's book list # Return a list with [(first_author, first_book), (last_author, last_book)] @@ -4201,16 +4134,6 @@ then rebuild the catalog.\n''').format(author[0]) divTag.insert(dtc,pAuthorTag) dtc += 1 - ''' - # Insert an <hr /> between non-series and series - if not current_series and non_series_books and book['series']: - # Insert an <hr /> - hrTag = Tag(soup,'hr') - hrTag['class'] = "series_divider" - divTag.insert(dtc,hrTag) - dtc += 1 - ''' - # Check for series if book['series'] and book['series'] != current_series: # Start a new series @@ -4235,17 +4158,6 @@ then rebuild the catalog.\n''').format(author[0]) pBookTag = Tag(soup, "p") ptc = 0 - ''' - # This if clause does not display MISSING_SYMBOL for wishlist items - # If this is the wishlist_tag genre, don't show missing symbols - # normalized_wishlist_tag = self.genre_tags_dict[self.opts.wishlist_tag] - if self.opts.wishlist_tag in book['tags'] and \ - self.genre_tags_dict[self.opts.wishlist_tag] != genre: - pBookTag['class'] = "wishlist_item" - pBookTag.insert(ptc,NavigableString(self.MISSING_SYMBOL)) - ptc += 1 - ''' - # book with read|reading|unread symbol or wishlist item if self.opts.wishlist_tag in book.get('tags', []): pBookTag['class'] = "wishlist_item" @@ -4271,12 +4183,18 @@ then rebuild the catalog.\n''').format(author[0]) aTag = Tag(soup, "a") if self.opts.generate_descriptions: aTag['href'] = "book_%d.html" % (int(float(book['id']))) - # Use series, series index if avail else just title + + # Generate the title from the template + args = self.generateFormatArgs(book) if current_series: - aTag.insert(0,escape(book['title'][len(book['series'])+1:])) + #aTag.insert(0,escape(book['title'][len(book['series'])+1:])) + formatted_title = self.by_genres_series_title_template.format(**args).rstrip() else: - aTag.insert(0,escape(book['title'])) + #aTag.insert(0,escape(book['title'])) + formatted_title = self.by_genres_normal_title_template.format(**args).rstrip() non_series_books += 1 + aTag.insert(0,NavigableString(escape(formatted_title))) + pBookTag.insert(ptc, aTag) ptc += 1 @@ -4322,36 +4240,21 @@ then rebuild the catalog.\n''').format(author[0]) xmlns=XHTML_NS, ) - generated_html = P('catalog/template.xhtml', + generated_html = P(os.path.join('catalog','template.xhtml'), data=True).decode('utf-8').format(**args) generated_html = substitute_entites(generated_html) return BeautifulSoup(generated_html) - if False: - print "title metadata:\n%s" % ', '.join(sorted(book.keys())) - if False: - for item in sorted(book.keys()): - try: - print "%s: %s%s" % (item, book[item][:50], '...' if len(book[item])>50 else '') - except: - print "%s: %s" % (item, book[item]) - # Generate the template arguments - css = P('catalog/stylesheet.css', data=True).decode('utf-8') - title_str = escape(book['title']) - - # Title/series + css = P(os.path.join('catalog','stylesheet.css'), data=True).decode('utf-8') + title_str = title = escape(book['title']) + series = '' + series_index = '' if book['series']: - series_id, _, title = book['title'].partition(':') - title = escape(title.strip()) series = escape(book['series']) series_index = str(book['series_index']) if series_index.endswith('.0'): series_index = series_index[:-2] - else: - title = escape(book['title']) - series = '' - series_index = '' # Author, author_prefix (read|reading|none symbol or missing symbol) author = book['author'] @@ -4392,7 +4295,10 @@ then rebuild the catalog.\n''').format(author[0]) # Date of publication pubdate = book['date'] - pubmonth, pubyear = pubdate.split(' ') + if pubdate: + pubmonth, pubyear = pubdate.split(' ') + else: + pubmonth = pubyear = '' # Thumb _soup = BeautifulSoup('<html>',selfClosingTags=['img']) @@ -4525,7 +4431,7 @@ then rebuild the catalog.\n''').format(author[0]) def generateMastheadImage(self, out_path): from calibre.ebooks.conversion.config import load_defaults from calibre.utils.fonts import fontconfig - font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf') + font_path = default_font = P(os.path.join('fonts','liberation','LiberationSerif-Bold.ttf')) recs = load_defaults('mobi_output') masthead_font_family = recs.get('masthead_font', 'Default') @@ -4562,16 +4468,15 @@ then rebuild the catalog.\n''').format(author[0]) draw.text((left, top), text, fill=(0,0,0), font=font) img.save(open(out_path, 'wb'), 'GIF') - def generateSeriesTitle(self, title): - if float(title['series_index']) - int(title['series_index']): - series_title = '%s %4.2f: %s' % (title['series'], - title['series_index'], - title['title']) - else: - series_title = '%s %d: %s' % (title['series'], - title['series_index'], - title['title']) - return series_title + def generateRatingString(self, book): + rating = '' + if 'rating' in book: + stars = int(book['rating']) / 2 + if stars: + star_string = self.FULL_RATING_SYMBOL * stars + empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + rating = '%s%s' % (star_string,empty_stars) + return rating def generateShortDescription(self, description, dest=None): # Truncate the description, on word boundaries if necessary @@ -4610,9 +4515,11 @@ then rebuild the catalog.\n''').format(author[0]) raise RuntimeError def generateSortTitle(self, title): - # Generate a string suitable for sorting from the title - # Ignore leading stop words - # Optionally convert leading numbers to strings + ''' + Generate a string suitable for sorting from the title + Ignore leading stop words + Optionally convert leading numbers to strings + ''' from calibre.ebooks.metadata import title_sort # Strip stop words @@ -4912,10 +4819,10 @@ then rebuild the catalog.\n''').format(author[0]) class NotImplementedError: def __init__(self, error): - self.error = error + self.error.append(error) def logerror(self): - self.opts.log.info('%s not implemented' % self.error) + self.opts.log.info('%s not implemented' % error) def run(self, path_to_output, opts, db, notification=DummyReporter()): opts.log = log @@ -4982,11 +4889,12 @@ then rebuild the catalog.\n''').format(author[0]) if opts_dict['ids']: build_log.append(" book count: %d" % len(opts_dict['ids'])) - sections_list = ['Authors'] ''' + sections_list = [] if opts.generate_authors: sections_list.append('Authors') ''' + sections_list = ['Authors'] if opts.generate_titles: sections_list.append('Titles') if opts.generate_genres: @@ -5042,7 +4950,7 @@ then rebuild the catalog.\n''').format(author[0]) if catalog_source_built: log.info(" Completed catalog source generation\n") else: - log.warn(" No database hits with supplied criteria") + log.warn(" *** Errors during catalog generation, check log for details ***") if catalog_source_built: recommendations = [] @@ -5072,8 +4980,6 @@ then rebuild the catalog.\n''').format(author[0]) abort_after_input_dump=False) plumber.merge_ui_recommendations(recommendations) plumber.run() - # returns to gui2.actions.catalog:catalog_generated() - return None - else: - # returns to gui2.actions.catalog:catalog_generated() - return catalog.error + + # returns to gui2.actions.catalog:catalog_generated() + return catalog.error From 61d365c25b35a265175f1c9d26ae573fb4998965 Mon Sep 17 00:00:00 2001 From: GRiker <griker@hotmail.com> Date: Mon, 17 Jan 2011 09:59:46 -0700 Subject: [PATCH 02/10] GwR catalog 1.0 revisions --- src/calibre/library/catalog.py | 46 ++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index f1c5e3ae65..cd50cf4378 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -1429,12 +1429,29 @@ class EPUB_MOBI(CatalogPlugin): self.updateProgressFullStep("Sorting database") self.booksByAuthor = list(self.booksByTitle) - self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) -# for book in self.booksByAuthor: -# print "{0:<30} {1:<30} {2:<30}".format(book['title'],book['author'],book['author_sort']) -# print -# stop + # Test for author_sort mismatches + self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) + # Build the unique_authors set from existing data + authors = [(record['author'], record['author_sort']) for record in self.booksByAuthor] + current_author = authors[0] + for (i,author) in enumerate(authors): + if author != current_author and i: + # Exit if author matches previous, but author_sort doesn't match + if author[0] == current_author[0]: + error_msg = _(''' +Inconsistent Author Sort values for Author '{0}' ('{1}' <> '{2}'), unable to build catalog.\n +Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, +then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) + self.opts.log.warn('\n*** Metadata error ***') + self.opts.log.warn(error_msg) + + self.error.append('Metadata error') + self.error.append(error_msg) + return False + + + self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author_sort) # Build the unique_authors set from existing data authors = [(record['author'], record['author_sort'].capitalize()) for record in self.booksByAuthor] @@ -1450,20 +1467,6 @@ class EPUB_MOBI(CatalogPlugin): # Note that current_author and author are tuples: (friendly, sort) multiple_authors = True - if author != current_author and i: - # Exit if author matches previous, but author_sort doesn't match - if author[0] == current_author[0]: - error_msg = _(''' -Inconsistent Author Sort values for Author '{0}', unable to continue building catalog.\n -Select all books by '{0}', apply correct Author Sort value in Edit Metadata dialog, -then rebuild the catalog.\n''').format(author[0]) - self.opts.log.warn('\n*** Metadata error ***') - self.opts.log.warn(error_msg) - - self.error.append('Metadata error') - self.error.append(error_msg) - return False - # New author, save the previous author/sort/count unique_authors.append((current_author[0], icu_title(current_author[1]), books_by_current_author)) @@ -1939,7 +1942,8 @@ then rebuild the catalog.\n''').format(author[0]) current_author = '' current_letter = '' current_series = None - for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): + #for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): + for book in self.booksByAuthor: book_count += 1 if self.letter_or_symbol(book['author_sort'][0].upper()) != current_letter : @@ -2118,7 +2122,7 @@ then rebuild the catalog.\n''').format(author[0]) def add_books_to_HTML_by_month(this_months_list, dtc): if len(this_months_list): - this_months_list = sorted(this_months_list, key=self.booksByAuthorSorter_author_sort) + #this_months_list = sorted(this_months_list, key=self.booksByAuthorSorter_author_sort) # Create a new month anchor date_string = strftime(u'%B %Y', current_date.timetuple()) From e18e5a5db9fc1682fafed636d2d6187cebbf8a0e Mon Sep 17 00:00:00 2001 From: GRiker <griker@hotmail.com> Date: Mon, 17 Jan 2011 10:29:38 -0700 Subject: [PATCH 03/10] GwR catalog 1.0 revisions --- src/calibre/gui2/actions/catalog.py | 2 ++ src/calibre/library/catalog.py | 14 +++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/calibre/gui2/actions/catalog.py b/src/calibre/gui2/actions/catalog.py index 1650c80d70..be6e7bfe60 100644 --- a/src/calibre/gui2/actions/catalog.py +++ b/src/calibre/gui2/actions/catalog.py @@ -56,6 +56,8 @@ class GenerateCatalogAction(InterfaceAction): def catalog_generated(self, job): if job.result: # Problems during catalog generation + # jobs.results is a list - the first entry is the intended title for the dialog + # Subsequent strings are error messages dialog_title = job.result.pop(0) if re.match('warning:', job.result[0].lower()): job.result.append("Catalog generation complete.") diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index cd50cf4378..cf02e9b792 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -1637,7 +1637,10 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) title['title_sort'][0:40])).decode('mac-roman')) return True else: - self.error.append( _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.")) + error_msg = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.\n") + self.opts.log.error('*** ' + error_msg + ' ***') + self.error.append(_('No books available to include in catalog')) + self.error.append(error_msg) return False def fetchBookmarks(self): @@ -3164,8 +3167,13 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) # Add the author tag cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') cmTag['name'] = "author" - navStr = '%s | %s' % (self.formatNCXText(book['author'], dest='author'), - book['date'].split()[1]) + + if book['date']: + navStr = '%s | %s' % (self.formatNCXText(book['author'], dest='author'), + book['date'].split()[1]) + else: + navStr = '%s' % (self.formatNCXText(book['author'], dest='author')) + if 'tags' in book and len(book['tags']): navStr = self.formatNCXText(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') cmTag.insert(0, NavigableString(navStr)) From f965037fb44184957d9bc20dc3efb1ce1adee6a9 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 17 Jan 2011 12:22:30 -0700 Subject: [PATCH 04/10] MOBI Output: Fix bug that could cause a link pointing to the start of a section to go to a point later in the section is the section contained an empty id attribute --- src/calibre/ebooks/mobi/writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/calibre/ebooks/mobi/writer.py b/src/calibre/ebooks/mobi/writer.py index cd6674c2e2..ed102ecc80 100644 --- a/src/calibre/ebooks/mobi/writer.py +++ b/src/calibre/ebooks/mobi/writer.py @@ -251,7 +251,7 @@ class Serializer(object): tag = prefixname(elem.tag, nsrmap) # Previous layers take care of @name id = elem.attrib.pop('id', None) - if id is not None: + if id: href = '#'.join((item.href, id)) offset = self.anchor_offset or buffer.tell() self.id_offsets[urlnormalize(href)] = offset From aa28b379517028b5e7951c8aa94f2226a99c918d Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 17 Jan 2011 13:03:34 -0700 Subject: [PATCH 05/10] Fix #8424 (Dilbert retrieval fails) --- resources/recipes/dilbert.recipe | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/recipes/dilbert.recipe b/resources/recipes/dilbert.recipe index 2c3268da2f..56aa4af8c9 100644 --- a/resources/recipes/dilbert.recipe +++ b/resources/recipes/dilbert.recipe @@ -28,7 +28,7 @@ class DilbertBig(BasicNewsRecipe): ,'publisher' : publisher } - feeds = [(u'Dilbert', u'http://feeds.dilbert.com/DilbertDailyStrip' )] + feeds = [(u'Dilbert', u'http://feed.dilbert.com/dilbert/daily_strip' )] def get_article_url(self, article): return article.get('feedburner_origlink', None) From 84d1dd94d23db7b53f47051ecb3cfd5c47965f0b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 17 Jan 2011 13:10:10 -0700 Subject: [PATCH 06/10] Make postprocess_html in the NY Times recipes more robust --- resources/recipes/nytimes.recipe | 204 +++++++++++++------------ resources/recipes/nytimes_sub.recipe | 214 +++++++++++++++------------ 2 files changed, 229 insertions(+), 189 deletions(-) diff --git a/resources/recipes/nytimes.recipe b/resources/recipes/nytimes.recipe index 6f80f4f85f..7e313e5727 100644 --- a/resources/recipes/nytimes.recipe +++ b/resources/recipes/nytimes.recipe @@ -586,105 +586,125 @@ class NYTimes(BasicNewsRecipe): return self.strip_anchors(soup) def postprocess_html(self,soup, True): + try: + if self.one_picture_per_article: + # Remove all images after first + largeImg = soup.find(True, {'class':'articleSpanImage'}) + inlineImgs = soup.findAll(True, {'class':'inlineImage module'}) + if largeImg: + for inlineImg in inlineImgs: + inlineImg.extract() + else: + if inlineImgs: + firstImg = inlineImgs[0] + for inlineImg in inlineImgs[1:]: + inlineImg.extract() + # Move firstImg before article body + cgFirst = soup.find(True, {'class':re.compile('columnGroup *first')}) + if cgFirst: + # Strip all sibling NavigableStrings: noise + navstrings = cgFirst.findAll(text=True, recursive=False) + [ns.extract() for ns in navstrings] + headline_found = False + tag = cgFirst.find(True) + insertLoc = 0 + while True: + insertLoc += 1 + if hasattr(tag,'class') and tag['class'] == 'articleHeadline': + headline_found = True + break + tag = tag.nextSibling + if not tag: + headline_found = False + break + if headline_found: + cgFirst.insert(insertLoc,firstImg) + else: + self.log(">>> No class:'columnGroup first' found <<<") + except: + self.log("ERROR: One picture per article in postprocess_html") - if self.one_picture_per_article: - # Remove all images after first - largeImg = soup.find(True, {'class':'articleSpanImage'}) - inlineImgs = soup.findAll(True, {'class':'inlineImage module'}) - if largeImg: - for inlineImg in inlineImgs: - inlineImg.extract() - else: - if inlineImgs: - firstImg = inlineImgs[0] - for inlineImg in inlineImgs[1:]: - inlineImg.extract() - # Move firstImg before article body - cgFirst = soup.find(True, {'class':re.compile('columnGroup *first')}) - if cgFirst: - # Strip all sibling NavigableStrings: noise - navstrings = cgFirst.findAll(text=True, recursive=False) - [ns.extract() for ns in navstrings] - headline_found = False - tag = cgFirst.find(True) - insertLoc = 0 - while True: - insertLoc += 1 - if hasattr(tag,'class') and tag['class'] == 'articleHeadline': - headline_found = True - break - tag = tag.nextSibling - if not tag: - headline_found = False - break - if headline_found: - cgFirst.insert(insertLoc,firstImg) - else: - self.log(">>> No class:'columnGroup first' found <<<") + try: + # Change captions to italic + for caption in soup.findAll(True, {'class':'caption'}) : + if caption and len(caption) > 0: + cTag = Tag(soup, "p", [("class", "caption")]) + c = self.fixChars(self.tag_to_string(caption,use_alt=False)).strip() + mp_off = c.find("More Photos") + if mp_off >= 0: + c = c[:mp_off] + cTag.insert(0, c) + caption.replaceWith(cTag) + except: + self.log("ERROR: Problem in change captions to italic") - # Change captions to italic - for caption in soup.findAll(True, {'class':'caption'}) : - if caption and caption.contents[0]: - cTag = Tag(soup, "p", [("class", "caption")]) - c = self.fixChars(self.tag_to_string(caption,use_alt=False)).strip() - mp_off = c.find("More Photos") - if mp_off >= 0: - c = c[:mp_off] - cTag.insert(0, c) - caption.replaceWith(cTag) + try: + # Change <nyt_headline> to <h2> + h1 = soup.find('h1') + if h1: + headline = h1.find("nyt_headline") + if headline: + tag = Tag(soup, "h2") + tag['class'] = "headline" + tag.insert(0, self.fixChars(headline.contents[0])) + h1.replaceWith(tag) + else: + # Blog entry - replace headline, remove <hr> tags + headline = soup.find('title') + if headline: + tag = Tag(soup, "h2") + tag['class'] = "headline" + tag.insert(0, self.fixChars(headline.contents[0])) + soup.insert(0, tag) + hrs = soup.findAll('hr') + for hr in hrs: + hr.extract() + except: + self.log("ERROR: Problem in Change <nyt_headline> to <h2>") - # Change <nyt_headline> to <h2> - h1 = soup.find('h1') - if h1: - headline = h1.find("nyt_headline") - if headline: - tag = Tag(soup, "h2") - tag['class'] = "headline" - tag.insert(0, self.fixChars(headline.contents[0])) - h1.replaceWith(tag) - else: - # Blog entry - replace headline, remove <hr> tags - headline = soup.find('title') - if headline: - tag = Tag(soup, "h2") - tag['class'] = "headline" - tag.insert(0, self.fixChars(headline.contents[0])) - soup.insert(0, tag) - hrs = soup.findAll('hr') - for hr in hrs: - hr.extract() + try: + # Change <h1> to <h3> - used in editorial blogs + masthead = soup.find("h1") + if masthead: + # Nuke the href + if masthead.a: + del(masthead.a['href']) + tag = Tag(soup, "h3") + tag.insert(0, self.fixChars(masthead.contents[0])) + masthead.replaceWith(tag) + except: + self.log("ERROR: Problem in Change <h1> to <h3> - used in editorial blogs") - # Change <h1> to <h3> - used in editorial blogs - masthead = soup.find("h1") - if masthead: - # Nuke the href - if masthead.a: - del(masthead.a['href']) - tag = Tag(soup, "h3") - tag.insert(0, self.fixChars(masthead.contents[0])) - masthead.replaceWith(tag) + try: + # Change <span class="bold"> to <b> + for subhead in soup.findAll(True, {'class':'bold'}) : + if subhead.contents: + bTag = Tag(soup, "b") + bTag.insert(0, subhead.contents[0]) + subhead.replaceWith(bTag) + except: + self.log("ERROR: Problem in Change <h1> to <h3> - used in editorial blogs") - # Change <span class="bold"> to <b> - for subhead in soup.findAll(True, {'class':'bold'}) : - if subhead.contents: - bTag = Tag(soup, "b") - bTag.insert(0, subhead.contents[0]) - subhead.replaceWith(bTag) + try: + divTag = soup.find('div',attrs={'id':'articleBody'}) + if divTag: + divTag['class'] = divTag['id'] + except: + self.log("ERROR: Problem in soup.find(div,attrs={id:articleBody})") - divTag = soup.find('div',attrs={'id':'articleBody'}) - if divTag: - divTag['class'] = divTag['id'] + try: + # Add class="authorId" to <div> so we can format with CSS + divTag = soup.find('div',attrs={'id':'authorId'}) + if divTag and divTag.contents[0]: + tag = Tag(soup, "p") + tag['class'] = "authorId" + tag.insert(0, self.fixChars(self.tag_to_string(divTag.contents[0], + use_alt=False))) + divTag.replaceWith(tag) + except: + self.log("ERROR: Problem in Add class=authorId to <div> so we can format with CSS") - # Add class="authorId" to <div> so we can format with CSS - divTag = soup.find('div',attrs={'id':'authorId'}) - if divTag and divTag.contents[0]: - tag = Tag(soup, "p") - tag['class'] = "authorId" - tag.insert(0, self.fixChars(self.tag_to_string(divTag.contents[0], - use_alt=False))) - divTag.replaceWith(tag) - - return soup + return soup def populate_article_metadata(self, article, soup, first): shortparagraph = "" diff --git a/resources/recipes/nytimes_sub.recipe b/resources/recipes/nytimes_sub.recipe index 8ac7c735f7..8f92852237 100644 --- a/resources/recipes/nytimes_sub.recipe +++ b/resources/recipes/nytimes_sub.recipe @@ -586,105 +586,125 @@ class NYTimes(BasicNewsRecipe): return self.strip_anchors(soup) def postprocess_html(self,soup, True): + try: + if self.one_picture_per_article: + # Remove all images after first + largeImg = soup.find(True, {'class':'articleSpanImage'}) + inlineImgs = soup.findAll(True, {'class':'inlineImage module'}) + if largeImg: + for inlineImg in inlineImgs: + inlineImg.extract() + else: + if inlineImgs: + firstImg = inlineImgs[0] + for inlineImg in inlineImgs[1:]: + inlineImg.extract() + # Move firstImg before article body + cgFirst = soup.find(True, {'class':re.compile('columnGroup *first')}) + if cgFirst: + # Strip all sibling NavigableStrings: noise + navstrings = cgFirst.findAll(text=True, recursive=False) + [ns.extract() for ns in navstrings] + headline_found = False + tag = cgFirst.find(True) + insertLoc = 0 + while True: + insertLoc += 1 + if hasattr(tag,'class') and tag['class'] == 'articleHeadline': + headline_found = True + break + tag = tag.nextSibling + if not tag: + headline_found = False + break + if headline_found: + cgFirst.insert(insertLoc,firstImg) + else: + self.log(">>> No class:'columnGroup first' found <<<") + except: + self.log("ERROR: One picture per article in postprocess_html") + + try: + # Change captions to italic + for caption in soup.findAll(True, {'class':'caption'}) : + if caption and len(caption) > 0: + cTag = Tag(soup, "p", [("class", "caption")]) + c = self.fixChars(self.tag_to_string(caption,use_alt=False)).strip() + mp_off = c.find("More Photos") + if mp_off >= 0: + c = c[:mp_off] + cTag.insert(0, c) + caption.replaceWith(cTag) + except: + self.log("ERROR: Problem in change captions to italic") + + try: + # Change <nyt_headline> to <h2> + h1 = soup.find('h1') + if h1: + headline = h1.find("nyt_headline") + if headline: + tag = Tag(soup, "h2") + tag['class'] = "headline" + tag.insert(0, self.fixChars(headline.contents[0])) + h1.replaceWith(tag) + else: + # Blog entry - replace headline, remove <hr> tags + headline = soup.find('title') + if headline: + tag = Tag(soup, "h2") + tag['class'] = "headline" + tag.insert(0, self.fixChars(headline.contents[0])) + soup.insert(0, tag) + hrs = soup.findAll('hr') + for hr in hrs: + hr.extract() + except: + self.log("ERROR: Problem in Change <nyt_headline> to <h2>") - if self.one_picture_per_article: - # Remove all images after first - largeImg = soup.find(True, {'class':'articleSpanImage'}) - inlineImgs = soup.findAll(True, {'class':'inlineImage module'}) - if largeImg: - for inlineImg in inlineImgs: - inlineImg.extract() - else: - if inlineImgs: - firstImg = inlineImgs[0] - for inlineImg in inlineImgs[1:]: - inlineImg.extract() - # Move firstImg before article body - cgFirst = soup.find(True, {'class':re.compile('columnGroup *first')}) - if cgFirst: - # Strip all sibling NavigableStrings: noise - navstrings = cgFirst.findAll(text=True, recursive=False) - [ns.extract() for ns in navstrings] - headline_found = False - tag = cgFirst.find(True) - insertLoc = 0 - while True: - insertLoc += 1 - if hasattr(tag,'class') and tag['class'] == 'articleHeadline': - headline_found = True - break - tag = tag.nextSibling - if not tag: - headline_found = False - break - if headline_found: - cgFirst.insert(insertLoc,firstImg) - else: - self.log(">>> No class:'columnGroup first' found <<<") + try: + # Change <h1> to <h3> - used in editorial blogs + masthead = soup.find("h1") + if masthead: + # Nuke the href + if masthead.a: + del(masthead.a['href']) + tag = Tag(soup, "h3") + tag.insert(0, self.fixChars(masthead.contents[0])) + masthead.replaceWith(tag) + except: + self.log("ERROR: Problem in Change <h1> to <h3> - used in editorial blogs") - # Change captions to italic - for caption in soup.findAll(True, {'class':'caption'}) : - if caption and caption.contents[0]: - cTag = Tag(soup, "p", [("class", "caption")]) - c = self.fixChars(self.tag_to_string(caption,use_alt=False)).strip() - mp_off = c.find("More Photos") - if mp_off >= 0: - c = c[:mp_off] - cTag.insert(0, c) - caption.replaceWith(cTag) - - # Change <nyt_headline> to <h2> - h1 = soup.find('h1') - if h1: - headline = h1.find("nyt_headline") - if headline: - tag = Tag(soup, "h2") - tag['class'] = "headline" - tag.insert(0, self.fixChars(headline.contents[0])) - h1.replaceWith(tag) - else: - # Blog entry - replace headline, remove <hr> tags - headline = soup.find('title') - if headline: - tag = Tag(soup, "h2") - tag['class'] = "headline" - tag.insert(0, self.fixChars(headline.contents[0])) - soup.insert(0, tag) - hrs = soup.findAll('hr') - for hr in hrs: - hr.extract() - - # Change <h1> to <h3> - used in editorial blogs - masthead = soup.find("h1") - if masthead: - # Nuke the href - if masthead.a: - del(masthead.a['href']) - tag = Tag(soup, "h3") - tag.insert(0, self.fixChars(masthead.contents[0])) - masthead.replaceWith(tag) - - # Change <span class="bold"> to <b> - for subhead in soup.findAll(True, {'class':'bold'}) : - if subhead.contents: - bTag = Tag(soup, "b") - bTag.insert(0, subhead.contents[0]) - subhead.replaceWith(bTag) - - divTag = soup.find('div',attrs={'id':'articleBody'}) - if divTag: - divTag['class'] = divTag['id'] - - # Add class="authorId" to <div> so we can format with CSS - divTag = soup.find('div',attrs={'id':'authorId'}) - if divTag and divTag.contents[0]: - tag = Tag(soup, "p") - tag['class'] = "authorId" - tag.insert(0, self.fixChars(self.tag_to_string(divTag.contents[0], - use_alt=False))) - divTag.replaceWith(tag) - - return soup + try: + # Change <span class="bold"> to <b> + for subhead in soup.findAll(True, {'class':'bold'}) : + if subhead.contents: + bTag = Tag(soup, "b") + bTag.insert(0, subhead.contents[0]) + subhead.replaceWith(bTag) + except: + self.log("ERROR: Problem in Change <h1> to <h3> - used in editorial blogs") + + try: + divTag = soup.find('div',attrs={'id':'articleBody'}) + if divTag: + divTag['class'] = divTag['id'] + except: + self.log("ERROR: Problem in soup.find(div,attrs={id:articleBody})") + + try: + # Add class="authorId" to <div> so we can format with CSS + divTag = soup.find('div',attrs={'id':'authorId'}) + if divTag and divTag.contents[0]: + tag = Tag(soup, "p") + tag['class'] = "authorId" + tag.insert(0, self.fixChars(self.tag_to_string(divTag.contents[0], + use_alt=False))) + divTag.replaceWith(tag) + except: + self.log("ERROR: Problem in Add class=authorId to <div> so we can format with CSS") + + return soup def populate_article_metadata(self, article, soup, first): shortparagraph = "" try: From 927c389e91ffe47880ed1b0949b421d449f7ad4b Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 17 Jan 2011 13:20:12 -0700 Subject: [PATCH 07/10] Fix #8436 (Add tag when using Add ISBN Dialog) --- src/calibre/gui2/actions/add.py | 9 ++-- src/calibre/gui2/dialogs/add_from_isbn.py | 7 +++ src/calibre/gui2/dialogs/add_from_isbn.ui | 52 +++++++++++++++++++---- 3 files changed, 56 insertions(+), 12 deletions(-) diff --git a/src/calibre/gui2/actions/add.py b/src/calibre/gui2/actions/add.py index 9917c542ae..6fa53d6290 100644 --- a/src/calibre/gui2/actions/add.py +++ b/src/calibre/gui2/actions/add.py @@ -91,13 +91,14 @@ class AddAction(InterfaceAction): self.gui.library_view.model().db.import_book(MetaInformation(None), []) self.gui.library_view.model().books_added(num) - def add_isbns(self, books): + def add_isbns(self, books, add_tags=[]): from calibre.ebooks.metadata import MetaInformation ids = set([]) + db = self.gui.library_view.model().db + for x in books: mi = MetaInformation(None) mi.isbn = x['isbn'] - db = self.gui.library_view.model().db if x['path'] is not None: ids.add(db.import_book(mi, [x['path']])) else: @@ -109,6 +110,8 @@ class AddAction(InterfaceAction): self.gui.iactions['Edit Metadata'].do_download_metadata(ids) finally: config['overwrite_author_title_metadata'] = orig + if add_tags and ids: + db.bulk_modify_tags(ids, add=add_tags) def files_dropped(self, paths): @@ -166,7 +169,7 @@ class AddAction(InterfaceAction): from calibre.gui2.dialogs.add_from_isbn import AddFromISBN d = AddFromISBN(self.gui) if d.exec_() == d.Accepted: - self.add_isbns(d.books) + self.add_isbns(d.books, add_tags=d.set_tags) def add_books(self, *args): ''' diff --git a/src/calibre/gui2/dialogs/add_from_isbn.py b/src/calibre/gui2/dialogs/add_from_isbn.py index f93cddecd5..433b70291c 100644 --- a/src/calibre/gui2/dialogs/add_from_isbn.py +++ b/src/calibre/gui2/dialogs/add_from_isbn.py @@ -12,6 +12,7 @@ from PyQt4.Qt import QDialog, QApplication from calibre.gui2.dialogs.add_from_isbn_ui import Ui_Dialog from calibre.ebooks.metadata import check_isbn from calibre.constants import iswindows +from calibre.gui2 import gprefs class AddFromISBN(QDialog, Ui_Dialog): @@ -25,7 +26,9 @@ class AddFromISBN(QDialog, Ui_Dialog): self.isbns = [] self.books = [] + self.set_tags = [] self.paste_button.clicked.connect(self.paste) + self.add_tags.setText(', '.join(gprefs.get('add from ISBN tags', []))) def paste(self, *args): app = QApplication.instance() @@ -37,6 +40,10 @@ class AddFromISBN(QDialog, Ui_Dialog): self.isbn_box.setPlainText(new) def accept(self, *args): + tags = unicode(self.add_tags.text()).strip().split(',') + tags = list(filter(None, [x.strip() for x in tags])) + gprefs['add from ISBN tags'] = tags + self.set_tags = tags for line in unicode(self.isbn_box.toPlainText()).strip().splitlines(): line = line.strip() if not line: diff --git a/src/calibre/gui2/dialogs/add_from_isbn.ui b/src/calibre/gui2/dialogs/add_from_isbn.ui index e37c4ed769..f598e6f1d8 100644 --- a/src/calibre/gui2/dialogs/add_from_isbn.ui +++ b/src/calibre/gui2/dialogs/add_from_isbn.ui @@ -18,8 +18,19 @@ <normaloff>:/images/add_book.png</normaloff>:/images/add_book.png</iconset> </property> <layout class="QGridLayout" name="gridLayout"> - <item row="0" column="0"> - <widget class="QPlainTextEdit" name="isbn_box"/> + <item row="0" column="0" rowspan="2"> + <layout class="QVBoxLayout" name="verticalLayout_2"> + <item> + <widget class="QPlainTextEdit" name="isbn_box"/> + </item> + <item> + <widget class="QPushButton" name="paste_button"> + <property name="text"> + <string>&Paste from clipboard</string> + </property> + </widget> + </item> + </layout> </item> <item row="0" column="1"> <widget class="QLabel" name="label"> @@ -34,6 +45,36 @@ </property> </widget> </item> + <item row="1" column="1"> + <layout class="QVBoxLayout" name="verticalLayout"> + <item> + <widget class="QLabel" name="label_2"> + <property name="text"> + <string>&Tags to set on created book entries:</string> + </property> + <property name="buddy"> + <cstring>add_tags</cstring> + </property> + </widget> + </item> + <item> + <widget class="QLineEdit" name="add_tags"/> + </item> + <item> + <spacer name="verticalSpacer"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>40</height> + </size> + </property> + </spacer> + </item> + </layout> + </item> <item row="2" column="0" colspan="2"> <widget class="QDialogButtonBox" name="buttonBox"> <property name="orientation"> @@ -44,13 +85,6 @@ </property> </widget> </item> - <item row="1" column="0"> - <widget class="QPushButton" name="paste_button"> - <property name="text"> - <string>&Paste from clipboard</string> - </property> - </widget> - </item> </layout> </widget> <resources> From e1fd50d72f2eefb70928d20dab094f0402172bae Mon Sep 17 00:00:00 2001 From: GRiker <griker@hotmail.com> Date: Mon, 17 Jan 2011 15:39:23 -0700 Subject: [PATCH 08/10] GwR fix for TOC discontinuity, default cover swapping --- src/calibre/library/catalog.py | 37 ++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index fa5041bfec..13df6625d4 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -1360,6 +1360,7 @@ class EPUB_MOBI(CatalogPlugin): return False self.fetchBookmarks() if self.opts.generate_descriptions: + self.generateThumbnails() self.generateHTMLDescriptions() self.generateHTMLByAuthor() if self.opts.generate_titles: @@ -1372,8 +1373,7 @@ class EPUB_MOBI(CatalogPlugin): self.generateHTMLByDateAdded() if self.generateRecentlyRead: self.generateHTMLByDateRead() - if self.opts.generate_descriptions: - self.generateThumbnails() + self.generateOPF() self.generateNCXHeader() self.generateNCXByAuthor("Authors") @@ -1452,6 +1452,12 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author_sort) + +# for book in self.booksByAuthor: +# print '{0:<10} {1:<5} {2:<20} {3:<20} {4:<20} {5:<20}'.format(book['series'], book['series_index'], book['title'], +# book['author'], book['authors'],book['author_sort']) +# print + # Build the unique_authors set from existing data authors = [(record['author'], record['author_sort'].capitalize()) for record in self.booksByAuthor] @@ -2848,23 +2854,26 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) thumb_generated = True valid_cover = True try: - thumbs.append("thumbnail_%d.jpg" % int(title['id'])) self.generateThumbnail(title, image_dir, thumb_file) + thumbs.append("thumbnail_%d.jpg" % int(title['id'])) except: if 'cover' in title and os.path.exists(title['cover']): valid_cover = False - self.opts.log.warn(" *** Invalid cover file for '%s' ***" % (title['title'])) + self.opts.log.warn(" *** Invalid cover file for '%s'***" % + (title['title'])) if not self.error: self.error.append('Invalid cover files') self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) + thumb_generated = False if not thumb_generated: - self.opts.log.warn(" using default cover for '%s'" % (title['title'])) - # Check to make sure default is current - # Check to see if thumbnail exists + self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) + # Confirm thumb exists, default is current default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") cover = os.path.join(self.catalogPath, "DefaultCover.png") + title['cover'] = cover + if not os.path.exists(cover): shutil.copyfile(I('book.png'), cover) @@ -2877,17 +2886,15 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) if thumb_timestamp < cover_timestamp: if False and self.verbose: self.opts.log.warn("updating thumbnail_default for %s" % title['title']) - #title['cover'] = os.path.join(self.catalogPath,"DefaultCover.jpg") - title['cover'] = cover self.generateThumbnail(title, image_dir, "thumbnail_default.jpg" if valid_cover else thumb_file) else: if False and self.verbose: self.opts.log.warn(" generating new thumbnail_default.jpg") - #title['cover'] = os.path.join(self.catalogPath,"DefaultCover.jpg") - title['cover'] = cover self.generateThumbnail(title, image_dir, "thumbnail_default.jpg" if valid_cover else thumb_file) + # Clear the book's cover property + title['cover'] = None # Write thumb_width to the file, validating cache contents @@ -3881,7 +3888,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) outfile.write(self.ncxSoup.prettify()) - # --------------- Helpers --------------- + # ======================== Helpers ======================== def author_to_author_sort(self, author): tokens = author.split() tokens = tokens[-1:] + tokens[:-1] @@ -3894,14 +3901,14 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) Sort non-series books before series books ''' if not book['series']: - key = '%s %s' % (book['author_sort'], + key = '%s %s' % (book['author_sort'].capitalize(), book['title_sort'].capitalize()) else: index = book['series_index'] integer = int(index) fraction = index-integer series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s ~%s %s' % (book['author_sort'], + key = '%s ~%s %s' % (book['author_sort'].capitalize(), self.generateSortTitle(book['series']), series_index) return key @@ -4315,7 +4322,7 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) # Thumb _soup = BeautifulSoup('<html>',selfClosingTags=['img']) thumb = Tag(_soup,"img") - if 'cover' in book: + if 'cover' in book and book['cover']: thumb['src'] = "../images/thumbnail_%d.jpg" % int(book['id']) else: thumb['src'] = "../images/thumbnail_default.jpg" From af04ca87cefa556c025e5cecaf5c92934b1264ff Mon Sep 17 00:00:00 2001 From: GRiker <griker@hotmail.com> Date: Mon, 17 Jan 2011 15:52:25 -0700 Subject: [PATCH 09/10] GwR fix for TOC discontinuity, default cover swapping --- src/calibre/library/catalog.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/calibre/library/catalog.py b/src/calibre/library/catalog.py index 13df6625d4..8edf266cfb 100644 --- a/src/calibre/library/catalog.py +++ b/src/calibre/library/catalog.py @@ -4314,10 +4314,9 @@ then rebuild the catalog.\n''').format(author[0],author[1],current_author[1]) # Date of publication pubdate = book['date'] - if pubdate: - pubmonth, pubyear = pubdate.split(' ') - else: - pubmonth = pubyear = '' + pubmonth, pubyear = pubdate.split() + if pubyear == '101': + pubdate = pubmonth = pubyear = '' # Thumb _soup = BeautifulSoup('<html>',selfClosingTags=['img']) From e82dd54242eb86382f55a2840338acea526b4fb1 Mon Sep 17 00:00:00 2001 From: Kovid Goyal <kovid@kovidgoyal.net> Date: Mon, 17 Jan 2011 17:42:40 -0700 Subject: [PATCH 10/10] Fix #8409 (bulk edit date) --- src/calibre/gui2/dialogs/metadata_bulk.py | 24 ++++- src/calibre/gui2/dialogs/metadata_bulk.ui | 117 +++++++++++++++------- 2 files changed, 100 insertions(+), 41 deletions(-) diff --git a/src/calibre/gui2/dialogs/metadata_bulk.py b/src/calibre/gui2/dialogs/metadata_bulk.py index 2b3a319663..6e6b553dba 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.py +++ b/src/calibre/gui2/dialogs/metadata_bulk.py @@ -49,7 +49,6 @@ def get_cover_data(path): # {{{ return cdata, area # }}} - class MyBlockingBusy(QDialog): # {{{ do_one_signal = pyqtSignal() @@ -134,7 +133,7 @@ class MyBlockingBusy(QDialog): # {{{ do_autonumber, do_remove_format, remove_format, do_swap_ta, \ do_remove_conv, do_auto_author, series, do_series_restart, \ series_start_value, do_title_case, cover_action, clear_series, \ - pubdate = self.args + pubdate, adddate = self.args # first loop: do author and title. These will commit at the end of each @@ -214,6 +213,9 @@ class MyBlockingBusy(QDialog): # {{{ if pubdate is not None: self.db.set_pubdate(id, pubdate, notify=False, commit=False) + if adddate is not None: + self.db.set_timestamp(id, adddate, notify=False, commit=False) + if do_series: if do_series_restart: if self.series_start_value is None: @@ -300,6 +302,10 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): self.pubdate.setSpecialValueText(_('Undefined')) self.clear_pubdate_button.clicked.connect(self.clear_pubdate) self.pubdate.dateChanged.connect(self.do_apply_pubdate) + self.adddate.setMinimumDate(UNDEFINED_QDATE) + self.adddate.setSpecialValueText(_('Undefined')) + self.clear_adddate_button.clicked.connect(self.clear_adddate) + self.adddate.dateChanged.connect(self.do_apply_adddate) if len(self.db.custom_field_keys(include_composites=False)) == 0: self.central_widget.removeTab(1) @@ -322,6 +328,12 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): def clear_pubdate(self, *args): self.pubdate.setDate(UNDEFINED_QDATE) + def do_apply_adddate(self, *args): + self.apply_adddate.setChecked(True) + + def clear_adddate(self, *args): + self.adddate.setDate(UNDEFINED_QDATE) + def button_clicked(self, which): if which == self.button_box.button(QDialogButtonBox.Apply): self.do_again = True @@ -726,7 +738,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): name = name.strip().replace('|', ',') self.authors.addItem(name) self.authors.setEditText('') - + self.authors.set_separator('&') self.authors.set_space_before_sep(True) self.authors.update_items_cache(self.db.all_author_names()) @@ -805,9 +817,11 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): do_remove_conv = self.remove_conversion_settings.isChecked() do_auto_author = self.auto_author_sort.isChecked() do_title_case = self.change_title_to_title_case.isChecked() - pubdate = None + pubdate = adddate = None if self.apply_pubdate.isChecked(): pubdate = qt_to_dt(self.pubdate.date()) + if self.apply_adddate.isChecked(): + adddate = qt_to_dt(self.adddate.date()) cover_action = None if self.cover_remove.isChecked(): @@ -821,7 +835,7 @@ class MetadataBulkDialog(ResizableDialog, Ui_MetadataBulkDialog): do_autonumber, do_remove_format, remove_format, do_swap_ta, do_remove_conv, do_auto_author, series, do_series_restart, series_start_value, do_title_case, cover_action, clear_series, - pubdate) + pubdate, adddate) bb = MyBlockingBusy(_('Applying changes to %d books.\nPhase {0} {1}%%.') %len(self.ids), args, self.db, self.ids, diff --git a/src/calibre/gui2/dialogs/metadata_bulk.ui b/src/calibre/gui2/dialogs/metadata_bulk.ui index 8db74b343d..f8ae926be6 100644 --- a/src/calibre/gui2/dialogs/metadata_bulk.ui +++ b/src/calibre/gui2/dialogs/metadata_bulk.ui @@ -347,6 +347,51 @@ from the value in the box</string> </item> </layout> </item> + <item row="9" column="0"> + <widget class="QLabel" name="label_10"> + <property name="text"> + <string>&Date:</string> + </property> + <property name="alignment"> + <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set> + </property> + <property name="buddy"> + <cstring>adddate</cstring> + </property> + </widget> + </item> + <item row="9" column="1"> + <layout class="QHBoxLayout" name="horizontalLayout_5"> + <item> + <widget class="QDateEdit" name="adddate"> + <property name="displayFormat"> + <string>d MMM yyyy</string> + </property> + <property name="calendarPopup"> + <bool>true</bool> + </property> + </widget> + </item> + <item> + <widget class="QToolButton" name="clear_adddate_button"> + <property name="text"> + <string>...</string> + </property> + <property name="icon"> + <iconset resource="../../../../resources/images.qrc"> + <normaloff>:/images/trash.png</normaloff>:/images/trash.png</iconset> + </property> + </widget> + </item> + </layout> + </item> + <item row="9" column="2"> + <widget class="QCheckBox" name="apply_adddate"> + <property name="text"> + <string>&Apply date</string> + </property> + </widget> + </item> <item row="10" column="0"> <widget class="QLabel" name="label_9"> <property name="text"> @@ -395,6 +440,42 @@ from the value in the box</string> </property> </widget> </item> + <item row="11" column="0"> + <widget class="QLabel" name="label_5"> + <property name="text"> + <string>Remove &format:</string> + </property> + <property name="buddy"> + <cstring>remove_format</cstring> + </property> + </widget> + </item> + <item row="11" column="1"> + <widget class="QComboBox" name="remove_format"> + <property name="maximumSize"> + <size> + <width>120</width> + <height>16777215</height> + </size> + </property> + </widget> + </item> + <item row="12" column="0"> + <spacer name="verticalSpacer"> + <property name="orientation"> + <enum>Qt::Vertical</enum> + </property> + <property name="sizeType"> + <enum>QSizePolicy::Fixed</enum> + </property> + <property name="sizeHint" stdset="0"> + <size> + <width>20</width> + <height>15</height> + </size> + </property> + </spacer> + </item> <item row="13" column="0" colspan="3"> <layout class="QHBoxLayout" name="horizontalLayout_3"> <item> @@ -478,42 +559,6 @@ Future conversion of these books will use the default settings.</string> </property> </spacer> </item> - <item row="12" column="0"> - <spacer name="verticalSpacer"> - <property name="orientation"> - <enum>Qt::Vertical</enum> - </property> - <property name="sizeType"> - <enum>QSizePolicy::Fixed</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>20</width> - <height>15</height> - </size> - </property> - </spacer> - </item> - <item row="11" column="0"> - <widget class="QLabel" name="label_5"> - <property name="text"> - <string>Remove &format:</string> - </property> - <property name="buddy"> - <cstring>remove_format</cstring> - </property> - </widget> - </item> - <item row="11" column="1"> - <widget class="QComboBox" name="remove_format"> - <property name="maximumSize"> - <size> - <width>120</width> - <height>16777215</height> - </size> - </property> - </widget> - </item> </layout> </widget> <widget class="QWidget" name="tab">