From d4027a823019735b705bc21639e6c582fa9710cd Mon Sep 17 00:00:00 2001 From: GRiker Date: Fri, 31 Aug 2012 03:31:24 -0600 Subject: [PATCH 1/7] refactoring of catalog building code wip --- src/calibre/gui2/catalog/catalog_epub_mobi.py | 87 +- src/calibre/gui2/catalog/catalog_epub_mobi.ui | 299 +- .../library/catalogs/epub_mobi_builder.py | 5633 +++++++++-------- 3 files changed, 3380 insertions(+), 2639 deletions(-) diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index f9708f292e..05c2b8c8b3 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -6,7 +6,8 @@ __license__ = 'GPL v3' __copyright__ = '2009, Kovid Goyal ' __docformat__ = 'restructuredtext en' -from copy import copy +import re, sys + from functools import partial from calibre.ebooks.conversion.config import load_defaults @@ -16,7 +17,7 @@ from calibre.utils.icu import sort_key from catalog_epub_mobi_ui import Ui_Form from PyQt4.Qt import (Qt, QAbstractItemView, QCheckBox, QComboBox, QDoubleSpinBox, QIcon, QLineEdit, QObject, QRadioButton, QSize, QSizePolicy, - QTableWidget, QTableWidgetItem, QToolButton, QVBoxLayout, QWidget, + QTableWidget, QTableWidgetItem, QTextEdit, QToolButton, QVBoxLayout, QWidget, SIGNAL) class PluginWidget(QWidget,Ui_Form): @@ -44,6 +45,7 @@ class PluginWidget(QWidget,Ui_Form): LineEditControls = [] RadioButtonControls = [] TableWidgetControls = [] + TextEditControls = [] for item in self.__dict__: if type(self.__dict__[item]) is QCheckBox: @@ -58,6 +60,8 @@ class PluginWidget(QWidget,Ui_Form): RadioButtonControls.append(str(self.__dict__[item].objectName())) elif type(self.__dict__[item]) is QTableWidget: TableWidgetControls.append(str(self.__dict__[item].objectName())) + elif type(self.__dict__[item]) is QTextEdit: + TextEditControls.append(str(self.__dict__[item].objectName())) option_fields = zip(CheckBoxControls, [True for i in CheckBoxControls], @@ -71,21 +75,25 @@ class PluginWidget(QWidget,Ui_Form): # LineEditControls option_fields += zip(['exclude_genre'],['\[.+\]|\+'],['line_edit']) + #option_fields += zip(['exclude_genre_results'],['excluded genres will appear here'],['line_edit']) + + # TextEditControls + #option_fields += zip(['exclude_genre_results'],['excluded genres will appear here'],['text_edit']) # SpinBoxControls option_fields += zip(['thumb_width'],[1.00],['spin_box']) # Exclusion rules - option_fields += zip(['exclusion_rules_tw','exclusion_rules_tw'], + option_fields += zip(['exclusion_rules_tw'], [{'ordinal':0, 'enabled':True, 'name':'Catalogs', 'field':'Tags', 'pattern':'Catalog'},], - ['table_widget','table_widget']) + ['table_widget']) # Prefix rules - option_fields += zip(['prefix_rules_tw','prefix_rules_tw','prefix_rules_tw'], + option_fields += zip(['prefix_rules_tw','prefix_rules_tw'], [{'ordinal':0, 'enabled':True, 'name':'Read book', @@ -98,7 +106,7 @@ class PluginWidget(QWidget,Ui_Form): 'field':'Tags', 'pattern':'Wishlist', 'prefix':u'\u00d7'},], - ['table_widget','table_widget','table_widget']) + ['table_widget','table_widget']) self.OPTION_FIELDS = option_fields @@ -110,20 +118,20 @@ class PluginWidget(QWidget,Ui_Form): ''' rule_set = [] for stored_rule in opt_value: - rule = copy(stored_rule) + rule = stored_rule.copy() # Skip disabled and incomplete rules if not rule['enabled']: continue elif not rule['field'] or not rule['pattern']: continue - elif 'prefix' in rule and not rule['prefix']: + elif 'prefix' in rule and rule['prefix'] is None: continue else: if rule['field'] != 'Tags': # Look up custom column friendly name rule['field'] = self.eligible_custom_fields[rule['field']]['field'] if rule['pattern'] in [_('any value'),_('any date')]: - rule['pattern'] = '.*' + rule_pattern = '.*' elif rule['pattern'] == _('unspecified'): rule['pattern'] = 'None' if 'prefix' in rule: @@ -135,6 +143,48 @@ class PluginWidget(QWidget,Ui_Form): # Strip off the trailing '_tw' opts_dict[c_name[:-3]] = opt_value + def exclude_genre_changed(self, regex): + """ Dynamically compute excluded genres. + + Run exclude_genre regex against db.all_tags() to show excluded tags. + PROVISIONAL CODE, NEEDS TESTING + + Args: + regex (QLineEdit.text()): regex to compile, compute + + Output: + self.exclude_genre_results (QLabel): updated to show tags to be excluded as genres + """ + if not regex: + self.exclude_genre_results.clear() + self.exclude_genre_results.setText(_('No genres will be excluded')) + return + + results = _('Regex does not match any tags in database') + try: + pattern = re.compile((str(regex))) + except: + results = _("regex error: %s") % sys.exc_info()[1] + else: + excluded_tags = [] + for tag in self.all_tags: + hit = pattern.search(tag) + if hit: + excluded_tags.append(hit.string) + if excluded_tags: + results = ', '.join(excluded_tags) + finally: + if self.DEBUG: + print(results) + self.exclude_genre_results.clear() + self.exclude_genre_results.setText(results) + + def exclude_genre_reset(self): + for default in self.OPTION_FIELDS: + if default[0] == 'exclude_genre': + self.exclude_genre.setText(default[1]) + break + def fetchEligibleCustomFields(self): self.all_custom_fields = self.db.custom_field_keys() custom_fields = {} @@ -163,10 +213,13 @@ class PluginWidget(QWidget,Ui_Form): ['thumb_width'] TableWidgetControls (c_type: table_widget): ['exclusion_rules_tw','prefix_rules_tw'] + TextEditControls (c_type: text_edit): + ['exclude_genre_results'] ''' self.name = name self.db = db + self.all_tags = db.all_tags() self.fetchEligibleCustomFields() self.populate_combo_boxes() @@ -200,9 +253,12 @@ class PluginWidget(QWidget,Ui_Form): if opt_value not in prefix_rules: prefix_rules.append(opt_value) - # Add icon to the reset button + # Add icon to the reset button, hook textChanged signal self.reset_exclude_genres_tb.setIcon(QIcon(I('trash.png'))) - self.reset_exclude_genres_tb.clicked.connect(self.reset_exclude_genres) + self.reset_exclude_genres_tb.clicked.connect(self.exclude_genre_reset) + + # Hook textChanged event for exclude_genre QLineEdit + self.exclude_genre.textChanged.connect(self.exclude_genre_changed) # Init self.merge_source_field_name self.merge_source_field_name = '' @@ -226,6 +282,9 @@ class PluginWidget(QWidget,Ui_Form): self.prefix_rules_table = PrefixRules(self.prefix_rules_gb, "prefix_rules_tw",prefix_rules, self.eligible_custom_fields,self.db) + # Initialize excluded genres preview + self.exclude_genre_changed(unicode(getattr(self, 'exclude_genre').text()).strip()) + def options(self): # Save/return the current options # exclude_genre stores literally @@ -377,12 +436,6 @@ class PluginWidget(QWidget,Ui_Form): self.merge_after.setEnabled(False) self.include_hr.setEnabled(False) - def reset_exclude_genres(self): - for default in self.OPTION_FIELDS: - if default[0] == 'exclude_genre': - self.exclude_genre.setText(default[1]) - break - class CheckableTableWidgetItem(QTableWidgetItem): ''' Borrowed from kiwidude diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.ui b/src/calibre/gui2/catalog/catalog_epub_mobi.ui index 5891d7a0cc..f3bffb4930 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.ui +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.ui @@ -41,151 +41,74 @@ Included sections - - - - Books by &Genre - - - - - - - Recently &Added - - - - - - - &Descriptions - - - - - - - Books by &Series - - - - - - - Books by &Title - - - true - Books by Author + &Authors false + + + + &Titles + + + + + + + &Series + + + + + + + &Genres + + + + + + + &Recently Added + + + + + + + &Descriptions + + + - + 0 0 - - - 0 - 0 - - - A regular expression describing genres to be excluded from the generated catalog. Genres are derived from the tags applied to your books. -The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book], and '+', the default tag for a read book. + The first matching prefix rule applies a prefix to book listings in the generated catalog. - Excluded genres + Prefixes - - - QFormLayout::FieldsStayAtSizeHint - - - - - -1 - - - 0 - - - - - - 175 - 0 - - - - - 200 - 16777215 - - - - Tags to &exclude - - - Qt::AutoText - - - Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter - - - true - - - exclude_genre - - - - - - - - 0 - 0 - - - - - 0 - 0 - - - - - - - - - - - Reset to default - - - ... - - - - + + + @@ -218,22 +141,148 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - + 0 0 + + + 0 + 0 + + - The first matching prefix rule applies a prefix to book listings in the generated catalog. + A regular expression describing genres to be excluded from the generated catalog. Genres are derived from the tags applied to your books. +The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book], and '+', the default tag for a read book. - Prefixes + Excluded genres - - - + + + + + + 175 + 0 + + + + + 200 + 16777215 + + + + Tags to &exclude (regex) + + + Qt::AutoText + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + true + + + exclude_genre + + + + + + + + 0 + 0 + + + + + 0 + 0 + + + + + + + + + + + Reset to default + + + ... + + + + + + + + 175 + 0 + + + + + 200 + 16777215 + + + + Results of regex + + + Qt::AutoText + + + Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter + + + true + + + exclude_genre + + + + + + + + 0 + 0 + + + + + 0 + 0 + + + + List of tags that will be excluded as genres + + + QFrame::StyledPanel + + + + + + Qt::PlainText + + + true + + diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 1860dcfe47..bfee271ed4 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -19,7 +19,6 @@ from calibre.utils.icu import capitalize, collation_order, sort_key from calibre.utils.magick.draw import thumbnail from calibre.utils.zipfile import ZipFile - class CatalogBuilder(object): ''' Generates catalog source files from calibre database @@ -29,15 +28,15 @@ class CatalogBuilder(object): gui2.tools:generate_catalog() or library.cli:command_catalog() called from gui2.convert.gui_conversion:gui_catalog() catalog = Catalog(notification=Reporter()) - catalog.createDirectoryStructure() - catalog.copyResources() - catalog.buildSources() + catalog.build_sources() Options managed in gui2.catalog.catalog_epub_mobi.py ''' + DEBUG = True + # A single number creates 'Last x days' only. # Multiple numbers create 'Last x days', 'x to y days ago' ... - # e.g, [7,15,30,60], [30] + # e.g, [7,15,30,60] or [30] # [] = No date ranges added DATE_RANGE=[30] @@ -46,8 +45,8 @@ class CatalogBuilder(object): # basename output file basename # creator dc:creator in OPF metadata - # descriptionClip limits size of NCX descriptions (Kindle only) - # includeSources Used in processSpecialTags to skip tags like '[SPL]' + # description_clip limits size of NCX descriptions (Kindle only) + # includeSources Used in filter_excluded_tags to skip tags like '[SPL]' # notification Used to check for cancel, report progress # stylesheet CSS stylesheet # title dc:title in OPF metadata, NCX periodical @@ -55,451 +54,475 @@ class CatalogBuilder(object): def __init__(self, db, opts, plugin, report_progress=DummyReporter(), - stylesheet="content/stylesheet.css"): - self.__opts = opts - self.__authorClip = opts.authorClip - self.__authors = None - self.__basename = opts.basename - self.__bookmarked_books = None - self.__booksByAuthor = None - self.__booksByDateRead = None - self.__booksByTitle = None - self.__booksByTitle_noSeriesPrefix = None - self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') - self.__archive_path = os.path.join(self.__cache_dir, "thumbs.zip") - self.__catalogPath = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') - self.__contentDir = os.path.join(self.catalogPath, "content") - self.__currentStep = 0.0 - self.__creator = opts.creator - self.__db = db - self.__defaultPrefix = None - self.__descriptionClip = opts.descriptionClip - self.__error = [] - self.__generateForKindle = True if (self.opts.fmt == 'mobi' and \ - self.opts.output_profile and \ - self.opts.output_profile.startswith("kindle")) else False - self.__generateRecentlyRead = True if self.opts.generate_recently_added \ - and self.opts.connected_kindle \ - and self.generateForKindle \ - else False - self.__genres = None - self.genres = [] - self.__genre_tags_dict = None - self.__htmlFileList_1 = [] - self.__htmlFileList_2 = [] - self.__markerTags = self.getMarkerTags() - self.__ncxSoup = None - self.__output_profile = None - self.__playOrder = 1 - self.__plugin = plugin - self.__prefixRules = [] - self.__progressInt = 0.0 - self.__progressString = '' - f, p, hr = self.opts.merge_comments.split(':') - self.__merge_comments = {'field':f, 'position':p, 'hr':hr} - self.__reporter = report_progress - self.__stylesheet = stylesheet - self.__thumbs = None - self.__thumbWidth = 0 - self.__thumbHeight = 0 - self.__title = opts.catalog_title - self.__totalSteps = 6.0 - self.__useSeriesPrefixInTitlesSection = False - self.__verbose = opts.verbose + stylesheet="content/stylesheet.css", + init_resources=True): + ''' active database ''' + @property + def db(self): + return self.__db + self.__db = db + + ''' opts passed from gui2.catalog.catalog_epub_mobi.py ''' + @property + def opts(self): + return self.__opts + self.__opts = opts + + ''' catalog??? device??? ''' + @property + def plugin(self): + return self.__plugin + self.__plugin = plugin + + ''' Progress Reporter for Jobs ''' + @property + def reporter(self): + return self.__reporter + self.__reporter = report_progress + + ''' stylesheet to include with catalog ''' + @property + def stylesheet(self): + return self.__stylesheet + self.__stylesheet = stylesheet + + # Initialize properties with dependents in _initialize() + ''' directory to store cached thumbs ''' + @property + def cache_dir(self): + return self.__cache_dir + self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') + + ''' temp dir to store generated catalog ''' + @property + def catalog_path(self): + return self.__catalog_path + self.__catalog_path = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') + + ''' True if generating for Kindle in MOBI format ''' + @property + def generate_for_kindle(self): + return self.__generate_for_kindle + self.__generate_for_kindle = True if (opts.fmt == 'mobi' and + opts.output_profile and + opts.output_profile.startswith("kindle")) else False + + self._initialize(init_resources) + + def _initialize(self,init_resources): + # continue with initialization + + ''' list of unique authors ''' + @property + def authors(self): + return self.__authors + self.__authors = None + + ''' dict of bookmarked books ''' + @property + def bookmarked_books(self): + return self.__bookmarked_books + @bookmarked_books.setter + def bookmarked_books(self, val): + self.__bookmarked_books = val + self.__bookmarked_books = None + + ''' list of bookmarked books, sorted by date read ''' + @property + def bookmarked_books_by_date_read(self): + return self.__bookmarked_books_by_date_read + @bookmarked_books_by_date_read.setter + def bookmarked_books_by_date_read(self, val): + self.__bookmarked_books_by_date_read = val + self.__bookmarked_books_by_date_read = None + + ''' list of books, sorted by author ''' + @property + def books_by_author(self): + return self.__books_by_author + @books_by_author.setter + def books_by_author(self, val): + self.__books_by_author = val + self.__books_by_author = None + + ''' list of books, grouped by date range (30 days) ''' + @property + def books_by_date_range(self): + return self.__books_by_date_range + @books_by_date_range.setter + def books_by_date_range(self, val): + self.__books_by_date_range = val + self.__books_by_date_range = None + + ''' list of books, by date added reverse (most recent first) ''' + @property + def books_by_month(self): + return self.__books_by_month + @books_by_month.setter + def books_by_month(self, val): + self.__books_by_month = val + self.__books_by_month = None + + ''' list of books in series ''' + @property + def books_by_series(self): + return self.__books_by_series + @books_by_series.setter + def books_by_series(self, val): + self.__books_by_series = val + self.__books_by_series = None + + ''' list of books, sorted by title ''' + @property + def books_by_title(self): + return self.__books_by_title + @books_by_title.setter + def books_by_title(self, val): + self.__books_by_title = val + self.__books_by_title = None + + ''' list of books in series, without series prefix ''' + @property + def books_by_title_no_series_prefix(self): + return books_by_title_no_series_prefix.__prop + @books_by_title_no_series_prefix.setter + def books_by_title_no_series_prefix(self, val): + self.__books_by_title_no_series_prefix = val + self.__books_by_title_no_series_prefix = None + + ''' content dir in generated catalog ''' + @property + def content_dir(self): + return self.__content_dir + self.__content_dir = os.path.join(self.catalog_path, "content") + + ''' track Job progress ''' + @property + def current_step(self): + return self.__current_step + @current_step.setter + def current_step(self, val): + self.__current_step = val + self.__current_step = 0.0 + + ''' cumulative error messages to report at conclusion ''' + @property + def error(self): + return self.__error + @error.setter + def error(self, val): + self.__error = val + self.__error = [] + + ''' tags to exclude as genres ''' + @property + def excluded_tags(self): + return self.__excluded_tags + self.__excluded_tags = self.get_excluded_tags() + + ''' True if connected Kindle and generating for Kindle ''' + @property + def generate_recently_read(self): + return self.__generate_recently_read + self.__generate_recently_read = True if (opts.generate_recently_added and + opts.connected_kindle and + self.generate_for_kindle) else False + + ''' list of dicts with books by genre ''' + @property + def genres(self): + return self.__genres + @genres.setter + def genres(self, val): + self.__genres = val + self.__genres = [] + + ''' dict of enabled genre tags ''' + @property + def genre_tags_dict(self): + return self.__genre_tags_dict + @genre_tags_dict.setter + def genre_tags_dict(self, val): + self.__genre_tags_dict = val + self.__genre_tags_dict = None + + ''' Author, Title, Series sections ''' + @property + def html_filelist_1(self): + return self.__html_filelist_1 + @html_filelist_1.setter + def html_filelist_1(self, val): + self.__html_filelist_1 = val + self.__html_filelist_1 = [] + + ''' Date Added, Date Read ''' + @property + def html_filelist_2(self): + return self.__html_filelist_2 + @html_filelist_2.setter + def html_filelist_2(self, val): + self.__html_filelist_2 = val + self.__html_filelist_2 = [] + + ''' additional field to include before/after comments ''' + @property + def merge_comments_rule(self): + return self.__merge_comments_rule + #f, p, hr = opts.merge_comments_rule.split(':') + #self.__merge_comments_rule = {'field':f, 'position':p, 'hr':hr} + self.__merge_comments_rule = dict(zip(['field','position','hr'],opts.merge_comments_rule.split(':'))) + + ''' cumulative HTML for NCX file ''' + @property + def ncx_soup(self): + return self.__ncx_soup + @ncx_soup.setter + def ncx_soup(self, val): + self.__ncx_soup = val + self.__ncx_soup = None + + ''' output_profile declares special symbols ''' + @property + def output_profile(self): + return self.__output_profile + self.__output_profile = None from calibre.customize.ui import output_profiles for profile in output_profiles(): - if profile.short_name == self.opts.output_profile: + if profile.short_name == opts.output_profile: self.__output_profile = profile break - # Process prefix rules - self.processPrefixRules() + ''' playOrder value for building NCX ''' + @property + def play_order(self): + return self.__play_order + @play_order.setter + def play_order(self, val): + self.__play_order = val + self.__play_order = 1 - # Confirm/create thumbs archive. - if self.opts.generate_descriptions: - if not os.path.exists(self.__cache_dir): - self.opts.log.info(" creating new thumb cache '%s'" % self.__cache_dir) - os.makedirs(self.__cache_dir) - if not os.path.exists(self.__archive_path): - self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % - float(self.opts.thumb_width)) - with ZipFile(self.__archive_path, mode='w') as zfw: - zfw.writestr("Catalog Thumbs Archive",'') + ''' dict of prefix rules ''' + @property + def prefix_rules(self): + return self.__prefix_rules + @prefix_rules.setter + def prefix_rules(self, val): + self.__prefix_rules = val + self.__prefix_rules = self.get_prefix_rules() + + ''' used with ProgressReporter() ''' + @property + def progress_int(self): + return self.__progress_int + @progress_int.setter + def progress_int(self, val): + self.__progress_int = val + self.__progress_int = 0.0 + + ''' used with ProgressReporter() ''' + @property + def progress_string(self): + return self.__progress_string + @progress_string.setter + def progress_string(self, val): + self.__progress_string = val + self.__progress_string = '' + + ''' device-specific symbol (default empty star) ''' + @property + def SYMBOL_EMPTY_RATING(self): + return self.output_profile.empty_ratings_char + + ''' device-specific symbol (default filled star) ''' + @property + def SYMBOL_FULL_RATING(self): + return self.output_profile.ratings_char + + ''' device-specific symbol for reading progress ''' + @property + def SYMBOL_PROGRESS_READ(self): + psr = '+' + if self.generate_for_kindle: + psr = '▪' + return psr + + ''' device-specific symbol for reading progress ''' + @property + def SYMBOL_PROGRESS_UNREAD(self): + psu = '-' + if self.generate_for_kindle: + psu = '▫' + return psu + + ''' device-specific symbol for reading progress ''' + @property + def SYMBOL_READING(self): + if self.generate_for_kindle: + return self.format_prefix('▷') else: - try: - with ZipFile(self.__archive_path, mode='r') as zfr: - try: - cached_thumb_width = zfr.read('thumb_width') - except: - cached_thumb_width = "-1" - except: - os.remove(self.__archive_path) - cached_thumb_width = '-1' + return self.format_prefix(' ') - if float(cached_thumb_width) != float(self.opts.thumb_width): - self.opts.log.warning(" invalidating cache at '%s'" % self.__archive_path) - self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % - (float(cached_thumb_width),float(self.opts.thumb_width))) - with ZipFile(self.__archive_path, mode='w') as zfw: - zfw.writestr("Catalog Thumbs Archive",'') - else: - self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % - (self.__archive_path, float(cached_thumb_width))) + @property + def thumb_height(self): + return self.__thumb_height + @thumb_height.setter + def thumb_height(self, val): + self.__thumb_height = val + self.__thumb_height = 0 - # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX - incremental_jobs = 0 - if self.opts.generate_authors: - incremental_jobs += 2 - if self.opts.generate_titles: - incremental_jobs += 2 - if self.opts.generate_recently_added: - incremental_jobs += 2 - if self.generateRecentlyRead: - incremental_jobs += 2 - if self.opts.generate_series: - incremental_jobs += 2 - if self.opts.generate_descriptions: - # +1 thumbs - incremental_jobs += 3 - self.__totalSteps += incremental_jobs + @property + def thumb_width(self): + return self.__thumb_width + @thumb_width.setter + def thumb_width(self, val): + self.__thumb_width = val + self.__thumb_width = 0 - # Load section list templates - templates = {} - execfile(P('catalog/section_list_templates.py'), templates) - for name, template in templates.iteritems(): - if name.startswith('by_') and name.endswith('_template'): - setattr(self, name, force_unicode(template, 'utf-8')) - - # Accessors - if True: - ''' - @dynamic_property - def xxxx(self): - def fget(self): - return self.__ - def fset(self, val): - self.__ = val - return property(fget=fget, fset=fset) - ''' - @dynamic_property - def authorClip(self): - def fget(self): - return self.__authorClip - def fset(self, val): - self.__authorClip = val - return property(fget=fget, fset=fset) - @dynamic_property - def authors(self): - def fget(self): - return self.__authors - def fset(self, val): - self.__authors = val - return property(fget=fget, fset=fset) - @dynamic_property - def basename(self): - def fget(self): - return self.__basename - def fset(self, val): - self.__basename = val - return property(fget=fget, fset=fset) - @dynamic_property - def bookmarked_books(self): - def fget(self): - return self.__bookmarked_books - def fset(self, val): - self.__bookmarked_books = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByAuthor(self): - def fget(self): - return self.__booksByAuthor - def fset(self, val): - self.__booksByAuthor = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByDateRead(self): - def fget(self): - return self.__booksByDateRead - def fset(self, val): - self.__booksByDateRead = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByTitle(self): - def fget(self): - return self.__booksByTitle - def fset(self, val): - self.__booksByTitle = val - return property(fget=fget, fset=fset) - @dynamic_property - def booksByTitle_noSeriesPrefix(self): - def fget(self): - return self.__booksByTitle_noSeriesPrefix - def fset(self, val): - self.__booksByTitle_noSeriesPrefix = val - return property(fget=fget, fset=fset) - @dynamic_property - def catalogPath(self): - def fget(self): - return self.__catalogPath - def fset(self, val): - self.__catalogPath = val - return property(fget=fget, fset=fset) - @dynamic_property - def contentDir(self): - def fget(self): - return self.__contentDir - def fset(self, val): - self.__contentDir = val - return property(fget=fget, fset=fset) - @dynamic_property - def currentStep(self): - def fget(self): - return self.__currentStep - def fset(self, val): - self.__currentStep = val - return property(fget=fget, fset=fset) - @dynamic_property - def creator(self): - def fget(self): - return self.__creator - def fset(self, val): - self.__creator = val - return property(fget=fget, fset=fset) - @dynamic_property - def db(self): - def fget(self): - return self.__db - return property(fget=fget) - @dynamic_property - def defaultPrefix(self): - def fget(self): - return self.__defaultPrefix - def fset(self, val): - self.__defaultPrefix = val - return property(fget=fget, fset=fset) - @dynamic_property - def descriptionClip(self): - def fget(self): - return self.__descriptionClip - def fset(self, val): - self.__descriptionClip = val - return property(fget=fget, fset=fset) - @dynamic_property - def error(self): - def fget(self): - return self.__error - def fset(self, val): - self.__error = val - return property(fget=fget,fset=fset) - @dynamic_property - def generateForKindle(self): - def fget(self): - return self.__generateForKindle - def fset(self, val): - self.__generateForKindle = val - return property(fget=fget, fset=fset) - @dynamic_property - def generateRecentlyRead(self): - def fget(self): - return self.__generateRecentlyRead - def fset(self, val): - self.__generateRecentlyRead = val - return property(fget=fget, fset=fset) - @dynamic_property - def genres(self): - def fget(self): - return self.__genres - def fset(self, val): - self.__genres = val - return property(fget=fget, fset=fset) - @dynamic_property - def genre_tags_dict(self): - def fget(self): - return self.__genre_tags_dict - def fset(self, val): - self.__genre_tags_dict = val - return property(fget=fget, fset=fset) - @dynamic_property - def htmlFileList_1(self): - def fget(self): - return self.__htmlFileList_1 - def fset(self, val): - self.__htmlFileList_1 = val - return property(fget=fget, fset=fset) - @dynamic_property - def htmlFileList_2(self): - def fget(self): - return self.__htmlFileList_2 - def fset(self, val): - self.__htmlFileList_2 = val - return property(fget=fget, fset=fset) - @dynamic_property - def libraryPath(self): - def fget(self): - return self.__libraryPath - def fset(self, val): - self.__libraryPath = val - return property(fget=fget, fset=fset) - @dynamic_property - def markerTags(self): - def fget(self): - return self.__markerTags - def fset(self, val): - self.__markerTags = val - return property(fget=fget, fset=fset) - @dynamic_property - def ncxSoup(self): - def fget(self): - return self.__ncxSoup - def fset(self, val): - self.__ncxSoup = val - return property(fget=fget, fset=fset) - @dynamic_property - def opts(self): - def fget(self): - return self.__opts - return property(fget=fget) - @dynamic_property - def playOrder(self): - def fget(self): - return self.__playOrder - def fset(self,val): - self.__playOrder = val - return property(fget=fget, fset=fset) - @dynamic_property - def plugin(self): - def fget(self): - return self.__plugin - return property(fget=fget) - @dynamic_property - def prefixRules(self): - def fget(self): - return self.__prefixRules - def fset(self, val): - self.__prefixRules = val - return property(fget=fget, fset=fset) - @dynamic_property - def progressInt(self): - def fget(self): - return self.__progressInt - def fset(self, val): - self.__progressInt = val - return property(fget=fget, fset=fset) - @dynamic_property - def progressString(self): - def fget(self): - return self.__progressString - def fset(self, val): - self.__progressString = val - return property(fget=fget, fset=fset) - @dynamic_property - def reporter(self): - def fget(self): - return self.__reporter - def fset(self, val): - self.__reporter = val - return property(fget=fget, fset=fset) - @dynamic_property - def stylesheet(self): - def fget(self): - return self.__stylesheet - def fset(self, val): - self.__stylesheet = val - return property(fget=fget, fset=fset) - @dynamic_property + ''' list of generated thumbs ''' + @property def thumbs(self): - def fget(self): - return self.__thumbs - def fset(self, val): - self.__thumbs = val - return property(fget=fget, fset=fset) - def thumbWidth(self): - def fget(self): - return self.__thumbWidth - def fset(self, val): - self.__thumbWidth = val - return property(fget=fget, fset=fset) - def thumbHeight(self): - def fget(self): - return self.__thumbHeight - def fset(self, val): - self.__thumbHeight = val - return property(fget=fget, fset=fset) - @dynamic_property - def title(self): - def fget(self): - return self.__title - def fset(self, val): - self.__title = val - return property(fget=fget, fset=fset) - @dynamic_property - def totalSteps(self): - def fget(self): - return self.__totalSteps - return property(fget=fget) - @dynamic_property - def useSeriesPrefixInTitlesSection(self): - def fget(self): - return self.__useSeriesPrefixInTitlesSection - def fset(self, val): - self.__useSeriesPrefixInTitlesSection = val - return property(fget=fget, fset=fset) - @dynamic_property - def verbose(self): - def fget(self): - return self.__verbose - def fset(self, val): - self.__verbose = val - return property(fget=fget, fset=fset) + return self.__thumbs + @thumbs.setter + def thumbs(self, val): + self.__thumbs = val + self.__thumbs = None - @dynamic_property - def READING_SYMBOL(self): - def fget(self): - return '▷' if self.generateForKindle else \ - '+' - return property(fget=fget) - @dynamic_property - def FULL_RATING_SYMBOL(self): - def fget(self): - return self.__output_profile.ratings_char - return property(fget=fget) - @dynamic_property - def EMPTY_RATING_SYMBOL(self): - def fget(self): - return self.__output_profile.empty_ratings_char - return property(fget=fget) - @dynamic_property + ''' full path to thumbs archive ''' + @property + def thumbs_path(self): + return self.__thumbs_path + self.__thumbs_path = os.path.join(self.cache_dir, "thumbs.zip") - def READ_PROGRESS_SYMBOL(self): - def fget(self): - return "▪" if self.generateForKindle else '+' - return property(fget=fget) - @dynamic_property - def UNREAD_PROGRESS_SYMBOL(self): - def fget(self): - return "▫" if self.generateForKindle else '-' - return property(fget=fget) + ''' used with ProgressReporter() ''' + @property + def total_steps(self): + return self.__total_steps + self.__total_steps = 6.0 - # Methods - def buildSources(self): - if self.booksByTitle is None: - if not self.fetchBooksByTitle(): + ''' switch controlling format of series books in Titles section ''' + @property + def use_series_prefix_in_titles_section(self): + return self.__use_series_prefix_in_titles_section + self.__use_series_prefix_in_titles_section = False + + self.compute_total_steps() + self.confirm_thumbs_archive() + self.load_section_templates() + if init_resources: + self.copy_catalog_resources() + + """ key() functions """ + + def kf_author_to_author_sort(self, author): + """ Compute author_sort value from author + + Tokenize author string, return capitalized string with last token first + + Args: + author (str): author, e.g. 'John Smith' + + Return: + (str): 'Smith, john' + """ + tokens = author.split() + tokens = tokens[-1:] + tokens[:-1] + if len(tokens) > 1: + tokens[0] += ',' + return ' '.join(tokens).capitalize() + + def kf_books_by_author_sorter_author(self, book): + """ Generate book sort key with computed author_sort. + + Generate a sort key of computed author_sort, title. + Twiddle included to force series to sort after non-series books. + 'Smith, john Star Wars' + 'Smith, john ~Star Wars 0001.0000' + + Args: + book (dict): book metadata + + Return: + (str): sort key + """ + if not book['series']: + key = '%s %s' % (self.kf_author_to_author_sort(book['author']), + capitalize(book['title_sort'])) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s ~%s %s' % (self.kf_author_to_author_sort(book['author']), + self.generate_sort_title(book['series']), + series_index) + return key + + def kf_books_by_author_sorter_author_sort(self, book): + """ Generate book sort key with supplied author_sort. + + Generate a sort key of author_sort, title. + Twiddle included to force series to sort after non-series books. + 'Smith, john Star Wars' + 'Smith, john ~Star Wars 0001.0000' + + Args: + book (dict): book metadata + + Return: + (str): sort key + """ + if not book['series']: + key = '%s ~%s' % (capitalize(book['author_sort']), + capitalize(book['title_sort'])) + else: + index = book['series_index'] + integer = int(index) + fraction = index-integer + series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) + key = '%s %s %s' % (capitalize(book['author_sort']), + self.generate_sort_title(book['series']), + series_index) + return key + + + """ Methods """ + + def build_sources(self): + """ Generate catalog source files. + + Assemble OPF, HTML and NCX files reflecting catalog options. + Generated source is OEB compliant. + Called from gui2.convert.gui_conversion:gui_catalog() + + Args: + (none) + + Touches: + error: problems reported during build + + Return: + (bool): True: successful build, possibly with warnings + False: failed to build + """ + + if self.books_by_title is None: + if not self.fetch_books_by_title(): return False - if not self.fetchBooksByAuthor(): + if not self.fetch_books_by_author(): return False - self.fetchBookmarks() + self.fetch_bookmarks() if self.opts.generate_descriptions: - self.generateThumbnails() - self.generateHTMLDescriptions() + self.generate_thumbnails() + self.generate_html_descriptions() if self.opts.generate_authors: - self.generateHTMLByAuthor() + self.generate_html_by_author() if self.opts.generate_titles: - self.generateHTMLByTitle() + self.generate_html_by_title() if self.opts.generate_series: - self.generateHTMLBySeries() + self.generate_html_by_series() if self.opts.generate_genres: - self.generateHTMLByTags() + self.generate_html_by_genres() # If this is the only Section, and there are no genres, bail if self.opts.section_list == ['Genres'] and not self.genres: error_msg = _("No enabled genres found to catalog.\n") @@ -510,35 +533,188 @@ class CatalogBuilder(object): self.error.append(error_msg) return False if self.opts.generate_recently_added: - self.generateHTMLByDateAdded() - if self.generateRecentlyRead: - self.generateHTMLByDateRead() + self.generate_html_by_date_added() + if self.generate_recently_read: + self.generate_html_by_date_read() - self.generateOPF() - self.generateNCXHeader() + self.generate_opf() + self.generate_ncx_header() if self.opts.generate_authors: - self.generateNCXByAuthor(_("Authors")) + self.generate_ncx_by_author(_("Authors")) if self.opts.generate_titles: - self.generateNCXByTitle(_("Titles")) + self.generate_ncx_by_title(_("Titles")) if self.opts.generate_series: - self.generateNCXBySeries(_("Series")) + self.generate_ncx_by_series(_("Series")) if self.opts.generate_genres: - self.generateNCXByGenre(_("Genres")) + self.generate_ncx_by_genre(_("Genres")) if self.opts.generate_recently_added: - self.generateNCXByDateAdded(_("Recently Added")) - if self.generateRecentlyRead: - self.generateNCXByDateRead(_("Recently Read")) + self.generate_ncx_by_date_added(_("Recently Added")) + if self.generate_recently_read: + self.generate_ncx_by_date_read(_("Recently Read")) if self.opts.generate_descriptions: - self.generateNCXDescriptions(_("Descriptions")) + self.generate_ncx_descriptions(_("Descriptions")) - self.writeNCX() + self.write_ncx() return True - def cleanUp(self): - pass + ''' + def calculate_thumbnail_dimensions(self): + """ Calculate thumb dimensions based on device DPI. - def copyResources(self): - '''Move resource files to self.catalogPath''' + Using the specified output profile, calculate thumb_width + in pixels, then set height to width * 1.33. Special-case for + Kindle/MOBI, as rendering off by 2. + *** dead code? *** + + Inputs: + opts.thumb_width (str|float): specified thumb_width + opts.output_profile.dpi (int): device DPI + + Outputs: + thumb_width (float): calculated thumb_width + thumb_height (float): calculated thumb_height + """ + + from calibre.customize.ui import output_profiles + for x in output_profiles(): + if x.short_name == self.opts.output_profile: + # aspect ratio: 3:4 + self.thumb_width = x.dpi * float(self.opts.thumb_width) + self.thumb_height = self.thumb_width * 1.33 + if 'kindle' in x.short_name and self.opts.fmt == 'mobi': + # Kindle DPI appears to be off by a factor of 2 + self.thumb_width = self.thumb_width/2 + self.thumb_height = self.thumb_height/2 + break + if self.opts.verbose: + self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ + (x.dpi, self.thumb_width, self.thumb_height)) + ''' + + def compute_total_steps(self): + """ Calculate number of build steps to generate catalog. + + Calculate total number of build steps based on enabled sections. + + Inputs: + opts.generate_* (bool): enabled sections + + Outputs: + total_steps (int): updated + """ + # Tweak build steps based on optional sections: 1 call for HTML, 1 for NCX + incremental_jobs = 0 + if self.opts.generate_authors: + incremental_jobs += 2 + if self.opts.generate_titles: + incremental_jobs += 2 + if self.opts.generate_recently_added: + incremental_jobs += 2 + if self.generate_recently_read: + incremental_jobs += 2 + if self.opts.generate_series: + incremental_jobs += 2 + if self.opts.generate_descriptions: + # +1 thumbs + incremental_jobs += 3 + self.total_steps += incremental_jobs + + def confirm_thumbs_archive(self): + """ Validate thumbs archive. + + Confirm existence of thumbs archive, or create if absent. + Confirm stored thumb_width matches current opts.thumb_width, + or invalidate archive. + generate_thumbnails() writes current thumb_width to archive. + + Inputs: + opts.thumb_width (float): requested thumb_width + thumbs_path (file): existing thumbs archive + + Outputs: + thumbs_path (file): new (non_existent or invalidated), or + validated existing thumbs archive + """ + if self.opts.generate_descriptions: + if not os.path.exists(self.cache_dir): + self.opts.log.info(" creating new thumb cache '%s'" % self.cache_dir) + os.makedirs(self.cache_dir) + if not os.path.exists(self.thumbs_path): + self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % + float(self.opts.thumb_width)) + with ZipFile(self.thumbs_path, mode='w') as zfw: + zfw.writestr("Catalog Thumbs Archive",'') + else: + try: + with ZipFile(self.thumbs_path, mode='r') as zfr: + try: + cached_thumb_width = zfr.read('thumb_width') + except: + cached_thumb_width = "-1" + except: + os.remove(self.thumbs_path) + cached_thumb_width = '-1' + + if float(cached_thumb_width) != float(self.opts.thumb_width): + self.opts.log.warning(" invalidating cache at '%s'" % self.thumbs_path) + self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % + (float(cached_thumb_width),float(self.opts.thumb_width))) + with ZipFile(self.thumbs_path, mode='w') as zfw: + zfw.writestr("Catalog Thumbs Archive",'') + else: + self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % + (self.thumbs_path, float(cached_thumb_width))) + + + def convert_html_entities(self, s): + """ Convert string containing HTML entities to its unicode equivalent. + + Convert a string containing HTML entities of the form '&' or '&97;' + to a normalized unicode string. E.g., 'AT&T' converted to 'AT&T'. + + Args: + s (str): str containing one or more HTML entities. + + Return: + s (str): converted string + """ + matches = re.findall("&#\d+;", s) + if len(matches) > 0: + hits = set(matches) + for hit in hits: + name = hit[2:-1] + try: + entnum = int(name) + s = s.replace(hit, unichr(entnum)) + except ValueError: + pass + + matches = re.findall("&\w+;", s) + hits = set(matches) + amp = "&" + if amp in hits: + hits.remove(amp) + for hit in hits: + name = hit[1:-1] + if htmlentitydefs.name2codepoint.has_key(name): + s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) + s = s.replace(amp, "&") + return s + + def copy_catalog_resources(self): + """ Copy resources from calibre source to self.catalog_path. + + Copy basic resources - default cover, stylesheet, and masthead (Kindle only) + from calibre resource directory to self.catalog_path, a temporary directory + for constructing the catalog. Files stored to specified destination dirs. + + Inputs: + files_to_copy (files): resource files from calibre resources, which may be overridden locally + + Output: + resource files copied to self.catalog_path/* + """ + self.create_catalog_directory_structure() catalog_resources = P("catalog") files_to_copy = [('','DefaultCover.jpg'), @@ -548,32 +724,169 @@ class CatalogBuilder(object): for file in files_to_copy: if file[0] == '': shutil.copy(os.path.join(catalog_resources,file[1]), - self.catalogPath) + self.catalog_path) else: shutil.copy(os.path.join(catalog_resources,file[1]), - os.path.join(self.catalogPath, file[0])) + os.path.join(self.catalog_path, file[0])) - # Create the custom masthead image overwriting default - # If failure, default mastheadImage.gif should still be in place - if self.generateForKindle: + if self.generate_for_kindle: try: - self.generateMastheadImage(os.path.join(self.catalogPath, + self.generate_masthead_image(os.path.join(self.catalog_path, 'images/mastheadImage.gif')) except: pass - def fetchBooksByAuthor(self): - ''' - Generate a list of titles sorted by author from the database - return = Success - ''' + def create_catalog_directory_structure(self): + """ Create subdirs in catalog output dir. - self.updateProgressFullStep("Sorting database") - self.booksByAuthor = list(self.booksByTitle) - self.booksByAuthor = sorted(self.booksByAuthor, key=self.booksByAuthorSorter_author) + Create /content and /images in self.catalog_path + + Inputs: + catalog_path (path): path to catalog output dir + + Output: + /content, /images created + """ + if not os.path.isdir(self.catalog_path): + os.makedirs(self.catalog_path) + + content_path = self.catalog_path + "/content" + if not os.path.isdir(content_path): + os.makedirs(content_path) + images_path = self.catalog_path + "/images" + if not os.path.isdir(images_path): + os.makedirs(images_path) + + def discover_prefix(self, record): + """ Return a prefix for record. + + Evaluate record against self.prefix_rules. Return assigned prefix + if matched. + + Args: + record (dict): book metadata + + Return: + prefix (str): matched a prefix_rule + None: no match + """ + def _log_prefix_rule_match_info(rule, record): + self.opts.log.info(" %s %s by %s (Prefix rule '%s': %s:%s)" % + (rule['prefix'],record['title'], + record['authors'][0], rule['name'], + rule['field'],rule['pattern'])) + + # Compare the record to each rule looking for a match + for rule in self.prefix_rules: + # Literal comparison for Tags field + if rule['field'].lower() == 'tags': + if rule['pattern'].lower() in map(unicode.lower,record['tags']): + if self.opts.verbose: + _log_prefix_rule_match_info(rule, record) + return rule['prefix'] + + # Regex match for custom field + elif rule['field'].startswith('#'): + field_contents = self.db.get_field(record['id'], + rule['field'], + index_is_id=True) + if field_contents == '': + field_contents = None + + if field_contents is not None: + try: + if re.search(rule['pattern'], unicode(field_contents), + re.IGNORECASE) is not None: + if self.opts.verbose: + _log_prefix_rule_match_info(rule, record) + return rule['prefix'] + except: + if self.opts.verbose: + self.opts.log.error("pattern failed to compile: %s" % rule['pattern']) + pass + elif field_contents is None and rule['pattern'] == 'None': + if self.opts.verbose: + _log_prefix_rule_match_info(rule, record) + return rule['prefix'] + + return None + + def establish_equivalencies(self, item_list, key=None): + """ Return icu equivalent sort letter. + + Returns base sort letter for accented characters. Code provided by + chaley, modified to force unaccented base letters for A, O & U when + an accented version would otherwise be returned. + + Args: + item_list (list): list of items, sorted by icu_sort + + Return: + cl_list (list): list of equivalent leading chars, 1:1 correspondence to item_list + """ + + # Hack to force the cataloged leading letter to be + # an unadorned character if the accented version sorts before the unaccented + exceptions = { + u'Ä':u'A', + u'Ö':u'O', + u'Ü':u'U' + } + + if key is not None: + sort_field = key + + cl_list = [None] * len(item_list) + last_ordnum = 0 + + for idx, item in enumerate(item_list): + if key: + c = item[sort_field] + else: + c = item + + ordnum, ordlen = collation_order(c) + if last_ordnum != ordnum: + last_c = icu_upper(c[0:ordlen]) + if last_c in exceptions.keys(): + last_c = exceptions[unicode(last_c)] + last_ordnum = ordnum + cl_list[idx] = last_c + + if self.DEBUG and self.opts.verbose: + if key: + for idx, item in enumerate(item_list): + print("%s %s" % (cl_list[idx],item[sort_field])) + else: + print("%s %s" % (cl_list[0], item)) + + return cl_list + + def fetch_books_by_author(self): + """ Generate a list of books sorted by author. + + Sort the database by author. Report author_sort inconsistencies as warning when + building EPUB or MOBI, error when building MOBI. Collect a list of unique authors + to self.authors. + + Inputs: + self.books_by_title (list): database, sorted by title + + Outputs: + books_by_author: database, sorted by author + authors: list of unique authors + error: author_sort mismatches + + Return: + True: no errors + False: author_sort mismatch detected while building MOBI + """ + + self.update_progress_full_step(_("Sorting database")) + self.books_by_author = sorted(list(self.books_by_title), key=self.kf_books_by_author_sorter_author) # Build the unique_authors set from existing data, test for author_sort mismatches - authors = [(record['author'], record['author_sort']) for record in self.booksByAuthor] + authors = [(record['author'], record['author_sort']) for record in self.books_by_author] current_author = authors[0] for (i,author) in enumerate(authors): if author != current_author and i: @@ -607,11 +920,11 @@ Author '{0}': current_author = author - self.booksByAuthor = sorted(self.booksByAuthor, - key=lambda x: sort_key(self.booksByAuthorSorter_author_sort(x))) + self.books_by_author = sorted(self.books_by_author, + key=lambda x: sort_key(self.kf_books_by_author_sorter_author_sort(x))) # Build the unique_authors set from existing data - authors = [(record['author'], capitalize(record['author_sort'])) for record in self.booksByAuthor] + authors = [(record['author'], capitalize(record['author_sort'])) for record in self.books_by_author] # authors[] contains a list of all book authors, with multiple entries for multiple books by author # authors[]: (([0]:friendly [1]:sort)) @@ -642,8 +955,8 @@ Author '{0}': unique_authors.append((current_author[0], icu_title(current_author[1]), books_by_current_author)) - if False and self.verbose: - self.opts.log.info("\nfetchBooksByauthor(): %d unique authors" % len(unique_authors)) + if self.DEBUG and self.opts.verbose: + self.opts.log.info("\nfetch_books_by_author(): %d unique authors" % len(unique_authors)) for author in unique_authors: self.opts.log.info((u" %-50s %-25s %2d" % (author[0][0:45], author[1][0:20], author[2])).encode('utf-8')) @@ -651,60 +964,50 @@ Author '{0}': self.authors = unique_authors return True - def fetchBooksByTitle(self): - self.updateProgressFullStep("Fetching database") + def fetch_books_by_title(self): + """ Populate self.books_by_title from database - self.opts.sort_by = 'title' + Create self.books_by_title from filtered database. + Keys: + authors massaged + author_sort record['author_sort'] or computed + cover massaged record['cover'] + date massaged record['pubdate'] + description massaged record['comments'] + merge_comments + id record['id'] + formats massaged record['formats'] + notes from opts.header_note_source_field + prefix from self.discover_prefix() + publisher massaged record['publisher'] + rating record['rating'] (0 if None) + series record['series'] or None + series_index record['series_index'] or 0.0 + short_description truncated description + tags filtered record['tags'] + timestamp record['timestamp'] + title massaged record['title'] + title_sort computed from record['title'] + uuid record['uuid'] - # Merge opts.exclude_tags with opts.search_text - # Updated to use exact match syntax + Inputs: + data (list): filtered list of book metadata dicts - exclude_tags = [] - for rule in self.opts.exclusion_rules: - if rule[1].lower() == 'tags': - exclude_tags.extend(rule[2].split(',')) + Outputs: + (list) books_by_title - # Remove dups - self.exclude_tags = exclude_tags = list(set(exclude_tags)) + Returns: + True: Successful + False: Empty data, (check filter restrictions) + """ - # Report tag exclusions - if self.opts.verbose and self.exclude_tags: - data = self.db.get_data_as_dict(ids=self.opts.ids) - for record in data: - matched = list(set(record['tags']) & set(exclude_tags)) - if matched : - self.opts.log.info(" - %s by %s (Exclusion rule Tags: '%s')" % - (record['title'], record['authors'][0], str(matched[0]))) - - search_phrase = '' - if exclude_tags: - search_terms = [] - for tag in exclude_tags: - search_terms.append("tag:=%s" % tag) - search_phrase = "not (%s)" % " or ".join(search_terms) - - # If a list of ids are provided, don't use search_text - if self.opts.ids: - self.opts.search_text = search_phrase - else: - if self.opts.search_text: - self.opts.search_text += " " + search_phrase - else: - self.opts.search_text = search_phrase - - # Fetch the database as a dictionary - data = self.plugin.search_sort_db(self.db, self.opts) - data = self.processExclusions(data) - - # Populate this_title{} from data[{},{}] - titles = [] - for record in data: + def _populate_title(record): + ''' populate this_title with massaged metadata ''' this_title = {} this_title['id'] = record['id'] this_title['uuid'] = record['uuid'] - this_title['title'] = self.convertHTMLEntities(record['title']) + this_title['title'] = self.convert_html_entities(record['title']) if record['series']: this_title['series'] = record['series'] this_title['series_index'] = record['series_index'] @@ -712,7 +1015,7 @@ Author '{0}': this_title['series'] = None this_title['series_index'] = 0.0 - this_title['title_sort'] = self.generateSortTitle(this_title['title']) + this_title['title_sort'] = self.generate_sort_title(this_title['title']) if 'authors' in record: # from calibre.ebooks.metadata import authors_to_string # return authors_to_string(self.authors) @@ -726,7 +1029,7 @@ Author '{0}': if 'author_sort' in record and record['author_sort'].strip(): this_title['author_sort'] = record['author_sort'] else: - this_title['author_sort'] = self.author_to_author_sort(this_title['author']) + this_title['author_sort'] = self.kf_author_to_author_sort(this_title['author']) if record['publisher']: this_title['publisher'] = re.sub('&', '&', record['publisher']) @@ -749,7 +1052,7 @@ Author '{0}': if ad_offset >= 0: record['comments'] = record['comments'][:ad_offset] - this_title['description'] = self.markdownComments(record['comments']) + this_title['description'] = self.massage_comments(record['comments']) # Create short description paras = BeautifulSoup(this_title['description']).findAll('p') @@ -758,34 +1061,34 @@ Author '{0}': for token in p.contents: if token.string is not None: tokens.append(token.string) - this_title['short_description'] = self.generateShortDescription(' '.join(tokens), dest="description") + this_title['short_description'] = self.generate_short_description(' '.join(tokens), dest="description") else: this_title['description'] = None this_title['short_description'] = None # Merge with custom field/value - if self.__merge_comments['field']: - this_title['description'] = self.mergeComments(this_title) + if self.merge_comments_rule['field']: + this_title['description'] = self.merge_comments(this_title) if record['cover']: this_title['cover'] = re.sub('&', '&', record['cover']) - this_title['prefix'] = self.discoverPrefix(record) + this_title['prefix'] = self.discover_prefix(record) if record['tags']: - this_title['tags'] = self.processSpecialTags(record['tags'], - this_title, self.opts) + this_title['tags'] = self.filter_excluded_tags(record['tags'], + self.opts.exclude_genre) if record['formats']: formats = [] for format in record['formats']: - formats.append(self.convertHTMLEntities(format)) + formats.append(self.convert_html_entities(format)) this_title['formats'] = formats # Add user notes to be displayed in header # Special case handling for datetime fields and lists if self.opts.header_note_source_field: - field_md = self.__db.metadata_for_field(self.opts.header_note_source_field) - notes = self.__db.get_field(record['id'], + field_md = self.db.metadata_for_field(self.opts.header_note_source_field) + notes = self.db.get_field(record['id'], self.opts.header_note_source_field, index_is_id=True) if notes: @@ -795,21 +1098,50 @@ Author '{0}': elif field_md['datatype'] == 'datetime': notes = format_date(notes,'dd MMM yyyy') this_title['notes'] = {'source':field_md['name'], - 'content':notes} + return this_title + + # Entry point + self.update_progress_full_step(_("Fetching database")) + + self.opts.sort_by = 'title' + search_phrase = '' + if self.excluded_tags: + search_terms = [] + for tag in self.excluded_tags: + search_terms.append("tag:=%s" % tag) + search_phrase = "not (%s)" % " or ".join(search_terms) + + # If a list of ids are provided, don't use search_text + if self.opts.ids: + self.opts.search_text = search_phrase + else: + if self.opts.search_text: + self.opts.search_text += " " + search_phrase + else: + self.opts.search_text = search_phrase + + # Fetch the database as a dictionary + data = self.plugin.search_sort_db(self.db, self.opts) + data = self.process_exclusions(data) + + # Populate this_title{} from data[{},{}] + titles = [] + for record in data: + this_title = _populate_title(record) titles.append(this_title) # Re-sort based on title_sort if len(titles): - #self.booksByTitle = sorted(titles, + #self.books_by_title = sorted(titles, # key=lambda x:(x['title_sort'].upper(), x['title_sort'].upper())) - self.booksByTitle = sorted(titles, key=lambda x: sort_key(x['title_sort'].upper())) + self.books_by_title = sorted(titles, key=lambda x: sort_key(x['title_sort'].upper())) - if False and self.verbose: - self.opts.log.info("fetchBooksByTitle(): %d books" % len(self.booksByTitle)) + if self.DEBUG and self.opts.verbose: + self.opts.log.info("fetch_books_by_title(): %d books" % len(self.books_by_title)) self.opts.log.info(" %-40s %-40s" % ('title', 'title_sort')) - for title in self.booksByTitle: + for title in self.books_by_title: self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], title['title_sort'][0:40])).decode('mac-roman')) return True @@ -820,13 +1152,23 @@ Author '{0}': self.error.append(error_msg) return False - def fetchBookmarks(self): - ''' - Collect bookmarks for catalog entries - This will use the system default save_template specified in + def fetch_bookmarks(self): + """ Interrogate connected Kindle for bookmarks. + + Discover bookmarks associated with books on Kindle downloaded by calibre. + Used in Descriptions to show reading progress, Last Read section showing date + last read. Kindle-specific, for AZW, MOBI, TAN and TXT formats. + Uses the system default save_template specified in Preferences|Add/Save|Sending to device, not a customized one specified in - the Kindle plugin - ''' + the Kindle plugin. + + Inputs: + (): bookmarks from connected Kindle + + Output: + bookmarked_books (dict): dict of Bookmarks + """ + from calibre.devices.usbms.device import Device from calibre.devices.kindle.driver import Bookmark from calibre.ebooks.metadata import MetaInformation @@ -845,7 +1187,7 @@ Author '{0}': def save_template(self): return self._save_template - def resolve_bookmark_paths(storage, path_map): + def _resolve_bookmark_paths(storage, path_map): pop_list = [] book_ext = {} for id in path_map: @@ -881,14 +1223,14 @@ Author '{0}': path_map.pop(id) return path_map, book_ext - if self.generateRecentlyRead: + if self.generate_recently_read: self.opts.log.info(" Collecting Kindle bookmarks matching catalog entries") d = BookmarkDevice(None) d.initialize(self.opts.connected_device['save_template']) bookmarks = {} - for book in self.booksByTitle: + for book in self.books_by_title: if 'formats' in book: path_map = {} id = book['id'] @@ -900,7 +1242,7 @@ Author '{0}': a_path = d.create_upload_path('/', myMeta, 'x.bookmark', create_dirs=False) path_map[id] = dict(path=a_path, fmts=[x.rpartition('.')[2] for x in book['formats']]) - path_map, book_ext = resolve_bookmark_paths(self.opts.connected_device['storage'], path_map) + path_map, book_ext = _resolve_bookmark_paths(self.opts.connected_device['storage'], path_map) if path_map: bookmark_ext = path_map[id].rpartition('.')[2] myBookmark = Bookmark(path_map[id], id, book_ext[id], bookmark_ext) @@ -909,8 +1251,8 @@ Author '{0}': except: book['percent_read'] = 0 dots = int((book['percent_read'] + 5)/10) - dot_string = self.READ_PROGRESS_SYMBOL * dots - empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) + dot_string = self.SYMBOL_PROGRESS_READ * dots + empty_dots = self.SYMBOL_PROGRESS_UNREAD * (10 - dots) book['reading_progress'] = '%s%s' % (dot_string,empty_dots) bookmarks[id] = ((myBookmark,book)) @@ -918,176 +1260,215 @@ Author '{0}': else: self.bookmarked_books = {} - def generateHTMLDescriptions(self): - ''' - Write each title to a separate HTML file in contentdir - ''' - self.updateProgressFullStep("'Descriptions'") + def filter_db_tags(self): + """ Remove excluded tags from data set, return normalized genre list. - for (title_num, title) in enumerate(self.booksByTitle): - self.updateProgressMicroStep("Description %d of %d" % \ - (title_num, len(self.booksByTitle)), - float(title_num*100/len(self.booksByTitle))/100) + Filter all db tags, removing excluded tags supplied in opts. + Test for multiple tags resolving to same normalized form. Normalized + tags are flattened to alphanumeric ascii_text. - # Generate the header from user-customizable template - soup = self.generateHTMLDescriptionHeader(title) + Args: + (none) - # Write the book entry to contentdir - outfile = open("%s/book_%d.html" % (self.contentDir, int(title['id'])), 'w') - outfile.write(soup.prettify()) - outfile.close() + Return: + genre_tags_dict (dict): dict of filtered, normalized tags in data set + """ - def generateHTMLByTitle(self): - ''' - Write books by title A-Z to HTML file - ''' - self.updateProgressFullStep("'Titles'") + def _format_tag_list(tags, indent=5, line_break=70, header='Tag list'): + def _next_tag(sorted_tags): + for (i, tag) in enumerate(sorted_tags): + if i < len(tags) - 1: + yield tag + ", " + else: + yield tag - soup = self.generateHTMLEmptyHeader("Books By Alpha Title") - body = soup.find('body') - btc = 0 + ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) + ans += ' ' * (indent + 1) + out_str = '' + sorted_tags = sorted(tags, key=sort_key) + for tag in _next_tag(sorted_tags): + out_str += tag + if len(out_str) >= line_break: + ans += out_str + '\n' + out_str = ' ' * (indent + 1) + return ans + out_str - pTag = Tag(soup, "p") - pTag['class'] = 'title' - ptc = 0 - aTag = Tag(soup,'a') - aTag['id'] = 'section_start' - pTag.insert(ptc, aTag) - ptc += 1 + # Entry point + normalized_tags = [] + friendly_tags = [] + excluded_tags = [] + for tag in self.db.all_tags(): + if tag in self.excluded_tags: + excluded_tags.append(tag) + continue + try: + if re.search(self.opts.exclude_genre, tag): + excluded_tags.append(tag) + continue + except: + self.opts.log.error("\tfilterDbTags(): malformed --exclude-genre regex pattern: %s" % self.opts.exclude_genre) - if not self.__generateForKindle: - # Kindle don't need this because it shows section titles in Periodical format - aTag = Tag(soup, "a") - aTag['id'] = "bytitle" - pTag.insert(ptc,aTag) - ptc += 1 - pTag.insert(ptc,NavigableString(_('Titles'))) + if tag == ' ': + continue - body.insert(btc,pTag) - btc += 1 + normalized_tags.append(re.sub('\W','',ascii_text(tag)).lower()) + friendly_tags.append(tag) - divTag = Tag(soup, "div") - dtc = 0 - current_letter = "" + genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) - # Re-sort title list without leading series/series_index - # Incoming title : - if not self.useSeriesPrefixInTitlesSection: - nspt = deepcopy(self.booksByTitle) - nspt = sorted(nspt, key=lambda x: sort_key(x['title_sort'].upper())) - self.booksByTitle_noSeriesPrefix = nspt + # Test for multiple genres resolving to same normalized form + normalized_set = set(normalized_tags) + for normalized in normalized_set: + if normalized_tags.count(normalized) > 1: + self.opts.log.warn(" Warning: multiple tags resolving to genre '%s':" % normalized) + for key in genre_tags_dict: + if genre_tags_dict[key] == normalized: + self.opts.log.warn(" %s" % key) + if self.opts.verbose: + self.opts.log.info('%s' % _format_tag_list(genre_tags_dict, header="enabled genre tags in database")) + self.opts.log.info('%s' % _format_tag_list(excluded_tags, header="excluded genre tags")) - # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksByTitle, key='title_sort') + return genre_tags_dict - # Loop through the books by title - # Generate one divRunningTag per initial letter for the purposes of - # minimizing widows and orphans on readers that can handle large - # <divs> styled as inline-block - title_list = self.booksByTitle - if not self.useSeriesPrefixInTitlesSection: - title_list = self.booksByTitle_noSeriesPrefix - drtc = 0 - divRunningTag = None - for idx, book in enumerate(title_list): - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: - # Start a new letter - if drtc and divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 - divRunningTag = Tag(soup, 'div') - if dtc > 0: - divRunningTag['class'] = "initial_letter" - drtc = 0 - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "author_title_letter_index" - aTag = Tag(soup, "a") - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - if current_letter == self.SYMBOLS: - aTag['id'] = self.SYMBOLS + "_titles" - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.SYMBOLS)) + def filter_excluded_tags(self, tags, regex): + """ Remove excluded tags from a tag list + + Run regex against list of tags, remove matching tags. Return filtered list. + + Args: + tags (list): list of tags + + Return: + tag_list(list): filtered list of tags + """ + + tag_list = [] + + try: + for tag in tags: + tag = self.convert_html_entities(tag) + if re.search(regex, tag): + continue else: - aTag['id'] = self.generateUnicodeName(current_letter) + "_titles" - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) - divRunningTag.insert(dtc,pIndexTag) - drtc += 1 + tag_list.append(tag) + except: + self.opts.log.error("\tfilter_excluded_tags(): malformed --exclude-genre regex pattern: %s" % regex) + return tags - # Add books - pBookTag = Tag(soup, "p") - pBookTag['class'] = "line_item" - ptc = 0 + return tag_list - pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup)) - ptc += 1 + def format_ncx_text(self, description, dest=None): + """ Massage NCX text for Kindle. - spanTag = Tag(soup, "span") - spanTag['class'] = "entry" - stc = 0 + Convert HTML entities for proper display on Kindle, convert + '&' to '&' (Kindle fails). + Args: + description (str): string, possibly with HTM entities + dest (kwarg): author, title or description - # Link to book - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) + Return: + (str): massaged, possibly truncated description + """ + # Kindle TOC descriptions won't render certain characters + # Fix up + massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - # Generate the title from the template - args = self.generateFormatArgs(book) - if book['series']: - formatted_title = self.by_titles_series_title_template.format(**args).rstrip() + # Replace '&' with '&' + massaged = re.sub("&","&", massaged) + + if massaged.strip() and dest: + #print traceback.print_stack(limit=3) + return self.generate_short_description(massaged.strip(), dest=dest) + else: + return None + + def format_prefix(self, prefix_char): + """ Generate HTML snippet with prefix character. + + Return a <code> snippet for Kindle, <span> snippet for EPUB. + Optimized to preserve first-column alignment for MOBI, EPUB. + + Args: + prefix_char (str): prefix character or None + + Return: + (str): BeautifulSoup HTML snippet to be inserted into <p> line item entry. + """ + + soup = BeautifulSoup('') + if self.opts.fmt == 'mobi': + codeTag = Tag(soup, "code") + if prefix_char is None: + codeTag.insert(0,NavigableString(' ')) else: - formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - spanTag.insert(stc, aTag) - stc += 1 + codeTag.insert(0,NavigableString(prefix_char)) + return codeTag + else: + spanTag = Tag(soup, "span") + spanTag['class'] = "prefix" + if prefix_char is None: + prefix_char = " " + spanTag.insert(0,NavigableString(prefix_char)) + return spanTag - # Dot - spanTag.insert(stc, NavigableString(" · ")) - stc += 1 + def generate_author_anchor(self, author): + """ Generate legal XHTML anchor. - # Link to author - emTag = Tag(soup, "em") - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) - aTag.insert(0, NavigableString(book['author'])) - emTag.insert(0,aTag) - spanTag.insert(stc, emTag) - stc += 1 + Convert author to legal XHTML (may contain unicode chars), stripping + non-alphanumeric chars. - pBookTag.insert(ptc, spanTag) - ptc += 1 + Args: + author (str): author name - if divRunningTag is not None: - divRunningTag.insert(drtc, pBookTag) - drtc += 1 + Return: + (str): asciized version of author + """ + return re.sub("\W","", ascii_text(author)) - # Add the last divRunningTag to divTag - if divRunningTag is not None: - divTag.insert(dtc, divRunningTag) - dtc += 1 + def generate_format_args(self, book): + """ Generate the format args for template substitution. - # Add the divTag to the body - body.insert(btc, divTag) - btc += 1 + self.load_section_templates imports string formatting templates of the form + 'by_*_template.py' for use in the various sections. The templates are designed to use + format args, supplied by this method. - # Write the volume to contentdir - outfile_spec = "%s/ByAlphaTitle.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/ByAlphaTitle.html") + Args: + book (dict): book metadata - def generateHTMLByAuthor(self): - ''' - Write books by author A-Z - ''' - self.updateProgressFullStep("'Authors'") + Return: + (dict): formatted args for templating + """ + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + args = dict( + title = book['title'], + series = book['series'], + series_index = series_index, + rating = self.generate_rating_string(book), + rating_parens = '(%s)' % self.generate_rating_string(book) if 'rating' in book else '', + pubyear = book['date'].split()[1] if book['date'] else '', + pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') + return args + + def generate_html_by_author(self): + """ Generate content/ByAlphaAuthor.html. + + Loop through self.books_by_author, generate HTML + with anchors for author and index letters. + + Input: + books_by_author (list): books, sorted by author + + Output: + content/ByAlphaAuthor.html (file) + """ friendly_name = _("Authors") + self.update_progress_full_step("%s HTML" % friendly_name) - soup = self.generateHTMLEmptyHeader(friendly_name) + soup = self.generate_html_empty_header(friendly_name) body = soup.find('body') btc = 0 @@ -1099,7 +1480,7 @@ Author '{0}': divRunningTag = None drtc = 0 - # Loop through booksByAuthor + # Loop through books_by_author # Each author/books group goes in an openingTag div (first) or # a runningTag div (subsequent) book_count = 0 @@ -1107,11 +1488,11 @@ Author '{0}': current_letter = '' current_series = None # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksByAuthor,key='author_sort') + sort_equivalents = self.establish_equivalencies(self.books_by_author,key='author_sort') - #for book in sorted(self.booksByAuthor, key = self.booksByAuthorSorter_author_sort): - #for book in self.booksByAuthor: - for idx, book in enumerate(self.booksByAuthor): + #for book in sorted(self.books_by_author, key = self.kf_books_by_author_sorter_author_sort): + #for book in self.books_by_author: + for idx, book in enumerate(self.books_by_author): book_count += 1 if self.letter_or_symbol(sort_equivalents[idx]) != current_letter : # Start a new letter with Index letter @@ -1140,14 +1521,14 @@ Author '{0}': pIndexTag.insert(0,aTag) pIndexTag.insert(1,NavigableString(self.SYMBOLS)) else: - aTag['id'] = self.generateUnicodeName(current_letter) + '_authors' + aTag['id'] = self.generate_unicode_name(current_letter) + '_authors' pIndexTag.insert(0,aTag) pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) divOpeningTag.insert(dotc,pIndexTag) dotc += 1 if book['author'] != current_author: - # Start a new author + # Start a new authorl current_author = book['author'] author_count += 1 if author_count >= 2: @@ -1172,7 +1553,7 @@ Author '{0}': pAuthorTag = Tag(soup, "p") pAuthorTag['class'] = "author_index" aTag = Tag(soup, "a") - aTag['id'] = "%s" % self.generateAuthorAnchor(current_author) + aTag['id'] = "%s" % self.generate_author_anchor(current_author) aTag.insert(0,NavigableString(current_author)) pAuthorTag.insert(0,aTag) if author_count == 1: @@ -1192,7 +1573,7 @@ Author '{0}': pSeriesTag['class'] = "series_mobi" if self.opts.generate_series: aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(book['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries',self.generate_series_anchor(book['series'])) aTag.insert(0, book['series']) pSeriesTag.insert(0, aTag) else: @@ -1212,7 +1593,7 @@ Author '{0}': pBookTag['class'] = "line_item" ptc = 0 - pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup)) + pBookTag.insert(ptc, self.format_prefix(book['prefix'])) ptc += 1 spanTag = Tag(soup, "span") @@ -1224,7 +1605,7 @@ Author '{0}': aTag['href'] = "book_%d.html" % (int(float(book['id']))) # Generate the title from the template - args = self.generateFormatArgs(book) + args = self.generate_format_args(book) if current_series: #aTag.insert(0,'%s%s' % (escape(book['title'][len(book['series'])+1:]),pubyear)) formatted_title = self.by_authors_series_title_template.format(**args).rstrip() @@ -1246,7 +1627,7 @@ Author '{0}': divRunningTag.insert(drtc,pBookTag) drtc += 1 - # Loop ends here + # loop ends here pTag = Tag(soup, "p") pTag['class'] = 'title' @@ -1256,7 +1637,7 @@ Author '{0}': pTag.insert(ptc, aTag) ptc += 1 - if not self.__generateForKindle: + if not self.generate_for_kindle: # Kindle don't need this because it shows section titles in Periodical format aTag = Tag(soup, "a") anchor_name = friendly_name.lower() @@ -1278,23 +1659,29 @@ Author '{0}': # Add the divTag to the body body.insert(btc, divTag) - # Write the generated file to contentdir - outfile_spec = "%s/ByAlphaAuthor.html" % (self.contentDir) + # Write the generated file to content_dir + outfile_spec = "%s/ByAlphaAuthor.html" % (self.content_dir) outfile = open(outfile_spec, 'w') outfile.write(soup.prettify()) outfile.close() - self.htmlFileList_1.append("content/ByAlphaAuthor.html") + self.html_filelist_1.append("content/ByAlphaAuthor.html") - def generateHTMLByDateAdded(self): - ''' - Write books by reverse chronological order - ''' - self.updateProgressFullStep("'Recently Added'") + def generate_html_by_date_added(self): + """ Generate content/ByDateAdded.html. - def add_books_to_HTML_by_month(this_months_list, dtc): + Loop through self.books_by_title sorted by reverse date, generate HTML. + + Input: + books_by_title (list): books, sorted by title + + Output: + content/ByDateAdded.html (file) + """ + + def _add_books_to_html_by_month(this_months_list, dtc): if len(this_months_list): - this_months_list = sorted(this_months_list, key=lambda x: sort_key(self.booksByAuthorSorter_author_sort(x))) + this_months_list = sorted(this_months_list, key=lambda x: sort_key(self.kf_books_by_author_sorter_author_sort)(x))) # Create a new month anchor date_string = strftime(u'%B %Y', current_date.timetuple()) @@ -1319,7 +1706,7 @@ Author '{0}': pAuthorTag['class'] = "author_index" aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(current_author)) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(current_author)) aTag.insert(0,NavigableString(current_author)) pAuthorTag.insert(0,aTag) divTag.insert(dtc,pAuthorTag) @@ -1337,7 +1724,7 @@ Author '{0}': aTag = Tag(soup,'a') if self.letter_or_symbol(new_entry['series']) == self.SYMBOLS: - aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(new_entry['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries',self.generate_series_anchor(new_entry['series'])) aTag.insert(0, new_entry['series']) pSeriesTag.insert(0, aTag) else: @@ -1352,7 +1739,7 @@ Author '{0}': pBookTag['class'] = "line_item" ptc = 0 - pBookTag.insert(ptc, self.formatPrefix(new_entry['prefix'],soup)) + pBookTag.insert(ptc, self.format_prefix(new_entry['prefix'])) ptc += 1 spanTag = Tag(soup, "span") @@ -1364,7 +1751,7 @@ Author '{0}': aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) # Generate the title from the template - args = self.generateFormatArgs(new_entry) + args = self.generate_format_args(new_entry) if current_series: formatted_title = self.by_month_added_series_title_template.format(**args).rstrip() else: @@ -1381,7 +1768,7 @@ Author '{0}': dtc += 1 return dtc - def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): + def _add_books_to_html_by_date_range(date_range_list, date_range, dtc): if len(date_range_list): pIndexTag = Tag(soup, "p") pIndexTag['class'] = "date_index" @@ -1398,7 +1785,7 @@ Author '{0}': pBookTag['class'] = "line_item" ptc = 0 - pBookTag.insert(ptc, self.formatPrefix(new_entry['prefix'],soup)) + pBookTag.insert(ptc, self.format_prefix(new_entry['prefix'])) ptc += 1 spanTag = Tag(soup, "span") @@ -1410,7 +1797,7 @@ Author '{0}': aTag['href'] = "book_%d.html" % (int(float(new_entry['id']))) # Generate the title from the template - args = self.generateFormatArgs(new_entry) + args = self.generate_format_args(new_entry) if new_entry['series']: formatted_title = self.by_recently_added_series_title_template.format(**args).rstrip() else: @@ -1427,7 +1814,7 @@ Author '{0}': emTag = Tag(soup, "em") aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(new_entry['author'])) aTag.insert(0, NavigableString(new_entry['author'])) emTag.insert(0,aTag) spanTag.insert(stc, emTag) @@ -1441,8 +1828,9 @@ Author '{0}': return dtc friendly_name = _("Recently Added") + self.update_progress_full_step("%s HTML" % friendly_name) - soup = self.generateHTMLEmptyHeader(friendly_name) + soup = self.generate_html_empty_header(friendly_name) body = soup.find('body') btc = 0 @@ -1456,7 +1844,7 @@ Author '{0}': pTag.insert(ptc, aTag) ptc += 1 - if not self.__generateForKindle: + if not self.generate_for_kindle: # Kindle don't need this because it shows section titles in Periodical format aTag = Tag(soup, "a") anchor_name = friendly_name.lower() @@ -1473,12 +1861,12 @@ Author '{0}': dtc = 0 # >>> Books by date range <<< - if self.useSeriesPrefixInTitlesSection: - self.booksByDateRange = sorted(self.booksByTitle, + if self.use_series_prefix_in_titles_section: + self.books_by_date_range = sorted(self.books_by_title, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) else: - nspt = deepcopy(self.booksByTitle) - self.booksByDateRange = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) + nspt = deepcopy(self.books_by_title) + self.books_by_date_range = sorted(nspt, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) date_range_list = [] today_time = nowf().replace(hour=23, minute=59, second=59) @@ -1489,7 +1877,7 @@ Author '{0}': else: date_range = 'Last %d days' % (self.DATE_RANGE[i]) - for book in self.booksByDateRange: + for book in self.books_by_date_range: book_time = book['timestamp'] delta = today_time-book_time if delta.days <= date_range_limit: @@ -1497,48 +1885,52 @@ Author '{0}': else: break - dtc = add_books_to_HTML_by_date_range(date_range_list, date_range, dtc) + dtc = _add_books_to_html_by_date_range(date_range_list, date_range, dtc) date_range_list = [book] # >>>> Books by month <<<< # Sort titles case-insensitive for by month using series prefix - self.booksByMonth = sorted(self.booksByTitle, + self.books_by_month = sorted(self.books_by_title, key=lambda x:(x['timestamp'], x['timestamp']),reverse=True) # Loop through books by date current_date = datetime.date.fromordinal(1) this_months_list = [] - for book in self.booksByMonth: + for book in self.books_by_month: if book['timestamp'].month != current_date.month or \ book['timestamp'].year != current_date.year: - dtc = add_books_to_HTML_by_month(this_months_list, dtc) + dtc = _add_books_to_html_by_month(this_months_list, dtc) this_months_list = [] current_date = book['timestamp'].date() this_months_list.append(book) # Add the last month's list - add_books_to_HTML_by_month(this_months_list, dtc) + _add_books_to_html_by_month(this_months_list, dtc) # Add the divTag to the body body.insert(btc, divTag) - # Write the generated file to contentdir - outfile_spec = "%s/ByDateAdded.html" % (self.contentDir) + # Write the generated file to content_dir + outfile_spec = "%s/ByDateAdded.html" % (self.content_dir) outfile = open(outfile_spec, 'w') outfile.write(soup.prettify()) outfile.close() - self.htmlFileList_2.append("content/ByDateAdded.html") + self.html_filelist_2.append("content/ByDateAdded.html") - def generateHTMLByDateRead(self): - ''' - Write books by active bookmarks - ''' - friendly_name = _('Recently Read') - self.updateProgressFullStep("'%s'" % friendly_name) - if not self.bookmarked_books: - return + def generate_html_by_date_read(self): + """ Generate content/ByDateRead.html. - def add_books_to_HTML_by_day(todays_list, dtc): + Create self.bookmarked_books_by_date_read from self.bookmarked_books. + Loop through self.bookmarked_books_by_date_read, generate HTML. + + Input: + bookmarked_books_by_date_read (list) + + Output: + content/ByDateRead.html (file) + """ + + def _add_books_to_html_by_day(todays_list, dtc): if len(todays_list): # Create a new day anchor date_string = strftime(u'%A, %B %d', current_date.timetuple()) @@ -1575,7 +1967,7 @@ Author '{0}': emTag = Tag(soup, "em") aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(new_entry['author'])) aTag.insert(0, NavigableString(new_entry['author'])) emTag.insert(0,aTag) pBookTag.insert(ptc, emTag) @@ -1585,7 +1977,7 @@ Author '{0}': dtc += 1 return dtc - def add_books_to_HTML_by_date_range(date_range_list, date_range, dtc): + def _add_books_to_html_by_date_range(date_range_list, date_range, dtc): if len(date_range_list): pIndexTag = Tag(soup, "p") pIndexTag['class'] = "date_index" @@ -1604,8 +1996,8 @@ Author '{0}': # Percent read dots = int((new_entry['percent_read'] + 5)/10) - dot_string = self.READ_PROGRESS_SYMBOL * dots - empty_dots = self.UNREAD_PROGRESS_SYMBOL * (10 - dots) + dot_string = self.SYMBOL_PROGRESS_READ * dots + empty_dots = self.SYMBOL_PROGRESS_UNREAD * (10 - dots) pBookTag.insert(ptc, NavigableString('%s%s' % (dot_string,empty_dots))) ptc += 1 @@ -1624,7 +2016,7 @@ Author '{0}': emTag = Tag(soup, "em") aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(new_entry['author'])) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(new_entry['author'])) aTag.insert(0, NavigableString(new_entry['author'])) emTag.insert(0,aTag) pBookTag.insert(ptc, emTag) @@ -1634,7 +2026,13 @@ Author '{0}': dtc += 1 return dtc - soup = self.generateHTMLEmptyHeader(friendly_name) + friendly_name = _('Recently Read') + self.update_progress_full_step("%s HTML" % friendly_name) + + if not self.bookmarked_books: + return + + soup = self.generate_html_empty_header(friendly_name) body = soup.find('body') btc = 0 @@ -1667,223 +2065,61 @@ Author '{0}': book[1]['percent_read'] = 0 bookmarked_books.append(book[1]) - self.booksByDateRead = sorted(bookmarked_books, + self.bookmarked_books_by_date_read = sorted(bookmarked_books, key=lambda x:(x['bookmark_timestamp'], x['bookmark_timestamp']),reverse=True) # >>>> Recently read by day <<<< current_date = datetime.date.fromordinal(1) todays_list = [] - for book in self.booksByDateRead: + for book in self.bookmarked_books_by_date_read: bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) if bookmark_time.day != current_date.day or \ bookmark_time.month != current_date.month or \ bookmark_time.year != current_date.year: - dtc = add_books_to_HTML_by_day(todays_list, dtc) + dtc = _add_books_to_html_by_day(todays_list, dtc) todays_list = [] current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() todays_list.append(book) # Add the last day's list - add_books_to_HTML_by_day(todays_list, dtc) + _add_books_to_html_by_day(todays_list, dtc) # Add the divTag to the body body.insert(btc, divTag) - # Write the generated file to contentdir - outfile_spec = "%s/ByDateRead.html" % (self.contentDir) + # Write the generated file to content_dir + outfile_spec = "%s/ByDateRead.html" % (self.content_dir) outfile = open(outfile_spec, 'w') outfile.write(soup.prettify()) outfile.close() - self.htmlFileList_2.append("content/ByDateRead.html") + self.html_filelist_2.append("content/ByDateRead.html") - def generateHTMLBySeries(self): - ''' - Generate a list of series - ''' - self.updateProgressFullStep("Fetching series") + def generate_html_by_genres(self): + """ Generate individual HTML files per tag. - self.opts.sort_by = 'series' + Filter out excluded tags. For each tag qualifying as a genre, + create a separate HTML file. Normalize tags to flatten synonymous tags. - # Merge self.exclude_tags with opts.search_text - # Updated to use exact match syntax + Inputs: + db.all_tags() (list): all database tags - search_phrase = 'series:true ' - if self.exclude_tags: - search_terms = [] - for tag in self.exclude_tags: - search_terms.append("tag:=%s" % tag) - search_phrase += "not (%s)" % " or ".join(search_terms) + Output: + (files): HTML file per genre + """ - # If a list of ids are provided, don't use search_text - if self.opts.ids: - self.opts.search_text = search_phrase - else: - if self.opts.search_text: - self.opts.search_text += " " + search_phrase - else: - self.opts.search_text = search_phrase + self.update_progress_full_step(_("Genres HTML")) - # Fetch the database as a dictionary - data = self.plugin.search_sort_db(self.db, self.opts) + self.genre_tags_dict = self.filter_db_tags() - # Remove exclusions - self.booksBySeries = self.processExclusions(data) - - if not self.booksBySeries: - self.opts.generate_series = False - self.opts.log(" no series found in selected books, cancelling series generation") - return - - # Generate series_sort - for book in self.booksBySeries: - book['series_sort'] = self.generateSortTitle(book['series']) - - friendly_name = _("Series") - - soup = self.generateHTMLEmptyHeader(friendly_name) - body = soup.find('body') - - btc = 0 - divTag = Tag(soup, "div") - dtc = 0 - current_letter = "" - current_series = None - - # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksBySeries, key='series_sort') - - # Loop through booksBySeries - series_count = 0 - for idx, book in enumerate(self.booksBySeries): - # Check for initial letter change - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter : - # Start a new letter with Index letter - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - pIndexTag = Tag(soup, "p") - pIndexTag['class'] = "series_letter_index" - aTag = Tag(soup, "a") - if current_letter == self.SYMBOLS: - aTag['id'] = self.SYMBOLS + "_series" - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(self.SYMBOLS)) - else: - aTag['id'] = self.generateUnicodeName(current_letter) + "_series" - pIndexTag.insert(0,aTag) - pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) - divTag.insert(dtc,pIndexTag) - dtc += 1 - # Check for series change - if book['series'] != current_series: - # Start a new series - series_count += 1 - current_series = book['series'] - pSeriesTag = Tag(soup,'p') - pSeriesTag['class'] = "series" - if self.opts.fmt == 'mobi': - pSeriesTag['class'] = "series_mobi" - aTag = Tag(soup, 'a') - aTag['id'] = self.generateSeriesAnchor(book['series']) - pSeriesTag.insert(0,aTag) - pSeriesTag.insert(1,NavigableString('%s' % book['series'])) - divTag.insert(dtc,pSeriesTag) - dtc += 1 - - # Add books - pBookTag = Tag(soup, "p") - pBookTag['class'] = "line_item" - ptc = 0 - - book['prefix'] = self.discoverPrefix(book) - pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup)) - ptc += 1 - - spanTag = Tag(soup, "span") - spanTag['class'] = "entry" - stc = 0 - - aTag = Tag(soup, "a") - if self.opts.generate_descriptions: - aTag['href'] = "book_%d.html" % (int(float(book['id']))) - # Use series, series index if avail else just title - #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) - - if is_date_undefined(book['pubdate']): - book['date'] = None - else: - book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) - - args = self.generateFormatArgs(book) - formatted_title = self.by_series_title_template.format(**args).rstrip() - aTag.insert(0,NavigableString(escape(formatted_title))) - - spanTag.insert(stc, aTag) - stc += 1 - - # · - spanTag.insert(stc, NavigableString(' · ')) - stc += 1 - - # Link to author - aTag = Tag(soup, "a") - if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", - self.generateAuthorAnchor(escape(' & '.join(book['authors'])))) - aTag.insert(0, NavigableString(' & '.join(book['authors']))) - spanTag.insert(stc, aTag) - stc += 1 - - pBookTag.insert(ptc, spanTag) - ptc += 1 - - divTag.insert(dtc, pBookTag) - dtc += 1 - - pTag = Tag(soup, "p") - pTag['class'] = 'title' - ptc = 0 - aTag = Tag(soup,'a') - aTag['id'] = 'section_start' - pTag.insert(ptc, aTag) - ptc += 1 - - if not self.__generateForKindle: - # Insert the <h2> tag with book_count at the head - aTag = Tag(soup, "a") - anchor_name = friendly_name.lower() - aTag['id'] = anchor_name.replace(" ","") - pTag.insert(0,aTag) - pTag.insert(1,NavigableString('%s' % friendly_name)) - body.insert(btc,pTag) - btc += 1 - - # Add the divTag to the body - body.insert(btc, divTag) - - # Write the generated file to contentdir - outfile_spec = "%s/BySeries.html" % (self.contentDir) - outfile = open(outfile_spec, 'w') - outfile.write(soup.prettify()) - outfile.close() - self.htmlFileList_1.append("content/BySeries.html") - - def generateHTMLByTags(self): - ''' - Generate individual HTML files for each tag, e.g. Fiction, Nonfiction ... - Note that special tags - have already been filtered from books[] - There may be synonomous tags - ''' - self.updateProgressFullStep("'Genres'") - - self.genre_tags_dict = self.filterDbTags(self.db.all_tags()) # Extract books matching filtered_tags genre_list = [] for friendly_tag in sorted(self.genre_tags_dict, key=sort_key): - #print("\ngenerateHTMLByTags(): looking for books with friendly_tag '%s'" % friendly_tag) + #print("\ngenerate_html_by_genres(): looking for books with friendly_tag '%s'" % friendly_tag) # tag_list => { normalized_genre_tag : [{book},{},{}], # normalized_genre_tag : [{book},{},{}] } tag_list = {} - for book in self.booksByAuthor: + for book in self.books_by_author: # Scan each book for tag matching friendly_tag if 'tags' in book and friendly_tag in book['tags']: this_book = {} @@ -1918,11 +2154,11 @@ Author '{0}': if self.opts.verbose: if len(genre_list): self.opts.log.info(" Genre summary: %d active genre tags used in generating catalog with %d titles" % - (len(genre_list), len(self.booksByTitle))) + (len(genre_list), len(self.books_by_title))) for genre in genre_list: for key in genre: - self.opts.log.info(" %s: %d %s" % (self.getFriendlyGenreTag(key), + self.opts.log.info(" %s: %d %s" % (self.get_friendly_genre_tag(key), len(genre[key]), 'titles' if len(genre[key]) > 1 else 'title')) @@ -1956,9 +2192,9 @@ Author '{0}': books_by_current_author += 1 # Write the genre book list as an article - titles_spanned = self.generateHTMLByGenre(genre, True if index==0 else False, + titles_spanned = self.generate_html_by_genre(genre, True if index==0 else False, genre_tag_set[genre], - "%s/Genre_%s.html" % (self.contentDir, + "%s/Genre_%s.html" % (self.content_dir, genre)) tag_file = "content/Genre_%s.html" % genre @@ -1970,1428 +2206,31 @@ Author '{0}': self.genres = master_genre_list - def generateThumbnails(self): - ''' - Generate a thumbnail per cover. If a current thumbnail exists, skip - If a cover doesn't exist, use default - Return list of active thumbs - ''' - self.updateProgressFullStep("'Thumbnails'") - thumbs = ['thumbnail_default.jpg'] - image_dir = "%s/images" % self.catalogPath - for (i,title) in enumerate(self.booksByTitle): - # Update status - self.updateProgressMicroStep("Thumbnail %d of %d" % \ - (i,len(self.booksByTitle)), - i/float(len(self.booksByTitle))) + def generate_html_by_genre(self, genre, section_head, books, outfile): + """ Generate individual genre HTML file. - thumb_file = 'thumbnail_%d.jpg' % int(title['id']) - thumb_generated = True - valid_cover = True - try: - self.generateThumbnail(title, image_dir, thumb_file) - thumbs.append("thumbnail_%d.jpg" % int(title['id'])) - except: - if 'cover' in title and os.path.exists(title['cover']): - valid_cover = False - self.opts.log.warn(" *** Invalid cover file for '%s'***" % - (title['title'])) - if not self.error: - self.error.append('Invalid cover files') - self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) + Generate an individual genre HTML file. Called from generate_html_by_genres() - thumb_generated = False + Args: + genre (str): genre name + section_head (bool): True if starting section + books (dict): list of books in genre + outfile (str): full pathname to output file - if not thumb_generated: - self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) - # Confirm thumb exists, default is current - default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") - cover = os.path.join(self.catalogPath, "DefaultCover.png") - title['cover'] = cover + Results: + (file): Genre HTML file written - if not os.path.exists(cover): - shutil.copyfile(I('book.png'), cover) + Returns: + titles_spanned (list): [(first_author, first_book), (last_author, last_book)] + """ - if os.path.isfile(default_thumb_fp): - # Check to see if default cover is newer than thumbnail - # os.path.getmtime() = modified time - # os.path.ctime() = creation time - cover_timestamp = os.path.getmtime(cover) - thumb_timestamp = os.path.getmtime(default_thumb_fp) - if thumb_timestamp < cover_timestamp: - if False and self.verbose: - self.opts.log.warn("updating thumbnail_default for %s" % title['title']) - self.generateThumbnail(title, image_dir, - "thumbnail_default.jpg" if valid_cover else thumb_file) - else: - if False and self.verbose: - self.opts.log.warn(" generating new thumbnail_default.jpg") - self.generateThumbnail(title, image_dir, - "thumbnail_default.jpg" if valid_cover else thumb_file) - # Clear the book's cover property - title['cover'] = None - - - # Write thumb_width to the file, validating cache contents - # Allows detection of aborted catalog builds - with ZipFile(self.__archive_path, mode='a') as zfw: - zfw.writestr('thumb_width', self.opts.thumb_width) - - self.thumbs = thumbs - - def generateOPF(self): - - self.updateProgressFullStep("Generating OPF") - - header = ''' - <?xml version="1.0" encoding="UTF-8"?> - <package xmlns="http://www.idpf.org/2007/opf" version="2.0" unique-identifier="calibre_id"> - <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"> - <dc:language>en-US</dc:language> - <meta name="calibre:publication_type" content="periodical:default"/> - </metadata> - <manifest></manifest> - <spine toc="ncx"></spine> - <guide></guide> - </package> - ''' - # Add the supplied metadata tags - soup = BeautifulStoneSoup(header, selfClosingTags=['item','itemref', 'reference']) - metadata = soup.find('metadata') - mtc = 0 - - titleTag = Tag(soup, "dc:title") - titleTag.insert(0,self.title) - metadata.insert(mtc, titleTag) - mtc += 1 - - creatorTag = Tag(soup, "dc:creator") - creatorTag.insert(0, self.creator) - metadata.insert(mtc, creatorTag) - mtc += 1 - - # Create the OPF tags - manifest = soup.find('manifest') - mtc = 0 - spine = soup.find('spine') - stc = 0 - guide = soup.find('guide') - - itemTag = Tag(soup, "item") - itemTag['id'] = "ncx" - itemTag['href'] = '%s.ncx' % self.basename - itemTag['media-type'] = "application/x-dtbncx+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - itemTag = Tag(soup, "item") - itemTag['id'] = 'stylesheet' - itemTag['href'] = self.stylesheet - itemTag['media-type'] = 'text/css' - manifest.insert(mtc, itemTag) - mtc += 1 - - itemTag = Tag(soup, "item") - itemTag['id'] = 'mastheadimage-image' - itemTag['href'] = "images/mastheadImage.gif" - itemTag['media-type'] = 'image/gif' - manifest.insert(mtc, itemTag) - mtc += 1 - - # Write the thumbnail images, descriptions to the manifest - sort_descriptions_by = [] - if self.opts.generate_descriptions: - for thumb in self.thumbs: - itemTag = Tag(soup, "item") - itemTag['href'] = "images/%s" % (thumb) - end = thumb.find('.jpg') - itemTag['id'] = "%s-image" % thumb[:end] - itemTag['media-type'] = 'image/jpeg' - manifest.insert(mtc, itemTag) - mtc += 1 - - # HTML files - add descriptions to manifest and spine - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - # Add html_files to manifest and spine - - for file in self.htmlFileList_1: - # By Author, By Title, By Series, - itemTag = Tag(soup, "item") - start = file.find('/') + 1 - end = file.find('.') - itemTag['href'] = file - itemTag['id'] = file[start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = file[start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - # Add genre files to manifest and spine - for genre in self.genres: - if False: self.opts.log.info("adding %s to manifest and spine" % genre['tag']) - itemTag = Tag(soup, "item") - start = genre['file'].find('/') + 1 - end = genre['file'].find('.') - itemTag['href'] = genre['file'] - itemTag['id'] = genre['file'][start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = genre['file'][start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - for file in self.htmlFileList_2: - # By Date Added, By Date Read - itemTag = Tag(soup, "item") - start = file.find('/') + 1 - end = file.find('.') - itemTag['href'] = file - itemTag['id'] = file[start:end].lower() - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = file[start:end].lower() - spine.insert(stc, itemrefTag) - stc += 1 - - for book in sort_descriptions_by: - # manifest - itemTag = Tag(soup, "item") - itemTag['href'] = "content/book_%d.html" % int(book['id']) - itemTag['id'] = "book%d" % int(book['id']) - itemTag['media-type'] = "application/xhtml+xml" - manifest.insert(mtc, itemTag) - mtc += 1 - - # spine - itemrefTag = Tag(soup, "itemref") - itemrefTag['idref'] = "book%d" % int(book['id']) - spine.insert(stc, itemrefTag) - stc += 1 - - # Guide - referenceTag = Tag(soup, "reference") - referenceTag['type'] = 'masthead' - referenceTag['title'] = 'mastheadimage-image' - referenceTag['href'] = 'images/mastheadImage.gif' - guide.insert(0,referenceTag) - - # Write the OPF file - outfile = open("%s/%s.opf" % (self.catalogPath, self.basename), 'w') - outfile.write(soup.prettify()) - - def generateNCXHeader(self): - - self.updateProgressFullStep("NCX header") - - header = ''' - <?xml version="1.0" encoding="utf-8"?> - <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata" version="2005-1" xml:lang="en"> - </ncx> - ''' - soup = BeautifulStoneSoup(header, selfClosingTags=['content','calibre:meta-img']) - - ncx = soup.find('ncx') - navMapTag = Tag(soup, 'navMap') - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "periodical" - navPointTag['id'] = "title" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(self.title)) - navLabelTag.insert(0, textTag) - navPointTag.insert(0, navLabelTag) - - if self.opts.generate_authors: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByAlphaAuthor.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_titles: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByAlphaTitle.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_series: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/BySeries.html" - navPointTag.insert(1, contentTag) - elif self.opts.generate_genres: - contentTag = Tag(soup, 'content') - #contentTag['src'] = "content/ByGenres.html" - contentTag['src'] = "%s" % self.genres[0]['file'] - navPointTag.insert(1, contentTag) - elif self.opts.generate_recently_added: - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/ByDateAdded.html" - navPointTag.insert(1, contentTag) - else: - # Descriptions only - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - contentTag = Tag(soup, 'content') - contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) - navPointTag.insert(1, contentTag) - - cmiTag = Tag(soup, '%s' % 'calibre:meta-img') - cmiTag['id'] = "mastheadImage" - cmiTag['src'] = "images/mastheadImage.gif" - navPointTag.insert(2,cmiTag) - navMapTag.insert(0,navPointTag) - - ncx.insert(0,navMapTag) - self.ncxSoup = soup - - def generateNCXDescriptions(self, tocTitle): - - self.updateProgressFullStep("NCX 'Descriptions'") - - # --- Construct the 'Books by Title' section --- - ncx_soup = self.ncxSoup - body = ncx_soup.find("navPoint") - btc = len(body.contents) - - # Add the section navPoint - navPointTag = Tag(ncx_soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "bytitle-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, 'navLabel') - textTag = Tag(ncx_soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(ncx_soup,"content") - contentTag['src'] = "content/book_%d.html" % int(self.booksByTitle[0]['id']) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Loop over the titles - sort_descriptions_by = self.booksByAuthor if self.opts.sort_descriptions_by_author \ - else self.booksByTitle - - for book in sort_descriptions_by: - navPointVolumeTag = Tag(ncx_soup, 'navPoint') - navPointVolumeTag['class'] = "article" - navPointVolumeTag['id'] = "book%dID" % int(book['id']) - navPointVolumeTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, "navLabel") - textTag = Tag(ncx_soup, "text") - if book['series']: - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - if self.generateForKindle: - # Don't include Author for Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s])' % - (book['title'], book['series'], series_index), dest='title'))) - else: - # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s (%s [%s]) · %s ' % - (book['title'], book['series'], series_index, book['author']), dest='title'))) - else: - if self.generateForKindle: - # Don't include Author for Kindle - title_str = self.formatNCXText('%s' % (book['title']), dest='title') - if self.opts.connected_kindle and book['id'] in self.bookmarked_books: - ''' - dots = int((book['percent_read'] + 5)/10) - dot_string = '+' * dots - empty_dots = '-' * (10 - dots) - title_str += ' %s%s' % (dot_string,empty_dots) - ''' - title_str += '*' - textTag.insert(0, NavigableString(title_str)) - else: - # Include Author for non-Kindle - textTag.insert(0, NavigableString(self.formatNCXText('%s · %s' % \ - (book['title'], book['author']), dest='title'))) - navLabelTag.insert(0,textTag) - navPointVolumeTag.insert(0,navLabelTag) - - contentTag = Tag(ncx_soup, "content") - contentTag['src'] = "content/book_%d.html#book%d" % (int(book['id']), int(book['id'])) - navPointVolumeTag.insert(1, contentTag) - - if self.generateForKindle: - # Add the author tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - - if book['date']: - navStr = '%s | %s' % (self.formatNCXText(book['author'], dest='author'), - book['date'].split()[1]) - else: - navStr = '%s' % (self.formatNCXText(book['author'], dest='author')) - - if 'tags' in book and len(book['tags']): - navStr = self.formatNCXText(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') - cmTag.insert(0, NavigableString(navStr)) - navPointVolumeTag.insert(2, cmTag) - - # Add the description tag - if book['short_description']: - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(book['short_description'], dest='description'))) - navPointVolumeTag.insert(3, cmTag) - - # Add this volume to the section tag - navPointTag.insert(nptc, navPointVolumeTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = ncx_soup - - def generateNCXBySeries(self, tocTitle): - self.updateProgressFullStep("NCX 'Series'") - - def add_to_series_by_letter(current_series_list): - current_series_list = " • ".join(current_series_list) - current_series_list = self.formatNCXText(current_series_list, dest="description") - series_by_letter.append(current_series_list) - - soup = self.ncxSoup - output = "BySeries" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Series' section --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "byseries-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "content/%s.html#section_start" % (output) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - series_by_letter = [] - # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksBySeries, key='series_sort') - - # Loop over the series titles, find start of each letter, add description_preview_count books - # Special switch for using different title list - - title_list = self.booksBySeries - - # Prime the pump - current_letter = self.letter_or_symbol(sort_equivalents[0]) - - title_letters = [current_letter] - current_series_list = [] - current_series = "" - for idx, book in enumerate(title_list): - sort_title = self.generateSortTitle(book['series']) - self.establish_equivalencies([sort_title])[0] - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: - - # Save the old list - add_to_series_by_letter(current_series_list) - - # Start the new list - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - title_letters.append(current_letter) - current_series = book['series'] - current_series_list = [book['series']] - else: - if len(current_series_list) < self.descriptionClip and \ - book['series'] != current_series : - current_series = book['series'] - current_series_list.append(book['series']) - - # Add the last book list - add_to_series_by_letter(current_series_list) - - # Add *article* entries for each populated series title letter - for (i,books) in enumerate(series_by_letter): - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sSeries-ID" % (title_letters[i].upper()) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(_(u"Series beginning with %s") % \ - (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - #contentTag['src'] = "content/%s.html#%s_series" % (output, title_letters[i]) - if title_letters[i] == self.SYMBOLS: - contentTag['src'] = "content/%s.html#%s_series" % (output, self.SYMBOLS) - else: - contentTag['src'] = "content/%s.html#%s_series" % (output, self.generateUnicodeName(title_letters[i])) - - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByTitle(self, tocTitle): - self.updateProgressFullStep("NCX 'Titles'") - - def add_to_books_by_letter(current_book_list): - current_book_list = " • ".join(current_book_list) - current_book_list = self.formatNCXText(current_book_list, dest="description") - books_by_letter.append(current_book_list) - - soup = self.ncxSoup - output = "ByAlphaTitle" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Title' section --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - navPointTag['id'] = "byalphatitle-ID" - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "content/%s.html#section_start" % (output) - navPointTag.insert(nptc, contentTag) - nptc += 1 - - books_by_letter = [] - - # Establish initial letter equivalencies - sort_equivalents = self.establish_equivalencies(self.booksByTitle, key='title_sort') - - # Loop over the titles, find start of each letter, add description_preview_count books - # Special switch for using different title list - if self.useSeriesPrefixInTitlesSection: - title_list = self.booksByTitle - else: - title_list = self.booksByTitle_noSeriesPrefix - - # Prime the list - current_letter = self.letter_or_symbol(sort_equivalents[0]) - title_letters = [current_letter] - current_book_list = [] - current_book = "" - for idx, book in enumerate(title_list): - #if self.letter_or_symbol(book['title_sort'][0]) != current_letter: - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: - - # Save the old list - add_to_books_by_letter(current_book_list) - - # Start the new list - #current_letter = self.letter_or_symbol(book['title_sort'][0]) - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - title_letters.append(current_letter) - current_book = book['title'] - current_book_list = [book['title']] - else: - if len(current_book_list) < self.descriptionClip and \ - book['title'] != current_book : - current_book = book['title'] - current_book_list.append(book['title']) - - # Add the last book list - add_to_books_by_letter(current_book_list) - - # Add *article* entries for each populated title letter - for (i,books) in enumerate(books_by_letter): - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sTitles-ID" % (title_letters[i].upper()) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(_(u"Titles beginning with %s") % \ - (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - if title_letters[i] == self.SYMBOLS: - contentTag['src'] = "content/%s.html#%s_titles" % (output, self.SYMBOLS) - else: - contentTag['src'] = "content/%s.html#%s_titles" % (output, self.generateUnicodeName(title_letters[i])) - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(self.formatNCXText(books, dest='description'))) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByAuthor(self, tocTitle): - self.updateProgressFullStep("NCX 'Authors'") - - def add_to_author_list(current_author_list, current_letter): - current_author_list = " • ".join(current_author_list) - current_author_list = self.formatNCXText(current_author_list, dest="description") - master_author_list.append((current_author_list, current_letter)) - - soup = self.ncxSoup - HTML_file = "content/ByAlphaAuthor.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Author' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each populated author index letter - # Loop over the sorted_authors list, find start of each letter, - # add description_preview_count artists - # self.authors[0]:friendly [1]:author_sort [2]:book_count - # (<friendly name>, author_sort, book_count) - - # Need to extract a list of author_sort, generate sort_equivalents from that - sort_equivalents = self.establish_equivalencies([x[1] for x in self.authors]) - - master_author_list = [] - # Prime the pump - current_letter = self.letter_or_symbol(sort_equivalents[0]) - current_author_list = [] - for idx, author in enumerate(self.authors): - if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: - # Save the old list - add_to_author_list(current_author_list, current_letter) - - # Start the new list - current_letter = self.letter_or_symbol(sort_equivalents[idx]) - current_author_list = [author[0]] - else: - if len(current_author_list) < self.descriptionClip: - current_author_list.append(author[0]) - - # Add the last author list - add_to_author_list(current_author_list, current_letter) - - # Add *article* entries for each populated author initial letter - # master_author_list{}: [0]:author list [1]:Initial letter - for authors_by_letter in master_author_list: - navPointByLetterTag = Tag(soup, 'navPoint') - navPointByLetterTag['class'] = "article" - navPointByLetterTag['id'] = "%sauthors-ID" % (authors_by_letter[1]) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(_("Authors beginning with '%s'") % (authors_by_letter[1]))) - navLabelTag.insert(0, textTag) - navPointByLetterTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - if authors_by_letter[1] == self.SYMBOLS: - contentTag['src'] = "%s#%s_authors" % (HTML_file, authors_by_letter[1]) - else: - contentTag['src'] = "%s#%s_authors" % (HTML_file, self.generateUnicodeName(authors_by_letter[1])) - navPointByLetterTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(authors_by_letter[0])) - navPointByLetterTag.insert(2, cmTag) - - navPointTag.insert(nptc, navPointByLetterTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - - self.ncxSoup = soup - - def generateNCXByDateAdded(self, tocTitle): - self.updateProgressFullStep("NCX 'Recently Added'") - - def add_to_master_month_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_month_list.append((current_titles_list, current_date, book_count)) - - def add_to_master_date_range_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_date_range_list.append((current_titles_list, date_range, book_count)) - - soup = self.ncxSoup - HTML_file = "content/ByDateAdded.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Recently Added' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each date range - current_titles_list = [] - master_date_range_list = [] - today = datetime.datetime.now() - today_time = datetime.datetime(today.year, today.month, today.day) - for (i,date) in enumerate(self.DATE_RANGE): - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - date_range_limit = self.DATE_RANGE[i] - for book in self.booksByDateRange: - book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) - if (today_time-book_time).days <= date_range_limit: - #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) - current_titles_list.append(book['title']) - else: - break - if current_titles_list: - add_to_master_date_range_list(current_titles_list) - current_titles_list = [book['title']] - - # Add *article* entries for each populated date range - # master_date_range_list{}: [0]:titles list [1]:datestr - for books_by_date_range in master_date_range_list: - navPointByDateRangeTag = Tag(soup, 'navPoint') - navPointByDateRangeTag['class'] = "article" - navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(books_by_date_range[1])) - navLabelTag.insert(0, textTag) - navPointByDateRangeTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bda_%s" % (HTML_file, - books_by_date_range[1].replace(' ','')) - - navPointByDateRangeTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_date_range[0])) - navPointByDateRangeTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ - '%d title' % books_by_date_range[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDateRangeTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDateRangeTag) - nptc += 1 - - - - # Create an NCX article entry for each populated month - # Loop over the booksByDate list, find start of each month, - # add description_preview_count titles - # master_month_list(list,date,count) - current_titles_list = [] - master_month_list = [] - current_date = self.booksByMonth[0]['timestamp'] - - for book in self.booksByMonth: - if book['timestamp'].month != current_date.month or \ - book['timestamp'].year != current_date.year: - # Save the old lists - add_to_master_month_list(current_titles_list) - - # Start the new list - current_date = book['timestamp'].date() - current_titles_list = [book['title']] - else: - current_titles_list.append(book['title']) - - # Add the last month list - add_to_master_month_list(current_titles_list) - - # Add *article* entries for each populated month - # master_months_list{}: [0]:titles list [1]:date - for books_by_month in master_month_list: - datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) - navPointByMonthTag = Tag(soup, 'navPoint') - navPointByMonthTag['class'] = "article" - navPointByMonthTag['id'] = "bda_%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(datestr)) - navLabelTag.insert(0, textTag) - navPointByMonthTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bda_%s-%s" % (HTML_file, - books_by_month[1].year,books_by_month[1].month) - - navPointByMonthTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_month[0])) - navPointByMonthTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ - '%d title' % books_by_month[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByMonthTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByMonthTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = soup - - def generateNCXByDateRead(self, tocTitle): - self.updateProgressFullStep("NCX 'Recently Read'") - if not self.booksByDateRead: - return - - def add_to_master_day_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_day_list.append((current_titles_list, current_date, book_count)) - - def add_to_master_date_range_list(current_titles_list): - book_count = len(current_titles_list) - current_titles_list = " • ".join(current_titles_list) - current_titles_list = self.formatNCXText(current_titles_list, dest='description') - master_date_range_list.append((current_titles_list, date_range, book_count)) - - soup = self.ncxSoup - HTML_file = "content/ByDateRead.html" - body = soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Recently Read' *section* --- - navPointTag = Tag(soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(soup,"content") - contentTag['src'] = "%s#section_start" % HTML_file - navPointTag.insert(nptc, contentTag) - nptc += 1 - - # Create an NCX article entry for each date range - current_titles_list = [] - master_date_range_list = [] - today = datetime.datetime.now() - today_time = datetime.datetime(today.year, today.month, today.day) - for (i,date) in enumerate(self.DATE_RANGE): - if i: - date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) - else: - date_range = 'Last %d days' % (self.DATE_RANGE[i]) - date_range_limit = self.DATE_RANGE[i] - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if (today_time-bookmark_time).days <= date_range_limit: - #print "generateNCXByDateAdded: %s added %d days ago" % (book['title'], (today_time-book_time).days) - current_titles_list.append(book['title']) - else: - break - if current_titles_list: - add_to_master_date_range_list(current_titles_list) - current_titles_list = [book['title']] - - # Create an NCX article entry for each populated day - # Loop over the booksByDate list, find start of each month, - # add description_preview_count titles - # master_month_list(list,date,count) - current_titles_list = [] - master_day_list = [] - current_date = datetime.datetime.utcfromtimestamp(self.booksByDateRead[0]['bookmark_timestamp']) - - for book in self.booksByDateRead: - bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) - if bookmark_time.day != current_date.day or \ - bookmark_time.month != current_date.month or \ - bookmark_time.year != current_date.year: - # Save the old lists - add_to_master_day_list(current_titles_list) - - # Start the new list - current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() - current_titles_list = [book['title']] - else: - current_titles_list.append(book['title']) - - # Add the last day list - add_to_master_day_list(current_titles_list) - - # Add *article* entries for each populated day - # master_day_list{}: [0]:titles list [1]:date - for books_by_day in master_day_list: - datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) - navPointByDayTag = Tag(soup, 'navPoint') - navPointByDayTag['class'] = "article" - navPointByDayTag['id'] = "bdr_%s-%s-%sID" % (books_by_day[1].year, - books_by_day[1].month, - books_by_day[1].day ) - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(soup, 'navLabel') - textTag = Tag(soup, 'text') - textTag.insert(0, NavigableString(datestr)) - navLabelTag.insert(0, textTag) - navPointByDayTag.insert(0,navLabelTag) - contentTag = Tag(soup, 'content') - contentTag['src'] = "%s#bdr_%s-%s-%s" % (HTML_file, - books_by_day[1].year, - books_by_day[1].month, - books_by_day[1].day) - - navPointByDayTag.insert(1,contentTag) - - if self.generateForKindle: - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - cmTag.insert(0, NavigableString(books_by_day[0])) - navPointByDayTag.insert(2, cmTag) - - cmTag = Tag(soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - navStr = '%d titles' % books_by_day[2] if books_by_day[2] > 1 else \ - '%d title' % books_by_day[2] - cmTag.insert(0, NavigableString(navStr)) - navPointByDayTag.insert(3, cmTag) - - navPointTag.insert(nptc, navPointByDayTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = soup - - def generateNCXByGenre(self, tocTitle): - # Create an NCX section for 'By Genre' - # Add each genre as an article - # 'tag', 'file', 'authors' - - self.updateProgressFullStep("NCX 'Genres'") - - if not len(self.genres): - self.opts.log.warn(" No genres found in tags.\n" - " No Genre section added to Catalog") - return - - ncx_soup = self.ncxSoup - body = ncx_soup.find("navPoint") - btc = len(body.contents) - - # --- Construct the 'Books By Genre' *section* --- - navPointTag = Tag(ncx_soup, 'navPoint') - navPointTag['class'] = "section" - file_ID = "%s" % tocTitle.lower() - file_ID = file_ID.replace(" ","") - navPointTag['id'] = "%s-ID" % file_ID - navPointTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, 'navLabel') - textTag = Tag(ncx_soup, 'text') - # textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) - textTag.insert(0, NavigableString('%s' % tocTitle)) - navLabelTag.insert(0, textTag) - nptc = 0 - navPointTag.insert(nptc, navLabelTag) - nptc += 1 - contentTag = Tag(ncx_soup,"content") - contentTag['src'] = "content/Genre_%s.html#section_start" % self.genres[0]['tag'] - navPointTag.insert(nptc, contentTag) - nptc += 1 - - for genre in self.genres: - # Add an article for each genre - navPointVolumeTag = Tag(ncx_soup, 'navPoint') - navPointVolumeTag['class'] = "article" - navPointVolumeTag['id'] = "genre-%s-ID" % genre['tag'] - navPointVolumeTag['playOrder'] = self.playOrder - self.playOrder += 1 - navLabelTag = Tag(ncx_soup, "navLabel") - textTag = Tag(ncx_soup, "text") - - # GwR *** Can this be optimized? - normalized_tag = None - for friendly_tag in self.genre_tags_dict: - if self.genre_tags_dict[friendly_tag] == genre['tag']: - normalized_tag = self.genre_tags_dict[friendly_tag] - break - textTag.insert(0, self.formatNCXText(NavigableString(friendly_tag), dest='description')) - navLabelTag.insert(0,textTag) - navPointVolumeTag.insert(0,navLabelTag) - contentTag = Tag(ncx_soup, "content") - contentTag['src'] = "content/Genre_%s.html#Genre_%s" % (normalized_tag, normalized_tag) - navPointVolumeTag.insert(1, contentTag) - - if self.generateForKindle: - # Build the author tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "author" - # First - Last author - - if len(genre['titles_spanned']) > 1 : - author_range = "%s - %s" % (genre['titles_spanned'][0][0], genre['titles_spanned'][1][0]) - else : - author_range = "%s" % (genre['titles_spanned'][0][0]) - - cmTag.insert(0, NavigableString(author_range)) - navPointVolumeTag.insert(2, cmTag) - - # Build the description tag - cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') - cmTag['name'] = "description" - - if False: - # Form 1: Titles spanned - if len(genre['titles_spanned']) > 1: - title_range = "%s -\n%s" % (genre['titles_spanned'][0][1], genre['titles_spanned'][1][1]) - else: - title_range = "%s" % (genre['titles_spanned'][0][1]) - cmTag.insert(0, NavigableString(self.formatNCXText(title_range, dest='description'))) - else: - # Form 2: title • title • title ... - titles = [] - for title in genre['books']: - titles.append(title['title']) - titles = sorted(titles, key=lambda x:(self.generateSortTitle(x),self.generateSortTitle(x))) - titles_list = self.generateShortDescription(u" • ".join(titles), dest="description") - cmTag.insert(0, NavigableString(self.formatNCXText(titles_list, dest='description'))) - - navPointVolumeTag.insert(3, cmTag) - - # Add this volume to the section tag - navPointTag.insert(nptc, navPointVolumeTag) - nptc += 1 - - # Add this section to the body - body.insert(btc, navPointTag) - btc += 1 - self.ncxSoup = ncx_soup - - def writeNCX(self): - self.updateProgressFullStep("Saving NCX") - - outfile = open("%s/%s.ncx" % (self.catalogPath, self.basename), 'w') - outfile.write(self.ncxSoup.prettify()) - - - # ======================== Helpers ======================== - def author_to_author_sort(self, author): - tokens = author.split() - tokens = tokens[-1:] + tokens[:-1] - if len(tokens) > 1: - tokens[0] += ',' - return ' '.join(tokens).capitalize() - - def booksByAuthorSorter_author_sort(self, book): - ''' - Sort non-series books before series books - ''' - if not book['series']: - key = '%s ~%s' % (capitalize(book['author_sort']), - capitalize(book['title_sort'])) - else: - index = book['series_index'] - integer = int(index) - fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s %s %s' % (capitalize(book['author_sort']), - self.generateSortTitle(book['series']), - series_index) - return key - - def booksByAuthorSorter_author(self, book): - ''' - Sort non-series books before series books - ''' - if not book['series']: - key = '%s %s' % (self.author_to_author_sort(book['author']), - capitalize(book['title_sort'])) - else: - index = book['series_index'] - integer = int(index) - fraction = index-integer - series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s ~%s %s' % (self.author_to_author_sort(book['author']), - self.generateSortTitle(book['series']), - series_index) - return key - - def calculateThumbnailSize(self): - ''' Calculate thumbnail dimensions based on device DPI. Scale Kindle by 50% ''' - from calibre.customize.ui import output_profiles - for x in output_profiles(): - if x.short_name == self.opts.output_profile: - # aspect ratio: 3:4 - self.thumbWidth = x.dpi * float(self.opts.thumb_width) - self.thumbHeight = self.thumbWidth * 1.33 - if 'kindle' in x.short_name and self.opts.fmt == 'mobi': - # Kindle DPI appears to be off by a factor of 2 - self.thumbWidth = self.thumbWidth/2 - self.thumbHeight = self.thumbHeight/2 - break - if True and self.verbose: - self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ - (x.dpi, self.thumbWidth, self.thumbHeight)) - - def convertHTMLEntities(self, s): - matches = re.findall("&#\d+;", s) - if len(matches) > 0: - hits = set(matches) - for hit in hits: - name = hit[2:-1] - try: - entnum = int(name) - s = s.replace(hit, unichr(entnum)) - except ValueError: - pass - - matches = re.findall("&\w+;", s) - hits = set(matches) - amp = "&" - if amp in hits: - hits.remove(amp) - for hit in hits: - name = hit[1:-1] - if htmlentitydefs.name2codepoint.has_key(name): - s = s.replace(hit, unichr(htmlentitydefs.name2codepoint[name])) - s = s.replace(amp, "&") - return s - - def createDirectoryStructure(self): - catalogPath = self.catalogPath - self.cleanUp() - - if not os.path.isdir(catalogPath): - os.makedirs(catalogPath) - - # Create /content and /images - content_path = catalogPath + "/content" - if not os.path.isdir(content_path): - os.makedirs(content_path) - images_path = catalogPath + "/images" - if not os.path.isdir(images_path): - os.makedirs(images_path) - - def discoverPrefix(self, record): - ''' - Evaluate conditions for including prefixes in various listings - ''' - def log_prefix_rule_match_info(rule, record): - self.opts.log.info(" %s %s by %s (Prefix rule '%s': %s:%s)" % - (rule['prefix'],record['title'], - record['authors'][0], rule['name'], - rule['field'],rule['pattern'])) - - # Compare the record to each rule looking for a match - for rule in self.prefixRules: - # Literal comparison for Tags field - if rule['field'].lower() == 'tags': - if rule['pattern'].lower() in map(unicode.lower,record['tags']): - if self.opts.verbose: - log_prefix_rule_match_info(rule, record) - return rule['prefix'] - - # Regex match for custom field - elif rule['field'].startswith('#'): - field_contents = self.__db.get_field(record['id'], - rule['field'], - index_is_id=True) - if field_contents == '': - field_contents = None - - if field_contents is not None: - try: - if re.search(rule['pattern'], unicode(field_contents), - re.IGNORECASE) is not None: - if self.opts.verbose: - log_prefix_rule_match_info(rule, record) - return rule['prefix'] - except: - # Compiling of pat failed, ignore it - if self.opts.verbose: - self.opts.log.error("pattern failed to compile: %s" % rule['pattern']) - pass - elif field_contents is None and rule['pattern'] == 'None': - if self.opts.verbose: - log_prefix_rule_match_info(rule, record) - return rule['prefix'] - - return None - - def establish_equivalencies(self, item_list, key=None): - # Filter for leading letter equivalencies - - # Hack to force the cataloged leading letter to be - # an unadorned character if the accented version sorts before the unaccented - exceptions = { - u'Ä':u'A', - u'Ö':u'O', - u'Ü':u'U' - } - - if key is not None: - sort_field = key - - cl_list = [None] * len(item_list) - last_ordnum = 0 - - for idx, item in enumerate(item_list): - if key: - c = item[sort_field] - else: - c = item - - ordnum, ordlen = collation_order(c) - if last_ordnum != ordnum: - last_c = icu_upper(c[0:ordlen]) - if last_c in exceptions.keys(): - last_c = exceptions[unicode(last_c)] - last_ordnum = ordnum - cl_list[idx] = last_c - - if False: - if key: - for idx, item in enumerate(item_list): - print("%s %s" % (cl_list[idx],item[sort_field])) - else: - print("%s %s" % (cl_list[0], item)) - - return cl_list - - def filterDbTags(self, tags): - # Remove the special marker tags from the database's tag list, - # return sorted list of normalized genre tags - - def format_tag_list(tags, indent=5, line_break=70, header='Tag list'): - def next_tag(sorted_tags): - for (i, tag) in enumerate(sorted_tags): - if i < len(tags) - 1: - yield tag + ", " - else: - yield tag - - ans = '%s%d %s:\n' % (' ' * indent, len(tags), header) - ans += ' ' * (indent + 1) - out_str = '' - sorted_tags = sorted(tags, key=sort_key) - for tag in next_tag(sorted_tags): - out_str += tag - if len(out_str) >= line_break: - ans += out_str + '\n' - out_str = ' ' * (indent + 1) - return ans + out_str - - normalized_tags = [] - friendly_tags = [] - excluded_tags = [] - for tag in tags: - if tag in self.markerTags: - excluded_tags.append(tag) - continue - try: - if re.search(self.opts.exclude_genre, tag): - excluded_tags.append(tag) - continue - except: - self.opts.log.error("\tfilterDbTags(): malformed --exclude-genre regex pattern: %s" % self.opts.exclude_genre) - - if tag == ' ': - continue - - normalized_tags.append(re.sub('\W','',ascii_text(tag)).lower()) - friendly_tags.append(tag) - - genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) - - # Test for multiple genres resolving to same normalized form - normalized_set = set(normalized_tags) - for normalized in normalized_set: - if normalized_tags.count(normalized) > 1: - self.opts.log.warn(" Warning: multiple tags resolving to genre '%s':" % normalized) - for key in genre_tags_dict: - if genre_tags_dict[key] == normalized: - self.opts.log.warn(" %s" % key) - if self.verbose: - self.opts.log.info('%s' % format_tag_list(genre_tags_dict, header="enabled genre tags in database")) - self.opts.log.info('%s' % format_tag_list(excluded_tags, header="excluded genre tags")) - - return genre_tags_dict - - def formatNCXText(self, description, dest=None): - # Kindle TOC descriptions won't render certain characters - # Fix up - massaged = unicode(BeautifulStoneSoup(description, convertEntities=BeautifulStoneSoup.HTML_ENTITIES)) - - # Replace '&' with '&' - massaged = re.sub("&","&", massaged) - - if massaged.strip() and dest: - #print traceback.print_stack(limit=3) - return self.generateShortDescription(massaged.strip(), dest=dest) - else: - return None - - def formatPrefix(self,prefix_char,soup): - # Generate the HTML for the prefix portion of the listing - # Kindle Previewer doesn't properly handle style=color:white - # MOBI does a better job allocating blank space with <code> - if self.opts.fmt == 'mobi': - codeTag = Tag(soup, "code") - if prefix_char is None: - codeTag.insert(0,NavigableString(' ')) - else: - codeTag.insert(0,NavigableString(prefix_char)) - return codeTag - else: - spanTag = Tag(soup, "span") - spanTag['class'] = "prefix" - - # color:white was the original technique used to align columns. - # The new technique is to float the prefix left with CSS. - if prefix_char is None: - if True: - prefix_char = " " - else: - del spanTag['class'] - spanTag['style'] = "color:white" - prefix_char = self.defaultPrefix - spanTag.insert(0,NavigableString(prefix_char)) - return spanTag - - def generateAuthorAnchor(self, author): - # Generate a legal XHTML id/href string - return re.sub("\W","", ascii_text(author)) - - def generateFormatArgs(self, book): - series_index = str(book['series_index']) - if series_index.endswith('.0'): - series_index = series_index[:-2] - args = dict( - title = book['title'], - series = book['series'], - series_index = series_index, - rating = self.generateRatingString(book), - rating_parens = '(%s)' % self.generateRatingString(book) if 'rating' in book else '', - pubyear = book['date'].split()[1] if book['date'] else '', - pubyear_parens = "(%s)" % book['date'].split()[1] if book['date'] else '') - return args - - def generateHTMLByGenre(self, genre, section_head, books, outfile): - # Write an HTML file of this genre's book list - # Return a list with [(first_author, first_book), (last_author, last_book)] - - soup = self.generateHTMLGenreHeader(genre) + soup = self.generate_html_genre_header(genre) body = soup.find('body') btc = 0 divTag = Tag(soup, 'div') dtc = 0 - # Insert section tag if this is the section start - first article only if section_head: aTag = Tag(soup,'a') @@ -3409,7 +2248,7 @@ Author '{0}': btc += 1 titleTag = body.find(attrs={'class':'title'}) - titleTag.insert(0,NavigableString('%s' % escape(self.getFriendlyGenreTag(genre)))) + titleTag.insert(0,NavigableString('%s' % escape(self.get_friendly_genre_tag(genre)))) # Insert the books by author list divTag = body.find(attrs={'class':'authors'}) @@ -3427,7 +2266,7 @@ Author '{0}': pAuthorTag['class'] = "author_index" aTag = Tag(soup, "a") if self.opts.generate_authors: - aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generateAuthorAnchor(book['author'])) + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(book['author'])) aTag.insert(0, book['author']) pAuthorTag.insert(0,aTag) divTag.insert(dtc,pAuthorTag) @@ -3443,7 +2282,7 @@ Author '{0}': pSeriesTag['class'] = "series_mobi" if self.opts.generate_series: aTag = Tag(soup,'a') - aTag['href'] = "%s.html#%s" % ('BySeries', self.generateSeriesAnchor(book['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries', self.generate_series_anchor(book['series'])) aTag.insert(0, book['series']) pSeriesTag.insert(0, aTag) else: @@ -3459,7 +2298,7 @@ Author '{0}': pBookTag['class'] = "line_item" ptc = 0 - pBookTag.insert(ptc, self.formatPrefix(book['prefix'],soup)) + pBookTag.insert(ptc, self.format_prefix(book['prefix'])) ptc += 1 spanTag = Tag(soup, "span") @@ -3472,7 +2311,7 @@ Author '{0}': aTag['href'] = "book_%d.html" % (int(float(book['id']))) # Generate the title from the template - args = self.generateFormatArgs(book) + args = self.generate_format_args(book) if current_series: #aTag.insert(0,escape(book['title'][len(book['series'])+1:])) formatted_title = self.by_genres_series_title_template.format(**args).rstrip() @@ -3491,7 +2330,7 @@ Author '{0}': divTag.insert(dtc, pBookTag) dtc += 1 - # Write the generated file to contentdir + # Write the generated file to content_dir outfile = open(outfile, 'w') outfile.write(soup.prettify()) outfile.close() @@ -3503,13 +2342,347 @@ Author '{0}': return titles_spanned - def generateHTMLDescriptionHeader(self, book): - ''' - Generate description header from template - ''' + def generate_html_by_series(self): + """ Generate content/BySeries.html. + + Search database for books in series. + + Input: + database + + Output: + content/BySeries.html (file) + """ + friendly_name = _("Series") + self.update_progress_full_step("%s HTML" % friendly_name) + + self.opts.sort_by = 'series' + + # Merge self.excluded_tags with opts.search_text + # Updated to use exact match syntax + + search_phrase = 'series:true ' + if self.excluded_tags: + search_terms = [] + for tag in self.excluded_tags: + search_terms.append("tag:=%s" % tag) + search_phrase += "not (%s)" % " or ".join(search_terms) + + # If a list of ids are provided, don't use search_text + if self.opts.ids: + self.opts.search_text = search_phrase + else: + if self.opts.search_text: + self.opts.search_text += " " + search_phrase + else: + self.opts.search_text = search_phrase + + # Fetch the database as a dictionary + data = self.plugin.search_sort_db(self.db, self.opts) + + # Remove exclusions + self.books_by_series = self.process_exclusions(data) + + if not self.books_by_series: + self.opts.generate_series = False + self.opts.log(" no series found in selected books, cancelling series generation") + return + + # Generate series_sort + for book in self.books_by_series: + book['series_sort'] = self.generate_sort_title(book['series']) + + soup = self.generate_html_empty_header(friendly_name) + body = soup.find('body') + + btc = 0 + divTag = Tag(soup, "div") + dtc = 0 + current_letter = "" + current_series = None + + # Establish initial letter equivalencies + sort_equivalents = self.establish_equivalencies(self.books_by_series, key='series_sort') + + # Loop through books_by_series + series_count = 0 + for idx, book in enumerate(self.books_by_series): + # Check for initial letter change + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter : + # Start a new letter with Index letter + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "series_letter_index" + aTag = Tag(soup, "a") + if current_letter == self.SYMBOLS: + aTag['id'] = self.SYMBOLS + "_series" + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.SYMBOLS)) + else: + aTag['id'] = self.generate_unicode_name(current_letter) + "_series" + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) + divTag.insert(dtc,pIndexTag) + dtc += 1 + # Check for series change + if book['series'] != current_series: + # Start a new series + series_count += 1 + current_series = book['series'] + pSeriesTag = Tag(soup,'p') + pSeriesTag['class'] = "series" + if self.opts.fmt == 'mobi': + pSeriesTag['class'] = "series_mobi" + aTag = Tag(soup, 'a') + aTag['id'] = self.generate_series_anchor(book['series']) + pSeriesTag.insert(0,aTag) + pSeriesTag.insert(1,NavigableString('%s' % book['series'])) + divTag.insert(dtc,pSeriesTag) + dtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + pBookTag['class'] = "line_item" + ptc = 0 + + book['prefix'] = self.discover_prefix(book) + pBookTag.insert(ptc, self.format_prefix(book['prefix'])) + ptc += 1 + + spanTag = Tag(soup, "span") + spanTag['class'] = "entry" + stc = 0 + + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + # Use series, series index if avail else just title + #aTag.insert(0,'%d. %s · %s' % (book['series_index'],escape(book['title']), ' & '.join(book['authors']))) + + if is_date_undefined(book['pubdate']): + book['date'] = None + else: + book['date'] = strftime(u'%B %Y', book['pubdate'].timetuple()) + + args = self.generate_format_args(book) + formatted_title = self.by_series_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + + spanTag.insert(stc, aTag) + stc += 1 + + # · + spanTag.insert(stc, NavigableString(' · ')) + stc += 1 + + # Link to author + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", + self.generate_author_anchor(escape(' & '.join(book['authors'])))) + aTag.insert(0, NavigableString(' & '.join(book['authors']))) + spanTag.insert(stc, aTag) + stc += 1 + + pBookTag.insert(ptc, spanTag) + ptc += 1 + + divTag.insert(dtc, pBookTag) + dtc += 1 + + pTag = Tag(soup, "p") + pTag['class'] = 'title' + ptc = 0 + aTag = Tag(soup,'a') + aTag['id'] = 'section_start' + pTag.insert(ptc, aTag) + ptc += 1 + + if not self.generate_for_kindle: + # Insert the <h2> tag with book_count at the head + aTag = Tag(soup, "a") + anchor_name = friendly_name.lower() + aTag['id'] = anchor_name.replace(" ","") + pTag.insert(0,aTag) + pTag.insert(1,NavigableString('%s' % friendly_name)) + body.insert(btc,pTag) + btc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + + # Write the generated file to content_dir + outfile_spec = "%s/BySeries.html" % (self.content_dir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.html_filelist_1.append("content/BySeries.html") + + def generate_html_by_title(self): + """ Generate content/ByAlphaTitle.html. + + Generate HTML of books sorted by title. + + Input: + books_by_title + + Output: + content/ByAlphaTitle.html (file) + """ + + self.update_progress_full_step(_("Titles HTML")) + + soup = self.generate_html_empty_header("Books By Alpha Title") + body = soup.find('body') + btc = 0 + + pTag = Tag(soup, "p") + pTag['class'] = 'title' + ptc = 0 + aTag = Tag(soup,'a') + aTag['id'] = 'section_start' + pTag.insert(ptc, aTag) + ptc += 1 + + if not self.generate_for_kindle: + # Kindle don't need this because it shows section titles in Periodical format + aTag = Tag(soup, "a") + aTag['id'] = "bytitle" + pTag.insert(ptc,aTag) + ptc += 1 + pTag.insert(ptc,NavigableString(_('Titles'))) + + body.insert(btc,pTag) + btc += 1 + + divTag = Tag(soup, "div") + dtc = 0 + current_letter = "" + + # Re-sort title list without leading series/series_index + # Incoming title <series> <series_index>: <title> + if not self.use_series_prefix_in_titles_section: + nspt = deepcopy(self.books_by_title) + nspt = sorted(nspt, key=lambda x: sort_key(x['title_sort'].upper())) + self.books_by_title_no_series_prefix = nspt + + # Establish initial letter equivalencies + sort_equivalents = self.establish_equivalencies(self.books_by_title, key='title_sort') + + # Loop through the books by title + # Generate one divRunningTag per initial letter for the purposes of + # minimizing widows and orphans on readers that can handle large + # <divs> styled as inline-block + title_list = self.books_by_title + if not self.use_series_prefix_in_titles_section: + title_list = self.books_by_title_no_series_prefix + drtc = 0 + divRunningTag = None + for idx, book in enumerate(title_list): + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: + # Start a new letter + if drtc and divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + divRunningTag = Tag(soup, 'div') + if dtc > 0: + divRunningTag['class'] = "initial_letter" + drtc = 0 + pIndexTag = Tag(soup, "p") + pIndexTag['class'] = "author_title_letter_index" + aTag = Tag(soup, "a") + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + if current_letter == self.SYMBOLS: + aTag['id'] = self.SYMBOLS + "_titles" + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(self.SYMBOLS)) + else: + aTag['id'] = self.generate_unicode_name(current_letter) + "_titles" + pIndexTag.insert(0,aTag) + pIndexTag.insert(1,NavigableString(sort_equivalents[idx])) + divRunningTag.insert(dtc,pIndexTag) + drtc += 1 + + # Add books + pBookTag = Tag(soup, "p") + pBookTag['class'] = "line_item" + ptc = 0 + + pBookTag.insert(ptc, self.format_prefix(book['prefix'])) + ptc += 1 + + spanTag = Tag(soup, "span") + spanTag['class'] = "entry" + stc = 0 + + # Link to book + aTag = Tag(soup, "a") + if self.opts.generate_descriptions: + aTag['href'] = "book_%d.html" % (int(float(book['id']))) + + # Generate the title from the template + args = self.generate_format_args(book) + if book['series']: + formatted_title = self.by_titles_series_title_template.format(**args).rstrip() + else: + formatted_title = self.by_titles_normal_title_template.format(**args).rstrip() + aTag.insert(0,NavigableString(escape(formatted_title))) + spanTag.insert(stc, aTag) + stc += 1 + + # Dot + spanTag.insert(stc, NavigableString(" · ")) + stc += 1 + + # Link to author + emTag = Tag(soup, "em") + aTag = Tag(soup, "a") + if self.opts.generate_authors: + aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", self.generate_author_anchor(book['author'])) + aTag.insert(0, NavigableString(book['author'])) + emTag.insert(0,aTag) + spanTag.insert(stc, emTag) + stc += 1 + + pBookTag.insert(ptc, spanTag) + ptc += 1 + + if divRunningTag is not None: + divRunningTag.insert(drtc, pBookTag) + drtc += 1 + + # Add the last divRunningTag to divTag + if divRunningTag is not None: + divTag.insert(dtc, divRunningTag) + dtc += 1 + + # Add the divTag to the body + body.insert(btc, divTag) + btc += 1 + + # Write the volume to content_dir + outfile_spec = "%s/ByAlphaTitle.html" % (self.content_dir) + outfile = open(outfile_spec, 'w') + outfile.write(soup.prettify()) + outfile.close() + self.html_filelist_1.append("content/ByAlphaTitle.html") + + def generate_html_description_header(self, book): + """ Generate the HTML Description header from template. + + Create HTML Description from book metadata and template. + Called by generate_html_descriptions() + + Args: + book (dict): book metadata + + Return: + soup (BeautifulSoup): HTML Description for book + """ + from calibre.ebooks.oeb.base import XHTML_NS - def generate_html(): + def _generate_html(): args = dict( author=author, author_prefix=author_prefix, @@ -3554,7 +2727,7 @@ Author '{0}': if book['prefix']: author_prefix = book['prefix'] + ' ' + _("by ") elif self.opts.connected_kindle and book['id'] in self.bookmarked_books: - author_prefix = self.READING_SYMBOL + ' ' + _("by ") + author_prefix = self.SYMBOL_READING + ' ' + _("by ") else: author_prefix = _("by ") @@ -3608,8 +2781,8 @@ Author '{0}': stars = int(book['rating']) / 2 rating = '' if stars: - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + star_string = self.SYMBOL_FULL_RATING * stars + empty_stars = self.SYMBOL_EMPTY_RATING * (5 - stars) rating = '%s%s <br/>' % (star_string,empty_stars) # Notes @@ -3624,10 +2797,8 @@ Author '{0}': if 'description' in book and book['description'] > '': comments = book['description'] - # >>>> Populate the template <<<< - soup = generate_html() - + soup = _generate_html() # >>>> Post-process the template <<<< body = soup.find('body') @@ -3645,7 +2816,7 @@ Author '{0}': if aTag: if book['series']: if self.opts.generate_series: - aTag['href'] = "%s.html#%s" % ('BySeries',self.generateSeriesAnchor(book['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries',self.generate_series_anchor(book['series'])) else: aTag.extract() @@ -3653,7 +2824,7 @@ Author '{0}': aTag = body.find('a', attrs={'class':'author'}) if self.opts.generate_authors and aTag: aTag['href'] = "%s.html#%s" % ("ByAlphaAuthor", - self.generateAuthorAnchor(book['author'])) + self.generate_author_anchor(book['author'])) if publisher == ' ': publisherTag = body.find('td', attrs={'class':'publisher'}) @@ -3681,11 +2852,47 @@ Author '{0}': newEmptyTag.insert(0,NavigableString(' ')) mt.replaceWith(newEmptyTag) - if False: - print soup.prettify() - return soup - def generateHTMLEmptyHeader(self, title): + def generate_html_descriptions(self): + """ Generate Description HTML for each book. + + Loop though books, write Description HTML for each book. + + Inputs: + books_by_title (list) + + Output: + (files): Description HTML for each book + """ + + self.update_progress_full_step(_("Descriptions HTML")) + + for (title_num, title) in enumerate(self.books_by_title): + self.update_progress_micro_step("%s %d of %d" % + (_("Description HTML"), + title_num, len(self.books_by_title)), + float(title_num*100/len(self.books_by_title))/100) + + # Generate the header from user-customizable template + soup = self.generate_html_description_header(title) + + # Write the book entry to content_dir + outfile = open("%s/book_%d.html" % (self.content_dir, int(title['id'])), 'w') + outfile.write(soup.prettify()) + outfile.close() + + def generate_html_empty_header(self, title): + """ Return a boilerplate HTML header. + + Generate an HTML header with document title. + + Args: + title (str): document title + + Return: + soup (BeautifulSoup): HTML header with document title inserted + """ + header = ''' <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata"> @@ -3704,30 +2911,51 @@ Author '{0}': titleTag.insert(0,NavigableString(title)) return soup - def generateHTMLGenreHeader(self, title): - header = ''' - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> - <html xmlns="http://www.w3.org/1999/xhtml" xmlns:calibre="http://calibre.kovidgoyal.net/2009/metadata"> - <head> - <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> - <link rel="stylesheet" type="text/css" href="stylesheet.css" media="screen" /> - <title> - - -

-
- - - ''' - # Insert the supplied title - soup = BeautifulSoup(header) - titleTag = soup.find('title') - titleTag.insert(0,escape(NavigableString(title))) + def generate_html_genre_header(self, title): + """ Generate HTML header with initial body content + + Start with a generic HTML header, add

and

+ + Args: + title (str): document title + + Return: + soup (BeautifulSoup): HTML with initial

and

tags + """ + + soup = self.generate_html_empty_header(title) + bodyTag = soup.find('body') + pTag = Tag(soup, 'p') + pTag['class'] = 'title' + bodyTag.insert(0,pTag) + divTag = Tag(soup, 'div') + divTag['class'] = 'authors' + bodyTag.insert(1,divTag) return soup - def generateMastheadImage(self, out_path): + + def generate_masthead_image(self, out_path): + """ Generate a Kindle masthead image. + + Generate a Kindle masthead image, used with Kindle periodical format. + + Args: + out_path (str): path to write generated masthead image + + Input: + opts.catalog_title (str): Title to render + masthead_font: User-specified font preference (MOBI output option) + + Output: + out_path (file): masthead image (GIF) + """ + from calibre.ebooks.conversion.config import load_defaults from calibre.utils.fonts import fontconfig + + MI_WIDTH = 600 + MI_HEIGHT = 60 + font_path = default_font = P('fonts/liberation/LiberationSerif-Bold.ttf') recs = load_defaults('mobi_output') masthead_font_family = recs.get('masthead_font', 'Default') @@ -3742,9 +2970,6 @@ Author '{0}': if not font_path or not os.access(font_path, os.R_OK): font_path = default_font - MI_WIDTH = 600 - MI_HEIGHT = 60 - try: from PIL import Image, ImageDraw, ImageFont Image, ImageDraw, ImageFont @@ -3758,42 +2983,1227 @@ Author '{0}': except: self.opts.log.error(" Failed to load user-specifed font '%s'" % font_path) font = ImageFont.truetype(default_font, 48) - text = self.title.encode('utf-8') + text = self.opts.catalog_title.encode('utf-8') width, height = draw.textsize(text, font=font) left = max(int((MI_WIDTH - width)/2.), 0) top = max(int((MI_HEIGHT - height)/2.), 0) draw.text((left, top), text, fill=(0,0,0), font=font) img.save(open(out_path, 'wb'), 'GIF') - def generateRatingString(self, book): + def generate_ncx_header(self): + """ Generate the basic NCX file. + + Generate the initial NCX, which is added to depending on included Sections. + + Inputs: + None + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): NCX foundation + """ + + self.update_progress_full_step(_("NCX header")) + + header = ''' + + + + ''' + soup = BeautifulStoneSoup(header, selfClosingTags=['content','calibre:meta-img']) + + ncx = soup.find('ncx') + navMapTag = Tag(soup, 'navMap') + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "periodical" + navPointTag['id'] = "title" + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(self.opts.catalog_title)) + navLabelTag.insert(0, textTag) + navPointTag.insert(0, navLabelTag) + + if self.opts.generate_authors: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByAlphaAuthor.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_titles: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByAlphaTitle.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_series: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/BySeries.html" + navPointTag.insert(1, contentTag) + elif self.opts.generate_genres: + contentTag = Tag(soup, 'content') + #contentTag['src'] = "content/ByGenres.html" + contentTag['src'] = "%s" % self.genres[0]['file'] + navPointTag.insert(1, contentTag) + elif self.opts.generate_recently_added: + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/ByDateAdded.html" + navPointTag.insert(1, contentTag) + else: + # Descriptions only + sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \ + else self.books_by_title + contentTag = Tag(soup, 'content') + contentTag['src'] = "content/book_%d.html" % int(sort_descriptions_by[0]['id']) + navPointTag.insert(1, contentTag) + + cmiTag = Tag(soup, '%s' % 'calibre:meta-img') + cmiTag['id'] = "mastheadImage" + cmiTag['src'] = "images/mastheadImage.gif" + navPointTag.insert(2,cmiTag) + navMapTag.insert(0,navPointTag) + + ncx.insert(0,navMapTag) + self.ncx_soup = soup + + def generate_ncx_descriptions(self, tocTitle): + """ Add Descriptions to the basic NCX file. + + Generate the Descriptions NCX content, add to self.ncx_soup. + + Inputs: + books_by_author (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Descriptions")) + + # --- Construct the 'Books by Title' section --- + ncx_soup = self.ncx_soup + body = ncx_soup.find("navPoint") + btc = len(body.contents) + + # Add the section navPoint + navPointTag = Tag(ncx_soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "bytitle-ID" + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(ncx_soup, 'navLabel') + textTag = Tag(ncx_soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(ncx_soup,"content") + contentTag['src'] = "content/book_%d.html" % int(self.books_by_title[0]['id']) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Loop over the titles + sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \ + else self.books_by_title + + for book in sort_descriptions_by: + navPointVolumeTag = Tag(ncx_soup, 'navPoint') + navPointVolumeTag['class'] = "article" + navPointVolumeTag['id'] = "book%dID" % int(book['id']) + navPointVolumeTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(ncx_soup, "navLabel") + textTag = Tag(ncx_soup, "text") + if book['series']: + series_index = str(book['series_index']) + if series_index.endswith('.0'): + series_index = series_index[:-2] + if self.generate_for_kindle: + # Don't include Author for Kindle + textTag.insert(0, NavigableString(self.format_ncx_text('%s (%s [%s])' % + (book['title'], book['series'], series_index), dest='title'))) + else: + # Include Author for non-Kindle + textTag.insert(0, NavigableString(self.format_ncx_text('%s (%s [%s]) · %s ' % + (book['title'], book['series'], series_index, book['author']), dest='title'))) + else: + if self.generate_for_kindle: + # Don't include Author for Kindle + title_str = self.format_ncx_text('%s' % (book['title']), dest='title') + if self.opts.connected_kindle and book['id'] in self.bookmarked_books: + ''' + dots = int((book['percent_read'] + 5)/10) + dot_string = '+' * dots + empty_dots = '-' * (10 - dots) + title_str += ' %s%s' % (dot_string,empty_dots) + ''' + title_str += '*' + textTag.insert(0, NavigableString(title_str)) + else: + # Include Author for non-Kindle + textTag.insert(0, NavigableString(self.format_ncx_text('%s · %s' % \ + (book['title'], book['author']), dest='title'))) + navLabelTag.insert(0,textTag) + navPointVolumeTag.insert(0,navLabelTag) + + contentTag = Tag(ncx_soup, "content") + contentTag['src'] = "content/book_%d.html#book%d" % (int(book['id']), int(book['id'])) + navPointVolumeTag.insert(1, contentTag) + + if self.generate_for_kindle: + # Add the author tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + + if book['date']: + navStr = '%s | %s' % (self.format_ncx_text(book['author'], dest='author'), + book['date'].split()[1]) + else: + navStr = '%s' % (self.format_ncx_text(book['author'], dest='author')) + + if 'tags' in book and len(book['tags']): + navStr = self.format_ncx_text(navStr + ' | ' + ' · '.join(sorted(book['tags'])), dest='author') + cmTag.insert(0, NavigableString(navStr)) + navPointVolumeTag.insert(2, cmTag) + + # Add the description tag + if book['short_description']: + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.format_ncx_text(book['short_description'], dest='description'))) + navPointVolumeTag.insert(3, cmTag) + + # Add this volume to the section tag + navPointTag.insert(nptc, navPointVolumeTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncx_soup = ncx_soup + + def generate_ncx_by_series(self, tocTitle): + """ Add Series to the basic NCX file. + + Generate the Series NCX content, add to self.ncx_soup. + + Inputs: + books_by_series (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Series")) + + def _add_to_series_by_letter(current_series_list): + current_series_list = " • ".join(current_series_list) + current_series_list = self.format_ncx_text(current_series_list, dest="description") + series_by_letter.append(current_series_list) + + soup = self.ncx_soup + output = "BySeries" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Series' section --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "byseries-ID" + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "content/%s.html#section_start" % (output) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + series_by_letter = [] + # Establish initial letter equivalencies + sort_equivalents = self.establish_equivalencies(self.books_by_series, key='series_sort') + + # Loop over the series titles, find start of each letter, add description_preview_count books + # Special switch for using different title list + + title_list = self.books_by_series + + # Prime the pump + current_letter = self.letter_or_symbol(sort_equivalents[0]) + + title_letters = [current_letter] + current_series_list = [] + current_series = "" + for idx, book in enumerate(title_list): + sort_title = self.generate_sort_title(book['series']) + sort_title_equivalent = self.establish_equivalencies([sort_title])[0] + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: + + # Save the old list + _add_to_series_by_letter(current_series_list) + + # Start the new list + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + title_letters.append(current_letter) + current_series = book['series'] + current_series_list = [book['series']] + else: + if len(current_series_list) < self.opts.description_clip and \ + book['series'] != current_series : + current_series = book['series'] + current_series_list.append(book['series']) + + # Add the last book list + _add_to_series_by_letter(current_series_list) + + # Add *article* entries for each populated series title letter + for (i,books) in enumerate(series_by_letter): + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sSeries-ID" % (title_letters[i].upper()) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(_(u"Series beginning with %s") % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + #contentTag['src'] = "content/%s.html#%s_series" % (output, title_letters[i]) + if title_letters[i] == self.SYMBOLS: + contentTag['src'] = "content/%s.html#%s_series" % (output, self.SYMBOLS) + else: + contentTag['src'] = "content/%s.html#%s_series" % (output, self.generate_unicode_name(title_letters[i])) + + navPointByLetterTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.format_ncx_text(books, dest='description'))) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncx_soup = soup + + def generate_ncx_by_title(self, tocTitle): + """ Add Titles to the basic NCX file. + + Generate the Titles NCX content, add to self.ncx_soup. + + Inputs: + books_by_title (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Titles")) + + def _add_to_books_by_letter(current_book_list): + current_book_list = " • ".join(current_book_list) + current_book_list = self.format_ncx_text(current_book_list, dest="description") + books_by_letter.append(current_book_list) + + soup = self.ncx_soup + output = "ByAlphaTitle" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Title' section --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + navPointTag['id'] = "byalphatitle-ID" + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "content/%s.html#section_start" % (output) + navPointTag.insert(nptc, contentTag) + nptc += 1 + + books_by_letter = [] + + # Establish initial letter equivalencies + sort_equivalents = self.establish_equivalencies(self.books_by_title, key='title_sort') + + # Loop over the titles, find start of each letter, add description_preview_count books + # Special switch for using different title list + if self.use_series_prefix_in_titles_section: + title_list = self.books_by_title + else: + title_list = self.books_by_title_no_series_prefix + + # Prime the list + current_letter = self.letter_or_symbol(sort_equivalents[0]) + title_letters = [current_letter] + current_book_list = [] + current_book = "" + for idx, book in enumerate(title_list): + #if self.letter_or_symbol(book['title_sort'][0]) != current_letter: + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: + + # Save the old list + _add_to_books_by_letter(current_book_list) + + # Start the new list + #current_letter = self.letter_or_symbol(book['title_sort'][0]) + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + title_letters.append(current_letter) + current_book = book['title'] + current_book_list = [book['title']] + else: + if len(current_book_list) < self.opts.description_clip and \ + book['title'] != current_book : + current_book = book['title'] + current_book_list.append(book['title']) + + # Add the last book list + _add_to_books_by_letter(current_book_list) + + # Add *article* entries for each populated title letter + for (i,books) in enumerate(books_by_letter): + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sTitles-ID" % (title_letters[i].upper()) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(_(u"Titles beginning with %s") % \ + (title_letters[i] if len(title_letters[i])>1 else "'" + title_letters[i] + "'"))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + if title_letters[i] == self.SYMBOLS: + contentTag['src'] = "content/%s.html#%s_titles" % (output, self.SYMBOLS) + else: + contentTag['src'] = "content/%s.html#%s_titles" % (output, self.generate_unicode_name(title_letters[i])) + navPointByLetterTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(self.format_ncx_text(books, dest='description'))) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncx_soup = soup + + def generate_ncx_by_author(self, tocTitle): + """ Add Authors to the basic NCX file. + + Generate the Authors NCX content, add to self.ncx_soup. + + Inputs: + authors (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Authors")) + + def _add_to_author_list(current_author_list, current_letter): + current_author_list = " • ".join(current_author_list) + current_author_list = self.format_ncx_text(current_author_list, dest="description") + master_author_list.append((current_author_list, current_letter)) + + soup = self.ncx_soup + HTML_file = "content/ByAlphaAuthor.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Author' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each populated author index letter + # Loop over the sorted_authors list, find start of each letter, + # add description_preview_count artists + # self.authors[0]:friendly [1]:author_sort [2]:book_count + # (, author_sort, book_count) + + # Need to extract a list of author_sort, generate sort_equivalents from that + sort_equivalents = self.establish_equivalencies([x[1] for x in self.authors]) + + master_author_list = [] + # Prime the pump + current_letter = self.letter_or_symbol(sort_equivalents[0]) + current_author_list = [] + for idx, author in enumerate(self.authors): + if self.letter_or_symbol(sort_equivalents[idx]) != current_letter: + # Save the old list + _add_to_author_list(current_author_list, current_letter) + + # Start the new list + current_letter = self.letter_or_symbol(sort_equivalents[idx]) + current_author_list = [author[0]] + else: + if len(current_author_list) < self.opts.description_clip: + current_author_list.append(author[0]) + + # Add the last author list + _add_to_author_list(current_author_list, current_letter) + + # Add *article* entries for each populated author initial letter + # master_author_list{}: [0]:author list [1]:Initial letter + for authors_by_letter in master_author_list: + navPointByLetterTag = Tag(soup, 'navPoint') + navPointByLetterTag['class'] = "article" + navPointByLetterTag['id'] = "%sauthors-ID" % (authors_by_letter[1]) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(_("Authors beginning with '%s'") % (authors_by_letter[1]))) + navLabelTag.insert(0, textTag) + navPointByLetterTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + if authors_by_letter[1] == self.SYMBOLS: + contentTag['src'] = "%s#%s_authors" % (HTML_file, authors_by_letter[1]) + else: + contentTag['src'] = "%s#%s_authors" % (HTML_file, self.generate_unicode_name(authors_by_letter[1])) + navPointByLetterTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(authors_by_letter[0])) + navPointByLetterTag.insert(2, cmTag) + + navPointTag.insert(nptc, navPointByLetterTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + + self.ncx_soup = soup + + def generate_ncx_by_date_added(self, tocTitle): + """ Add Recently Added to the basic NCX file. + + Generate the Recently Added NCX content, add to self.ncx_soup. + + Inputs: + books_by_date_range (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Recently Added")) + + def _add_to_master_month_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.format_ncx_text(current_titles_list, dest='description') + master_month_list.append((current_titles_list, current_date, book_count)) + + def _add_to_master_date_range_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.format_ncx_text(current_titles_list, dest='description') + master_date_range_list.append((current_titles_list, date_range, book_count)) + + soup = self.ncx_soup + HTML_file = "content/ByDateAdded.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Added' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each date range + current_titles_list = [] + master_date_range_list = [] + today = datetime.datetime.now() + today_time = datetime.datetime(today.year, today.month, today.day) + for (i,date) in enumerate(self.DATE_RANGE): + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + date_range_limit = self.DATE_RANGE[i] + for book in self.books_by_date_range: + book_time = datetime.datetime(book['timestamp'].year, book['timestamp'].month, book['timestamp'].day) + if (today_time-book_time).days <= date_range_limit: + #print "generate_ncx_by_date_added: %s added %d days ago" % (book['title'], (today_time-book_time).days) + current_titles_list.append(book['title']) + else: + break + if current_titles_list: + _add_to_master_date_range_list(current_titles_list) + current_titles_list = [book['title']] + + # Add *article* entries for each populated date range + # master_date_range_list{}: [0]:titles list [1]:datestr + for books_by_date_range in master_date_range_list: + navPointByDateRangeTag = Tag(soup, 'navPoint') + navPointByDateRangeTag['class'] = "article" + navPointByDateRangeTag['id'] = "%s-ID" % books_by_date_range[1].replace(' ','') + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(books_by_date_range[1])) + navLabelTag.insert(0, textTag) + navPointByDateRangeTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bda_%s" % (HTML_file, + books_by_date_range[1].replace(' ','')) + + navPointByDateRangeTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_date_range[0])) + navPointByDateRangeTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_date_range[2] if books_by_date_range[2] > 1 else \ + '%d title' % books_by_date_range[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByDateRangeTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByDateRangeTag) + nptc += 1 + + # Create an NCX article entry for each populated month + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_month_list = [] + current_date = self.books_by_month[0]['timestamp'] + + for book in self.books_by_month: + if book['timestamp'].month != current_date.month or \ + book['timestamp'].year != current_date.year: + # Save the old lists + _add_to_master_month_list(current_titles_list) + + # Start the new list + current_date = book['timestamp'].date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last month list + _add_to_master_month_list(current_titles_list) + + # Add *article* entries for each populated month + # master_months_list{}: [0]:titles list [1]:date + for books_by_month in master_month_list: + datestr = strftime(u'%B %Y', books_by_month[1].timetuple()) + navPointByMonthTag = Tag(soup, 'navPoint') + navPointByMonthTag['class'] = "article" + navPointByMonthTag['id'] = "bda_%s-%s-ID" % (books_by_month[1].year,books_by_month[1].month ) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByMonthTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bda_%s-%s" % (HTML_file, + books_by_month[1].year,books_by_month[1].month) + + navPointByMonthTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_month[0])) + navPointByMonthTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_month[2] if books_by_month[2] > 1 else \ + '%d title' % books_by_month[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByMonthTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByMonthTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncx_soup = soup + + def generate_ncx_by_date_read(self, tocTitle): + """ Add By Date Read to the basic NCX file. + + Generate the By Date Read NCX content (Kindle only), add to self.ncx_soup. + + Inputs: + bookmarked_books_by_date_read (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + def _add_to_master_day_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.format_ncx_text(current_titles_list, dest='description') + master_day_list.append((current_titles_list, current_date, book_count)) + + def _add_to_master_date_range_list(current_titles_list): + book_count = len(current_titles_list) + current_titles_list = " • ".join(current_titles_list) + current_titles_list = self.format_ncx_text(current_titles_list, dest='description') + master_date_range_list.append((current_titles_list, date_range, book_count)) + + self.update_progress_full_step(_("NCX for Recently Read")) + + if not self.bookmarked_books_by_date_read: + return + + soup = self.ncx_soup + HTML_file = "content/ByDateRead.html" + body = soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Recently Read' *section* --- + navPointTag = Tag(soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(soup,"content") + contentTag['src'] = "%s#section_start" % HTML_file + navPointTag.insert(nptc, contentTag) + nptc += 1 + + # Create an NCX article entry for each date range + current_titles_list = [] + master_date_range_list = [] + today = datetime.datetime.now() + today_time = datetime.datetime(today.year, today.month, today.day) + for (i,date) in enumerate(self.DATE_RANGE): + if i: + date_range = '%d to %d days ago' % (self.DATE_RANGE[i-1], self.DATE_RANGE[i]) + else: + date_range = 'Last %d days' % (self.DATE_RANGE[i]) + date_range_limit = self.DATE_RANGE[i] + for book in self.bookmarked_books_by_date_read: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if (today_time-bookmark_time).days <= date_range_limit: + #print "generate_ncx_by_date_added: %s added %d days ago" % (book['title'], (today_time-book_time).days) + current_titles_list.append(book['title']) + else: + break + if current_titles_list: + _add_to_master_date_range_list(current_titles_list) + current_titles_list = [book['title']] + + # Create an NCX article entry for each populated day + # Loop over the booksByDate list, find start of each month, + # add description_preview_count titles + # master_month_list(list,date,count) + current_titles_list = [] + master_day_list = [] + current_date = datetime.datetime.utcfromtimestamp(self.bookmarked_books_by_date_read[0]['bookmark_timestamp']) + + for book in self.bookmarked_books_by_date_read: + bookmark_time = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']) + if bookmark_time.day != current_date.day or \ + bookmark_time.month != current_date.month or \ + bookmark_time.year != current_date.year: + # Save the old lists + _add_to_master_day_list(current_titles_list) + + # Start the new list + current_date = datetime.datetime.utcfromtimestamp(book['bookmark_timestamp']).date() + current_titles_list = [book['title']] + else: + current_titles_list.append(book['title']) + + # Add the last day list + _add_to_master_day_list(current_titles_list) + + # Add *article* entries for each populated day + # master_day_list{}: [0]:titles list [1]:date + for books_by_day in master_day_list: + datestr = strftime(u'%A, %B %d', books_by_day[1].timetuple()) + navPointByDayTag = Tag(soup, 'navPoint') + navPointByDayTag['class'] = "article" + navPointByDayTag['id'] = "bdr_%s-%s-%sID" % (books_by_day[1].year, + books_by_day[1].month, + books_by_day[1].day ) + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(soup, 'navLabel') + textTag = Tag(soup, 'text') + textTag.insert(0, NavigableString(datestr)) + navLabelTag.insert(0, textTag) + navPointByDayTag.insert(0,navLabelTag) + contentTag = Tag(soup, 'content') + contentTag['src'] = "%s#bdr_%s-%s-%s" % (HTML_file, + books_by_day[1].year, + books_by_day[1].month, + books_by_day[1].day) + + navPointByDayTag.insert(1,contentTag) + + if self.generate_for_kindle: + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + cmTag.insert(0, NavigableString(books_by_day[0])) + navPointByDayTag.insert(2, cmTag) + + cmTag = Tag(soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + navStr = '%d titles' % books_by_day[2] if books_by_day[2] > 1 else \ + '%d title' % books_by_day[2] + cmTag.insert(0, NavigableString(navStr)) + navPointByDayTag.insert(3, cmTag) + + navPointTag.insert(nptc, navPointByDayTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncx_soup = soup + + def generate_ncx_by_genre(self, tocTitle): + """ Add Genres to the basic NCX file. + + Generate the Genre NCX content, add to self.ncx_soup. + + Inputs: + genres (list) + + Updated: + play_order (int) + + Outputs: + ncx_soup (file): updated + """ + + self.update_progress_full_step(_("NCX for Genres")) + + if not len(self.genres): + self.opts.log.warn(" No genres found in tags.\n" + " No Genre section added to Catalog") + return + + ncx_soup = self.ncx_soup + body = ncx_soup.find("navPoint") + btc = len(body.contents) + + # --- Construct the 'Books By Genre' *section* --- + navPointTag = Tag(ncx_soup, 'navPoint') + navPointTag['class'] = "section" + file_ID = "%s" % tocTitle.lower() + file_ID = file_ID.replace(" ","") + navPointTag['id'] = "%s-ID" % file_ID + navPointTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(ncx_soup, 'navLabel') + textTag = Tag(ncx_soup, 'text') + # textTag.insert(0, NavigableString('%s (%d)' % (section_title, len(genre_list)))) + textTag.insert(0, NavigableString('%s' % tocTitle)) + navLabelTag.insert(0, textTag) + nptc = 0 + navPointTag.insert(nptc, navLabelTag) + nptc += 1 + contentTag = Tag(ncx_soup,"content") + contentTag['src'] = "content/Genre_%s.html#section_start" % self.genres[0]['tag'] + navPointTag.insert(nptc, contentTag) + nptc += 1 + + for genre in self.genres: + # Add an article for each genre + navPointVolumeTag = Tag(ncx_soup, 'navPoint') + navPointVolumeTag['class'] = "article" + navPointVolumeTag['id'] = "genre-%s-ID" % genre['tag'] + navPointVolumeTag['playOrder'] = self.play_order + self.play_order += 1 + navLabelTag = Tag(ncx_soup, "navLabel") + textTag = Tag(ncx_soup, "text") + + # GwR *** Can this be optimized? + normalized_tag = None + for friendly_tag in self.genre_tags_dict: + if self.genre_tags_dict[friendly_tag] == genre['tag']: + normalized_tag = self.genre_tags_dict[friendly_tag] + break + textTag.insert(0, self.format_ncx_text(NavigableString(friendly_tag), dest='description')) + navLabelTag.insert(0,textTag) + navPointVolumeTag.insert(0,navLabelTag) + contentTag = Tag(ncx_soup, "content") + contentTag['src'] = "content/Genre_%s.html#Genre_%s" % (normalized_tag, normalized_tag) + navPointVolumeTag.insert(1, contentTag) + + if self.generate_for_kindle: + # Build the author tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "author" + # First - Last author + + if len(genre['titles_spanned']) > 1 : + author_range = "%s - %s" % (genre['titles_spanned'][0][0], genre['titles_spanned'][1][0]) + else : + author_range = "%s" % (genre['titles_spanned'][0][0]) + + cmTag.insert(0, NavigableString(author_range)) + navPointVolumeTag.insert(2, cmTag) + + # Build the description tag + cmTag = Tag(ncx_soup, '%s' % 'calibre:meta') + cmTag['name'] = "description" + + if False: + # Form 1: Titles spanned + if len(genre['titles_spanned']) > 1: + title_range = "%s -\n%s" % (genre['titles_spanned'][0][1], genre['titles_spanned'][1][1]) + else: + title_range = "%s" % (genre['titles_spanned'][0][1]) + cmTag.insert(0, NavigableString(self.format_ncx_text(title_range, dest='description'))) + else: + # Form 2: title • title • title ... + titles = [] + for title in genre['books']: + titles.append(title['title']) + titles = sorted(titles, key=lambda x:(self.generate_sort_title(x),self.generate_sort_title(x))) + titles_list = self.generate_short_description(u" • ".join(titles), dest="description") + cmTag.insert(0, NavigableString(self.format_ncx_text(titles_list, dest='description'))) + + navPointVolumeTag.insert(3, cmTag) + + # Add this volume to the section tag + navPointTag.insert(nptc, navPointVolumeTag) + nptc += 1 + + # Add this section to the body + body.insert(btc, navPointTag) + btc += 1 + self.ncx_soup = ncx_soup + + def generate_opf(self): + """ Generate the OPF file. + + Start with header template, construct manifest, spine and guide. + + Inputs: + genres (list) + html_filelist_1 (list) + html_filelist_2 (list) + thumbs (list) + + Updated: + play_order (int) + + Outputs: + opts.basename + '.opf' (file): written + """ + + self.update_progress_full_step(_("Generating OPF")) + + header = ''' + + + + en-US + + + + + + + ''' + # Add the supplied metadata tags + soup = BeautifulStoneSoup(header, selfClosingTags=['item','itemref', 'reference']) + metadata = soup.find('metadata') + mtc = 0 + + titleTag = Tag(soup, "dc:title") + titleTag.insert(0,self.opts.catalog_title) + metadata.insert(mtc, titleTag) + mtc += 1 + + creatorTag = Tag(soup, "dc:creator") + creatorTag.insert(0, self.opts.creator) + metadata.insert(mtc, creatorTag) + mtc += 1 + + # Create the OPF tags + manifest = soup.find('manifest') + mtc = 0 + spine = soup.find('spine') + stc = 0 + guide = soup.find('guide') + + itemTag = Tag(soup, "item") + itemTag['id'] = "ncx" + itemTag['href'] = '%s.ncx' % self.opts.basename + itemTag['media-type'] = "application/x-dtbncx+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + itemTag = Tag(soup, "item") + itemTag['id'] = 'stylesheet' + itemTag['href'] = self.stylesheet + itemTag['media-type'] = 'text/css' + manifest.insert(mtc, itemTag) + mtc += 1 + + itemTag = Tag(soup, "item") + itemTag['id'] = 'mastheadimage-image' + itemTag['href'] = "images/mastheadImage.gif" + itemTag['media-type'] = 'image/gif' + manifest.insert(mtc, itemTag) + mtc += 1 + + # Write the thumbnail images, descriptions to the manifest + sort_descriptions_by = [] + if self.opts.generate_descriptions: + for thumb in self.thumbs: + itemTag = Tag(soup, "item") + itemTag['href'] = "images/%s" % (thumb) + end = thumb.find('.jpg') + itemTag['id'] = "%s-image" % thumb[:end] + itemTag['media-type'] = 'image/jpeg' + manifest.insert(mtc, itemTag) + mtc += 1 + + # HTML files - add descriptions to manifest and spine + sort_descriptions_by = self.books_by_author if self.opts.sort_descriptions_by_author \ + else self.books_by_title + # Add html_files to manifest and spine + + for file in self.html_filelist_1: + # By Author, By Title, By Series, + itemTag = Tag(soup, "item") + start = file.find('/') + 1 + end = file.find('.') + itemTag['href'] = file + itemTag['id'] = file[start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = file[start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + # Add genre files to manifest and spine + for genre in self.genres: + itemTag = Tag(soup, "item") + start = genre['file'].find('/') + 1 + end = genre['file'].find('.') + itemTag['href'] = genre['file'] + itemTag['id'] = genre['file'][start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = genre['file'][start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + for file in self.html_filelist_2: + # By Date Added, By Date Read + itemTag = Tag(soup, "item") + start = file.find('/') + 1 + end = file.find('.') + itemTag['href'] = file + itemTag['id'] = file[start:end].lower() + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = file[start:end].lower() + spine.insert(stc, itemrefTag) + stc += 1 + + for book in sort_descriptions_by: + # manifest + itemTag = Tag(soup, "item") + itemTag['href'] = "content/book_%d.html" % int(book['id']) + itemTag['id'] = "book%d" % int(book['id']) + itemTag['media-type'] = "application/xhtml+xml" + manifest.insert(mtc, itemTag) + mtc += 1 + + # spine + itemrefTag = Tag(soup, "itemref") + itemrefTag['idref'] = "book%d" % int(book['id']) + spine.insert(stc, itemrefTag) + stc += 1 + + # Guide + referenceTag = Tag(soup, "reference") + referenceTag['type'] = 'masthead' + referenceTag['title'] = 'mastheadimage-image' + referenceTag['href'] = 'images/mastheadImage.gif' + guide.insert(0,referenceTag) + + # Write the OPF file + outfile = open("%s/%s.opf" % (self.catalog_path, self.opts.basename), 'w') + outfile.write(soup.prettify()) + + def generate_rating_string(self, book): + """ Generate rating string for Descriptions. + + Starting with database rating (0-10), return 5 stars, with 0-5 filled, + balance empty. + + Args: + book (dict): book metadata + + Return: + rating (str): 5 stars, 1-5 solid, balance empty. Empty str for no rating. + """ + rating = '' try: if 'rating' in book: stars = int(book['rating']) / 2 if stars: - star_string = self.FULL_RATING_SYMBOL * stars - empty_stars = self.EMPTY_RATING_SYMBOL * (5 - stars) + star_string = self.SYMBOL_FULL_RATING * stars + empty_stars = self.SYMBOL_EMPTY_RATING * (5 - stars) rating = '%s%s' % (star_string,empty_stars) except: # Rating could be None pass return rating - def generateSeriesAnchor(self, series): + def generate_series_anchor(self, series): + """ Generate legal XHTML anchor for series names. + + Flatten series name to ascii_legal text. + + Args: + series (str): series name + + Return: + (str): asciized version of series name + """ + # Generate a legal XHTML id/href string if self.letter_or_symbol(series) == self.SYMBOLS: return "symbol_%s_series" % re.sub('\W','',series).lower() else: return "%s_series" % re.sub('\W','',ascii_text(series)).lower() - def generateShortDescription(self, description, dest=None): - # Truncate the description, on word boundaries if necessary - # Possible destinations: - # description NCX summary - # title NCX title - # author NCX author + def generate_short_description(self, description, dest=None): + """ Generate a truncated version of the supplied string. - def shortDescription(description, limit): + Given a string and NCX destination, truncate string to length specified + in self.opts. + + Args: + description (str): string to truncate + dest (str): NCX destination + description NCX summary + title NCX title + author NCX author + + Return: + (str): truncated description + """ + + def _short_description(description, limit): short_description = "" words = description.split() for word in words: @@ -3809,26 +4219,35 @@ Author '{0}': # No truncation for titles, let the device deal with it return description elif dest == 'author': - if self.authorClip and len(description) < self.authorClip: + if self.opts.author_clip and len(description) < self.opts.author_clip: return description else: - return shortDescription(description, self.authorClip) + return _short_description(description, self.opts.author_clip) elif dest == 'description': - if self.descriptionClip and len(description) < self.descriptionClip: + if self.opts.description_clip and len(description) < self.opts.description_clip: return description else: - return shortDescription(description, self.descriptionClip) + return _short_description(description, self.opts.description_clip) else: print " returning description with unspecified destination '%s'" % description raise RuntimeError - def generateSortTitle(self, title): - ''' - Generate a string suitable for sorting from the title - Ignore leading stop words - Optionally convert leading numbers to strings - ''' - from calibre.ebooks.metadata import title_sort + def generate_sort_title(self, title): + """ Generates a sort string from title. + + Based on trunk title_sort algorithm, but also accommodates series + numbers by padding with leading zeroes to force proper numeric + sorting. Option to sort numbers alphabetically, e.g. '1942' sorts + as 'Nineteen forty two'. + + Args: + title (str): + + Return: + (str): sort string + """ + + from calibre.ebooks.metadata import title_sort from calibre.library.catalogs.utils import NumberToText # Strip stop words @@ -3870,29 +4289,41 @@ Author '{0}': translated.append(word) return ' '.join(translated) - def generateThumbnail(self, title, image_dir, thumb_file): - ''' - Thumbs are cached with the full cover's crc. If the crc doesn't - match, the cover has been changed since the thumb was cached and needs - to be replaced. - ''' + def generate_thumbnail(self, title, image_dir, thumb_file): + """ Create thumbnail of cover or return previously cached thumb. - def open_archive(mode='r'): + Test thumb archive for currently cached cover. Return cached version, or create + and cache new version. Uses calibre.utils.magick.draw to generate thumbnail from + cover. + + Args: + title (dict): book metadata + image_dir (str): directory to write thumb data to + thumb_file (str): filename to save thumb as + + Output: + (file): thumb written to /images + (archive): current thumb archived under cover crc + """ + + def _open_archive(mode='r'): try: - return ZipFile(self.__archive_path, mode=mode, allowZip64=True) + return ZipFile(self.thumbs_path, mode=mode, allowZip64=True) except: - # Happens on windows if the file is opened by another + # occurs under windows if the file is opened by another # process pass + if self.DEBUG and self.opts.verbose: + self.opts.log.info(" generate_thumbnail():") + # Generate crc for current cover - #self.opts.log.info(" generateThumbnail():") with open(title['cover'], 'rb') as f: data = f.read() cover_crc = hex(zlib.crc32(data)) # Test cache for uuid - zf = open_archive() + zf = _open_archive() if zf is not None: with zf: try: @@ -3906,10 +4337,9 @@ Author '{0}': f.write(thumb_data) return - - # Save thumb for catalog + # Save thumb for catalog. If invalid data, error returns to generate_thumbnails() thumb_data = thumbnail(data, - width=self.thumbWidth, height=self.thumbHeight)[-1] + width=self.thumb_width, height=self.thumb_height)[-1] with open(os.path.join(image_dir, thumb_file), 'wb') as f: f.write(thumb_data) @@ -3917,44 +4347,218 @@ Author '{0}': if zf is not None: # Ensure that the read succeeded # If we failed to open the zip file for reading, # we dont know if it contained the thumb or not - zf = open_archive('a') + zf = _open_archive('a') if zf is not None: with zf: zf.writestr(title['uuid']+cover_crc, thumb_data) - def generateUnicodeName(self, c): - ''' - Generate an anchor name string - ''' + def generate_thumbnails(self): + """ Generate a thumbnail cover for each book. + + Generate or retrieve a thumbnail for each cover. If nonexistent or faulty + cover data, substitute default cover. Checks for updated default cover. + At completion, writes self.opts.thumb_width to archive. + + Inputs: + books_by_title (list): books to catalog + + Output: + thumbs (list): list of referenced thumbnails + """ + + self.update_progress_full_step(_("Thumbnails")) + thumbs = ['thumbnail_default.jpg'] + image_dir = "%s/images" % self.catalog_path + for (i,title) in enumerate(self.books_by_title): + # Update status + self.update_progress_micro_step("%s %d of %d" % + (_("Thumbnail"), i, len(self.books_by_title)), + i/float(len(self.books_by_title))) + + thumb_file = 'thumbnail_%d.jpg' % int(title['id']) + thumb_generated = True + valid_cover = True + try: + self.generate_thumbnail(title, image_dir, thumb_file) + thumbs.append("thumbnail_%d.jpg" % int(title['id'])) + except: + if 'cover' in title and os.path.exists(title['cover']): + valid_cover = False + self.opts.log.warn(" *** Invalid cover file for '%s'***" % + (title['title'])) + if not self.error: + self.error.append('Invalid cover files') + self.error.append("Warning: invalid cover file for '%s', default cover substituted.\n" % (title['title'])) + + thumb_generated = False + + if not thumb_generated: + self.opts.log.warn(" using default cover for '%s' (%d)" % (title['title'], title['id'])) + # Confirm thumb exists, default is current + default_thumb_fp = os.path.join(image_dir,"thumbnail_default.jpg") + cover = os.path.join(self.catalog_path, "DefaultCover.png") + title['cover'] = cover + + if not os.path.exists(cover): + shutil.copyfile(I('book.png'), cover) + + if os.path.isfile(default_thumb_fp): + # Check to see if default cover is newer than thumbnail + # os.path.getmtime() = modified time + # os.path.ctime() = creation time + cover_timestamp = os.path.getmtime(cover) + thumb_timestamp = os.path.getmtime(default_thumb_fp) + if thumb_timestamp < cover_timestamp: + if self.DEBUG and self.opts.verbose: + self.opts.log.warn("updating thumbnail_default for %s" % title['title']) + self.generate_thumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) + else: + if self.DEBUG and self.opts.verbose: + self.opts.log.warn(" generating new thumbnail_default.jpg") + self.generate_thumbnail(title, image_dir, + "thumbnail_default.jpg" if valid_cover else thumb_file) + # Clear the book's cover property + title['cover'] = None + + + # Write thumb_width to the file, validating cache contents + # Allows detection of aborted catalog builds + with ZipFile(self.thumbs_path, mode='a') as zfw: + zfw.writestr('thumb_width', self.opts.thumb_width) + + self.thumbs = thumbs + + def generate_unicode_name(self, c): + """ Generate a legal XHTML anchor from unicode character. + + Generate a legal XHTML anchor from unicode character. + + Args: + c (unicode): character + + Return: + (str): legal XHTML anchor string of unicode charactar name + """ fullname = unicodedata.name(unicode(c)) terms = fullname.split() return "_".join(terms) - def getFriendlyGenreTag(self, genre): + def get_excluded_tags(self): + """ Get excluded_tags from opts.exclusion_rules. + + Parse opts.exclusion_rules for tags to be excluded, return list. + Log books that will be excluded by excluded_tags. + + Inputs: + opts.excluded_tags (tuples): exclusion rules + + Return: + excluded_tags (list): excluded tags + """ + excluded_tags = [] + for rule in self.opts.exclusion_rules: + if rule[1].lower() == 'tags': + excluded_tags.extend(rule[2].split(',')) + + # Remove dups + excluded_tags = list(set(excluded_tags)) + + # Report excluded books + if self.opts.verbose and excluded_tags: + data = self.db.get_data_as_dict(ids=self.opts.ids) + for record in data: + matched = list(set(record['tags']) & set(excluded_tags)) + if matched : + self.opts.log.info(" - %s by %s (Exclusion rule Tags: '%s')" % + (record['title'], record['authors'][0], str(matched[0]))) + return excluded_tags + + def get_friendly_genre_tag(self, genre): + """ Return the first friendly_tag matching genre. + + Scan self.genre_tags_dict[] for first friendly_tag matching genre. + genre_tags_dict[] populated in filter_db_tags(). + + Args: + genre (str): genre to match + + Return: + friendly_tag (str): friendly_tag matching genre + """ # Find the first instance of friendly_tag matching genre for friendly_tag in self.genre_tags_dict: if self.genre_tags_dict[friendly_tag] == genre: return friendly_tag - def getMarkerTags(self): - ''' - Return a list of special marker tags to be excluded from genre list - exclusion_rules = ('name','Tags|#column','[]|pattern') - ''' - markerTags = [] - for rule in self.opts.exclusion_rules: - if rule[1].lower() == 'tags': - markerTags.extend(rule[2].split(',')) - return markerTags + def get_prefix_rules(self): + """ Convert opts.prefix_rules to dict. + + Convert opts.prefix_rules to dict format. + + Input: + opts.prefix_rules (tuples): (name, field, pattern, prefix) + + Return: + (list): list of prefix_rules dicts + """ + + pr = [] + if self.opts.prefix_rules: + try: + for rule in self.opts.prefix_rules: + prefix_rule = {} + prefix_rule['name'] = rule[0] + prefix_rule['field'] = rule[1] + prefix_rule['pattern'] = rule[2] + prefix_rule['prefix'] = rule[3] + pr.append(prefix_rule) + except: + self.opts.log.error("malformed prefix_rules: %s" % repr(self.opts.prefix_rules)) + raise + return pr def letter_or_symbol(self,char): + """ Test asciized char for A-z. + + Convert char to ascii, test for A-z. + + Args: + char (chr): character to test + + Return: + (str): char if A-z, else SYMBOLS + """ if not re.search('[a-zA-Z]', ascii_text(char)): return self.SYMBOLS else: return char - def markdownComments(self, comments): - ''' + def load_section_templates(self): + """ Add section templates to local namespace. + + Load section templates from resource directory. If user has made local copies, + these will be used for individual section generation. + generate_format_args() builds args that populate templates. + Templates referenced in individual section builders, e.g. + generate_html_by_title(). + + Inputs: + (files): section template files from resource dir + + Results: + (strs): section templates added to local namespace + """ + + templates = {} + execfile(P('catalog/section_list_templates.py'), templates) + for name, template in templates.iteritems(): + if name.startswith('by_') and name.endswith('_template'): + setattr(self, name, force_unicode(template, 'utf-8')) + + def massage_comments(self, comments): + """ Massage comments to somewhat consistent format. + Convert random comment text to normalized, xml-legal block of

s 'plain text' returns as

plain text

@@ -3977,7 +4581,13 @@ Author '{0}': Deprecated HTML returns as HTML via BeautifulSoup() - ''' + Args: + comments (str): comments from metadata, possibly HTML + + Return: + result (BeautifulSoup): massaged comments in HTML form + """ + # Hackish - ignoring sentences ending or beginning in numbers to avoid # confusion with decimal points. @@ -4067,19 +4677,29 @@ Author '{0}': return result.renderContents(encoding=None) - def mergeComments(self, record): - ''' - merge ['description'] with custom field contents to be displayed in Descriptions - ''' + def merge_comments(self, record): + """ Merge comments with custom column content. + + Merge comments from book metadata with user-specified custom column + content, optionally before or after. Optionally insert
between + fields. + + Args: + record (dict): book metadata + + Return: + merged (str): comments merged with addendum + """ + merged = '' if record['description']: - addendum = self.__db.get_field(record['id'], - self.__merge_comments['field'], + addendum = self.db.get_field(record['id'], + self.merge_comments_rule['field'], index_is_id=True) if addendum is None: addendum = '' - include_hr = eval(self.__merge_comments['hr']) - if self.__merge_comments['position'] == 'before': + include_hr = eval(self.merge_comments_rule['hr']) + if self.merge_comments_rule['position'] == 'before': merged = addendum if include_hr: merged += '
' @@ -4095,33 +4715,25 @@ Author '{0}': merged += addendum else: # Return the custom field contents - merged = self.__db.get_field(record['id'], - self.__merge_comments['field'], + merged = self.db.get_field(record['id'], + self.merge_comments_rule['field'], index_is_id=True) return merged - def processPrefixRules(self): - if self.opts.prefix_rules: - # Put the prefix rules into an ordered list of dicts - try: - for rule in self.opts.prefix_rules: - prefix_rule = {} - prefix_rule['name'] = rule[0] - prefix_rule['field'] = rule[1] - prefix_rule['pattern'] = rule[2] - prefix_rule['prefix'] = rule[3] - self.prefixRules.append(prefix_rule) - except: - self.opts.log.error("malformed self.opts.prefix_rules: %s" % repr(self.opts.prefix_rules)) - raise - # Use the highest order prefix symbol as default - self.defaultPrefix = self.opts.prefix_rules[0][3] + def process_exclusions(self, data_set): + """ Filter data_set based on exclusion_rules. + + Compare each book in data_set to each exclusion_rule. Remove + books matching exclusion criteria. + + Args: + data_set (list): all candidate books + + Return: + (list): filtered data_set + """ - def processExclusions(self, data_set): - ''' - Remove excluded entries - ''' filtered_data_set = [] exclusion_pairs = [] exclusion_set = [] @@ -4137,7 +4749,7 @@ Author '{0}': for record in data_set: for exclusion_pair in exclusion_pairs: field,pat = exclusion_pair - field_contents = self.__db.get_field(record['id'], + field_contents = self.db.get_field(record['id'], field, index_is_id=True) if field_contents: @@ -4159,36 +4771,63 @@ Author '{0}': else: return data_set - def processSpecialTags(self, tags, this_title, opts): + def update_progress_full_step(self, description): + """ Update calibre's job status UI. - tag_list = [] + Call ProgessReporter() with updates. - try: - for tag in tags: - tag = self.convertHTMLEntities(tag) - if re.search(opts.exclude_genre, tag): - continue - else: - tag_list.append(tag) - except: - self.opts.log.error("\tprocessSpecialTags(): malformed --exclude-genre regex pattern: %s" % opts.exclude_genre) - return tags + Args: + description (str): text describing current step - return tag_list + Result: + (UI): Jobs UI updated + """ - def updateProgressFullStep(self, description): - self.currentStep += 1 - self.progressString = description - self.progressInt = float((self.currentStep-1)/self.totalSteps) - self.reporter(self.progressInt, self.progressString) + self.current_step += 1 + self.progress_string = description + self.progress_int = float((self.current_step-1)/self.total_steps) + self.reporter(self.progress_int, self.progress_string) if self.opts.cli_environment: - self.opts.log(u"%3.0f%% %s" % (self.progressInt*100, self.progressString)) + self.opts.log(u"%3.0f%% %s" % (self.progress_int*100, self.progress_string)) - def updateProgressMicroStep(self, description, micro_step_pct): - step_range = 100/self.totalSteps - self.progressString = description - coarse_progress = float((self.currentStep-1)/self.totalSteps) + def update_progress_micro_step(self, description, micro_step_pct): + """ Update calibre's job status UI. + + Called from steps requiring more time: + generate_html_descriptions() + generate_thumbnails() + + Args: + description (str): text describing microstep + micro_step_pct (float): percentage of full step + + Results: + (UI): Jobs UI updated + """ + + step_range = 100/self.total_steps + self.progress_string = description + coarse_progress = float((self.current_step-1)/self.total_steps) fine_progress = float((micro_step_pct*step_range)/100) - self.progressInt = coarse_progress + fine_progress - self.reporter(self.progressInt, self.progressString) + self.progress_int = coarse_progress + fine_progress + self.reporter(self.progress_int, self.progress_string) + + def write_ncx(self): + """ Write accumulated ncx_soup to file. + + Expanded description + + Inputs: + catalog_path (str): path to generated catalog + opts.basename (str): catalog basename + + Output: + (file): basename.NCX written + """ + + self.update_progress_full_step(_("Saving NCX")) + + outfile = open("%s/%s.ncx" % (self.catalog_path, self.opts.basename), 'w') + outfile.write(self.ncx_soup.prettify()) + From 2b85633a879e8bee5df6c7f30ccae8f9be910ba9 Mon Sep 17 00:00:00 2001 From: GRiker Date: Sat, 1 Sep 2012 06:04:42 -0600 Subject: [PATCH 2/7] Catalog refactoring, wip --- src/calibre/gui2/catalog/catalog_epub_mobi.py | 12 +- src/calibre/library/catalogs/epub_mobi.py | 30 +- .../library/catalogs/epub_mobi_builder.py | 360 ++++++++++-------- 3 files changed, 224 insertions(+), 178 deletions(-) diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 05c2b8c8b3..7c89d3e0dd 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -11,7 +11,7 @@ import re, sys from functools import partial from calibre.ebooks.conversion.config import load_defaults -from calibre.gui2 import gprefs, question_dialog +from calibre.gui2 import gprefs, info_dialog, question_dialog from calibre.utils.icu import sort_key from catalog_epub_mobi_ui import Ui_Form @@ -75,7 +75,6 @@ class PluginWidget(QWidget,Ui_Form): # LineEditControls option_fields += zip(['exclude_genre'],['\[.+\]|\+'],['line_edit']) - #option_fields += zip(['exclude_genre_results'],['excluded genres will appear here'],['line_edit']) # TextEditControls #option_fields += zip(['exclude_genre_results'],['excluded genres will appear here'],['text_edit']) @@ -172,7 +171,7 @@ class PluginWidget(QWidget,Ui_Form): if hit: excluded_tags.append(hit.string) if excluded_tags: - results = ', '.join(excluded_tags) + results = ', '.join(sorted(excluded_tags)) finally: if self.DEBUG: print(results) @@ -334,16 +333,21 @@ class PluginWidget(QWidget,Ui_Form): elif self.merge_after.isChecked(): checked = 'after' include_hr = self.include_hr.isChecked() - opts_dict['merge_comments'] = "%s:%s:%s" % \ + opts_dict['merge_comments_rule'] = "%s:%s:%s" % \ (self.merge_source_field_name, checked, include_hr) opts_dict['header_note_source_field'] = self.header_note_source_field_name + # Fix up exclude_genre regex if blank. Assume blank = no exclusions + if opts_dict['exclude_genre'] == '': + opts_dict['exclude_genre'] = 'a^' + # Append the output profile try: opts_dict['output_profile'] = [load_defaults('page_setup')['output_profile']] except: opts_dict['output_profile'] = ['default'] + if self.DEBUG: print "opts_dict" for opt in sorted(opts_dict.keys(), key=sort_key): diff --git a/src/calibre/library/catalogs/epub_mobi.py b/src/calibre/library/catalogs/epub_mobi.py index 25385e556c..6a0e4c83b4 100644 --- a/src/calibre/library/catalogs/epub_mobi.py +++ b/src/calibre/library/catalogs/epub_mobi.py @@ -120,9 +120,9 @@ class EPUB_MOBI(CatalogPlugin): help=_("Custom field containing note text to insert in Description header.\n" "Default: '%default'\n" "Applies to: AZW3, ePub, MOBI output formats")), - Option('--merge-comments', + Option('--merge-comments-rule', default='::', - dest='merge_comments', + dest='merge_comments_rule', action = None, help=_("#:[before|after]:[True|False] specifying:\n" " Custom field containing notes to merge with Comments\n" @@ -182,8 +182,8 @@ class EPUB_MOBI(CatalogPlugin): else: op = "kindle" - opts.descriptionClip = 380 if op.endswith('dx') or 'kindle' not in op else 100 - opts.authorClip = 100 if op.endswith('dx') or 'kindle' not in op else 60 + opts.description_clip = 380 if op.endswith('dx') or 'kindle' not in op else 100 + opts.author_clip = 100 if op.endswith('dx') or 'kindle' not in op else 60 opts.output_profile = op opts.basename = "Catalog" @@ -198,11 +198,12 @@ class EPUB_MOBI(CatalogPlugin): (self.name,self.fmt,'for %s ' % opts.output_profile if opts.output_profile else '', 'CLI' if opts.cli_environment else 'GUI')) - # If exclude_genre is blank, assume user wants all genre tags included + # If exclude_genre is blank, assume user wants all tags as genres if opts.exclude_genre.strip() == '': - opts.exclude_genre = '\[^.\]' - build_log.append(" converting empty exclude_genre to '\[^.\]'") - + #opts.exclude_genre = '\[^.\]' + #build_log.append(" converting empty exclude_genre to '\[^.\]'") + opts.exclude_genre = 'a^' + build_log.append(" converting empty exclude_genre to 'a^'") if opts.connected_device['is_device_connected'] and \ opts.connected_device['kind'] == 'device': if opts.connected_device['serial']: @@ -304,10 +305,10 @@ class EPUB_MOBI(CatalogPlugin): keys.sort() build_log.append(" opts:") for key in keys: - if key in ['catalog_title','authorClip','connected_kindle','descriptionClip', + if key in ['catalog_title','author_clip','connected_kindle','description_clip', 'exclude_book_marker','exclude_genre','exclude_tags', - 'exclusion_rules', - 'header_note_source_field','merge_comments', + 'exclusion_rules', 'fmt', + 'header_note_source_field','merge_comments_rule', 'output_profile','prefix_rules','read_book_marker', 'search_text','sort_by','sort_descriptions_by_author','sync', 'thumb_width','wishlist_tag']: @@ -323,10 +324,7 @@ class EPUB_MOBI(CatalogPlugin): if opts.verbose: log.info(" Begin catalog source generation") - catalog.createDirectoryStructure() - catalog.copyResources() - catalog.calculateThumbnailSize() - catalog_source_built = catalog.buildSources() + catalog_source_built = catalog.build_sources() if opts.verbose: if catalog_source_built: @@ -388,7 +386,7 @@ class EPUB_MOBI(CatalogPlugin): # Run ebook-convert from calibre.ebooks.conversion.plumber import Plumber - plumber = Plumber(os.path.join(catalog.catalogPath, + plumber = Plumber(os.path.join(catalog.catalog_path, opts.basename + '.opf'), path_to_output, log, report_progress=notification, abort_after_input_dump=False) plumber.merge_ui_recommendations(recommendations) diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index bfee271ed4..8c60686ffe 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -9,6 +9,7 @@ from xml.sax.saxutils import escape from calibre import (prepare_string_for_xml, strftime, force_unicode) from calibre.customize.conversion import DummyReporter +from calibre.customize.ui import output_profiles from calibre.ebooks.BeautifulSoup import BeautifulSoup, BeautifulStoneSoup, Tag, NavigableString from calibre.ebooks.chardet import substitute_entites from calibre.ptempfile import PersistentTemporaryDirectory @@ -32,7 +33,7 @@ class CatalogBuilder(object): Options managed in gui2.catalog.catalog_epub_mobi.py ''' - DEBUG = True + DEBUG = False # A single number creates 'Last x days' only. # Multiple numbers create 'Last x days', 'x to y days ago' ... @@ -46,78 +47,21 @@ class CatalogBuilder(object): # basename output file basename # creator dc:creator in OPF metadata # description_clip limits size of NCX descriptions (Kindle only) - # includeSources Used in filter_excluded_tags to skip tags like '[SPL]' + # includeSources Used in filter_excluded_genres to skip tags like '[SPL]' # notification Used to check for cancel, report progress # stylesheet CSS stylesheet # title dc:title in OPF metadata, NCX periodical # verbosity level of diagnostic printout - def __init__(self, db, opts, plugin, - report_progress=DummyReporter(), - stylesheet="content/stylesheet.css", - init_resources=True): - - ''' active database ''' - @property - def db(self): - return self.__db - self.__db = db - - ''' opts passed from gui2.catalog.catalog_epub_mobi.py ''' - @property - def opts(self): - return self.__opts - self.__opts = opts - - ''' catalog??? device??? ''' - @property - def plugin(self): - return self.__plugin - self.__plugin = plugin - - ''' Progress Reporter for Jobs ''' - @property - def reporter(self): - return self.__reporter - self.__reporter = report_progress - - ''' stylesheet to include with catalog ''' - @property - def stylesheet(self): - return self.__stylesheet - self.__stylesheet = stylesheet - - # Initialize properties with dependents in _initialize() - ''' directory to store cached thumbs ''' - @property - def cache_dir(self): - return self.__cache_dir - self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') - - ''' temp dir to store generated catalog ''' - @property - def catalog_path(self): - return self.__catalog_path - self.__catalog_path = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') - - ''' True if generating for Kindle in MOBI format ''' - @property - def generate_for_kindle(self): - return self.__generate_for_kindle - self.__generate_for_kindle = True if (opts.fmt == 'mobi' and - opts.output_profile and - opts.output_profile.startswith("kindle")) else False - - self._initialize(init_resources) - - def _initialize(self,init_resources): - # continue with initialization - + """ property decorators for attributes """ + if True: ''' list of unique authors ''' @property def authors(self): return self.__authors - self.__authors = None + @authors.setter + def authors(self, val): + self.__authors = val ''' dict of bookmarked books ''' @property @@ -126,7 +70,6 @@ class CatalogBuilder(object): @bookmarked_books.setter def bookmarked_books(self, val): self.__bookmarked_books = val - self.__bookmarked_books = None ''' list of bookmarked books, sorted by date read ''' @property @@ -135,7 +78,6 @@ class CatalogBuilder(object): @bookmarked_books_by_date_read.setter def bookmarked_books_by_date_read(self, val): self.__bookmarked_books_by_date_read = val - self.__bookmarked_books_by_date_read = None ''' list of books, sorted by author ''' @property @@ -144,7 +86,6 @@ class CatalogBuilder(object): @books_by_author.setter def books_by_author(self, val): self.__books_by_author = val - self.__books_by_author = None ''' list of books, grouped by date range (30 days) ''' @property @@ -153,7 +94,6 @@ class CatalogBuilder(object): @books_by_date_range.setter def books_by_date_range(self, val): self.__books_by_date_range = val - self.__books_by_date_range = None ''' list of books, by date added reverse (most recent first) ''' @property @@ -162,7 +102,6 @@ class CatalogBuilder(object): @books_by_month.setter def books_by_month(self, val): self.__books_by_month = val - self.__books_by_month = None ''' list of books in series ''' @property @@ -171,7 +110,6 @@ class CatalogBuilder(object): @books_by_series.setter def books_by_series(self, val): self.__books_by_series = val - self.__books_by_series = None ''' list of books, sorted by title ''' @property @@ -180,22 +118,29 @@ class CatalogBuilder(object): @books_by_title.setter def books_by_title(self, val): self.__books_by_title = val - self.__books_by_title = None ''' list of books in series, without series prefix ''' @property def books_by_title_no_series_prefix(self): - return books_by_title_no_series_prefix.__prop + return self.__books_by_title_no_series_prefix @books_by_title_no_series_prefix.setter def books_by_title_no_series_prefix(self, val): self.__books_by_title_no_series_prefix = val - self.__books_by_title_no_series_prefix = None + + ''' directory to store cached thumbs ''' + @property + def cache_dir(self): + return self.__cache_dir + + ''' temp dir to store generated catalog ''' + @property + def catalog_path(self): + return self.__catalog_path ''' content dir in generated catalog ''' @property def content_dir(self): return self.__content_dir - self.__content_dir = os.path.join(self.catalog_path, "content") ''' track Job progress ''' @property @@ -204,7 +149,11 @@ class CatalogBuilder(object): @current_step.setter def current_step(self, val): self.__current_step = val - self.__current_step = 0.0 + + ''' active database ''' + @property + def db(self): + return self.__db ''' cumulative error messages to report at conclusion ''' @property @@ -213,21 +162,21 @@ class CatalogBuilder(object): @error.setter def error(self, val): self.__error = val - self.__error = [] ''' tags to exclude as genres ''' @property def excluded_tags(self): return self.__excluded_tags - self.__excluded_tags = self.get_excluded_tags() + + ''' True if generating for Kindle in MOBI format ''' + @property + def generate_for_kindle(self): + return self.__generate_for_kindle ''' True if connected Kindle and generating for Kindle ''' - @property + @property def generate_recently_read(self): return self.__generate_recently_read - self.__generate_recently_read = True if (opts.generate_recently_added and - opts.connected_kindle and - self.generate_for_kindle) else False ''' list of dicts with books by genre ''' @property @@ -236,7 +185,6 @@ class CatalogBuilder(object): @genres.setter def genres(self, val): self.__genres = val - self.__genres = [] ''' dict of enabled genre tags ''' @property @@ -245,7 +193,6 @@ class CatalogBuilder(object): @genre_tags_dict.setter def genre_tags_dict(self, val): self.__genre_tags_dict = val - self.__genre_tags_dict = None ''' Author, Title, Series sections ''' @property @@ -254,7 +201,6 @@ class CatalogBuilder(object): @html_filelist_1.setter def html_filelist_1(self, val): self.__html_filelist_1 = val - self.__html_filelist_1 = [] ''' Date Added, Date Read ''' @property @@ -263,15 +209,11 @@ class CatalogBuilder(object): @html_filelist_2.setter def html_filelist_2(self, val): self.__html_filelist_2 = val - self.__html_filelist_2 = [] ''' additional field to include before/after comments ''' @property def merge_comments_rule(self): return self.__merge_comments_rule - #f, p, hr = opts.merge_comments_rule.split(':') - #self.__merge_comments_rule = {'field':f, 'position':p, 'hr':hr} - self.__merge_comments_rule = dict(zip(['field','position','hr'],opts.merge_comments_rule.split(':'))) ''' cumulative HTML for NCX file ''' @property @@ -280,18 +222,16 @@ class CatalogBuilder(object): @ncx_soup.setter def ncx_soup(self, val): self.__ncx_soup = val - self.__ncx_soup = None + + ''' opts passed from gui2.catalog.catalog_epub_mobi.py ''' + @property + def opts(self): + return self.__opts ''' output_profile declares special symbols ''' @property def output_profile(self): return self.__output_profile - self.__output_profile = None - from calibre.customize.ui import output_profiles - for profile in output_profiles(): - if profile.short_name == opts.output_profile: - self.__output_profile = profile - break ''' playOrder value for building NCX ''' @property @@ -300,7 +240,11 @@ class CatalogBuilder(object): @play_order.setter def play_order(self, val): self.__play_order = val - self.__play_order = 1 + + ''' catalog??? device??? ''' + @property + def plugin(self): + return self.__plugin ''' dict of prefix rules ''' @property @@ -309,7 +253,6 @@ class CatalogBuilder(object): @prefix_rules.setter def prefix_rules(self, val): self.__prefix_rules = val - self.__prefix_rules = self.get_prefix_rules() ''' used with ProgressReporter() ''' @property @@ -318,7 +261,6 @@ class CatalogBuilder(object): @progress_int.setter def progress_int(self, val): self.__progress_int = val - self.__progress_int = 0.0 ''' used with ProgressReporter() ''' @property @@ -327,7 +269,16 @@ class CatalogBuilder(object): @progress_string.setter def progress_string(self, val): self.__progress_string = val - self.__progress_string = '' + + ''' Progress Reporter for Jobs ''' + @property + def reporter(self): + return self.__reporter + + ''' stylesheet to include with catalog ''' + @property + def stylesheet(self): + return self.__stylesheet ''' device-specific symbol (default empty star) ''' @property @@ -369,7 +320,6 @@ class CatalogBuilder(object): @thumb_height.setter def thumb_height(self, val): self.__thumb_height = val - self.__thumb_height = 0 @property def thumb_width(self): @@ -377,7 +327,6 @@ class CatalogBuilder(object): @thumb_width.setter def thumb_width(self, val): self.__thumb_width = val - self.__thumb_width = 0 ''' list of generated thumbs ''' @property @@ -386,27 +335,78 @@ class CatalogBuilder(object): @thumbs.setter def thumbs(self, val): self.__thumbs = val - self.__thumbs = None ''' full path to thumbs archive ''' @property def thumbs_path(self): return self.__thumbs_path - self.__thumbs_path = os.path.join(self.cache_dir, "thumbs.zip") - ''' used with ProgressReporter() ''' + ''' used with ProgressReporter() ''' @property def total_steps(self): return self.__total_steps - self.__total_steps = 6.0 + @total_steps.setter + def total_steps(self, val): + self.__total_steps = val ''' switch controlling format of series books in Titles section ''' @property def use_series_prefix_in_titles_section(self): return self.__use_series_prefix_in_titles_section + + def __init__(self, db, _opts, plugin, + report_progress=DummyReporter(), + stylesheet="content/stylesheet.css", + init_resources=True): + + self.__db = db + self.__opts = _opts + self.__plugin = plugin + self.__reporter = report_progress + self.__stylesheet = stylesheet + self.__cache_dir = os.path.join(config_dir, 'caches', 'catalog') + self.__catalog_path = PersistentTemporaryDirectory("_epub_mobi_catalog", prefix='') + self.__generate_for_kindle = True if (_opts.fmt == 'mobi' and + _opts.output_profile and + _opts.output_profile.startswith("kindle")) else False + + self.__authors = None + self.__bookmarked_books = None + self.__bookmarked_books_by_date_read = None + self.__books_by_author = None + self.__books_by_date_range = None + self.__books_by_month = None + self.__books_by_series = None + self.__books_by_title = None + self.__books_by_title_no_series_prefix = None + self.__content_dir = os.path.join(self.catalog_path, "content") + self.__current_step = 0.0 + self.__error = [] + self.__excluded_tags = self.get_excluded_tags() + self.__generate_recently_read = True if (_opts.generate_recently_added and + _opts.connected_kindle and + self.generate_for_kindle) else False + self.__genres = [] + self.__genre_tags_dict = None + self.__html_filelist_1 = [] + self.__html_filelist_2 = [] + self.__merge_comments_rule = dict(zip(['field','position','hr'],_opts.merge_comments_rule.split(':'))) + self.__ncx_soup = None + self.__output_profile = None + self.__output_profile = self.get_output_profile(_opts) + self.__play_order = 1 + self.__prefix_rules = self.get_prefix_rules() + self.__progress_int = 0.0 + self.__progress_string = '' + self.__thumb_height = 0 + self.__thumb_width = 0 + self.__thumbs = None + self.__thumbs_path = os.path.join(self.cache_dir, "thumbs.zip") + self.__total_steps = 6.0 self.__use_series_prefix_in_titles_section = False self.compute_total_steps() + self.calculate_thumbnail_dimensions() self.confirm_thumbs_archive() self.load_section_templates() if init_resources: @@ -414,7 +414,7 @@ class CatalogBuilder(object): """ key() functions """ - def kf_author_to_author_sort(self, author): + def _kf_author_to_author_sort(self, author): """ Compute author_sort value from author Tokenize author string, return capitalized string with last token first @@ -431,10 +431,11 @@ class CatalogBuilder(object): tokens[0] += ',' return ' '.join(tokens).capitalize() - def kf_books_by_author_sorter_author(self, book): + def _kf_books_by_author_sorter_author(self, book): """ Generate book sort key with computed author_sort. - Generate a sort key of computed author_sort, title. + Generate a sort key of computed author_sort, title. Used to look for + author_sort mismatches. Twiddle included to force series to sort after non-series books. 'Smith, john Star Wars' 'Smith, john ~Star Wars 0001.0000' @@ -446,25 +447,23 @@ class CatalogBuilder(object): (str): sort key """ if not book['series']: - key = '%s %s' % (self.kf_author_to_author_sort(book['author']), + key = '%s %s' % (self._kf_author_to_author_sort(book['author']), capitalize(book['title_sort'])) else: index = book['series_index'] integer = int(index) fraction = index-integer series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s ~%s %s' % (self.kf_author_to_author_sort(book['author']), + key = '%s ~%s %s' % (self._kf_author_to_author_sort(book['author']), self.generate_sort_title(book['series']), series_index) return key - def kf_books_by_author_sorter_author_sort(self, book): + def _kf_books_by_author_sorter_author_sort(self, book, longest_author_sort=60): """ Generate book sort key with supplied author_sort. Generate a sort key of author_sort, title. - Twiddle included to force series to sort after non-series books. - 'Smith, john Star Wars' - 'Smith, john ~Star Wars 0001.0000' + Bang, tilde included to force series to sort after non-series books. Args: book (dict): book metadata @@ -473,19 +472,20 @@ class CatalogBuilder(object): (str): sort key """ if not book['series']: - key = '%s ~%s' % (capitalize(book['author_sort']), - capitalize(book['title_sort'])) + fs = '{:<%d}!{!s}' % longest_author_sort + key = fs.format(capitalize(book['author_sort']), + capitalize(book['title_sort'])) else: index = book['series_index'] integer = int(index) fraction = index-integer series_index = '%04d%s' % (integer, str('%0.4f' % fraction).lstrip('0')) - key = '%s %s %s' % (capitalize(book['author_sort']), - self.generate_sort_title(book['series']), - series_index) + fs = '{:<%d}~{!s}{!s}' % longest_author_sort + key = fs.format(capitalize(book['author_sort']), + self.generate_sort_title(book['series']), + series_index) return key - """ Methods """ def build_sources(self): @@ -557,7 +557,6 @@ class CatalogBuilder(object): self.write_ncx() return True - ''' def calculate_thumbnail_dimensions(self): """ Calculate thumb dimensions based on device DPI. @@ -587,9 +586,9 @@ class CatalogBuilder(object): self.thumb_height = self.thumb_height/2 break if self.opts.verbose: - self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ + self.opts.log(" Thumbnails:") + self.opts.log(" DPI = %d; thumbnail dimensions: %d x %d" % \ (x.dpi, self.thumb_width, self.thumb_height)) - ''' def compute_total_steps(self): """ Calculate number of build steps to generate catalog. @@ -637,10 +636,10 @@ class CatalogBuilder(object): """ if self.opts.generate_descriptions: if not os.path.exists(self.cache_dir): - self.opts.log.info(" creating new thumb cache '%s'" % self.cache_dir) + self.opts.log.info(" creating new thumb cache '%s'" % self.cache_dir) os.makedirs(self.cache_dir) if not os.path.exists(self.thumbs_path): - self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % + self.opts.log.info(' creating thumbnail archive, thumb_width: %1.2f"' % float(self.opts.thumb_width)) with ZipFile(self.thumbs_path, mode='w') as zfw: zfw.writestr("Catalog Thumbs Archive",'') @@ -656,16 +655,15 @@ class CatalogBuilder(object): cached_thumb_width = '-1' if float(cached_thumb_width) != float(self.opts.thumb_width): - self.opts.log.warning(" invalidating cache at '%s'" % self.thumbs_path) + self.opts.log.warning(" invalidating cache at '%s'" % self.thumbs_path) self.opts.log.warning(' thumb_width changed: %1.2f" => %1.2f"' % (float(cached_thumb_width),float(self.opts.thumb_width))) with ZipFile(self.thumbs_path, mode='w') as zfw: zfw.writestr("Catalog Thumbs Archive",'') else: - self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % + self.opts.log.info(' existing thumb cache at %s, cached_thumb_width: %1.2f"' % (self.thumbs_path, float(cached_thumb_width))) - def convert_html_entities(self, s): """ Convert string containing HTML entities to its unicode equivalent. @@ -854,11 +852,12 @@ class CatalogBuilder(object): cl_list[idx] = last_c if self.DEBUG and self.opts.verbose: + print(" establish_equivalencies():") if key: for idx, item in enumerate(item_list): - print("%s %s" % (cl_list[idx],item[sort_field])) + print(" %s %s" % (cl_list[idx],item[sort_field])) else: - print("%s %s" % (cl_list[0], item)) + print(" %s %s" % (cl_list[0], item)) return cl_list @@ -883,9 +882,10 @@ class CatalogBuilder(object): """ self.update_progress_full_step(_("Sorting database")) - self.books_by_author = sorted(list(self.books_by_title), key=self.kf_books_by_author_sorter_author) - # Build the unique_authors set from existing data, test for author_sort mismatches + # First pass: Sort by author, test for author_sort mismatches + self.books_by_author = sorted(list(self.books_by_title), key=self._kf_books_by_author_sorter_author) + authors = [(record['author'], record['author_sort']) for record in self.books_by_author] current_author = authors[0] for (i,author) in enumerate(authors): @@ -920,8 +920,20 @@ Author '{0}': current_author = author + # Second pass: Sort using sort_key to normalize accented letters + # Determine the longest author_sort length before sorting + asl = [i['author_sort'] for i in self.books_by_author] + las = max(asl, key=len) self.books_by_author = sorted(self.books_by_author, - key=lambda x: sort_key(self.kf_books_by_author_sorter_author_sort(x))) + key=lambda x: sort_key(self._kf_books_by_author_sorter_author_sort(x, len(las)))) + + if self.DEBUG and self.opts.verbose: + tl = [i['title'] for i in self.books_by_author] + lt = max(tl, key=len) + fs = '{:<6}{:<%d} {:<%d} {!s}' % (len(lt),len(las)) + print(fs.format('','Title','Author','Series')) + for i in self.books_by_author: + print(fs.format('', i['title'],i['author_sort'],i['series'])) # Build the unique_authors set from existing data authors = [(record['author'], capitalize(record['author_sort'])) for record in self.books_by_author] @@ -1029,7 +1041,7 @@ Author '{0}': if 'author_sort' in record and record['author_sort'].strip(): this_title['author_sort'] = record['author_sort'] else: - this_title['author_sort'] = self.kf_author_to_author_sort(this_title['author']) + this_title['author_sort'] = self._kf_author_to_author_sort(this_title['author']) if record['publisher']: this_title['publisher'] = re.sub('&', '&', record['publisher']) @@ -1076,7 +1088,7 @@ Author '{0}': this_title['prefix'] = self.discover_prefix(record) if record['tags']: - this_title['tags'] = self.filter_excluded_tags(record['tags'], + this_title['tags'] = self.filter_excluded_genres(record['tags'], self.opts.exclude_genre) if record['formats']: formats = [] @@ -1097,7 +1109,7 @@ Author '{0}': notes = ' · '.join(notes) elif field_md['datatype'] == 'datetime': notes = format_date(notes,'dd MMM yyyy') - this_title['notes'] = {'source':field_md['name'], + this_title['notes'] = {'source':field_md['name'],'content':notes} return this_title @@ -1143,7 +1155,7 @@ Author '{0}': self.opts.log.info(" %-40s %-40s" % ('title', 'title_sort')) for title in self.books_by_title: self.opts.log.info((u" %-40s %-40s" % (title['title'][0:40], - title['title_sort'][0:40])).decode('mac-roman')) + title['title_sort'][0:40])).encode('utf-8')) return True else: error_msg = _("No books found to catalog.\nCheck 'Excluded books' criteria in E-book options.\n") @@ -1311,7 +1323,7 @@ Author '{0}': if tag == ' ': continue - normalized_tags.append(re.sub('\W','',ascii_text(tag)).lower()) + normalized_tags.append(self.normalize_tag(tag)) friendly_tags.append(tag) genre_tags_dict = dict(zip(friendly_tags,normalized_tags)) @@ -1330,7 +1342,7 @@ Author '{0}': return genre_tags_dict - def filter_excluded_tags(self, tags, regex): + def filter_excluded_genres(self, tags, regex): """ Remove excluded tags from a tag list Run regex against list of tags, remove matching tags. Return filtered list. @@ -1352,7 +1364,7 @@ Author '{0}': else: tag_list.append(tag) except: - self.opts.log.error("\tfilter_excluded_tags(): malformed --exclude-genre regex pattern: %s" % regex) + self.opts.log.error("\tfilter_excluded_genres(): malformed --exclude-genre regex pattern: %s" % regex) return tags return tag_list @@ -1490,8 +1502,6 @@ Author '{0}': # Establish initial letter equivalencies sort_equivalents = self.establish_equivalencies(self.books_by_author,key='author_sort') - #for book in sorted(self.books_by_author, key = self.kf_books_by_author_sorter_author_sort): - #for book in self.books_by_author: for idx, book in enumerate(self.books_by_author): book_count += 1 if self.letter_or_symbol(sort_equivalents[idx]) != current_letter : @@ -1680,8 +1690,11 @@ Author '{0}': def _add_books_to_html_by_month(this_months_list, dtc): if len(this_months_list): - - this_months_list = sorted(this_months_list, key=lambda x: sort_key(self.kf_books_by_author_sorter_author_sort)(x))) + # Determine the longest author_sort_length before sorting + asl = [i['author_sort'] for i in this_months_list] + las = max(asl, key=len) + this_months_list = sorted(this_months_list, + key=lambda x: sort_key(self._kf_books_by_author_sorter_author_sort(x, len(las)))) # Create a new month anchor date_string = strftime(u'%B %Y', current_date.timetuple()) @@ -1722,9 +1735,7 @@ Author '{0}': pSeriesTag['class'] = "series_mobi" if self.opts.generate_series: aTag = Tag(soup,'a') - - if self.letter_or_symbol(new_entry['series']) == self.SYMBOLS: - aTag['href'] = "%s.html#%s" % ('BySeries',self.generate_series_anchor(new_entry['series'])) + aTag['href'] = "%s.html#%s" % ('BySeries',self.generate_series_anchor(new_entry['series'])) aTag.insert(0, new_entry['series']) pSeriesTag.insert(0, aTag) else: @@ -2740,7 +2751,7 @@ Author '{0}': for (i, tag) in enumerate(sorted(book.get('tags', []))): aTag = Tag(_soup,'a') if self.opts.generate_genres: - aTag['href'] = "Genre_%s.html" % re.sub("\W","",ascii_text(tag).lower()) + aTag['href'] = "Genre_%s.html" % self.normalize_tag(tag) aTag.insert(0,escape(NavigableString(tag))) genresTag.insert(gtc, aTag) gtc += 1 @@ -2852,6 +2863,7 @@ Author '{0}': newEmptyTag.insert(0,NavigableString(' ')) mt.replaceWith(newEmptyTag) + return soup def generate_html_descriptions(self): """ Generate Description HTML for each book. @@ -2933,7 +2945,6 @@ Author '{0}': bodyTag.insert(1,divTag) return soup - def generate_masthead_image(self, out_path): """ Generate a Kindle masthead image. @@ -4247,7 +4258,7 @@ Author '{0}': (str): sort string """ - from calibre.ebooks.metadata import title_sort + from calibre.ebooks.metadata import title_sort from calibre.library.catalogs.utils import NumberToText # Strip stop words @@ -4314,9 +4325,6 @@ Author '{0}': # process pass - if self.DEBUG and self.opts.verbose: - self.opts.log.info(" generate_thumbnail():") - # Generate crc for current cover with open(title['cover'], 'rb') as f: data = f.read() @@ -4415,7 +4423,7 @@ Author '{0}': "thumbnail_default.jpg" if valid_cover else thumb_file) else: if self.DEBUG and self.opts.verbose: - self.opts.log.warn(" generating new thumbnail_default.jpg") + self.opts.log.warn(" generating new thumbnail_default.jpg") self.generate_thumbnail(title, image_dir, "thumbnail_default.jpg" if valid_cover else thumb_file) # Clear the book's cover property @@ -4466,11 +4474,12 @@ Author '{0}': # Report excluded books if self.opts.verbose and excluded_tags: + self.opts.log.info(" Excluded books:") data = self.db.get_data_as_dict(ids=self.opts.ids) for record in data: matched = list(set(record['tags']) & set(excluded_tags)) if matched : - self.opts.log.info(" - %s by %s (Exclusion rule Tags: '%s')" % + self.opts.log.info(" - '%s' by %s (Exclusion rule Tags: '%s')" % (record['title'], record['authors'][0], str(matched[0]))) return excluded_tags @@ -4491,6 +4500,19 @@ Author '{0}': if self.genre_tags_dict[friendly_tag] == genre: return friendly_tag + def get_output_profile(self, _opts): + """ Return profile matching opts.output_profile + + Input: + _opts (object): build options object + + Return: + (profile): output profile matching name + """ + for profile in output_profiles(): + if profile.short_name == _opts.output_profile: + return profile + def get_prefix_rules(self): """ Convert opts.prefix_rules to dict. @@ -4502,7 +4524,6 @@ Author '{0}': Return: (list): list of prefix_rules dicts """ - pr = [] if self.opts.prefix_rules: try: @@ -4721,6 +4742,28 @@ Author '{0}': return merged + def normalize_tag(self, tag): + """ Generate an XHTML-legal anchor string from tag. + + Parse tag for non-ascii, convert to unicode name. + + Args: + tags (str): tag name possible containing symbols + + Return: + normalized (str): unicode names substituted for non-ascii chars + """ + + normalized = massaged = re.sub('\s','',ascii_text(tag).lower()) + if re.search('\W',normalized): + normalized = '' + for c in massaged: + if re.search('\W',c): + normalized += self.generate_unicode_name(c) + else: + normalized += c + return normalized + def process_exclusions(self, data_set): """ Filter data_set based on exclusion_rules. @@ -4744,7 +4787,6 @@ Author '{0}': exclusion_pairs.append((field,pat)) else: continue - if exclusion_pairs: for record in data_set: for exclusion_pair in exclusion_pairs: @@ -4757,7 +4799,7 @@ Author '{0}': re.IGNORECASE) is not None: if self.opts.verbose: field_md = self.db.metadata_for_field(field) - self.opts.log.info(" - %s (Exclusion rule '%s': %s:%s)" % + self.opts.log.info(" - %s (Exclusion rule '%s': %s:%s)" % (record['title'], field_md['name'], field,pat)) exclusion_set.append(record) if record in filtered_data_set: @@ -4786,6 +4828,8 @@ Author '{0}': self.current_step += 1 self.progress_string = description self.progress_int = float((self.current_step-1)/self.total_steps) + if not self.progress_int: + self.progress_int = 0.01 self.reporter(self.progress_int, self.progress_string) if self.opts.cli_environment: self.opts.log(u"%3.0f%% %s" % (self.progress_int*100, self.progress_string)) From 0ff236f71fb61d78414c64a939bbc793fd79a210 Mon Sep 17 00:00:00 2001 From: GRiker Date: Sun, 2 Sep 2012 05:02:15 -0600 Subject: [PATCH 3/7] 0.8.67+, added catalog help functionality --- resources/catalog/help_epub_mobi.html | 193 ++++++++++++++++++ src/calibre/devices/apple/driver.py | 24 ++- src/calibre/gui2/catalog/catalog_epub_mobi.py | 12 +- src/calibre/gui2/catalog/catalog_epub_mobi.ui | 4 +- src/calibre/gui2/dialogs/catalog.py | 31 ++- src/calibre/gui2/dialogs/catalog.ui | 8 +- 6 files changed, 256 insertions(+), 16 deletions(-) create mode 100644 resources/catalog/help_epub_mobi.html diff --git a/resources/catalog/help_epub_mobi.html b/resources/catalog/help_epub_mobi.html new file mode 100644 index 0000000000..9eb491697d --- /dev/null +++ b/resources/catalog/help_epub_mobi.html @@ -0,0 +1,193 @@ + + + + +Untitled Document + + + + +

AZW3 • EPUB • MOBI Catalogs

+

Selecting books to catalog

+

Included sections: Selecting which sections to include in the catalog

+

Prefixes: Adding a prefix indicating book status

+

Excluded books: Ignoring certain books when generating the catalog

+

Excluded genres: Ignoring certain tags as genres when generating the catalog

+

Other options: Specifying thumb size, adding extra information to Descriptions

+

Customizing the appearance of the generated catalog

+
+

Selecting books to catalog

+

If you want all of your library cataloged, remove any search or filtering criteria by clearing the Search: field in the main window. With a single book selected, all books in your library will be candidates for inclusion in the generated catalog. Individual books may be excluded by various criteria; see the Excluded genres section below for more information.

+

If you want only some of your library cataloged, you have two options:

+
    +
  1. Create a multiple selection of the books you want cataloged. With more than one book selected in calibre's main window, only the selected books will be cataloged.
    2. +
  2. Use the Search: field or calibre's Tag Browser to filter the displayed books. With a single book selected, only the displayed books will be cataloged.
    3. +
+
+

Included sections

+

<img>

+

Sections enabled by a checkmark will be included in the generated catalog:

+
    +
  • Authors - all books, sorted by author, presented in a list format.
    • +
  • Titles - all books, sorted by title, presented in a list format.
    • +
  • Series - all books in series, sorted by series, presented in a list format
    • +
  • Genres - individual genres presented in a list, sorted by Author and Series
    • +
  • Recently Added - all books, sorted in reverse chronological order. List includes books added in the last 30 days, then a month-by-month listing of added books.
    • +
  • Descriptions - detailed description page for each book, including a cover thumbnail and comments. Sorted by author, with non-series books listed before series books.
    • +
+
+

Prefixes

+

<img>

+

Prefix rules specify matching criteria and a prefix to use when the criteria is matched.

+

The checkbox in the first column enables the rule. Name is a rule name that you provide. Field is either Tags or a custom column in your library. Value is the content of Field to match. When a prefix rule is satisfied, the book will be marked with the selected symbol in the Prefix column.

+

Three prefix rules have been specified in the example above:

+
    +
  1. Read book specifies that a book with any date in a custom column named Last read will be prefixed with a checkmark symbol.
    2. +
  2. Wishlist item specifies that any book with a Wishlist tag will be prefixed with an X symbol.
    3. +
  3. Library books specifies that any book with a value of True (or Yes) in a custom column Available in Library will be prefixed with a double arrow symbol.
    4. +
+

The first matching rule supplies the prefix. Disabled or incomplete rules are ignored.

+
+

Excluded books

+

<img>

+

Exclusion rules specify matching criteria for books that will not be cataloged.

+

The checkbox in the first column enables the rule. Name is a rule name that you provide. Field is either Tags or a custom column in your library. Value is the content of Field to match. When an exclusion rule is satisfied, the book will be excluded from the generated catalog.

+

Two exclusion rules have been specified in the example above:

+
    +
  1. The Catalogs rule specifies that any book with a Catalog tag will be excluded from the generated catalog.
    2. +
  2. The Archived Books rule specifies that any book with a value of Archived in the custom column Status will be excluded from the generated catalog.
    3. +
+

All rules are evaluated for every book. Disabled or incomplete rules are ignored.

+
+

Excluded genres

+

<img>

+

When the catalog is generated, tags in your database are used as genres. For example, you may have the tags Fiction and Nonfiction applied to books in your library. These tags become genres in the generated catalog, with all tagged books showing up in their respective genre lists.

+

You may be using certain tags for other purposes, perhaps a + to indicate a read book, or a bracketed tag like [Amazon Freebie] to indicate a book's source. The Excluded genres regex allows you to specify tags that you don't want used as genres in the generated catalog. The default exclusion regex pattern \[.+\]\+ excludes any tags of the form [tag], as well as excluding +, the default tag for read books, from being used as genres in the generated catalog.

+

You can also use the exact tag name in a regex. For example, [Amazon Freebie] will exclude that tag from the generated catalog. If you want to list multiple exact tags for exclusion, put a pipe character between them: [Amazon Freebie]|[Project Gutenberg].

+

The Results field shows you which tags will be excluded when the catalog is built, based on the tags in your database and the regex you enter.

+
+

Other options

+

Thumb width specifies a width preference for cover thumbnails included in the Descriptions section. Thumbnails are cached to improve performance. If you want to experiment with different widths, try generating a catalog with just a few books until you've found your optimal width, then generate your full catalog. The first time a catalog is generated with a new thumbnail width, performance will be slower, but subsequent runs will take advantage of the cache.

+

Extra note specifies a custom column's contents to be inserted into the Description, opposite the cover. For example, you might want to display the date you last read a book using a Last Read custom column. For advanced use of the Description note feature, see this post in the calibre forum.

+

Merge with Comments specifies a custom column whose content will be non-destructively merged with the Comments metadata during catalog generation. For example, you might have a custom column Author Bio that you'd like to append to the Comments metadata. You can choose to insert the custom column contents before or after the Comments section, and optionally separate the appended content with a horizontal rule. Eligible custom column types include text, comments, and composite.

+
+

Customizing catalog appearance

+

If you wish to change the default appearance of the Description pages or Section lists, you can do so by modifying a local copy of the catalog's template and CSS files, overriding the default layout. This requires familiarity with HTML and CSS.
+

+
    +
  • Open your calibre configuration directory - Preferences|Advanced|Miscellaneous|Open calibre configuration directory.
    +
    • +
  • Within this directory, create a subdirectory resources (if it doesn't already exist).
    +
    • +
  • Within the resources directory, create a subdirectory catalog.
    +
    • +
  • Open a new file system window, navigate to the default catalog resources folder:
    +
      +
    • OSX: Applications/calibre/Contents/Resources/resources/catalog
      +
      • +
    • Windows: Program Files\Calibre2\resources\catalog
      +
      • +
    +
    • +
  • Copy all of the files from the default catalog resources folder to the newly created resources/catalog directory within your calibre configuration directory.
    +
    • +
  • Edit template.xhtml and stylesheet.css to your preferences. If you want to start over, copy from the default template and CSS files again. If you want to discard your customizations, delete the catalog subfolder from your calibre configuration resources directory.
    +
    • +
+

Customizing the Description page. Available fields in template.xhtml:
+

+
    +
  • {author} - Book author(s)
    +
    • +
  • {author_prefix} - Checkmark if read, 'X' if wishlist, else blank
    +
    • +
  • {comments} - Contents of the book's Comments field
    +
    • +
  • {formats} - List of installed formats
    +
    • +
  • {genres} - List of genres
    +
    • +
  • {note_content} - Note content as specified in options
    +
    • +
  • {note_source} - Note source field as specified in options
    +
    • +
  • {pubdate} - Month/Year published
    +
    • +
  • {publisher} - Publisher
    +
    • +
  • {pubmonth} - Month published
    +
    • +
  • {pubyear} - Year published
    +
    • +
  • {rating} - Graphical rating
    +
    • +
  • {series} - Series name
    +
    • +
  • {series_index} - Series index
    +
    • +
  • {thumb} - Cover thumb url
    +
    • +
  • {title} - Book title
    +
    • +
+

Example: Changing year of publication in Description header
+ Default: <td class="date">{pubyear}</td>
+ Removed: <td class="date"></td>
+ Augmented: <td class="date">Year of publication: {pubyear}</td>
+Modified: <td class="date">{pubmonth} {pubyear}</td>

+

Customizing the Section lists. Templates controlling the display of book titles in the various Section lists (Books by Author, Books by Title, etc) may be edited to taste.
+ Available fields in section_list_templates.py:
+ {title} - Title of the book
+ {series} - Series name
+ {series_index} - Number of the book in the series
+ {rating} - 0-5 stars
+ {rating_parens} - (0-5 stars)
+ {pubyear} - Year the book was published
+ {pubyear_parens} - (Year the book was published)
+ Example: Changing Books by Author to remove year of publication:
+Default:

+

Code:
+ by_authors_normal_title_template = '{title} {pubyear_parens}'
+ by_authors_series_title_template = '[{series_index}] {title} {pubyear_parens}'
+ Year of publication removed:

+

Code:
+ by_authors_normal_title_template = '{title}'
+ by_authors_series_title_template = '[{series_index}] {title}'
+ Rating added:

+

Code:
+ by_authors_normal_title_template = '{title} {rating}'
+ by_authors_series_title_template = '[{series_index}] {title} {rating}'

+

Tips for experimenting with customization:
+ Work with a small subset of your catalog, 5-10 books
+ If you are experimenting with Section list templates, disable Descriptions in E-book options - catalog generation will be much faster.
+ If you are experimenting with CSS, build an EPUB version of your catalog, explode it with the Tweak EPUB feature to find the class of the element you want to change.

+

 

+

 

+ + diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 838edb4e69..8b05fb5b8f 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -13,7 +13,8 @@ from calibre.constants import isosx, iswindows from calibre.devices.errors import OpenFeedback, UserFeedback from calibre.devices.usbms.deviceconfig import DeviceConfig from calibre.devices.interface import DevicePlugin -from calibre.ebooks.metadata import authors_to_string, MetaInformation, title_sort +from calibre.ebooks.metadata import (author_to_author_sort, authors_to_string, + MetaInformation, title_sort) from calibre.ebooks.metadata.book.base import Metadata from calibre.utils.config import config_dir, dynamic, prefs from calibre.utils.date import now, parse_date @@ -405,9 +406,9 @@ class ITUNES(DriverBase): @return: A BookList. Implementation notes: - iTunes does not sync purchased books, they are only on the device. They are visible, but - they are not backed up to iTunes. Since calibre can't manage them, don't show them in the - list of device books. + iTunes does not sync purchased books, they are only on the device. They are + visible, but they are not backed up to iTunes. Since calibre can't manage them, + don't show them in the list of device books. """ if not oncard: @@ -1637,9 +1638,19 @@ class ITUNES(DriverBase): logger().info('%s%s' % (' '*indent,'-' * len(msg))) for book in booklist: + tl = [i.title for i in booklist] + lt = max(tl, key=len) + al = [i.author for i in booklist] + la = max(al, key=len) + asl = [i.author_sort for i in booklist] + las = max(asl, key=len) if isosx: - logger().info("%s%-40.40s %-30.30s %-10.10s %s" % - (' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) + fs = '{!s}{:<%d} {:<%d} {:<%d} {:<10} {!s}' % (' ' * indent, len(lt), + len(la), len(las)) + logger.info(fs.format(book.title, book.author, book.author_sort, + str(book.library_id)[-9:], book.uuid)) + #logger().info("%s%-40.40s %-30.30s %-10.10s %s" % + # (' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) elif iswindows: logger().info("%s%-40.40s %-30.30s" % (' '*indent,book.title, book.author)) @@ -3478,6 +3489,7 @@ class Book(Metadata): ''' def __init__(self,title,author): Metadata.__init__(self, title, authors=author.split(' & ')) + self.author_sort = author_to_author_sort(author) @property def title_sorter(self): diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 7c89d3e0dd..4dc42b73bd 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -11,13 +11,14 @@ import re, sys from functools import partial from calibre.ebooks.conversion.config import load_defaults -from calibre.gui2 import gprefs, info_dialog, question_dialog +from calibre.gui2 import gprefs, info_dialog, open_url, question_dialog from calibre.utils.icu import sort_key from catalog_epub_mobi_ui import Ui_Form from PyQt4.Qt import (Qt, QAbstractItemView, QCheckBox, QComboBox, QDoubleSpinBox, QIcon, QLineEdit, QObject, QRadioButton, QSize, QSizePolicy, - QTableWidget, QTableWidgetItem, QTextEdit, QToolButton, QVBoxLayout, QWidget, + QTableWidget, QTableWidgetItem, QTextEdit, QToolButton, QUrl, + QVBoxLayout, QWidget, SIGNAL) class PluginWidget(QWidget,Ui_Form): @@ -440,6 +441,13 @@ class PluginWidget(QWidget,Ui_Form): self.merge_after.setEnabled(False) self.include_hr.setEnabled(False) + def show_help(self): + ''' + Display help file + ''' + url = 'file:///' + P('catalog/help_epub_mobi.html') + open_url(QUrl(url)) + class CheckableTableWidgetItem(QTableWidgetItem): ''' Borrowed from kiwidude diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.ui b/src/calibre/gui2/catalog/catalog_epub_mobi.ui index f3bffb4930..28a17725df 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.ui +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.ui @@ -268,7 +268,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - List of tags that will be excluded as genres + Tags that will be excluded as genres QFrame::StyledPanel @@ -379,7 +379,7 @@ The default pattern \[.+\]|\+ excludes tags of the form [tag], e.g., [Test book] - &Extra note + E&xtra note header_note_source_field diff --git a/src/calibre/gui2/dialogs/catalog.py b/src/calibre/gui2/dialogs/catalog.py index a8f7ed160f..6ec5dd6d13 100644 --- a/src/calibre/gui2/dialogs/catalog.py +++ b/src/calibre/gui2/dialogs/catalog.py @@ -10,7 +10,7 @@ import os, sys, importlib from calibre.customize.ui import config from calibre.gui2.dialogs.catalog_ui import Ui_Dialog -from calibre.gui2 import dynamic, ResizableDialog +from calibre.gui2 import dynamic, ResizableDialog, info_dialog from calibre.customize.ui import catalog_plugins class Catalog(ResizableDialog, Ui_Dialog): @@ -22,7 +22,6 @@ class Catalog(ResizableDialog, Ui_Dialog): from PyQt4.uic import compileUi ResizableDialog.__init__(self, parent) - self.dbspec, self.ids = dbspec, ids # Display the number of books we've been passed @@ -115,6 +114,7 @@ class Catalog(ResizableDialog, Ui_Dialog): self.format.currentIndexChanged.connect(self.show_plugin_tab) self.buttonBox.button(self.buttonBox.Apply).clicked.connect(self.apply) + self.buttonBox.button(self.buttonBox.Help).clicked.connect(self.help) self.show_plugin_tab(None) geom = dynamic.get('catalog_window_geom', None) @@ -129,6 +129,10 @@ class Catalog(ResizableDialog, Ui_Dialog): if cf in pw.formats: self.tabs.addTab(pw, pw.TITLE) break + if hasattr(self.tabs.widget(1),'show_help'): + self.buttonBox.button(self.buttonBox.Help).setVisible(True) + else: + self.buttonBox.button(self.buttonBox.Help).setVisible(False) def format_changed(self, idx): cf = unicode(self.format.currentText()) @@ -165,6 +169,29 @@ class Catalog(ResizableDialog, Ui_Dialog): self.save_catalog_settings() return ResizableDialog.accept(self) + def help(self): + ''' + To add help functionality for a specific format: + In gui2.catalog.catalog_.py, add the following: + from calibre.gui2 import open_url + from PyQt4.Qt import QUrl + + In the PluginWidget() class, add this method: + def show_help(self): + url = 'file:///' + P('catalog/help_.html') + open_url(QUrl(url)) + + Create the help file at resources/catalog/help_.html + ''' + if self.tabs.count() > 1 and hasattr(self.tabs.widget(1),'show_help'): + try: + self.tabs.widget(1).show_help() + except: + info_dialog(self, _('No help available'), + _('No help available for this output format.'), + show_copy_button=False, + show=True) + def reject(self): dynamic.set('catalog_window_geom', bytearray(self.saveGeometry())) ResizableDialog.reject(self) diff --git a/src/calibre/gui2/dialogs/catalog.ui b/src/calibre/gui2/dialogs/catalog.ui index cf51ac8848..a4366b26c2 100644 --- a/src/calibre/gui2/dialogs/catalog.ui +++ b/src/calibre/gui2/dialogs/catalog.ui @@ -14,7 +14,7 @@ Generate catalog - + :/images/lt.png:/images/lt.png @@ -37,7 +37,7 @@ Qt::Horizontal - QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Ok + QDialogButtonBox::Apply|QDialogButtonBox::Cancel|QDialogButtonBox::Help|QDialogButtonBox::Ok @@ -54,8 +54,8 @@ 0 0 - 666 - 599 + 650 + 575 From 58d31f0316c251da122a0ee49cb845e34aecdb6b Mon Sep 17 00:00:00 2001 From: GRiker Date: Sun, 2 Sep 2012 06:13:06 -0600 Subject: [PATCH 4/7] catalog help wip --- resources/catalog/help_epub_mobi.html | 66 +++++++++++++++++++-------- 1 file changed, 48 insertions(+), 18 deletions(-) diff --git a/resources/catalog/help_epub_mobi.html b/resources/catalog/help_epub_mobi.html index 9eb491697d..3acfe7b634 100644 --- a/resources/catalog/help_epub_mobi.html +++ b/resources/catalog/help_epub_mobi.html @@ -29,6 +29,16 @@ h3 { font-family: "Courier New", Courier, monospace; font-size: 90%; } +.code_header { + text-indent: 2em; +} +.code_body { + font-family: "Lucida Console", Monaco, monospace; + background-color: #EEE; + font-size: .9em; + margin-left: 2em; + margin-top: 0px; +} @@ -157,25 +167,45 @@ h3 {

Example: Changing year of publication in Description header
- Default: <td class="date">{pubyear}</td>
- Removed: <td class="date"></td>
- Augmented: <td class="date">Year of publication: {pubyear}</td>
-Modified: <td class="date">{pubmonth} {pubyear}</td>

+

+
    +
  • Default: <td class="date">{pubyear}</td>
    +
    • +
  • Removed: <td class="date"></td>
    +
    • +
  • Augmented: <td class="date">Year of publication: {pubyear}</td>
    +
    • +
  • Modified: <td class="date">{pubmonth} {pubyear}</td>
    • +

Customizing the Section lists. Templates controlling the display of book titles in the various Section lists (Books by Author, Books by Title, etc) may be edited to taste.
- Available fields in section_list_templates.py:
- {title} - Title of the book
- {series} - Series name
- {series_index} - Number of the book in the series
- {rating} - 0-5 stars
- {rating_parens} - (0-5 stars)
- {pubyear} - Year the book was published
- {pubyear_parens} - (Year the book was published)
- Example: Changing Books by Author to remove year of publication:
-Default:

-

Code:
- by_authors_normal_title_template = '{title} {pubyear_parens}'
- by_authors_series_title_template = '[{series_index}] {title} {pubyear_parens}'
- Year of publication removed:

+ Available fields in section_list_templates.py:
+

+
    +
  • {title} - Title of the book
    +
    • +
  • {series} - Series name
    +
    • +
  • {series_index} - Number of the book in the series
    +
    • +
  • {rating} - 0-5 stars
    +
    • +
  • {rating_parens} - (0-5 stars)
    +
    • +
  • {pubyear} - Year the book was published
    +
    • +
  • {pubyear_parens} - (Year the book was published)
    +
    • +
+

Example: Changing Books by Author to remove year of publication:
+ Default:

+

<img>

+

Code:
+

+

by_authors_normal_title_template = '{title} {pubyear_parens}'
+

+

by_authors_series_title_template = '[{series_index}] {title} {pubyear_parens}'
+

+

Year of publication removed:

Code:
by_authors_normal_title_template = '{title}'
by_authors_series_title_template = '[{series_index}] {title}'
From 1cd7b53bb23a56373ee55be1ea1ce26f9b27a4e0 Mon Sep 17 00:00:00 2001 From: GRiker Date: Sun, 2 Sep 2012 08:36:18 -0600 Subject: [PATCH 5/7] 0.8.67+ (KG reverts usb lib), GwR catalog wip, Apple driver changes --- src/calibre/devices/apple/driver.py | 4 ++-- src/calibre/gui2/catalog/catalog_epub_mobi.py | 5 ++++- src/calibre/library/catalogs/epub_mobi_builder.py | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index 8b05fb5b8f..aa17602942 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -1645,9 +1645,9 @@ class ITUNES(DriverBase): asl = [i.author_sort for i in booklist] las = max(asl, key=len) if isosx: - fs = '{!s}{:<%d} {:<%d} {:<%d} {:<10} {!s}' % (' ' * indent, len(lt), + fs = '{!s}{:<%d} {:<%d} {:<%d} {:<10} {!s}' % (len(lt), len(la), len(las)) - logger.info(fs.format(book.title, book.author, book.author_sort, + logger().info(fs.format(book.title, book.author, book.author_sort, str(book.library_id)[-9:], book.uuid)) #logger().info("%s%-40.40s %-30.30s %-10.10s %s" % # (' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) diff --git a/src/calibre/gui2/catalog/catalog_epub_mobi.py b/src/calibre/gui2/catalog/catalog_epub_mobi.py index 4dc42b73bd..84c46e2b36 100644 --- a/src/calibre/gui2/catalog/catalog_epub_mobi.py +++ b/src/calibre/gui2/catalog/catalog_epub_mobi.py @@ -172,7 +172,10 @@ class PluginWidget(QWidget,Ui_Form): if hit: excluded_tags.append(hit.string) if excluded_tags: - results = ', '.join(sorted(excluded_tags)) + if set(excluded_tags) == set(self.all_tags): + results = _("All genres will be excluded") + else: + results = ', '.join(sorted(excluded_tags)) finally: if self.DEBUG: print(results) diff --git a/src/calibre/library/catalogs/epub_mobi_builder.py b/src/calibre/library/catalogs/epub_mobi_builder.py index 8c60686ffe..8b4d241748 100644 --- a/src/calibre/library/catalogs/epub_mobi_builder.py +++ b/src/calibre/library/catalogs/epub_mobi_builder.py @@ -883,7 +883,7 @@ class CatalogBuilder(object): self.update_progress_full_step(_("Sorting database")) - # First pass: Sort by author, test for author_sort mismatches + # Test for author_sort mismatches self.books_by_author = sorted(list(self.books_by_title), key=self._kf_books_by_author_sorter_author) authors = [(record['author'], record['author_sort']) for record in self.books_by_author] From c0f08524a9c85e0cb79244a798cccba881d93c94 Mon Sep 17 00:00:00 2001 From: GRiker Date: Sun, 2 Sep 2012 08:46:58 -0600 Subject: [PATCH 6/7] iTunes author_sort fix wip --- src/calibre/devices/apple/driver.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/calibre/devices/apple/driver.py b/src/calibre/devices/apple/driver.py index aa17602942..90e5416d3c 100644 --- a/src/calibre/devices/apple/driver.py +++ b/src/calibre/devices/apple/driver.py @@ -1647,8 +1647,9 @@ class ITUNES(DriverBase): if isosx: fs = '{!s}{:<%d} {:<%d} {:<%d} {:<10} {!s}' % (len(lt), len(la), len(las)) - logger().info(fs.format(book.title, book.author, book.author_sort, - str(book.library_id)[-9:], book.uuid)) + logger().info(fs.format(' '*indent, book.title, book.author, + book.author_sort, str(book.library_id)[-9:], + book.uuid)) #logger().info("%s%-40.40s %-30.30s %-10.10s %s" % # (' '*indent,book.title, book.author, str(book.library_id)[-9:], book.uuid)) elif iswindows: From 43da0609f8dc12f61c3c08f5a6c0787219271358 Mon Sep 17 00:00:00 2001 From: GRiker Date: Wed, 5 Sep 2012 09:29:00 -0600 Subject: [PATCH 7/7] catalog builder updates wip --- resources/catalog/help/epub_mobi/help.html | 151 ++++++++ .../help/epub_mobi/images/catalog_options.png | Bin 0 -> 33552 bytes .../help/epub_mobi/images/custom_cover.png | Bin 0 -> 102346 bytes .../help/epub_mobi/images/excluded_books.png | Bin 0 -> 28293 bytes .../help/epub_mobi/images/excluded_genres.png | Bin 0 -> 26036 bytes .../epub_mobi/images/included_sections.png | Bin 0 -> 21115 bytes .../help/epub_mobi/images/other_options.png | Bin 0 -> 40608 bytes .../help/epub_mobi/images/prefix_rules.png | Bin 0 -> 38498 bytes .../help/epub_mobi/images/send_to_device.png | Bin 0 -> 13489 bytes resources/catalog/help_epub_mobi.html | 114 ------ src/calibre/gui2/catalog/catalog_epub_mobi.py | 28 +- src/calibre/gui2/catalog/catalog_epub_mobi.ui | 337 ++++++++++-------- src/calibre/library/catalogs/__init__.py | 2 + src/calibre/library/catalogs/epub_mobi.py | 31 +- .../library/catalogs/epub_mobi_builder.py | 76 ++-- 15 files changed, 413 insertions(+), 326 deletions(-) create mode 100644 resources/catalog/help/epub_mobi/help.html create mode 100644 resources/catalog/help/epub_mobi/images/catalog_options.png create mode 100644 resources/catalog/help/epub_mobi/images/custom_cover.png create mode 100644 resources/catalog/help/epub_mobi/images/excluded_books.png create mode 100644 resources/catalog/help/epub_mobi/images/excluded_genres.png create mode 100644 resources/catalog/help/epub_mobi/images/included_sections.png create mode 100644 resources/catalog/help/epub_mobi/images/other_options.png create mode 100644 resources/catalog/help/epub_mobi/images/prefix_rules.png create mode 100644 resources/catalog/help/epub_mobi/images/send_to_device.png delete mode 100644 resources/catalog/help_epub_mobi.html diff --git a/resources/catalog/help/epub_mobi/help.html b/resources/catalog/help/epub_mobi/help.html new file mode 100644 index 0000000000..8244b1a4d9 --- /dev/null +++ b/resources/catalog/help/epub_mobi/help.html @@ -0,0 +1,151 @@ + + + + +Creating AZW3 • EPUB • MOBI Catalogs + + + + +

Creating AZW3 • EPUB • MOBI Catalogs

+

Selecting books to catalog

+

Included sections: Selecting which sections to include in the catalog

+

Prefixes: Adding a prefix indicating book status

+

Excluded books: Ignoring certain books when generating the catalog

+

Excluded genres: Ignoring certain tags as genres when generating the catalog

+

Other options: Specifying thumb size, adding extra information to Descriptions

+

Custom catalog covers

+

Additional help resources

+

Calibre's Create catalog feature enables you to create a catalog of your library in a variety of formats. This help file describes cataloging options when generating a catalog in AZW3, EPUB and MOBI formats.

+
+

Selecting books to catalog

+

If you want all of your library cataloged, remove any search or filtering criteria in the main window. With a single book selected, all books in your library will be candidates for inclusion in the generated catalog. Individual books may be excluded by various criteria; see the Excluded genres section below for more information.

+

If you want only some of your library cataloged, you have two options:

+
    +
  1. Create a multiple selection of the books you want cataloged. With more than one book selected in calibre's main window, only the selected books will be cataloged.
    2. +
  2. Use the Search field or the Tag Browser to filter the displayed books. Only the displayed books will be cataloged.
    3. +
+

To begin catalog generation, select the menu item Convert books|Create a catalog of the books in your calibre library. You may also add a Create Catalog button to a toolbar in Preferences|Interface|Toolbar for easier access to the Generate catalog dialog.

+

+

In Catalog options, select AZW3, EPUB or MOBI as the Catalog format. In the Catalog title field, provide a name that will be used for the generated catalog. If a catalog of the same name and format already exists, it will be replaced with the newly-generated catalog.

+

+

Enabling Send catalog to device automatically will download the generated catalog to a connected device upon completion.

+
+

Included sections

+

+

Sections enabled by a checkmark will be included in the generated catalog:

+
    +
  • Authors - all books, sorted by author, presented in a list format. Non-series books are listed before series books.
    • +
  • Titles - all books, sorted by title, presented in a list format.
    • +
  • Series - all books that are part of a series, sorted by series, presented in a list format.
    • +
  • Genres - individual genres presented in a list, sorted by Author and Series.
    • +
  • Recently Added - all books, sorted in reverse chronological order. List includes books added in the last 30 days, then a month-by-month listing of added books.
    • +
  • Descriptions - detailed description page for each book, including a cover thumbnail and comments. Sorted by author, with non-series books listed before series books.
    • +
+
+
+

Prefixes

+

+

Prefix rules allow you to add a prefix to book listings when certain criteria are met. For example, you might want to mark books you've read with a checkmark, or books on your wishlist with an X.

+

The checkbox in the first column enables the rule. Name is a rule name that you provide. Field is either Tags or a custom column from your library. Value is the content of Field to match. When a prefix rule is satisfied, the book will be marked with the selected Prefix.

+

Three prefix rules have been specified in the example above:

+
    +
  1. Read book specifies that a book with any date in a custom column named Last read will be prefixed with a checkmark symbol.
    2. +
  2. Wishlist item specifies that any book with a Wishlist tag will be prefixed with an X symbol.
    3. +
  3. Library books specifies that any book with a value of True (or Yes) in a custom column Available in Library will be prefixed with a double arrow symbol.
    4. +
+

The first matching rule supplies the prefix. Disabled or incomplete rules are ignored.

+
+
+

Excluded books

+

+

Exclusion rules allow you to specify books that will not be cataloged.

+

The checkbox in the first column enables the rule. Name is a rule name that you provide. Field is either Tags or a custom column in your library. Value is the content of Field to match. When an exclusion rule is satisfied, the book will be excluded from the generated catalog.

+

Two exclusion rules have been specified in the example above:

+
    +
  1. The Catalogs rule specifies that any book with a Catalog tag will be excluded from the generated catalog.
    2. +
  2. The Archived Books rule specifies that any book with a value of Archived in the custom column Status will be excluded from the generated catalog.
    3. +
+

All rules are evaluated for every book. Disabled or incomplete rules are ignored.

+
+
+

Excluded genres

+

+

When the catalog is generated, tags in your database are used as genres. For example, you may use the tags Fiction and Nonfiction. These tags become genres in the generated catalog, with books listed under their respective genre lists based on their assigned tags. A book will be listed in every genre section for which it has a corresponding tag.

+

You may be using certain tags for other purposes, perhaps a + to indicate a read book, or a bracketed tag like [Amazon Freebie] to indicate a book's source. The Excluded genres regex allows you to specify tags that you don't want used as genres in the generated catalog. The default exclusion regex pattern \[.+\]\+ excludes any tags of the form [tag], as well as excluding +, the default tag for read books, from being used as genres in the generated catalog.

+

You can also use an exact tag name in a regex. For example, [Amazon Freebie] or [Project Gutenberg]. If you want to list multiple exact tags for exclusion, put a pipe (vertical bar) character between them: [Amazon Freebie]|[Project Gutenberg].

+

Results of regex shows you which tags will be excluded when the catalog is built, based on the tags in your database and the regex pattern you enter. The results are updated as you modify the regex pattern.

+
+
+

Other options

+

+

Catalog cover specifies whether to generate a new cover or use an existing cover. It is possible to create a custom cover for your catalogs - see Custom catalog covers for more information. If you have created a custom cover that you want to reuse, select Use existing cover. Otherwise, select Generate new cover.

+

Extra Description note specifies a custom column's contents to be inserted into the Description page, next to the cover thumbnail. For example, you might want to display the date you last read a book using a Last Read custom column. For advanced use of the Description note feature, see this post in the calibre forum.

+

Thumb width specifies a width preference for cover thumbnails included with Descriptions pages. Thumbnails are cached to improve performance.To experiment with different widths, try generating a catalog with just a few books until you've determined your preferred width, then generate your full catalog. The first time a catalog is generated with a new thumbnail width, performance will be slower, but subsequent builds of the catalog will take advantage of the thumbnail cache.

+

Merge with Comments specifies a custom column whose content will be non-destructively merged with the Comments metadata during catalog generation. For example, you might have a custom column Author Bio that you'd like to append to the Comments metadata. You can choose to insert the custom column contents before or after the Comments section, and optionally separate the appended content with a horizontal rule separator. Eligible custom column types include text, comments, and composite.

+
+
+

Custom catalog covers

+
+

With the Generate Cover plugin installed, you can create custom covers for your catalog.

+

To install the plugin, go to Preferences|Advanced|Plugins|Get new plugins.

+
+
+

Additional help resources

+

For more information on calibre's Catalog feature, see the MobileRead forum sticky Creating Catalogs - Start here, where you can find information on how to customize the catalog templates, and how to submit a bug report.

+

To ask questions or discuss calibre's Catalog feature with other users, visit the MobileRead forum Calibre Catalogs.

+

Calibre's online user manual can be found here.

+
+ + diff --git a/resources/catalog/help/epub_mobi/images/catalog_options.png b/resources/catalog/help/epub_mobi/images/catalog_options.png new file mode 100644 index 0000000000000000000000000000000000000000..44a97f8b12e7368c977b9e0a2c007a93847e5245 GIT binary patch literal 33552 zcmZU3b980P6Yhzf2`9E~+n6{xv1el2n%J7yw(W^++qRv&+A zRDbnVg(}ENAi!e70ssI6DM?W!001=RtDFiA`SrxFaV7Zrfw7g;Z~y?{Q2)6>0IBJi z003;hnTUvjf~k$8jf1I;EuoZ%2%)XLjj@@f5dh%2Qjut)@JHi&(Bz|623QFWI4LU^ ziBRp{(q6h~!LhaBY@;j(D7h#kIxZMBvA!^r7#c1ZA^BflGX)vUDG)61IXgyuVIv<# z);6s+fkt!r`3m=wIju{+!wmrjfE9xTDM~ghU|Benb7nIfJu7C2_yGi_9sq?XxNGYm znz_B*eU$|V^7I5i0{FIl$BmT)d}>kMJaK6QNO1v5u3T%DfIU)xQV)748DNPQFr`P0 zECs0k1<()MMMVK&gaI(l7P{gA7?lf~ApyxdN0K&xGfY6}z;?(ipx6Y&H(^vZ9-#0F z_$xg$iUc}k3fLNuJ-h;#b_4V?Gdhx?1V_ODyMKIcq+T&5(NZXahRbN~R~z4%xX z6x*BaPn~7Z<2yV}fZ*tG7%V7&3qTq@dL5a1;KcbEoa$iCkRWIPjDS2SV5>v@55t7ykECi|F=CL-^#1SmbmJQAstfE$ zU$dY3Qu_D}1Oy-&$T~ZE3Ao|}^QD@++o1!AG6gFR7`wX*2Ehyh6jeDC?5vgGBLV>5 z&|;v-i~YVI`7x>cA}smCx%x7I3uB6kfYO4~=Zmn17*Np$&nti$Rbw@Zpr84@+d@b6 z*qQp(O@XLwL9PY8UV*up9a&6S0So>;yd#cT1AS`I3^D zg&@TekH)e|Vkt$b5WU8@@2Tyv&iHAIxW`Bg0hq(uMLKgarA1PT$<0G1gzyyQS)n7v zF|y5$Q0f76;vBgNrgZM$R^mKUI7eRf1YUtrKYbOM&}E?N{^By((8B-dQ(|IjM4{=Y zuGTC^w&-oGZc`U{z<&KrvW3a{eFF6GH{9R;t>$L3=7eU1X6iK}ZI%m^3wmn+N6@QZ z(JtGSv8xF$4neehFGGLQ7R{B@lk(H()9Vw_n*^v|wlE_xClV+$On^)ea}Q(>y)bG{ z#*_#qX#{GhC|fSpbONUM9uip}%8uL)@(#of-VV|=OQ`e=`EtygWGE?7Nr;LNw+y#@ zs|Yr^Yz+Dk=l?juIljLO zzcaa;w1?;V!PST>Z=Imspk23y*TQl0%MHv;{yFdl;s*VOV$Wu{OO=oYliHQW7EP5} zg(h9~2#Bfni&B#&h{~A8T7kFRRn|^4wD?!?URj?qwdzWhqoP@iAJZM?20Z;wgk8Wc z^Un||eQc)XF&Sg5TgtjLuEO$sUNtUNF8LPC=XCon*h0hN9#!OGOC_$#8^Qfk+HsA{ zx~IB@N%M3klLd1o&v~VJ^7+?Ny?J%>PIC=&tvSuP%+lRC*?E$=q|!^(uJR9e zc8A7M%Zsy_HXz{#DnaDED*HdB_dQJm&75&5yZU)9Dlm( zk_|KrJjy_mN0Dogx5-$i@9v99RHjOE;c+|ny=@|1JxH?ccStc69jfiW8OMF+3Uz24J5Igr9hs2Uclvu0+7RcI?G)>jcF%*Oj8iGwFwUpd zT9I{5y{fv}44b9j;YIu&JrVc4`#IeXF8;oq^;Iqf|sv`Ms_Gmx_wdyIL}+z&e_Or5~a z$v4OMnK$gWfw%Lwq~|86EwJC9jo^2+Y|Gltd>*t@M7CmE1%#RkDhj}Iwt--pHP$fUHnL(Ete*iT05CC~f>@^!^Y{G$+oJpY>{Y zg+~-j157i?6Y9@P`AaWL@^<`o`*!cw4m)ano01ChLz1Slq~rtS3uIA~j~QKF9}00g zN>Vfa5aMt*-?g=hI~A(0)`*|Q(#0s^u7~Z0gX6pGdN>=>uhE>PWk)a<)7t!B!w$Pw z2XOY=ZbqMbODaqDOGwNr&9r94XV+(GX7o&4O;n89$Ki+8;-9H#XixPQ@(Jap{>YBV zP4LsHpcun2&@E`DH#JNABBw^aXRy<)^%_p*l+;f0-=iHeOJF1;9+oYsxuZB1tkkl0 zx;QMT`60hYbpUf~eUNrgHAz5!Me}5YWsT*cacG~hUzYT({7SpVhOCy^$mo!|1K3{{ z1p5y6t)j4eLi1fK7O3bp9n5BDu@c4>4i^sPbMsk}BU&k{J=3;ebe`$d?1XjNy0}`o z+43PlRdwf{0@M&&ZbDc?SoCT9Xt_BYjx51x<5;#?=}4*8XnyYg{zw-*f<6j20@V~` zX|*x1qS|`i`yjSj1stnbs`IZEY0c|mZmxAI`j{B!=>it0tyeviT9-~2Yv|pYVc0BY ztmC)gJepomt=>A>3~U6yzCE%nVlSwkQJhV#f9?pL`Bm!S`jk{DSJ9NcdMmhAe=6+d z{J2NFzYk_Zv@sXN$w$bOh>Kd|KXIsUn9jA+iu^;*v3%k%a>$-hl+M5};Fh!1 z8%H=coE;p!?)uwh)bGhD3;NEp8H0B@NA=UBz z&Fz~6J`8?`SNf~#j+6mGXVqOv&uo|p_N2KF$8$%?X6NUc@8sX{L&eD~czYMY2&vLh)ZwGsN_&)2n;RHDDG~PVV z*1aALk5B?*?^9g({2ymO9|l9^3$o_<30`)d3QjH77PmtSHbXY4yV5*uUXkzeY?wXl zmHLpsfFl45z{-ZcwJrp32L&jV%OEOyjLhzU$C~@qYq6!!IzooAiRD@*zD=G;}|My)o-}le|s!D=T z{ohugO2q$v5hDLf-oJHWCI1oqPZ2HV|Cjwqe%DpUET;HzQ+cEEKiO{*Azk(HPq%Hy zpZ#)FpI5K{Q{-dZQ1Q8~fB&<{SLI`ieP13{f{O9KZ*OVfJ2oBiZ$fs@P0+(Wnu63_ z*DC~Ge{vQ7*L8ds9d4NbH55|fS>1UyTFi(I^q^N3!I96$mcxJg#eD*&pR^|X`3)7% zBt$RTHhpM`KdC@k1g>2PbDliJU?u*mrzClyNBN<;tVa7`MSRoM09o6LsZ#ddMH3Oz zILV$MV4d*4IeO*@8T6`%bZ(FUl$}ghKInQ7tUFFs`l)>Qx|V;acTm2%@y6W$Z}Rqh zbOiB-PE0Qk6-&s;T{;RsfC8o3OSu*y=>kyQ1e7yI8TP~ks*k5*+vE>ZQUUt&oZ2N5O3 z&6i_+4I9!69gK^MD+tTogC>gei2Q>Dg%;e^Ap75}?fD>!DfW5mIwTir=$5Hg=mLLF zzh#(zuwOsP!%6M~h?T%t#T1#*#O!Fob`q0<{#)naERorIz*ad*bep59xquD&eErgg z@Ea~GsJ-5Gb61JQ|MtVMPmn(!KPePS^eY^*_f*jJau(EsH3(IJ0-D$tw*TLjglhKt z#dZ$V;q0mEs7L>Rgv)>V7Bb*Pfd1edH|?$0Ol{g4Wq@g#O3e_dqxQR65rIQPW7#zW zTR5aEPYG?Fkc1}f?937rA0w>#^R^#3>~|@3*MqZ?3Vg4iz+X3X;jL?vxRhfF^c?}~ zU?kAqa|#|BxS~pZI5#}0FAes5#4w;CY=P($W)}I1vKHs3#pPFnN)n&+pW|#X^K>yU ze&BHjS4dYL3W1SEaICT-A!yxUNQnra#(4#WJfTU}#>+j1wWn-3gtLL~S=7Y9!9zuj zn{|pYbyevy>yG0YZUVBDH#ox|zPY>C?si;`TNbIe%{+pK|qvG=_N}ZWm&G>vpu6BkL%uVmZRwwI6ilLsb~^+=ADD<09vqK zg=u&w`X-#DUO`1c=$Qp+9TA{YIz5M6ur8Z0C>q52cHg=4F6-TTUfBsGVz1yVLb@Cn z5Cq6#+p68HLyy}EDV%K%soO=+HR*3)>MNLSh>5snjUg9oCwN?0OdZslVqG;WD3o1{ zjcD?MDJSvLEbP1OxclwtsyU;$G_G+Dcz)`LX^qVCnkfbJMm4QDn zkxpx>VbaXAJv$)19WZjit+K}N5A&!!Mux#BycF-Zx!Yr=B9H}jWHax|lYZ=QUVKv4 zyKi*=M*_=r#g?2F7UhbE2^Y~(R?0<6Hd@g=tK*gA9}_s-EyD(vpQl-3?X|Xe^VAe5 z{8j|M_hGN87JXcNjuST+A1h)x`z_tglfv44--~y=mFX!Tq~0P?Y`W1ZAa3i?WQ*0m zT(>iQ+(?QSUm(a3Ih({m7kqtw2mu47nPg3f}8uw@B1snQ)aS@<0nzhdOju8zN{4%EwemlQEUHHas zgpkSY3&toTE_W+=@lvOsQFvl)9&^z4bv9MD`uCg>>E8)<3V7ge{E{lwp2x84R#kmz z$bXHFj^r@-qy2LuPoIq}kPEF)!o_RO%%4e|ph~VZ2l%){Vb#73$iF16N@R*ALJq5k zuzk7yEzM|20&vlze$}+O+l<0Msi09xT9~|}l?xh07pRcRtP#9#e1G3}H}p@=Ao>7F z9KJNQu{9WXhn>+2jCZ*|OpV74mV8;T>nPrKQNbOZt}T4N&*((5n{x z{&L#-AYk~xGjyM)sdrjJbfZ1JgTMpzDNqAOMvCFVQo~)VL*j0rTSxs@k8>SovA;8D zk1X^cshmdtYYc-!AnX_C8YVKS&Njn~-U+%&hUW&3WNz&s)XEmdNzjAxAqZKmEAdvQz%^ zc>UE?4%?kcuZpUB-}#asHFY#nqA&U3j_!`@8oKr2s9#FNrf6#4_`uzGy}$jvkPd6ty{;z}1d2`yY#615 zJI;l4Vc{n7TVvjX4wjc6xU(Kec?7SCe@Oh{NqS70^hAT{b+8yu#3%SY7)^Az68DV| zYuh39k%knKZ)TJ|YdRt%ZNFEw^9R}*i{!#}>d%$DbJob~MO}IjyYRk2+C>w{yQ2T3 z=gDB1;EUF+@UY#uC)gev>Qi(=Pnm}G9EZF*f!pOQLsqAPD}pq|HabmYy1tc7ckfMD z^LTG)zgsGsqe&ar%4!#pwA-F46_=-cy7la%VZWX-0y|zRMemU7jH^-;9~yojXzr4W z4~^dZ`_8y4$~GV3WW+7%Omjlywti-c{SUO0v?=@Pldzd19 zMCx6BFK(JaX?v*eH0IR$A9Q0sq)v!851!v$wa9-VFN*)?`w$q{%vqX&F-PilKp387 z#wS9veEuhs7h{OBE;4B* zj+j<&Q|~bine>(zDu!<+FCLS(L*W9m=W{GaDe)5i7e)5d`@MG7AAa%jJXs~%x|R$n zva@@O>2|}nXc9np9SE`B%imZ80)Oq)1R|EtP&(6@;!Q5N5Y(t6INIR8wX=q!&u-+L zH(s`!Y&TJ|Vly8-eKZdhUOI8i6p2`=$m`I1ne*PNeMnaI8R8SXf(QEMna}^`q50-< zPAZOo*P~WIm(1(({iodIVa@65^@7Wf=)Cye4Wki`sN1n}||1vo`PoCvkqOcTsiB1>CW#>C!QD=p~r%{eMG2e8Tdt`*D z^W#Tl%LV_zxvv$Eb8X#PU&H`W6%>OHYvySccPu2E1Jz1>h(*x1tuPM(P?fs?KZ+)j z@tfLWU>BG~eYuF?xprfpvI<(o_Q($#a{6nkd_XQRynn;f}DrE_uFcEqF1Rnu+`c8L@I0T7-_z}dB}9!(n|x)mWRMyKm5XosAbGxbuWvzHdz@lnC(7(JlLTEK z=bpO)PSudhEQR-BusQJlhRa79>@9g5Z)TGZTa9Y8nj>h1d9QCilE3%;zU&0ggTv6Z z#TrNC`CWE_gNWm8Fx`IsbYqd7P`;H~H?h@wzwY*N@#7l6g%_3U)1+zn@(W%hC}L-} zgy*V_O$AF=`X3Mg^3Fq_?JmEfDq?!L=rfeb(w z<5cx+tcgW6W`?!hTC!yM4%?N!O8z;ND@#DOEuB=1=!)BX7oyRMVNe<-JuIlVAQ!Ah zD_H1Id(gz;fbssAW_X^uB!aBFBv!gWS=^iBT1pPzjYVTT*nS=ecAcZ&j3>xtw2g$G z#{Lr|GeTwzjna-_Fw2hU()*sT&SXupl1Gf$Of9fq(ay*Sq@Px=j%Zy1NLKRZtyW5t$Z zNweR57aFJn@;78~SabI)qXWQxd8B0z?Pp3f#TZC%Ug~JV?^3G#I3a3fCKO|?eqY;G z1@xtM`t^&epbAl%9dOT$b#k#@knUnJBhCngvI;CTP!+tSQ8+tBTscsB1typvT5eni zE(Z=e{f7t)&b&!vQNQo^JaKu#BP*!8DnO?l7^r>FkP2fjF|sgSf846yKLy?^5-z)w z(hy37*{zmA1g9CaKq%;g{Z*KwE~>;fg}jQ5y!vFLc}1qc$5-Q+b`NiF`!KjhnCq3^VcLm0>bq?ghJl8QD=Am(&!HMaqd+h#(7h(e}wO>gDs4PMi2lgg6 ze!T-9XWKiwCF(AWuACZN--48z4fI{C-;QW>y^CP@aF%E_Ol`-1a?I2;risqet4)#7+Nk};YW8l@TtwI)x>}2fgK&Kwa$)EinBi(g_qM1^gj z*XG_6)htP=_cldw$#MbDI#?va1!D%rJ#6T{QM2;|%4&zA$_sZif??Xt(eQrkn+EB$ z_|@8MknNNY9-Y1#s!epu2x=Im=bP;=Q^5Le50f^#qkQYUg56ydCgzP+JdE5a53kDi zQHI*RXpRajDRjKE+1_TFoN2$@kHB`rC01-%4J)C9)s!|^J&sP4GdX(2n{{#X z#1A1m5|8J>=5#2-Wuv95ees095C~?wX4{KE0t7mptt>`J*zU0&qGVh#$)Z474W_G;J@-6nK&X+wKc`&izG~{hEtzQ=tf!@1Mbx1%t#**$2WgjB1 zn{E_yy}#ZU#TwU{uQOQE!c6Dx#^QRpT9p67HF;mSJD(5L=0KY>?=yJj9$m1`J)b^w zSYS#K)oCzWyfA&aGm{f@i%H(eQ;y!{djBm{KX!XHdQ)IY1+SEsjWdY9ir=NIzxrFW zhqy~aLXN+L?~je)`vCJ=3mez=SS;@LB(X|8a>X*Tw*55YS~ne<$vr|%B#4LA#&lejHa@VFKeub=T~qNU zQ7?IaY%VNwzTJPlhcZ7jX3?KZn4{0={uY=L!D&Mp$)eO3$4+@%%_&_V7XL74`)g7h zdR4wlSw7o9H91hAO0;U$A1c!;aJF@x6eSY52I9PpKz1>)K`7ay!a)UrBQ&nxlgES0 zi3d;Q@%ugWcdHz;!#vZ?TEw(P+O5;7PNVxm`_@cnJ_Z3dID`)5kzVq)IuxoHR=U(C z?%2c>9t~Af+ZEYpyTd>}Kbe<%fo-*SzKd$hcD7Wz8!Eiwwd#NZiymCJ0Vm$qEz^@$ zGu zX-8}Ez3!&XfQjGY9jSX&<{ipb^2E3i^ZPT)S=*FxKhWulL%NtQbb!{vMgqBZNx@T< zrpFgMJ3D5ng>H|HqAN%A1Rz-!m)g98Kz!V-b(XU8@nm~)qyu`R zH%q|2$J)lYjtDsy+Q=(75J5xGNNPGKAfYS?_wU|s+hG9**B5D^T*9o0o- ziO?%Z(@~z7nHd4AQK%dUyis>DU34>Yr2Co``ws9gYtjdOxl&MF$+cV zTn}8MiG?9rvG(*qJZtplcp$3_HuiPd%eU`!D?u)HTP zxGR1I=o_K+9}{go=4vpF(Bv%{UdpiaG){GZfLb5wR;9&C|CYju7C?QpPHU3o)t2J4 z2KL*p@%{PI&Guwol?ystp*k+W%&__K+~$u0=kEej*NzXqT^*ngrSZmJ#inzr8ZD%61J z0e?R#KB!?!g@MjVrH8;vHM9n4Tj00!irhE3jIi8n$`M^FAUXsQc&<54TN4%KN#